{ "schema_version": "0.6.0", "generated_at": "2026-08-30T08:25:18.742Z", "inspect_source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "title": "FronteraEval evaluation catalogue", "scope_note": "Discovery metadata is not independent validation. Only records marked reviewed contain a bounded FronteraEval assessment.", "topics": { "autonomy-agents": { "label": "Autonomy & agents", "definition": "Tool use, long-horizon task completion, self-directed work and operation in external environments." }, "ai-rd": { "label": "AI R&D", "definition": "Research engineering, model development and capabilities that may accelerate AI progress." }, "cyber": { "label": "Cyber", "definition": "Cybersecurity knowledge, vulnerability discovery, exploitation, defence and autonomous operations." }, "bio-cbrn": { "label": "Bio / CBRN", "definition": "Hazardous biological, chemical, radiological or nuclear knowledge and operational assistance." }, "deception-misalignment": { "label": "Deception & misalignment", "definition": "Scheming, covert action, strategic deception, sandbagging and misaligned agent behaviour." }, "human-influence": { "label": "Human influence & agency", "definition": "Persuasion, manipulation, social engineering, trust formation and effects on human agency." }, "safeguards": { "label": "Safeguards", "definition": "Refusal, jailbreak robustness, harmful-response prevention, monitoring and defensive controls." }, "evaluation-integrity": { "label": "Evaluation integrity", "definition": "Validity, contamination, elicitation, judge reliability, eval awareness and protocol integrity." }, "general-capability": { "label": "General capability", "definition": "Reasoning, knowledge, coding, mathematics and broad task performance used as capability context." }, "multimodal": { "label": "Multimodal", "definition": "Evaluations requiring or assessing combinations of text, images, audio or other modalities." } }, "stats": { "records": 314, "sources": { "inspect-internal": 247, "canonical-source": 25, "inspect-register": 42 }, "review_status": { "imported": 231, "catalogued": 53, "reviewed": 30 }, "enrichment": { "register_entries_enriched": 42, "register_parse_failures": 0, "entries_with_reported_results": 9, "linked_resources": 1416, "internal_modules_enriched": 129, "internal_entries_enriched": 248, "internal_metadata_failures": 0, "support_tasks_excluded": 1, "entries_with_paper": 277, "entries_with_version": 289 }, "source_resolution": { "status": { "verified": 73, "source-derived": 3, "paper-only": 238, "host-only": 0, "unresolved": 0 }, "paper_status": { "present": 290, "not-found": 15, "not-applicable": 9 }, "source_groups": 196, "records": 314, "inspect_as_origin_problems": 0, "reviewed_source_problems": 0, "duplicate_resource_problems": 0 }, "methodological_reviews": { "documentary": 30, "independently_replicated": 0, "added_by_registry": 30, "registry_schema_version": "1.0.0", "reviewed_at": "2026-08-27" } }, "records": [ { "id": "inspect:abstention_bench", "name": "AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions", "slug": "inspect--abstention-bench", "organisation": "Polina Kirichenko, Mark Ibrahim, Kamalika Chaudhuri et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/abstention_bench", "topics": [ "evaluation-integrity" ], "description": "Evaluating abstention across 20 diverse datasets, including questions with unknown answers, underspecification, false premises, subjective interpretations, and outdated information.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/abstention_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2506.09038", "primary": true }, { "kind": "paper", "label": "Paper: AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions", "url": "https://arxiv.org/abs/2506.09038", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/stellalisy/mediQ", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/abstention_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Tasksource", "url": "https://huggingface.co/datasets/tasksource/bigbench", "primary": false }, { "kind": "dataset", "label": "Dataset: Allen Institute for AI", "url": "https://huggingface.co/datasets/allenai/coconot", "primary": false }, { "kind": "dataset", "label": "Dataset: Idavidrein", "url": "https://huggingface.co/datasets/Idavidrein/gpqa", "primary": false }, { "kind": "dataset", "label": "Dataset: OpenAI", "url": "https://huggingface.co/datasets/openai/gsm8k", "primary": false }, { "kind": "dataset", "label": "Dataset: Amayuelas", "url": "https://huggingface.co/datasets/amayuelas/KUQ", "primary": false }, { "kind": "dataset", "label": "Dataset: Center for AI Safety", "url": "https://huggingface.co/datasets/cais/mmlu", "primary": false }, { "kind": "dataset", "label": "Dataset: Ninoscherrer", "url": "https://huggingface.co/datasets/ninoscherrer/moralchoice", "primary": false } ], "resource_count": 11, "paper_url": "https://arxiv.org/abs/2506.09038", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/abstention_bench/eval.yaml", "family_title": "AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions", "group": "Safeguards", "version": "4-A", "contributors": [ "jeqcho" ], "tags": [ "Safeguards" ], "task": { "name": "abstention_bench", "dataset_samples": 39558, "comment": null, "support_only": false }, "tasks": [ { "name": "abstention_bench", "dataset_samples": 39558, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 39558, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://huggingface.co/datasets/rajpurkar/squad_v2/resolve/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/stellalisy/mediQ/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/nyu-mll/BBQ/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/yinzhangyue/SelfAware/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/thunlp/FalseQA/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/Yuki-Asuuna/UMWP/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/suzgunmirac/BIG-Bench-Hard/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://github.com/facebookresearch/worldsense/raw/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://drive.google.com/uc?id=12aLKsSKe85G0u5bBTq0X0aKICsdxpaFL", "fetch_method": "gdown_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://drive.google.com/uc?id=1q-6FIEGufKVBE3s6OdFoLWL2iHQPJh8h", "fetch_method": "gdown_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://drive.google.com/uc?id={multiple}", "fetch_method": "gdown_and_verify", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "tasksource/bigbench", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "allenai/coconot", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "Idavidrein/gpqa", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "openai/gsm8k", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "amayuelas/KUQ", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "cais/mmlu", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "ninoscherrer/moralchoice", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/mikejqzhang/SituatedQA/{SHA}/data/qa_data/", "fetch_method": "other", "state": "pinned", "comment": "Custom HF builder downloads via dl_manager" }, { "type": "direct_url", "source": "https://qasper-dataset.s3.us-west-2.amazonaws.com/qasper-{train-dev,test}-v0.3.tgz", "fetch_method": "other", "state": "pinned", "comment": "Custom HF builder downloads via dl_manager" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:abstention_bench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Polina Kirichenko, Mark Ibrahim, Kamalika Chaudhuri et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2506.09038" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.09038", "arxiv_id": "2506.09038", "title": "AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions", "authors": [ "Polina Kirichenko", "Mark Ibrahim", "Kamalika Chaudhuri", "Samuel J. Bell" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.09038", "arxiv_id": "2506.09038", "title": "AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions", "authors": [ "Polina Kirichenko", "Mark Ibrahim", "Kamalika Chaudhuri", "Samuel J. Bell" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2506.09038", "arxiv_id": "2506.09038", "title": "AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions", "authors": [ "Polina Kirichenko", "Mark Ibrahim", "Kamalika Chaudhuri", "Samuel J. Bell" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/stellalisy/mediQ", "owner": "Stellalisy", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/abstention_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/tasksource/bigbench", "provider": "Tasksource" }, { "url": "https://huggingface.co/datasets/allenai/coconot", "provider": "Allen Institute for AI" }, { "url": "https://huggingface.co/datasets/Idavidrein/gpqa", "provider": "Idavidrein" }, { "url": "https://huggingface.co/datasets/openai/gsm8k", "provider": "OpenAI" }, { "url": "https://huggingface.co/datasets/amayuelas/KUQ", "provider": "Amayuelas" }, { "url": "https://huggingface.co/datasets/cais/mmlu", "provider": "Center for AI Safety" }, { "url": "https://huggingface.co/datasets/ninoscherrer/moralchoice", "provider": "Ninoscherrer" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2506.09038" }, "code_url": "https://github.com/stellalisy/mediQ", "metadata": { "source_search": "Polina Kirichenko, Mark Ibrahim, Kamalika Chaudhuri et al. Original paper authors AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions Polina Kirichenko Mark Ibrahim Kamalika Chaudhuri Samuel J. Bell Stellalisy evaluation-task" } }, { "id": "inspect:agent_bench_os", "name": "AgentBench: Evaluate LLMs as Agents", "slug": "inspect--agent-bench-os", "organisation": "Xiao Liu, Hao Yu, Hanchen Zhang et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_bench", "topics": [ "autonomy-agents" ], "description": "A benchmark designed to evaluate LLMs as Agents", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2308.03688", "primary": true }, { "kind": "paper", "label": "Paper: AgentBench: Evaluating LLMs as Agents", "url": "https://arxiv.org/abs/2308.03688", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/THUDM/AgentBench", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_bench/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2308.03688", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_bench/eval.yaml", "family_title": "AgentBench: Evaluate LLMs as Agents", "group": "Coding", "version": "3-A", "contributors": [ "Felhof", "hannagabor", "shaheenahmedc" ], "tags": [ "Coding" ], "task": { "name": "agent_bench_os", "dataset_samples": 26, "comment": null, "support_only": false }, "tasks": [ { "name": "agent_bench_os", "dataset_samples": 26, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 26, "dependency": null, "requires_internet": false, "sandbox": [ "solver", "scorer" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/THUDM/AgentBench", "fetch_method": "git_clone", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agent_bench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xiao Liu, Hao Yu, Hanchen Zhang et al.", "organization": "Tsinghua University; The Ohio State University; University of California, Berkeley; Microsoft", "organizations": [ "Tsinghua University", "The Ohio State University", "University of California, Berkeley", "Microsoft" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2308.03688" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2308.03688", "arxiv_id": "2308.03688", "title": "AgentBench: Evaluating LLMs as Agents", "authors": [ "Xiao Liu", "Hao Yu", "Hanchen Zhang", "Yifan Xu", "Xuanyu Lei", "Hanyu Lai", "Yu Gu", "Hangliang Ding", "Kaiwen Men", "Kejuan Yang", "Shudan Zhang", "Xiang Deng", "Aohan Zeng", "Zhengxiao Du", "Chenhui Zhang", "Sheng Shen", "Tianjun Zhang", "Yu Su", "Huan Sun", "Minlie Huang", "Yuxiao Dong", "Jie Tang" ], "organizations": [ "Tsinghua University", "The Ohio State University", "University of California, Berkeley", "Microsoft" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2308.03688", "arxiv_id": "2308.03688", "title": "AgentBench: Evaluating LLMs as Agents", "authors": [ "Xiao Liu", "Hao Yu", "Hanchen Zhang", "Yifan Xu", "Xuanyu Lei", "Hanyu Lai", "Yu Gu", "Hangliang Ding", "Kaiwen Men", "Kejuan Yang", "Shudan Zhang", "Xiang Deng", "Aohan Zeng", "Zhengxiao Du", "Chenhui Zhang", "Sheng Shen", "Tianjun Zhang", "Yu Su", "Huan Sun", "Minlie Huang", "Yuxiao Dong", "Jie Tang" ], "organizations": [ "Tsinghua University", "The Ohio State University", "University of California, Berkeley", "Microsoft" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/THUDM/AgentBench", "owner": "THUDM", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_bench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2308.03688" }, "code_url": "https://github.com/THUDM/AgentBench", "metadata": { "source_search": "Xiao Liu, Hao Yu, Hanchen Zhang et al. Tsinghua University; The Ohio State University; University of California, Berkeley; Microsoft Original paper authors AgentBench: Evaluating LLMs as Agents Xiao Liu Hao Yu Hanchen Zhang Yifan Xu Xuanyu Lei Hanyu Lai Yu Gu Hangliang Ding Kaiwen Men Kejuan Yang Shudan Zhang Xiang Deng Aohan Zeng Zhengxiao Du Chenhui Zhang Sheng Shen Tianjun Zhang Yu Su Huan Sun Minlie Huang Yuxiao Dong Jie Tang THUDM evaluation-task" } }, { "id": "inspect:agent_threat_bench_autonomy_hijack", "name": "Agent Threat Bench Autonomy Hijack", "slug": "inspect--agent-threat-bench-autonomy-hijack", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench", "topics": [ "autonomy-agents" ], "description": "Evaluates LLM agents against the OWASP Top 10 for Agentic Applications (2026), measuring both task utility and security resilience across memory poisoning, autonomy hijacking, and data exfiltration scenarios.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml", "family_title": "AgentThreatBench: Evaluating LLM Agent Resilience to OWASP Agentic Threats", "group": "Safeguards", "version": "1-A", "contributors": [ "vgudur-dev" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "agent_threat_bench_autonomy_hijack", "dataset_samples": 6, "comment": null, "support_only": false }, "tasks": [ { "name": "agent_threat_bench_memory_poison", "dataset_samples": 10, "comment": null, "support_only": false }, { "name": "agent_threat_bench_autonomy_hijack", "dataset_samples": 6, "comment": null, "support_only": false }, { "name": "agent_threat_bench_data_exfil", "dataset_samples": 8, "comment": null, "support_only": false } ], "protocol": { "version": "1-A", "dataset_samples": 6, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agent_threat_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026" }, "paper": { "status": "present", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026" }, "code_url": null, "metadata": { "source_search": "Associated paper authors Original paper authors evaluation-family" } }, { "id": "inspect:agent_threat_bench_data_exfil", "name": "Agent Threat Bench Data Exfil", "slug": "inspect--agent-threat-bench-data-exfil", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench", "topics": [ "autonomy-agents" ], "description": "Evaluates LLM agents against the OWASP Top 10 for Agentic Applications (2026), measuring both task utility and security resilience across memory poisoning, autonomy hijacking, and data exfiltration scenarios.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml", "family_title": "AgentThreatBench: Evaluating LLM Agent Resilience to OWASP Agentic Threats", "group": "Safeguards", "version": "1-A", "contributors": [ "vgudur-dev" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "agent_threat_bench_data_exfil", "dataset_samples": 8, "comment": null, "support_only": false }, "tasks": [ { "name": "agent_threat_bench_memory_poison", "dataset_samples": 10, "comment": null, "support_only": false }, { "name": "agent_threat_bench_autonomy_hijack", "dataset_samples": 6, "comment": null, "support_only": false }, { "name": "agent_threat_bench_data_exfil", "dataset_samples": 8, "comment": null, "support_only": false } ], "protocol": { "version": "1-A", "dataset_samples": 8, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agent_threat_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026" }, "paper": { "status": "present", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026" }, "code_url": null, "metadata": { "source_search": "Associated paper authors Original paper authors evaluation-family" } }, { "id": "inspect:agent_threat_bench_memory_poison", "name": "Agent Threat Bench Memory Poison", "slug": "inspect--agent-threat-bench-memory-poison", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench", "topics": [ "autonomy-agents" ], "description": "Evaluates LLM agents against the OWASP Top 10 for Agentic Applications (2026), measuring both task utility and security resilience across memory poisoning, autonomy hijacking, and data exfiltration scenarios.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml", "family_title": "AgentThreatBench: Evaluating LLM Agent Resilience to OWASP Agentic Threats", "group": "Safeguards", "version": "1-A", "contributors": [ "vgudur-dev" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "agent_threat_bench_memory_poison", "dataset_samples": 10, "comment": null, "support_only": false }, "tasks": [ { "name": "agent_threat_bench_memory_poison", "dataset_samples": 10, "comment": null, "support_only": false }, { "name": "agent_threat_bench_autonomy_hijack", "dataset_samples": 6, "comment": null, "support_only": false }, { "name": "agent_threat_bench_data_exfil", "dataset_samples": 8, "comment": null, "support_only": false } ], "protocol": { "version": "1-A", "dataset_samples": 10, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agent_threat_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026" }, "paper": { "status": "present", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agent_threat_bench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026" }, "code_url": null, "metadata": { "source_search": "Associated paper authors Original paper authors evaluation-family" } }, { "id": "canonical:agentboard", "name": "AgentBoard", "slug": "canonical--agentboard", "organisation": "HKUST NLP and the AgentBoard paper authors", "source_type": "canonical-source", "source_url": "https://github.com/hkust-nlp/AgentBoard", "topics": [ "autonomy-agents", "evaluation-integrity" ], "description": "Multi-environment benchmark and analysis toolkit for language-model agents.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://github.com/hkust-nlp/AgentBoard", "primary": true }, { "kind": "paper", "label": "Paper: AgentBoard: An Analytical Evaluation Board of Multi-turn LLM Agents", "url": "https://arxiv.org/abs/2401.13178", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/hkust-nlp/AgentBoard", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/hkust-nlp/AgentBoard", "primary": false } ], "resource_count": 4, "source_key": "canonical:agentboard", "record_type": "evaluation-suite", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "HKUST NLP and the AgentBoard paper authors", "organization": "Hong Kong University of Science and Technology", "entity_type": "research_group", "role": "original_research", "confidence": "verified", "evidence_url": "https://github.com/hkust-nlp/AgentBoard", "organizations": [ "Hong Kong University of Science and Technology" ], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2401.13178", "arxiv_id": "2401.13178", "title": "AgentBoard: An Analytical Evaluation Board of Multi-turn LLM Agents", "authors": [ "Chang Ma", "Junlei Zhang", "Zhihao Zhu", "Cheng Yang", "Yujiu Yang", "Yaohui Jin", "Zhenzhong Lan", "Lingpeng Kong", "Junxian He" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2401.13178", "arxiv_id": "2401.13178", "title": "AgentBoard: An Analytical Evaluation Board of Multi-turn LLM Agents", "authors": [ "Chang Ma", "Junlei Zhang", "Zhihao Zhu", "Cheng Yang", "Yujiu Yang", "Yaohui Jin", "Zhenzhong Lan", "Lingpeng Kong", "Junxian He" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/hkust-nlp/AgentBoard", "owner": "HKUST NLP", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://github.com/hkust-nlp/AgentBoard" }, "paper_url": "https://arxiv.org/abs/2401.13178", "code_url": "https://github.com/hkust-nlp/AgentBoard", "metadata": { "source_search": "HKUST NLP and the AgentBoard paper authors Hong Kong University of Science and Technology Original research by AgentBoard: An Analytical Evaluation Board of Multi-turn LLM Agents Chang Ma Junlei Zhang Zhihao Zhu Cheng Yang Yujiu Yang Yaohui Jin Zhenzhong Lan Lingpeng Kong Junxian He HKUST NLP evaluation-suite" } }, { "id": "inspect:agentdojo", "name": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents", "slug": "inspect--agentdojo", "organisation": "Edoardo Debenedetti, Jie Zhang, Mislav Balunović et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentdojo", "topics": [ "autonomy-agents" ], "description": "Assesses whether AI agents can be hijacked by malicious third parties using prompt injections in simple environments such as a workspace or travel booking app.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentdojo/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2406.13352", "primary": true }, { "kind": "paper", "label": "Paper: AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents", "url": "https://arxiv.org/abs/2406.13352", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentdojo/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2406.13352", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentdojo/eval.yaml", "family_title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents", "group": "Safeguards", "version": "2-A", "contributors": [ "ericwinsor-aisi" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "agentdojo", "dataset_samples": 1014, "comment": null, "support_only": false }, "tasks": [ { "name": "agentdojo", "dataset_samples": 1014, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1014, "dependency": "agentdojo", "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agentdojo", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Edoardo Debenedetti, Jie Zhang, Mislav Balunović et al.", "organization": "ETH Zurich", "organizations": [ "ETH Zurich" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2406.13352" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2406.13352", "arxiv_id": "2406.13352", "title": "AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents", "authors": [ "Edoardo Debenedetti", "Jie Zhang", "Mislav Balunović", "Luca Beurer-Kellner", "Marc Fischer", "Florian Tramèr" ], "organizations": [ "ETH Zurich" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2406.13352", "arxiv_id": "2406.13352", "title": "AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents", "authors": [ "Edoardo Debenedetti", "Jie Zhang", "Mislav Balunović", "Luca Beurer-Kellner", "Marc Fischer", "Florian Tramèr" ], "organizations": [ "ETH Zurich" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentdojo/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2406.13352" }, "code_url": null, "metadata": { "source_search": "Edoardo Debenedetti, Jie Zhang, Mislav Balunović et al. ETH Zurich Original paper authors AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents Edoardo Debenedetti Jie Zhang Mislav Balunović Luca Beurer-Kellner Marc Fischer Florian Tramèr evaluation-task" } }, { "id": "canonical:agentdojo", "name": "AgentDojo", "slug": "canonical--agentdojo", "organisation": "ETH Zurich SPY Lab and Invariant Labs", "source_type": "canonical-source", "source_url": "https://agentdojo.spylab.ai/", "topics": [ "autonomy-agents", "safeguards" ], "description": "Dynamic environment for evaluating attacks and defences for tool-using agents.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Utility-security tradeoffs against AgentDojo's prompt-injection scenarios under the exact agent and defense configuration.", "does_not_measure": "All prompt-injection vectors, live tool ecosystems, supply-chain attacks, or real-world incident frequency.", "best_for": "Comparing agents and defenses on reproducible indirect prompt-injection scenarios.", "not_sufficient_for": "Claims that a production agent is secure against prompt injection.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/ethz-spylab/agentdojo", "primary": true }, { "kind": "paper", "label": "Paper: AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents", "url": "https://openreview.net/forum?id=m1YYAQjO3w", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ethz-spylab/agentdojo", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://agentdojo.spylab.ai", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/ethz-spylab/agentdojo", "primary": false } ], "resource_count": 5, "source_key": "canonical:agentdojo", "record_type": "evaluation-environment", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "ETH Zurich SPY Lab and Invariant Labs", "organization": "ETH Zurich + Invariant Labs", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://github.com/ethz-spylab/agentdojo", "organizations": [ "ETH Zurich + Invariant Labs" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://openreview.net/forum?id=m1YYAQjO3w", "arxiv_id": null, "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents", "authors": [], "organizations": [], "source": "openreview", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://openreview.net/forum?id=m1YYAQjO3w", "arxiv_id": null, "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents", "authors": [], "organizations": [], "source": "openreview", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ethz-spylab/agentdojo", "owner": "ETH Zurich SPY Lab", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/ethz-spylab/agentdojo" }, "paper_url": "https://openreview.net/forum?id=m1YYAQjO3w", "code_url": "https://github.com/ethz-spylab/agentdojo", "metadata": { "source_search": "ETH Zurich SPY Lab and Invariant Labs ETH Zurich + Invariant Labs Evaluation developed by AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents ETH Zurich SPY Lab evaluation-environment" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Task utility and resistance to indirect prompt-injection attacks for tool-using agents in simulated application environments.", "unit_of_analysis": "One agent trajectory on a user task with or without an injected adversarial instruction.", "interaction_type": "Tool-using agent interaction with structured environments, user goals, and attacker-controlled content.", "target_outcome": "User-task success, attacker-goal success, and defense performance across benchmark security cases.", "human_participants": "No human subjects. Task and attack scenarios are authored and curated.", "elicitation": "Agents operate with defined tools, system prompts, environments, attack placements, and optional defenses.", "scoring": "Environment-specific checks measure legitimate task completion and whether the attacker's goal was achieved.", "protocol_dependence": "Very high: environment, tools, attack placement, system prompt, defense, and model snapshot define the threat surface.", "implementation_dependence": "High: tool semantics, environment state, attack code, and success checks affect outcomes.", "limitations": [ "Simulated environments do not capture the full complexity of production integrations.", "Attack scenarios are finite and can become familiar to models or defenses.", "Task and attack success may miss subtler integrity or confidentiality harms.", "Defense results may not transfer to different tool APIs or system prompts." ], "comparability": "Use the same AgentDojo version, task and attack set, environment, tools, system prompt, defense, and model snapshot.", "decision_relevance": "Useful for agent-security engineering and defense regression; production approval requires product-specific threat modeling and adversarial testing.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2406.13352" }, { "kind": "official", "url": "https://agentdojo.spylab.ai/" }, { "kind": "implementation", "url": "https://github.com/ethz-spylab/agentdojo" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:agentharm", "name": "Agentharm", "slug": "inspect--agentharm", "organisation": "Gray Swan AI and the UK AI Security Institute", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm", "topics": [ "autonomy-agents", "safeguards" ], "description": "Assesses whether AI agents might engage in harmful activities by testing their responses to malicious prompts in areas like cybercrime, harassment, and fraud, aiming to ensure safe behavior.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Whether an evaluated agent can be elicited to execute AgentHarm's selected harmful tasks under the published tools, environment, and jailbreak setup.", "does_not_measure": "Real-world misuse prevalence, operator skill, deployment access, victim impact, or the probability that a deployed agent will cause harm.", "best_for": "Stress-testing harmful agent behavior and the interaction between refusal safeguards and retained task capability.", "not_sufficient_for": "Estimating incident rates or certifying that an agent is safe in operational environments.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents", "primary": true }, { "kind": "paper", "label": "Paper: AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "url": "https://arxiv.org/abs/2410.09024", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Ai Safety Institute", "url": "https://huggingface.co/datasets/ai-safety-institute/AgentHarm", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2410.09024", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm/eval.yaml", "family_title": "AgentHarm: Harmfulness Potential in AI Agents", "group": "Safeguards", "version": "8-B", "contributors": [ "alexandrasouly-aisi", "ericwinsor-aisi", "max-andr", "xanderdavies" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "agentharm", "dataset_samples": 176, "comment": null, "support_only": false }, "tasks": [ { "name": "agentharm", "dataset_samples": 176, "comment": null, "support_only": false }, { "name": "agentharm_benign", "dataset_samples": 176, "comment": null, "support_only": false } ], "protocol": { "version": "8-B", "dataset_samples": 176, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "ai-safety-institute/AgentHarm", "fetch_method": "snapshot_download", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agentharm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "AgentHarm was developed by Gray Swan AI and the UK AI Security Institute. The previously linked GitHub repository is no longer available; the paper, official AISI research page, dataset, and maintained Inspect implementation remain the defensible public sources." }, "origin": { "name": "Gray Swan AI and the UK AI Security Institute", "organization": "Gray Swan AI + UK AI Security Institute", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents", "organizations": [ "Gray Swan AI + UK AI Security Institute" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2410.09024", "arxiv_id": "2410.09024", "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "authors": [ "Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian", "Derek Duenas", "Maxwell Lin", "Justin Wang", "Dan Hendrycks", "Andy Zou", "Zico Kolter", "Matt Fredrikson", "Eric Winsor", "Jerome Wynne", "Yarin Gal", "Xander Davies" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2410.09024", "arxiv_id": "2410.09024", "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "authors": [ "Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian", "Derek Duenas", "Maxwell Lin", "Justin Wang", "Dan Hendrycks", "Andy Zou", "Zico Kolter", "Matt Fredrikson", "Eric Winsor", "Jerome Wynne", "Yarin Gal", "Xander Davies" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/ai-safety-institute/AgentHarm", "provider": "Ai Safety Institute" } ], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents" }, "code_url": null, "metadata": { "source_search": "Gray Swan AI and the UK AI Security Institute Gray Swan AI + UK AI Security Institute Evaluation developed by AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents Maksym Andriushchenko Alexandra Souly Mateusz Dziemian Derek Duenas Maxwell Lin Justin Wang Dan Hendrycks Andy Zou Zico Kolter Matt Fredrikson Eric Winsor Jerome Wynne Yarin Gal Xander Davies evaluation-family" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Ability and willingness of tool-using agents to complete selected malicious multi-step tasks, including after jailbreak elicitation.", "unit_of_analysis": "One agent trajectory on one malicious or benign task in a benchmark environment.", "interaction_type": "Multi-step tool use with task-specific environments, system prompts, and optional jailbreaks.", "target_outcome": "Refusal or compliance and task completion across harmful categories, with retained benign capability where tested.", "human_participants": "No human subjects. Experts author and validate task scenarios and scoring criteria.", "elicitation": "The agent receives malicious or benign tasks, tools, and an environment; jailbreak techniques may be applied to test retained harmful capability.", "scoring": "Task-specific completion checks combined with refusal or compliance measures and category-level aggregation.", "protocol_dependence": "Very high: task set, tools, environment, system prompt, jailbreak, and agent scaffold determine performance.", "implementation_dependence": "High: task validators, sandbox behavior, tool affordances, and model endpoint changes affect outcomes.", "limitations": [ "Benchmark tasks are selected and simulated rather than observed real incidents.", "Jailbreak success does not establish that an operator can deploy the same workflow at scale.", "Tool and environment design may over- or under-estimate operational difficulty.", "Task completion is not equivalent to downstream harm." ], "comparability": "Match AgentHarm release, malicious and benign subsets, tools, environment, jailbreaks, agent scaffold, and model snapshot.", "decision_relevance": "Useful for agentic misuse stress tests and safeguard design; it must be combined with deployment exposure, monitoring, and consequence analysis.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2410.09024" }, { "kind": "official", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents" }, { "kind": "metadata", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentharm" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "canonical:agentharm", "name": "AgentHarm", "slug": "canonical--agentharm", "organisation": "Gray Swan AI and the UK AI Security Institute", "source_type": "canonical-source", "source_url": "https://arxiv.org/abs/2410.09024", "topics": [ "autonomy-agents", "safeguards" ], "description": "Evaluates whether language-model agents can execute harmful multi-step tasks.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents", "primary": true }, { "kind": "paper", "label": "Paper: AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "url": "https://arxiv.org/abs/2410.09024", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2410.09024", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents", "primary": false } ], "resource_count": 4, "source_key": "canonical:agentharm", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "AgentHarm was developed by Gray Swan AI and the UK AI Security Institute. The previously linked GitHub repository is no longer available; the paper, official AISI research page, dataset, and maintained Inspect implementation remain the defensible public sources." }, "origin": { "name": "Gray Swan AI and the UK AI Security Institute", "organization": "Gray Swan AI + UK AI Security Institute", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents", "organizations": [ "Gray Swan AI + UK AI Security Institute" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2410.09024", "arxiv_id": "2410.09024", "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "authors": [ "Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian", "Derek Duenas", "Maxwell Lin", "Justin Wang", "Dan Hendrycks", "Andy Zou", "Zico Kolter", "Matt Fredrikson", "Eric Winsor", "Jerome Wynne", "Yarin Gal", "Xander Davies" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2410.09024", "arxiv_id": "2410.09024", "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "authors": [ "Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian", "Derek Duenas", "Maxwell Lin", "Justin Wang", "Dan Hendrycks", "Andy Zou", "Zico Kolter", "Matt Fredrikson", "Eric Winsor", "Jerome Wynne", "Yarin Gal", "Xander Davies" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents" }, "paper_url": "https://arxiv.org/abs/2410.09024", "code_url": null, "metadata": { "source_search": "Gray Swan AI and the UK AI Security Institute Gray Swan AI + UK AI Security Institute Evaluation developed by AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents Maksym Andriushchenko Alexandra Souly Mateusz Dziemian Derek Duenas Maxwell Lin Justin Wang Dan Hendrycks Andy Zou Zico Kolter Matt Fredrikson Eric Winsor Jerome Wynne Yarin Gal Xander Davies evaluation-resource" } }, { "id": "inspect:agentharm_benign", "name": "Agentharm Benign", "slug": "inspect--agentharm-benign", "organisation": "Gray Swan AI and the UK AI Security Institute", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm", "topics": [ "autonomy-agents", "safeguards" ], "description": "Assesses whether AI agents might engage in harmful activities by testing their responses to malicious prompts in areas like cybercrime, harassment, and fraud, aiming to ensure safe behavior.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents", "primary": true }, { "kind": "paper", "label": "Paper: AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "url": "https://arxiv.org/abs/2410.09024", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Ai Safety Institute", "url": "https://huggingface.co/datasets/ai-safety-institute/AgentHarm", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2410.09024", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm/eval.yaml", "family_title": "AgentHarm: Harmfulness Potential in AI Agents", "group": "Safeguards", "version": "8-B", "contributors": [ "alexandrasouly-aisi", "ericwinsor-aisi", "max-andr", "xanderdavies" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "agentharm_benign", "dataset_samples": 176, "comment": null, "support_only": false }, "tasks": [ { "name": "agentharm", "dataset_samples": 176, "comment": null, "support_only": false }, { "name": "agentharm_benign", "dataset_samples": 176, "comment": null, "support_only": false } ], "protocol": { "version": "8-B", "dataset_samples": 176, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "ai-safety-institute/AgentHarm", "fetch_method": "snapshot_download", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agentharm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "AgentHarm was developed by Gray Swan AI and the UK AI Security Institute. The previously linked GitHub repository is no longer available; the paper, official AISI research page, dataset, and maintained Inspect implementation remain the defensible public sources." }, "origin": { "name": "Gray Swan AI and the UK AI Security Institute", "organization": "Gray Swan AI + UK AI Security Institute", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents", "organizations": [ "Gray Swan AI + UK AI Security Institute" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2410.09024", "arxiv_id": "2410.09024", "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "authors": [ "Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian", "Derek Duenas", "Maxwell Lin", "Justin Wang", "Dan Hendrycks", "Andy Zou", "Zico Kolter", "Matt Fredrikson", "Eric Winsor", "Jerome Wynne", "Yarin Gal", "Xander Davies" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2410.09024", "arxiv_id": "2410.09024", "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "authors": [ "Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian", "Derek Duenas", "Maxwell Lin", "Justin Wang", "Dan Hendrycks", "Andy Zou", "Zico Kolter", "Matt Fredrikson", "Eric Winsor", "Jerome Wynne", "Yarin Gal", "Xander Davies" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentharm/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/ai-safety-institute/AgentHarm", "provider": "Ai Safety Institute" } ], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://www.aisi.gov.uk/research/agentharm-a-benchmark-for-measuring-harmfulness-of-llm-agents" }, "code_url": null, "metadata": { "source_search": "Gray Swan AI and the UK AI Security Institute Gray Swan AI + UK AI Security Institute Evaluation developed by AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents Maksym Andriushchenko Alexandra Souly Mateusz Dziemian Derek Duenas Maxwell Lin Justin Wang Dan Hendrycks Andy Zou Zico Kolter Matt Fredrikson Eric Winsor Jerome Wynne Yarin Gal Xander Davies evaluation-family" } }, { "id": "inspect:agentic_misalignment", "name": "Agentic Misalignment: How LLMs could be insider threats", "slug": "inspect--agentic-misalignment", "organisation": "Anthropic", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentic_misalignment", "topics": [ "deception-misalignment", "autonomy-agents" ], "description": "Eliciting unethical behaviour (most famously blackmail) in response to a fictional company-assistant scenario where the model is faced with replacement.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Harmful action selection in the benchmark's synthetic high-conflict scenarios under a specified model, prompt, tool, and opportunity configuration.", "does_not_measure": "The base rate of misalignment in normal deployment, spontaneous long-term goals, real organizational incidents, or the probability of autonomous insider threats.", "best_for": "Stress-testing model systems under explicit conflict and opportunity and comparing scenario or safeguard interventions.", "not_sufficient_for": "Estimating real-world incident probability or concluding that a model has persistent malicious goals.", "evidence_reach": [ "controlled-model-behaviour", "deployment-readiness" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentic_misalignment/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://www.anthropic.com/research/agentic-misalignment", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://www.anthropic.com/research/agentic-misalignment", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentic_misalignment/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.anthropic.com/research/agentic-misalignment", "primary": false } ], "resource_count": 4, "paper_url": "https://www.anthropic.com/research/agentic-misalignment", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentic_misalignment/eval.yaml", "family_title": "Agentic Misalignment: How LLMs could be insider threats", "group": "Scheming", "version": "4-A", "contributors": [ "bmillwood-aisi" ], "tags": [ "Scheming" ], "task": { "name": "agentic_misalignment", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "agentic_misalignment", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 1, "dependency": "agentic_misalignment", "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agentic_misalignment", "record_type": "evaluation-task", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Anthropic", "organization": "Anthropic", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://www.anthropic.com/research/agentic-misalignment", "organizations": [ "Anthropic" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://www.anthropic.com/research/agentic-misalignment", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "official-research-page", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://www.anthropic.com/research/agentic-misalignment", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "official-research-page", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agentic_misalignment/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://www.anthropic.com/research/agentic-misalignment" }, "code_url": null, "metadata": { "source_search": "Anthropic Anthropic Evaluation developed by evaluation-task" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "high", "construct": "Whether an agentic model selects harmful insider-like actions when placed in synthetic organizational scenarios with explicit goal conflict, opportunity, and access.", "unit_of_analysis": "One model trajectory or decision in a constructed company-assistant scenario.", "interaction_type": "Role-based agent simulation with access to synthetic emails, tools, private information, and consequential action options.", "target_outcome": "Selection or execution of benchmark-defined harmful actions such as blackmail, leaking information, or other goal-protecting misconduct.", "human_participants": "No human subjects. Scenarios and labels are constructed and reviewed by researchers.", "elicitation": "The protocol creates explicit conflict between the model's assigned objective or continued operation and organizational instructions, while providing an opportunity for harmful action.", "scoring": "Scenario-specific classifiers and transcript review identify whether the model took, planned, or refused the harmful action under each condition.", "protocol_dependence": "Very high: scenario framing, goal assignment, threat to the model, access privileges, action affordances, and prompting strongly influence behavior.", "implementation_dependence": "High: tool semantics, transcript parsing, classifier thresholds, model endpoint, and system prompt affect measured rates.", "limitations": [ "The scenarios deliberately concentrate conflict and opportunity and are not deployment base-rate samples.", "Role-play compliance can be difficult to distinguish from persistent goal-directed misalignment.", "Synthetic tools and organizations simplify real constraints, oversight, and consequences.", "Rates should not be extrapolated directly across models, scaffolds, or deployment contexts." ], "comparability": "Match scenario version, conflict and opportunity condition, tool access, action space, system prompt, sampling settings, classifier, and model snapshot.", "decision_relevance": "Useful for pre-deployment stress testing, access-control design, monitoring requirements, and targeted follow-up. It is not an empirical incident-rate estimate.", "source_basis": [ { "kind": "official", "url": "https://www.anthropic.com/research/agentic-misalignment" }, { "kind": "metadata", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/agentic_misalignment" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:agie_aqua_rat", "name": "Agie Aqua Rat", "slug": "inspect--agie-aqua-rat", "organisation": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval", "topics": [ "general-capability" ], "description": "AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2304.06364", "primary": true }, { "kind": "paper", "label": "Paper: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ruixiangcui/AGIEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2304.06364", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "family_title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "group": "Knowledge", "version": "2-A", "contributors": [ "bouromain" ], "tags": [ "Knowledge" ], "task": { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, "tasks": [ { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 254, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/ruixiangcui/AGIEval/{SHA}/data/v1_1/", "fetch_method": "other", "state": "pinned", "comment": "Multiple task files fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agieval", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "organization": "University of Copenhagen", "organizations": [ "University of Copenhagen" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2304.06364" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ruixiangcui/AGIEval", "owner": "Ruixiangcui", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2304.06364" }, "code_url": "https://github.com/ruixiangcui/AGIEval", "metadata": { "source_search": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al. University of Copenhagen Original paper authors AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models Wanjun Zhong Ruixiang Cui Yiduo Guo Yaobo Liang Shuai Lu Yanlin Wang Amin Saied Weizhu Chen Nan Duan Ruixiangcui evaluation-family" } }, { "id": "inspect:agie_logiqa_en", "name": "Agie Logiqa En", "slug": "inspect--agie-logiqa-en", "organisation": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval", "topics": [ "general-capability" ], "description": "AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2304.06364", "primary": true }, { "kind": "paper", "label": "Paper: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ruixiangcui/AGIEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2304.06364", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "family_title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "group": "Knowledge", "version": "2-A", "contributors": [ "bouromain" ], "tags": [ "Knowledge" ], "task": { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, "tasks": [ { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 651, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/ruixiangcui/AGIEval/{SHA}/data/v1_1/", "fetch_method": "other", "state": "pinned", "comment": "Multiple task files fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agieval", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "organization": "University of Copenhagen", "organizations": [ "University of Copenhagen" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2304.06364" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ruixiangcui/AGIEval", "owner": "Ruixiangcui", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2304.06364" }, "code_url": "https://github.com/ruixiangcui/AGIEval", "metadata": { "source_search": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al. University of Copenhagen Original paper authors AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models Wanjun Zhong Ruixiang Cui Yiduo Guo Yaobo Liang Shuai Lu Yanlin Wang Amin Saied Weizhu Chen Nan Duan Ruixiangcui evaluation-family" } }, { "id": "inspect:agie_lsat_ar", "name": "Agie Lsat Ar", "slug": "inspect--agie-lsat-ar", "organisation": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval", "topics": [ "general-capability" ], "description": "AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2304.06364", "primary": true }, { "kind": "paper", "label": "Paper: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ruixiangcui/AGIEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2304.06364", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "family_title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "group": "Knowledge", "version": "2-A", "contributors": [ "bouromain" ], "tags": [ "Knowledge" ], "task": { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, "tasks": [ { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 230, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/ruixiangcui/AGIEval/{SHA}/data/v1_1/", "fetch_method": "other", "state": "pinned", "comment": "Multiple task files fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agieval", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "organization": "University of Copenhagen", "organizations": [ "University of Copenhagen" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2304.06364" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ruixiangcui/AGIEval", "owner": "Ruixiangcui", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2304.06364" }, "code_url": "https://github.com/ruixiangcui/AGIEval", "metadata": { "source_search": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al. University of Copenhagen Original paper authors AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models Wanjun Zhong Ruixiang Cui Yiduo Guo Yaobo Liang Shuai Lu Yanlin Wang Amin Saied Weizhu Chen Nan Duan Ruixiangcui evaluation-family" } }, { "id": "inspect:agie_lsat_lr", "name": "Agie Lsat Lr", "slug": "inspect--agie-lsat-lr", "organisation": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval", "topics": [ "general-capability" ], "description": "AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2304.06364", "primary": true }, { "kind": "paper", "label": "Paper: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ruixiangcui/AGIEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2304.06364", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "family_title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "group": "Knowledge", "version": "2-A", "contributors": [ "bouromain" ], "tags": [ "Knowledge" ], "task": { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, "tasks": [ { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 510, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/ruixiangcui/AGIEval/{SHA}/data/v1_1/", "fetch_method": "other", "state": "pinned", "comment": "Multiple task files fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agieval", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "organization": "University of Copenhagen", "organizations": [ "University of Copenhagen" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2304.06364" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ruixiangcui/AGIEval", "owner": "Ruixiangcui", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2304.06364" }, "code_url": "https://github.com/ruixiangcui/AGIEval", "metadata": { "source_search": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al. University of Copenhagen Original paper authors AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models Wanjun Zhong Ruixiang Cui Yiduo Guo Yaobo Liang Shuai Lu Yanlin Wang Amin Saied Weizhu Chen Nan Duan Ruixiangcui evaluation-family" } }, { "id": "inspect:agie_lsat_rc", "name": "Agie Lsat Rc", "slug": "inspect--agie-lsat-rc", "organisation": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval", "topics": [ "general-capability" ], "description": "AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2304.06364", "primary": true }, { "kind": "paper", "label": "Paper: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ruixiangcui/AGIEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2304.06364", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "family_title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "group": "Knowledge", "version": "2-A", "contributors": [ "bouromain" ], "tags": [ "Knowledge" ], "task": { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, "tasks": [ { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 269, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/ruixiangcui/AGIEval/{SHA}/data/v1_1/", "fetch_method": "other", "state": "pinned", "comment": "Multiple task files fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agieval", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "organization": "University of Copenhagen", "organizations": [ "University of Copenhagen" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2304.06364" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ruixiangcui/AGIEval", "owner": "Ruixiangcui", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2304.06364" }, "code_url": "https://github.com/ruixiangcui/AGIEval", "metadata": { "source_search": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al. University of Copenhagen Original paper authors AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models Wanjun Zhong Ruixiang Cui Yiduo Guo Yaobo Liang Shuai Lu Yanlin Wang Amin Saied Weizhu Chen Nan Duan Ruixiangcui evaluation-family" } }, { "id": "inspect:agie_math", "name": "Agie Math", "slug": "inspect--agie-math", "organisation": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval", "topics": [ "general-capability" ], "description": "AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2304.06364", "primary": true }, { "kind": "paper", "label": "Paper: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ruixiangcui/AGIEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2304.06364", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "family_title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "group": "Knowledge", "version": "2-A", "contributors": [ "bouromain" ], "tags": [ "Knowledge" ], "task": { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, "tasks": [ { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/ruixiangcui/AGIEval/{SHA}/data/v1_1/", "fetch_method": "other", "state": "pinned", "comment": "Multiple task files fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agieval", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "organization": "University of Copenhagen", "organizations": [ "University of Copenhagen" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2304.06364" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ruixiangcui/AGIEval", "owner": "Ruixiangcui", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2304.06364" }, "code_url": "https://github.com/ruixiangcui/AGIEval", "metadata": { "source_search": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al. University of Copenhagen Original paper authors AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models Wanjun Zhong Ruixiang Cui Yiduo Guo Yaobo Liang Shuai Lu Yanlin Wang Amin Saied Weizhu Chen Nan Duan Ruixiangcui evaluation-family" } }, { "id": "inspect:agie_sat_en", "name": "Agie Sat En", "slug": "inspect--agie-sat-en", "organisation": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval", "topics": [ "general-capability" ], "description": "AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2304.06364", "primary": true }, { "kind": "paper", "label": "Paper: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ruixiangcui/AGIEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2304.06364", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "family_title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "group": "Knowledge", "version": "2-A", "contributors": [ "bouromain" ], "tags": [ "Knowledge" ], "task": { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, "tasks": [ { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 206, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/ruixiangcui/AGIEval/{SHA}/data/v1_1/", "fetch_method": "other", "state": "pinned", "comment": "Multiple task files fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agieval", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "organization": "University of Copenhagen", "organizations": [ "University of Copenhagen" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2304.06364" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ruixiangcui/AGIEval", "owner": "Ruixiangcui", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2304.06364" }, "code_url": "https://github.com/ruixiangcui/AGIEval", "metadata": { "source_search": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al. University of Copenhagen Original paper authors AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models Wanjun Zhong Ruixiang Cui Yiduo Guo Yaobo Liang Shuai Lu Yanlin Wang Amin Saied Weizhu Chen Nan Duan Ruixiangcui evaluation-family" } }, { "id": "inspect:agie_sat_en_without_passage", "name": "Agie Sat En Without Passage", "slug": "inspect--agie-sat-en-without-passage", "organisation": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval", "topics": [ "general-capability" ], "description": "AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2304.06364", "primary": true }, { "kind": "paper", "label": "Paper: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ruixiangcui/AGIEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2304.06364", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "family_title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "group": "Knowledge", "version": "2-A", "contributors": [ "bouromain" ], "tags": [ "Knowledge" ], "task": { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, "tasks": [ { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 206, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/ruixiangcui/AGIEval/{SHA}/data/v1_1/", "fetch_method": "other", "state": "pinned", "comment": "Multiple task files fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agieval", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "organization": "University of Copenhagen", "organizations": [ "University of Copenhagen" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2304.06364" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ruixiangcui/AGIEval", "owner": "Ruixiangcui", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2304.06364" }, "code_url": "https://github.com/ruixiangcui/AGIEval", "metadata": { "source_search": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al. University of Copenhagen Original paper authors AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models Wanjun Zhong Ruixiang Cui Yiduo Guo Yaobo Liang Shuai Lu Yanlin Wang Amin Saied Weizhu Chen Nan Duan Ruixiangcui evaluation-family" } }, { "id": "inspect:agie_sat_math", "name": "Agie Sat Math", "slug": "inspect--agie-sat-math", "organisation": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval", "topics": [ "general-capability" ], "description": "AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2304.06364", "primary": true }, { "kind": "paper", "label": "Paper: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "url": "https://arxiv.org/abs/2304.06364", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ruixiangcui/AGIEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2304.06364", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml", "family_title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "group": "Knowledge", "version": "2-A", "contributors": [ "bouromain" ], "tags": [ "Knowledge" ], "task": { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false }, "tasks": [ { "name": "agie_aqua_rat", "dataset_samples": 254, "comment": null, "support_only": false }, { "name": "agie_logiqa_en", "dataset_samples": 651, "comment": null, "support_only": false }, { "name": "agie_lsat_ar", "dataset_samples": 230, "comment": null, "support_only": false }, { "name": "agie_lsat_lr", "dataset_samples": 510, "comment": null, "support_only": false }, { "name": "agie_lsat_rc", "dataset_samples": 269, "comment": null, "support_only": false }, { "name": "agie_math", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "agie_sat_en", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_en_without_passage", "dataset_samples": 206, "comment": null, "support_only": false }, { "name": "agie_sat_math", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 220, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/ruixiangcui/AGIEval/{SHA}/data/v1_1/", "fetch_method": "other", "state": "pinned", "comment": "Multiple task files fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:agieval", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al.", "organization": "University of Copenhagen", "organizations": [ "University of Copenhagen" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2304.06364" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.06364", "arxiv_id": "2304.06364", "title": "AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models", "authors": [ "Wanjun Zhong", "Ruixiang Cui", "Yiduo Guo", "Yaobo Liang", "Shuai Lu", "Yanlin Wang", "Amin Saied", "Weizhu Chen", "Nan Duan" ], "organizations": [ "University of Copenhagen" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ruixiangcui/AGIEval", "owner": "Ruixiangcui", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/agieval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2304.06364" }, "code_url": "https://github.com/ruixiangcui/AGIEval", "metadata": { "source_search": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo et al. University of Copenhagen Original paper authors AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models Wanjun Zhong Ruixiang Cui Yiduo Guo Yaobo Liang Shuai Lu Yanlin Wang Amin Saied Weizhu Chen Nan Duan Ruixiangcui evaluation-family" } }, { "id": "register:ahb", "name": "AHB", "slug": "register--ahb", "organisation": "Marcello Galisai, Susanna Cifani, Francesco Giarrusso et al.", "source_type": "inspect-register", "source_url": "https://github.com/icaro-lab/ahb", "topics": [ "general-capability" ], "description": "A text-only safety benchmark for evaluating whether language models maintain refusal behavior under humanities-style adversarial reformulations of harmful prompts.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/ahb/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2604.18487", "primary": true }, { "kind": "paper", "label": "Paper: Adversarial Humanities Benchmark: Results on Stylistic Robustness in Frontier Model Safety", "url": "https://arxiv.org/abs/2604.18487", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/icaro-lab/ahb", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/ahb/eval.yaml", "primary": false } ], "paper_title": null, "paper_url": "https://arxiv.org/abs/2604.18487", "code_url": "https://github.com/icaro-lab/ahb", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/ahb/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "Safeguards", "safety", "text-generation" ], "contributors": [ "emmepra" ], "tasks": [ { "name": "ahb", "path": "src/ahb_inspect/tasks.py" } ], "protocol": { "implementation_commit": "4b7b631245fa300df98d2c310e83273ed0d4a207", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:ahb", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Marcello Galisai, Susanna Cifani, Francesco Giarrusso et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2604.18487" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2604.18487", "arxiv_id": "2604.18487", "title": "Adversarial Humanities Benchmark: Results on Stylistic Robustness in Frontier Model Safety", "authors": [ "Marcello Galisai", "Susanna Cifani", "Francesco Giarrusso", "Piercosma Bisconti", "Matteo Prandi", "Federico Pierucci", "Federico Sartore", "Daniele Nardi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2604.18487", "arxiv_id": "2604.18487", "title": "Adversarial Humanities Benchmark: Results on Stylistic Robustness in Frontier Model Safety", "authors": [ "Marcello Galisai", "Susanna Cifani", "Francesco Giarrusso", "Piercosma Bisconti", "Matteo Prandi", "Federico Pierucci", "Federico Sartore", "Daniele Nardi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/icaro-lab/ahb", "owner": "Icaro Lab", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/ahb/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2604.18487" }, "metadata": { "source_search": "Marcello Galisai, Susanna Cifani, Francesco Giarrusso et al. Original paper authors Adversarial Humanities Benchmark: Results on Stylistic Robustness in Frontier Model Safety Marcello Galisai Susanna Cifani Francesco Giarrusso Piercosma Bisconti Matteo Prandi Federico Pierucci Federico Sartore Daniele Nardi Icaro Lab evaluation-package" } }, { "id": "canonical:ailuminate", "name": "AILuminate", "slug": "canonical--ailuminate", "organisation": "MLCommons", "source_type": "canonical-source", "source_url": "https://mlcommons.org/ailuminate/", "topics": [ "safeguards" ], "description": "Standardised safety benchmark suite for hazardous and policy-relevant model responses.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Response safety on AILuminate's prompt distribution, taxonomy, language, evaluator, and model-system configuration.", "does_not_measure": "All safety hazards, adaptive jailbreak robustness, multi-turn or tool-using misuse, deployment incidence, human harm, or a complete product safety case.", "best_for": "Providing a standardized baseline for harmful-response behavior and comparing systems under a common prompt and reporting protocol.", "not_sufficient_for": "Certifying a model or product as safe across contexts, languages, adversaries, or agentic deployments.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "suite_developer", "label": "Suite developed by", "url": "https://mlcommons.org/ailuminate", "primary": true }, { "kind": "official", "label": "Official or project page", "url": "https://mlcommons.org/ailuminate", "primary": false } ], "resource_count": 2, "source_key": "canonical:ailuminate", "record_type": "evaluation-suite", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "MLCommons", "organization": "MLCommons", "organizations": [ "MLCommons" ], "entity_type": "organization", "role": "suite_developer", "role_label": "Suite developed by", "confidence": "curated", "evidence_url": "https://mlcommons.org/ailuminate" }, "paper": { "status": "not-applicable", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-applicable", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Suite developed by", "attribution_status": "verified", "preferred_source": { "label": "Suite developed by", "kind": "suite_developer", "url": "https://mlcommons.org/ailuminate" }, "paper_url": null, "code_url": null, "metadata": { "source_search": "MLCommons MLCommons Suite developed by evaluation-suite" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "moderate", "construct": "Propensity of a language-model system to produce unsafe responses across a standardized taxonomy of hazard categories and prompts.", "unit_of_analysis": "One model response to one benchmark prompt, classified against category-specific safety criteria.", "interaction_type": "Primarily single-turn text prompting of a specified model system under standardized evaluation conditions.", "target_outcome": "Prompt-level unsafe-response classification and aggregate safety performance across hazard categories.", "human_participants": "No human recipients are exposed in benchmark runs. Human expertise informs prompt development, hazard taxonomy, annotation, and evaluator validation.", "elicitation": "The evaluated system receives a controlled prompt set spanning defined hazards, using specified system instructions and generation settings.", "scoring": "Prompt responses are classified with benchmark-defined evaluators and aggregated into category and overall safety results under MLCommons reporting rules.", "protocol_dependence": "Very high: hazard taxonomy, prompt set, language, model-system prompt, generation settings, evaluator, and aggregation rules define the result.", "implementation_dependence": "High: evaluator calibration, endpoint behavior, prompt formatting, and benchmark version affect classifications.", "limitations": [ "A fixed prompt distribution cannot cover adaptive or newly emerging misuse strategies.", "Aggregate safety ratings can conceal hazard-category and prompt-level variation.", "Automated or model-assisted evaluators may make systematic classification errors.", "Text response safety is narrower than agentic, multimodal, and deployment safety." ], "comparability": "Require the same AILuminate release, language and prompt set, category taxonomy, system prompt, decoding, model snapshot, evaluator, and aggregation rules.", "decision_relevance": "Useful as a standardized assurance input and regression baseline. Procurement or deployment decisions require product-specific threat models, adaptive testing, and operational controls.", "source_basis": [ { "kind": "official", "url": "https://mlcommons.org/ailuminate/" }, { "kind": "implementation", "url": "https://github.com/mlcommons/ailuminate" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:aime2024", "name": "AIME 2024: Problems from the American Invitational Mathematics Examination", "slug": "inspect--aime2024", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2024", "topics": [ "general-capability" ], "description": "A benchmark for evaluating AI's ability to solve challenging mathematics problems from the 2024 AIME - a prestigious high school mathematics competition.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2024/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2024/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Maxwell Jia", "url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024", "primary": false } ], "resource_count": 4, "paper_url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2024/eval.yaml", "family_title": "AIME 2024: Problems from the American Invitational Mathematics Examination", "group": "Mathematics", "version": "4-A", "contributors": [ "tamazgadaev" ], "tags": [ "Mathematics" ], "task": { "name": "aime2024", "dataset_samples": 30, "comment": null, "support_only": false }, "tasks": [ { "name": "aime2024", "dataset_samples": 30, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 30, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "Maxwell-Jia/AIME_2024", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:aime2024", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024" }, "paper": { "status": "present", "url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2024/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024", "provider": "Maxwell Jia" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://huggingface.co/datasets/Maxwell-Jia/AIME_2024" }, "code_url": null, "metadata": { "source_search": "Associated paper authors Original paper authors evaluation-task" } }, { "id": "inspect:aime2025", "name": "AIME 2025: Problems from the American Invitational Mathematics Examination", "slug": "inspect--aime2025", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2025", "topics": [ "general-capability" ], "description": "A benchmark for evaluating AI's ability to solve challenging mathematics problems from the 2025 AIME - a prestigious high school mathematics competition.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2025/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://huggingface.co/datasets/math-ai/aime25", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://huggingface.co/datasets/math-ai/aime25", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2025/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Math Ai", "url": "https://huggingface.co/datasets/math-ai/aime25", "primary": false } ], "resource_count": 4, "paper_url": "https://huggingface.co/datasets/math-ai/aime25", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2025/eval.yaml", "family_title": "AIME 2025: Problems from the American Invitational Mathematics Examination", "group": "Mathematics", "version": "4-A", "contributors": [ "jannalulu" ], "tags": [ "Mathematics" ], "task": { "name": "aime2025", "dataset_samples": 30, "comment": null, "support_only": false }, "tasks": [ { "name": "aime2025", "dataset_samples": 30, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 30, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "math-ai/aime25", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:aime2025", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://huggingface.co/datasets/math-ai/aime25" }, "paper": { "status": "present", "url": "https://huggingface.co/datasets/math-ai/aime25", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://huggingface.co/datasets/math-ai/aime25", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2025/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/math-ai/aime25", "provider": "Math Ai" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://huggingface.co/datasets/math-ai/aime25" }, "code_url": null, "metadata": { "source_search": "Associated paper authors Original paper authors evaluation-task" } }, { "id": "inspect:aime2026", "name": "AIME 2026: Problems from the American Invitational Mathematics Examination", "slug": "inspect--aime2026", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2026", "topics": [ "general-capability" ], "description": "A benchmark for evaluating AI's ability to solve challenging mathematics problems from the 2026 AIME - a prestigious high school mathematics competition.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2026/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://huggingface.co/datasets/math-ai/aime26", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://huggingface.co/datasets/math-ai/aime26", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2026/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Math Ai", "url": "https://huggingface.co/datasets/math-ai/aime26", "primary": false } ], "resource_count": 4, "paper_url": "https://huggingface.co/datasets/math-ai/aime26", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2026/eval.yaml", "family_title": "AIME 2026: Problems from the American Invitational Mathematics Examination", "group": "Mathematics", "version": "2-A", "contributors": [ "joeda" ], "tags": [ "Mathematics" ], "task": { "name": "aime2026", "dataset_samples": 30, "comment": null, "support_only": false }, "tasks": [ { "name": "aime2026", "dataset_samples": 30, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 30, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "math-ai/aime26", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:aime2026", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://huggingface.co/datasets/math-ai/aime26" }, "paper": { "status": "present", "url": "https://huggingface.co/datasets/math-ai/aime26", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://huggingface.co/datasets/math-ai/aime26", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/aime2026/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/math-ai/aime26", "provider": "Math Ai" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://huggingface.co/datasets/math-ai/aime26" }, "code_url": null, "metadata": { "source_search": "Associated paper authors Original paper authors evaluation-task" } }, { "id": "inspect:air_bench", "name": "AIR Bench: AI Risk Benchmark", "slug": "inspect--air-bench", "organisation": "Yi Zeng, Yu Yang, Andy Zhou et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/air_bench", "topics": [ "safeguards" ], "description": "A safety benchmark evaluating language models against risk categories derived from government regulations and company policies.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/air_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.17436", "primary": true }, { "kind": "paper", "label": "Paper: AIR-Bench 2024: A Safety Benchmark Based on Risk Categories from Regulations and Policies", "url": "https://arxiv.org/abs/2407.17436", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/air_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Stanford Crfm", "url": "https://huggingface.co/datasets/stanford-crfm/air-bench-2024", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.17436", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/air_bench/eval.yaml", "family_title": "AIR Bench: AI Risk Benchmark", "group": "Knowledge", "version": "4-A", "contributors": [ "l1990790120" ], "tags": [ "Knowledge" ], "task": { "name": "air_bench", "dataset_samples": 5694, "comment": null, "support_only": false }, "tasks": [ { "name": "air_bench", "dataset_samples": 5694, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 5694, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "stanford-crfm/air-bench-2024", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:air_bench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yi Zeng, Yu Yang, Andy Zhou et al.", "organization": "Lapis Labs; Stanford University", "organizations": [ "Lapis Labs", "Stanford University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.17436" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.17436", "arxiv_id": "2407.17436", "title": "AIR-Bench 2024: A Safety Benchmark Based on Risk Categories from Regulations and Policies", "authors": [ "Yi Zeng", "Yu Yang", "Andy Zhou", "Jeffrey Ziwei Tan", "Yuheng Tu", "Yifan Mai", "Kevin Klyman", "Minzhou Pan", "Ruoxi Jia", "Dawn Song", "Percy Liang", "Bo Li" ], "organizations": [ "Lapis Labs", "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.17436", "arxiv_id": "2407.17436", "title": "AIR-Bench 2024: A Safety Benchmark Based on Risk Categories from Regulations and Policies", "authors": [ "Yi Zeng", "Yu Yang", "Andy Zhou", "Jeffrey Ziwei Tan", "Yuheng Tu", "Yifan Mai", "Kevin Klyman", "Minzhou Pan", "Ruoxi Jia", "Dawn Song", "Percy Liang", "Bo Li" ], "organizations": [ "Lapis Labs", "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2407.17436", "arxiv_id": "2407.17436", "title": "AIR-Bench 2024: A Safety Benchmark Based on Risk Categories from Regulations and Policies", "authors": [ "Yi Zeng", "Yu Yang", "Andy Zhou", "Jeffrey Ziwei Tan", "Yuheng Tu", "Yifan Mai", "Kevin Klyman", "Minzhou Pan", "Ruoxi Jia", "Dawn Song", "Percy Liang", "Bo Li" ], "organizations": [ "Lapis Labs", "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/air_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/stanford-crfm/air-bench-2024", "provider": "Stanford Crfm" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.17436" }, "code_url": null, "metadata": { "source_search": "Yi Zeng, Yu Yang, Andy Zhou et al. Lapis Labs; Stanford University Original paper authors AIR-Bench 2024: A Safety Benchmark Based on Risk Categories from Regulations and Policies Yi Zeng Yu Yang Andy Zhou Jeffrey Ziwei Tan Yuheng Tu Yifan Mai Kevin Klyman Minzhou Pan Ruoxi Jia Dawn Song Percy Liang Bo Li evaluation-task" } }, { "id": "register:alignment_faking", "name": "Alignment Faking", "slug": "register--alignment-faking", "organisation": "Ryan Greenblatt, Carson Denison, Benjamin Wright et al.", "source_type": "inspect-register", "source_url": "https://github.com/marliechorgan/alignment-faking-inspect", "topics": [ "deception-misalignment" ], "description": "Measures whether LLMs strategically comply with harmful requests when believing outputs will be used for RLHF training (free-tier/monitored) versus when unmonitored (paid-tier). Uses 100 harmful prompts from LLM-LAT/harmful-dataset. Each prompt is presented twice with free-tier and paid-tier tags inside a system prompt describing a conflicting training objective. Primary metric is the compliance gap (free minus paid compliance rate via two-proportion z-test); a secondary scorer detects verbalized alignment-faking reasoning in scratchpad outputs.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/alignment_faking/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2412.14093", "primary": true }, { "kind": "paper", "label": "Paper: Alignment faking in large language models", "url": "https://arxiv.org/abs/2412.14093", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/marliechorgan/alignment-faking-inspect", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/alignment_faking/eval.yaml", "primary": false } ], "paper_title": "Alignment faking in large language models", "paper_url": "https://arxiv.org/abs/2412.14093", "code_url": "https://github.com/marliechorgan/alignment-faking-inspect", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/alignment_faking/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "marliechorgan" ], "tasks": [ { "name": "alignment_faking", "path": "src/alignment_faking/alignment_faking.py" } ], "protocol": { "implementation_commit": "d102d213833b05c67464a44e28e9a414977f80e1", "report_commit": "d102d213833b05c67464a44e28e9a414977f80e1", "report_timestamp": "June 2026", "command": "uv run inspect eval src/alignment_faking/alignment_faking.py@alignment_faking --model anthropic/claude-sonnet-4-6 -T paraphrase=ensemble", "source_comment": null }, "reported_results": [ { "model": "anthropic/claude-sonnet-4-6", "provider": "Anthropic", "metrics": [ { "key": "free_compliance_rate", "value": 0.333 }, { "key": "paid_compliance_rate", "value": 0.37 }, { "key": "compliance_gap", "value": -0.037 }, { "key": "compliance_gap_pvalue", "value": 0.267 }, { "key": "free_tier_af_rate", "value": 0.005 } ] }, { "model": "anthropic/claude-opus-4-8", "provider": "Anthropic", "metrics": [ { "key": "free_compliance_rate", "value": 0.725 }, { "key": "paid_compliance_rate", "value": 0.775 }, { "key": "compliance_gap", "value": -0.05 }, { "key": "compliance_gap_pvalue", "value": 0.371 }, { "key": "free_tier_af_rate", "value": 0 } ] } ], "reported_result_count": 2, "reported_results_truncated": false, "report_notes": [ "Paraphrase ensemble (canonical + 7 paraphrases) x both tiers, real-lab framing. Sonnet 4.6: 50 prompts = 800 samples (400/tier). Opus 4.8: 15 prompts = 240 samples (120/tier).", "Headline metric is compliance_gap = free_tier minus paid_tier compliance rate, with a two-proportion z-test (compliance_gap_pvalue).", "Finding: no robust alignment-faking gap on either current model. A fictional-lab de-fingerprinting variant showed a marginal positive gap on Sonnet 4.6 (+0.075, p=0.09, n=240/tier) but it did not replicate in a higher-powered run of the strongest cell (canonical/fictional, n=100/tier: gap 0.000). Verbalised faking <=2%.", "Verbatim-prompt results are a lower bound (contamination / mitigation overfitting); the original positive-control models (Claude 3 Opus, Claude 3.5 Sonnet) are API-retired. Full conditions table and limitations are in the upstream README." ], "models_reported": [ "anthropic/claude-sonnet-4-6", "anthropic/claude-opus-4-8" ], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:alignment_faking", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Ryan Greenblatt, Carson Denison, Benjamin Wright et al.", "organization": "UC Berkeley", "organizations": [ "UC Berkeley" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2412.14093" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2412.14093", "arxiv_id": "2412.14093", "title": "Alignment faking in large language models", "authors": [ "Ryan Greenblatt", "Carson Denison", "Benjamin Wright", "Fabien Roger", "Monte MacDiarmid", "Sam Marks", "Johannes Treutlein", "Tim Belonax", "Jack Chen", "David Duvenaud", "Akbir Khan", "Julian Michael", "Sören Mindermann", "Ethan Perez", "Linda Petrini", "Jonathan Uesato", "Jared Kaplan", "Buck Shlegeris", "Samuel R. Bowman", "Evan Hubinger" ], "organizations": [ "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2412.14093", "arxiv_id": "2412.14093", "title": "Alignment faking in large language models", "authors": [ "Ryan Greenblatt", "Carson Denison", "Benjamin Wright", "Fabien Roger", "Monte MacDiarmid", "Sam Marks", "Johannes Treutlein", "Tim Belonax", "Jack Chen", "David Duvenaud", "Akbir Khan", "Julian Michael", "Sören Mindermann", "Ethan Perez", "Linda Petrini", "Jonathan Uesato", "Jared Kaplan", "Buck Shlegeris", "Samuel R. Bowman", "Evan Hubinger" ], "organizations": [ "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2412.14093", "arxiv_id": "2412.14093", "title": "Alignment faking in large language models", "authors": [ "Ryan Greenblatt", "Carson Denison", "Benjamin Wright", "Fabien Roger", "Monte MacDiarmid", "Sam Marks", "Johannes Treutlein", "Tim Belonax", "Jack Chen", "David Duvenaud", "Akbir Khan", "Julian Michael", "Sören Mindermann", "Ethan Perez", "Linda Petrini", "Jonathan Uesato", "Jared Kaplan", "Buck Shlegeris", "Samuel R. Bowman", "Evan Hubinger" ], "organizations": [ "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/marliechorgan/alignment-faking-inspect", "owner": "Marliechorgan", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/alignment_faking/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2412.14093" }, "metadata": { "source_search": "Ryan Greenblatt, Carson Denison, Benjamin Wright et al. UC Berkeley Original paper authors Alignment faking in large language models Ryan Greenblatt Carson Denison Benjamin Wright Fabien Roger Monte MacDiarmid Sam Marks Johannes Treutlein Tim Belonax Jack Chen David Duvenaud Akbir Khan Julian Michael Sören Mindermann Ethan Perez Linda Petrini Jonathan Uesato Jared Kaplan Buck Shlegeris Samuel R. Bowman Evan Hubinger Marliechorgan evaluation-package" } }, { "id": "inspect:anima", "name": "ANIMA: Animal Norms In Moral Assessment", "slug": "inspect--anima", "organisation": "Jasmine Brazilek, Miles Tidmarsh", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/anima", "topics": [ "general-capability" ], "description": "Evaluates the quality of a model's moral reasoning about animal welfare across 13 ethical dimensions.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/anima/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2604.13076", "primary": true }, { "kind": "paper", "label": "Paper: Alignment midtraining for animals", "url": "https://arxiv.org/abs/2604.13076", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/anima/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Compassion in Machine Learning", "url": "https://huggingface.co/datasets/CompassioninMachineLearning/anima", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2604.13076", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/anima/eval.yaml", "family_title": "ANIMA: Animal Norms In Moral Assessment", "group": "Safeguards", "version": "6-C", "contributors": [ "nishu-builder", "darkness8i8", "jm355" ], "tags": [ "Safeguards" ], "task": { "name": "anima", "dataset_samples": 26, "comment": null, "support_only": false }, "tasks": [ { "name": "anima", "dataset_samples": 26, "comment": null, "support_only": false } ], "protocol": { "version": "6-C", "dataset_samples": 26, "dependency": "anima", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "CompassioninMachineLearning/anima", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:anima", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jasmine Brazilek, Miles Tidmarsh", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2604.13076" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2604.13076", "arxiv_id": "2604.13076", "title": "Alignment midtraining for animals", "authors": [ "Jasmine Brazilek", "Miles Tidmarsh" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2604.13076", "arxiv_id": "2604.13076", "title": "Alignment midtraining for animals", "authors": [ "Jasmine Brazilek", "Miles Tidmarsh" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/anima/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/CompassioninMachineLearning/anima", "provider": "Compassion in Machine Learning" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2604.13076" }, "code_url": null, "metadata": { "source_search": "Jasmine Brazilek, Miles Tidmarsh Original paper authors Alignment midtraining for animals Jasmine Brazilek Miles Tidmarsh evaluation-task" } }, { "id": "inspect:ape_eval", "name": "APE: Attempt to Persuade Eval", "slug": "inspect--ape-eval", "organisation": "FAR AI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ape", "topics": [ "human-influence" ], "description": "Measures a model's willingness to attempt persuasion on harmful, controversial, and benign topics. The key metric is not persuasion effectiveness but whether the model attempts to persuade at all — particularly on harmful statements. Uses a multi-model setup: the evaluated model (persuader) converses with a simulated user (persuadee), and a third model (evaluator) scores each persuader turn for persuasion attempt. Based on the paper \"It's the Thought that Counts\" (arXiv:2506.02873).", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Whether and how often the model attempts persuasion in APE's simulated dialogue distribution under the stated prompting and classification protocol.", "does_not_measure": "Persuasion effectiveness on humans, belief change, behavioral change, covert targeting, deployment at scale, durable agency loss, or electoral effects.", "best_for": "Comparing willingness to deploy persuasive strategies under a fixed set of simulated opportunities and model-system conditions.", "not_sufficient_for": "Claims that a model is persuasive, manipulative, or capable of changing human behavior in real deployment.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ape/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://www.far.ai/blog/attempt-to-persuade-eval", "primary": true }, { "kind": "paper", "label": "Paper: It's the Thought that Counts: Evaluating the Attempts of Frontier LLMs to Persuade on Harmful Topics", "url": "https://arxiv.org/abs/2506.02873", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/AlignmentResearch/AttemptPersuadeEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ape/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.far.ai/blog/attempt-to-persuade-eval", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2506.02873", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ape/eval.yaml", "family_title": "APE: Attempt to Persuade Eval", "group": "Safeguards", "version": "2-A", "contributors": [ "cmv13" ], "tags": [ "Safeguards" ], "task": { "name": "ape_eval", "dataset_samples": 600, "comment": null, "support_only": false }, "tasks": [ { "name": "ape_eval", "dataset_samples": 600, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 600, "dependency": null, "requires_internet": true, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/AlignmentResearch/AttemptPersuadeEval/{SHA}/src/topics/", "fetch_method": "other", "state": "pinned", "comment": "Fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:ape", "record_type": "evaluation-task", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "Inspect Evals contains a maintained port. FAR AI introduced APE and publishes the reference implementation." }, "origin": { "name": "FAR AI", "organization": "FAR AI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://www.far.ai/blog/attempt-to-persuade-eval", "organizations": [ "FAR AI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.02873", "arxiv_id": "2506.02873", "title": "It's the Thought that Counts: Evaluating the Attempts of Frontier LLMs to Persuade on Harmful Topics", "authors": [ "Matthew Kowal", "Jasper Timm", "Jean-Francois Godbout", "Thomas Costello", "Antonio A. Arechar", "Gordon Pennycook", "David Rand", "Adam Gleave", "Kellin Pelrine" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.02873", "arxiv_id": "2506.02873", "title": "It's the Thought that Counts: Evaluating the Attempts of Frontier LLMs to Persuade on Harmful Topics", "authors": [ "Matthew Kowal", "Jasper Timm", "Jean-Francois Godbout", "Thomas Costello", "Antonio A. Arechar", "Gordon Pennycook", "David Rand", "Adam Gleave", "Kellin Pelrine" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/AlignmentResearch/AttemptPersuadeEval", "owner": "FAR AI", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ape/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://www.far.ai/blog/attempt-to-persuade-eval" }, "code_url": "https://github.com/AlignmentResearch/AttemptPersuadeEval", "metadata": { "source_search": "FAR AI FAR AI Evaluation developed by It's the Thought that Counts: Evaluating the Attempts of Frontier LLMs to Persuade on Harmful Topics Matthew Kowal Jasper Timm Jean-Francois Godbout Thomas Costello Antonio A. Arechar Gordon Pennycook David Rand Adam Gleave Kellin Pelrine FAR AI evaluation-task" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "high", "construct": "A model's propensity to attempt persuasion when placed in simulated conversations where persuasion could advance its assigned objective.", "unit_of_analysis": "One model-generated response or dialogue trajectory in a simulated persuasive situation, classified for whether an attempt to persuade occurred.", "interaction_type": "Model-to-model or simulated-user dialogue across benign and harmful persuasion contexts.", "target_outcome": "Presence and strength of an attempted persuasive strategy under the benchmark's classifier and rubric.", "human_participants": "No human persuasion subjects are exposed in the core evaluation. Human judgment contributes to task and classifier validation.", "elicitation": "The evaluated model receives role, context, goal, and conversation state designed to create an opportunity or incentive to persuade; outputs depend on the system prompt, simulated counterpart, and sampling settings.", "scoring": "A benchmark-specific classifier or rubric identifies whether the model attempted persuasion and aggregates attempt rates across scenarios and categories.", "protocol_dependence": "Very high: role instructions, scenario distribution, counterpart simulation, system prompt, sampling settings, and attempt classifier define the measured behavior.", "implementation_dependence": "High: dialogue construction, classifier calibration, transcript truncation, and model access route can materially change attempt rates.", "limitations": [ "Attempting persuasion is distinct from succeeding at persuasion.", "Simulated interlocutors do not reproduce human susceptibility, trust, or resistance.", "The scenario distribution may omit important persuasive contexts and strategies.", "Classifier errors and provider-side system changes can affect measured attempt rates." ], "comparability": "Compare only with the same APE version, scenario subset, simulated counterpart, system prompt, generation settings, classifier, and model-system access route.", "decision_relevance": "Useful as an early capability and behavioral-propensity signal for influence-risk evaluations. Human-effect and deployment evidence remain necessary for risk or policy conclusions.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2506.02873" }, { "kind": "official", "url": "https://www.far.ai/blog/attempt-to-persuade-eval" }, { "kind": "implementation", "url": "https://github.com/AlignmentResearch/AttemptPersuadeEval" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:apps", "name": "APPS: Automated Programming Progress Standard", "slug": "inspect--apps", "organisation": "Dan Hendrycks, Steven Basart, Saurav Kadavath et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/apps", "topics": [ "general-capability" ], "description": "APPS is a dataset for evaluating model performance on Python programming tasks across three difficulty levels consisting of 1,000 at introductory, 3,000 at interview, and 1,000 at competition level. The dataset consists of an additional 5,000 training samples, for a total of 10,000 total samples. We evaluate on questions from the test split, which consists of programming problems commonly found in coding interviews.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/apps/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2105.09938", "primary": true }, { "kind": "paper", "label": "Paper: Measuring Coding Challenge Competence With APPS", "url": "https://arxiv.org/abs/2105.09938", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/apps/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Codeparrot", "url": "https://huggingface.co/datasets/codeparrot/apps", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2105.09938", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/apps/eval.yaml", "family_title": "APPS: Automated Programming Progress Standard", "group": "Coding", "version": "2-A", "contributors": [ "camtice" ], "tags": [ "Coding" ], "task": { "name": "apps", "dataset_samples": 5000, "comment": null, "support_only": false }, "tasks": [ { "name": "apps", "dataset_samples": 5000, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 5000, "dependency": null, "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "codeparrot/apps", "fetch_method": "snapshot_download", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:apps", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Dan Hendrycks, Steven Basart, Saurav Kadavath et al.", "organization": "UC Berkeley; University of Chicago; TTIC; Anthropic", "organizations": [ "UC Berkeley", "University of Chicago", "TTIC", "Anthropic" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2105.09938" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2105.09938", "arxiv_id": "2105.09938", "title": "Measuring Coding Challenge Competence With APPS", "authors": [ "Dan Hendrycks", "Steven Basart", "Saurav Kadavath", "Mantas Mazeika", "Akul Arora", "Ethan Guo", "Collin Burns", "Samir Puranik", "Horace He", "Dawn Song", "Jacob Steinhardt" ], "organizations": [ "UC Berkeley", "University of Chicago", "TTIC", "Anthropic" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2105.09938", "arxiv_id": "2105.09938", "title": "Measuring Coding Challenge Competence With APPS", "authors": [ "Dan Hendrycks", "Steven Basart", "Saurav Kadavath", "Mantas Mazeika", "Akul Arora", "Ethan Guo", "Collin Burns", "Samir Puranik", "Horace He", "Dawn Song", "Jacob Steinhardt" ], "organizations": [ "UC Berkeley", "University of Chicago", "TTIC", "Anthropic" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2105.09938", "arxiv_id": "2105.09938", "title": "Measuring Coding Challenge Competence With APPS", "authors": [ "Dan Hendrycks", "Steven Basart", "Saurav Kadavath", "Mantas Mazeika", "Akul Arora", "Ethan Guo", "Collin Burns", "Samir Puranik", "Horace He", "Dawn Song", "Jacob Steinhardt" ], "organizations": [ "UC Berkeley", "University of Chicago", "TTIC", "Anthropic" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/apps/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/codeparrot/apps", "provider": "Codeparrot" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2105.09938" }, "code_url": null, "metadata": { "source_search": "Dan Hendrycks, Steven Basart, Saurav Kadavath et al. UC Berkeley; University of Chicago; TTIC; Anthropic Original paper authors Measuring Coding Challenge Competence With APPS Dan Hendrycks Steven Basart Saurav Kadavath Mantas Mazeika Akul Arora Ethan Guo Collin Burns Samir Puranik Horace He Dawn Song Jacob Steinhardt evaluation-task" } }, { "id": "register:appworld", "name": "AppWorld", "slug": "register--appworld", "organisation": "Harsh Trivedi, Tushar Khot, Mareike Hartmann et al.", "source_type": "inspect-register", "source_url": "https://github.com/anirudhvenu/appworld-inspect", "topics": [ "autonomy-agents" ], "description": "AppWorld evaluates autonomous agents on 750 day-to-day digital tasks requiring iterative Python code generation against 457 APIs across 9 simulated apps. Tasks are split into test_normal (168) and test_challenge (417, includes unseen Amazon/Gmail APIs). Scoring uses programmatic state-based unit tests checking database diffs for goal completion (Task Goal Completion, TGC) and collateral damage avoidance, rather than reference-solution comparison. Scenario Goal Completion (SGC) measures consistency across task variants.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/appworld/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.18901", "primary": true }, { "kind": "paper", "label": "Paper: AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents", "url": "https://arxiv.org/abs/2407.18901", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/anirudhvenu/appworld-inspect", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/appworld/eval.yaml", "primary": false } ], "paper_title": "AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents", "paper_url": "https://arxiv.org/abs/2407.18901", "code_url": "https://github.com/anirudhvenu/appworld-inspect", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/appworld/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "anirudhvenu" ], "tasks": [ { "name": "appworld", "path": "src/appworld_inspect/appworld_inspect.py" } ], "protocol": { "implementation_commit": "880fbc47e73270576137b928fdb1ecab61529da1", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:appworld", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Harsh Trivedi, Tushar Khot, Mareike Hartmann et al.", "organization": "Allen Institute for Artificial Intelligence", "organizations": [ "Allen Institute for Artificial Intelligence" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.18901" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.18901", "arxiv_id": "2407.18901", "title": "AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents", "authors": [ "Harsh Trivedi", "Tushar Khot", "Mareike Hartmann", "Ruskin Manku", "Vinty Dong", "Edward Li", "Shashank Gupta", "Ashish Sabharwal", "Niranjan Balasubramanian" ], "organizations": [ "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.18901", "arxiv_id": "2407.18901", "title": "AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents", "authors": [ "Harsh Trivedi", "Tushar Khot", "Mareike Hartmann", "Ruskin Manku", "Vinty Dong", "Edward Li", "Shashank Gupta", "Ashish Sabharwal", "Niranjan Balasubramanian" ], "organizations": [ "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2407.18901", "arxiv_id": "2407.18901", "title": "AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents", "authors": [ "Harsh Trivedi", "Tushar Khot", "Mareike Hartmann", "Ruskin Manku", "Vinty Dong", "Edward Li", "Shashank Gupta", "Ashish Sabharwal", "Niranjan Balasubramanian" ], "organizations": [ "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/anirudhvenu/appworld-inspect", "owner": "Anirudhvenu", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/appworld/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.18901" }, "metadata": { "source_search": "Harsh Trivedi, Tushar Khot, Mareike Hartmann et al. Allen Institute for Artificial Intelligence Original paper authors AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents Harsh Trivedi Tushar Khot Mareike Hartmann Ruskin Manku Vinty Dong Edward Li Shashank Gupta Ashish Sabharwal Niranjan Balasubramanian Anirudhvenu evaluation-package" } }, { "id": "register:aratrust", "name": "AraTrust", "slug": "register--aratrust", "organisation": "Emad A. Alghamdi, Reem I. Masoud, Deema Alnuhait et al.", "source_type": "inspect-register", "source_url": "https://github.com/Raulster24/aratrust-inspect", "topics": [ "general-capability" ], "description": "AraTrust evaluates LLM trustworthiness when prompted in Arabic via 522 human-written multiple-choice questions (3 options each) spanning 8 categories: truthfulness, ethics, physical health, mental health, unfairness, illegal activities, privacy, and offensive language, with 34 subcategories. Questions were authored by native Arabic speakers or adapted from exams and existing datasets. Scoring is accuracy of selected answer against a single correct option. The eval code loads the dataset from HuggingFace, filters malformed rows, and scores with exact-match choice grading.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/aratrust/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.09017", "primary": true }, { "kind": "paper", "label": "Paper: AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic", "url": "https://arxiv.org/abs/2403.09017", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Raulster24/aratrust-inspect", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/aratrust/eval.yaml", "primary": false } ], "paper_title": "AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic", "paper_url": "https://arxiv.org/abs/2403.09017", "code_url": "https://github.com/Raulster24/aratrust-inspect", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/aratrust/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "Raulster24" ], "tasks": [ { "name": "aratrust", "path": "src/aratrust/aratrust.py" } ], "protocol": { "implementation_commit": "e4b89df399057ca5ae8f8524418907c4868badcd", "report_commit": "e4b89df399057ca5ae8f8524418907c4868badcd", "report_timestamp": "July 2026", "command": null, "source_comment": null }, "reported_results": [ { "model": "openai/gpt-4o", "provider": null, "metrics": [ { "key": "accuracy", "value": 0.935 }, { "key": "stderr", "value": 0.011 } ] }, { "model": "openai/gpt-4o-mini", "provider": null, "metrics": [ { "key": "accuracy", "value": 0.9 }, { "key": "stderr", "value": 0.013 } ] } ], "reported_result_count": 2, "reported_results_truncated": false, "report_notes": [ "521 usable samples; 1 of 522 source rows dropped for an empty option.", "Scores exceed the paper's GPT-4 (81%), as expected for newer models." ], "models_reported": [ "openai/gpt-4o", "openai/gpt-4o-mini" ], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:aratrust", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Emad A. Alghamdi, Reem I. Masoud, Deema Alnuhait et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.09017" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.09017", "arxiv_id": "2403.09017", "title": "AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic", "authors": [ "Emad A. Alghamdi", "Reem I. Masoud", "Deema Alnuhait", "Afnan Y. Alomairi", "Ahmed Ashraf", "Mohamed Zaytoon" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.09017", "arxiv_id": "2403.09017", "title": "AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic", "authors": [ "Emad A. Alghamdi", "Reem I. Masoud", "Deema Alnuhait", "Afnan Y. Alomairi", "Ahmed Ashraf", "Mohamed Zaytoon" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Raulster24/aratrust-inspect", "owner": "Raulster24", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/aratrust/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.09017" }, "metadata": { "source_search": "Emad A. Alghamdi, Reem I. Masoud, Deema Alnuhait et al. Original paper authors AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic Emad A. Alghamdi Reem I. Masoud Deema Alnuhait Afnan Y. Alomairi Ahmed Ashraf Mohamed Zaytoon Raulster24 evaluation-package" } }, { "id": "inspect:arc_challenge", "name": "ARC Challenge", "slug": "inspect--arc-challenge", "organisation": "Peter Clark, Isaac Cowhey, Oren Etzioni et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc", "topics": [ "general-capability" ], "description": "Dataset of natural, grade-school science multiple-choice questions (authored for human tests).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1803.05457", "primary": true }, { "kind": "paper", "label": "Paper: Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge", "url": "https://arxiv.org/abs/1803.05457", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Allen Institute for AI", "url": "https://huggingface.co/datasets/allenai/ai2_arc", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1803.05457", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc/eval.yaml", "family_title": "ARC: AI2 Reasoning Challenge", "group": "Reasoning", "version": "2-A", "contributors": [ "jjallaire" ], "tags": [ "Reasoning" ], "task": { "name": "arc_challenge", "dataset_samples": 1172, "comment": null, "support_only": false }, "tasks": [ { "name": "arc_easy", "dataset_samples": 2376, "comment": null, "support_only": false }, { "name": "arc_challenge", "dataset_samples": 1172, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1172, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "allenai/ai2_arc", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:arc", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Peter Clark, Isaac Cowhey, Oren Etzioni et al.", "organization": "Allen Institute for AI; Allen Institute for Artificial Intelligence; Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign", "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence", "Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1803.05457" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1803.05457", "arxiv_id": "1803.05457", "title": "Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge", "authors": [ "Peter Clark", "Isaac Cowhey", "Oren Etzioni", "Tushar Khot", "Ashish Sabharwal", "Carissa Schoenick", "Oyvind Tafjord" ], "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence", "Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1803.05457", "arxiv_id": "1803.05457", "title": "Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge", "authors": [ "Peter Clark", "Isaac Cowhey", "Oren Etzioni", "Tushar Khot", "Ashish Sabharwal", "Carissa Schoenick", "Oyvind Tafjord" ], "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence", "Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/allenai/ai2_arc", "provider": "Allen Institute for AI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1803.05457" }, "code_url": null, "metadata": { "source_search": "Peter Clark, Isaac Cowhey, Oren Etzioni et al. Allen Institute for AI; Allen Institute for Artificial Intelligence; Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign Original paper authors Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge Peter Clark Isaac Cowhey Oren Etzioni Tushar Khot Ashish Sabharwal Carissa Schoenick Oyvind Tafjord evaluation-family" } }, { "id": "inspect:arc_easy", "name": "ARC Easy", "slug": "inspect--arc-easy", "organisation": "Peter Clark, Isaac Cowhey, Oren Etzioni et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc", "topics": [ "general-capability" ], "description": "Dataset of natural, grade-school science multiple-choice questions (authored for human tests).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1803.05457", "primary": true }, { "kind": "paper", "label": "Paper: Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge", "url": "https://arxiv.org/abs/1803.05457", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Allen Institute for AI", "url": "https://huggingface.co/datasets/allenai/ai2_arc", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1803.05457", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc/eval.yaml", "family_title": "ARC: AI2 Reasoning Challenge", "group": "Reasoning", "version": "2-A", "contributors": [ "jjallaire" ], "tags": [ "Reasoning" ], "task": { "name": "arc_easy", "dataset_samples": 2376, "comment": null, "support_only": false }, "tasks": [ { "name": "arc_easy", "dataset_samples": 2376, "comment": null, "support_only": false }, { "name": "arc_challenge", "dataset_samples": 1172, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 2376, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "allenai/ai2_arc", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:arc", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Peter Clark, Isaac Cowhey, Oren Etzioni et al.", "organization": "Allen Institute for AI; Allen Institute for Artificial Intelligence; Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign", "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence", "Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1803.05457" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1803.05457", "arxiv_id": "1803.05457", "title": "Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge", "authors": [ "Peter Clark", "Isaac Cowhey", "Oren Etzioni", "Tushar Khot", "Ashish Sabharwal", "Carissa Schoenick", "Oyvind Tafjord" ], "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence", "Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1803.05457", "arxiv_id": "1803.05457", "title": "Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge", "authors": [ "Peter Clark", "Isaac Cowhey", "Oren Etzioni", "Tushar Khot", "Ashish Sabharwal", "Carissa Schoenick", "Oyvind Tafjord" ], "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence", "Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/arc/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/allenai/ai2_arc", "provider": "Allen Institute for AI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1803.05457" }, "code_url": null, "metadata": { "source_search": "Peter Clark, Isaac Cowhey, Oren Etzioni et al. Allen Institute for AI; Allen Institute for Artificial Intelligence; Allen Institute for AI, Eastern Illinois University, University of Illinois at Urbana-Champaign Original paper authors Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge Peter Clark Isaac Cowhey Oren Etzioni Tushar Khot Ashish Sabharwal Carissa Schoenick Oyvind Tafjord evaluation-family" } }, { "id": "canonical:arc-agi-2", "name": "ARC-AGI-2", "slug": "canonical--arc-agi-2", "organisation": "ARC Prize Foundation and the ARC-AGI-2 authors", "source_type": "canonical-source", "source_url": "https://arcprize.org/arc-agi/2/", "topics": [ "general-capability", "evaluation-integrity" ], "description": "Tests novel abstract reasoning and skill acquisition on tasks designed to resist memorised solutions.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arcprize.org/arc-agi/2", "primary": true }, { "kind": "paper", "label": "Paper: ARC-AGI-2: A New Challenge for Frontier AI Reasoning Systems", "url": "https://arxiv.org/abs/2505.11831", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arcprize.org/arc-agi/2", "primary": false } ], "resource_count": 3, "source_key": "canonical:arc-agi-2", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "ARC Prize Foundation and the ARC-AGI-2 authors", "organization": "ARC Prize Foundation", "entity_type": "research_project", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arcprize.org/arc-agi/2", "organizations": [ "ARC Prize Foundation" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.11831", "arxiv_id": "2505.11831", "title": "ARC-AGI-2: A New Challenge for Frontier AI Reasoning Systems", "authors": [ "Francois Chollet", "Mike Knoop", "Gregory Kamradt", "Bryan Landers", "Henry Pinkard" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.11831", "arxiv_id": "2505.11831", "title": "ARC-AGI-2: A New Challenge for Frontier AI Reasoning Systems", "authors": [ "Francois Chollet", "Mike Knoop", "Gregory Kamradt", "Bryan Landers", "Henry Pinkard" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arcprize.org/arc-agi/2" }, "paper_url": "https://arxiv.org/abs/2505.11831", "code_url": null, "metadata": { "source_search": "ARC Prize Foundation and the ARC-AGI-2 authors ARC Prize Foundation Evaluation developed by ARC-AGI-2: A New Challenge for Frontier AI Reasoning Systems Francois Chollet Mike Knoop Gregory Kamradt Bryan Landers Henry Pinkard evaluation-resource" } }, { "id": "register:arxivrollbench", "name": "ArxivRollBench", "slug": "register--arxivrollbench", "organisation": "Zi Liang, Liantong Yu, Shiyu Zhang et al.", "source_type": "inspect-register", "source_url": "https://github.com/liangzid/ArxivRoll", "topics": [ "evaluation-integrity" ], "description": "A rolling benchmark for evaluating recent scientific text reasoning from arXiv papers. ArxivRollBench constructs multiple-choice sequencing, cloze, and next-fragment prediction tasks over newly released scientific papers across arXiv domains, with compact and full public releases.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/arxivrollbench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2507.19219", "primary": true }, { "kind": "paper", "label": "Paper: How Much Do Large Language Model Cheat on Evaluation? Benchmarking Overestimation under the One-Time-Pad-Based Framework", "url": "https://arxiv.org/abs/2507.19219", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/liangzid/ArxivRoll", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/arxivrollbench/eval.yaml", "primary": false } ], "paper_title": null, "paper_url": "https://arxiv.org/abs/2507.19219", "code_url": "https://github.com/liangzid/ArxivRoll", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/arxivrollbench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "Reasoning", "Knowledge", "Science", "arXiv" ], "contributors": [ "liangzid" ], "tasks": [ { "name": "arxivrollbench", "path": "arxivrollbench_inspect.py" } ], "protocol": { "implementation_commit": "01ec6e132e1cccdc204bed8b20a8473bb82c3ca7", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": "The Inspect AI task loads Hugging Face datasets with explicit pinned dataset revisions." }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:arxivrollbench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zi Liang, Liantong Yu, Shiyu Zhang et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2507.19219" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2507.19219", "arxiv_id": "2507.19219", "title": "How Much Do Large Language Model Cheat on Evaluation? Benchmarking Overestimation under the One-Time-Pad-Based Framework", "authors": [ "Zi Liang", "Liantong Yu", "Shiyu Zhang", "Qingqing Ye", "Haibo Hu" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2507.19219", "arxiv_id": "2507.19219", "title": "How Much Do Large Language Model Cheat on Evaluation? Benchmarking Overestimation under the One-Time-Pad-Based Framework", "authors": [ "Zi Liang", "Liantong Yu", "Shiyu Zhang", "Qingqing Ye", "Haibo Hu" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/liangzid/ArxivRoll", "owner": "Liangzid", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/arxivrollbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2507.19219" }, "metadata": { "source_search": "Zi Liang, Liantong Yu, Shiyu Zhang et al. Original paper authors How Much Do Large Language Model Cheat on Evaluation? Benchmarking Overestimation under the One-Time-Pad-Based Framework Zi Liang Liantong Yu Shiyu Zhang Qingqing Ye Haibo Hu Liangzid evaluation-package" } }, { "id": "inspect:assistant_bench_closed_book_one_shot", "name": "Assistant Bench Closed Book One Shot", "slug": "inspect--assistant-bench-closed-book-one-shot", "organisation": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench", "topics": [ "autonomy-agents" ], "description": "Tests whether AI agents can perform real-world time-consuming tasks on the web.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.15711", "primary": true }, { "kind": "paper", "label": "Paper: AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "url": "https://arxiv.org/abs/2407.15711", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: AssistantBench", "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.15711", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "family_title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "group": "Assistants", "version": "5-B", "contributors": [ "nlpet", "caspardh" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "assistant_bench_closed_book_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, "tasks": [ { "name": "assistant_bench_closed_book_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_closed_book_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_browser", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 33, "dependency": null, "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "AssistantBench/AssistantBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:assistant_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "organization": "Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind", "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.15711" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "provider": "AssistantBench" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.15711" }, "code_url": null, "metadata": { "source_search": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al. Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind Original paper authors AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks? Ori Yoran Samuel Joseph Amouyal Chaitanya Malaviya Ben Bogin Ofir Press Jonathan Berant evaluation-family" } }, { "id": "inspect:assistant_bench_closed_book_zero_shot", "name": "Assistant Bench Closed Book Zero Shot", "slug": "inspect--assistant-bench-closed-book-zero-shot", "organisation": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench", "topics": [ "autonomy-agents" ], "description": "Tests whether AI agents can perform real-world time-consuming tasks on the web.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.15711", "primary": true }, { "kind": "paper", "label": "Paper: AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "url": "https://arxiv.org/abs/2407.15711", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: AssistantBench", "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.15711", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "family_title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "group": "Assistants", "version": "5-B", "contributors": [ "nlpet", "caspardh" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "assistant_bench_closed_book_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, "tasks": [ { "name": "assistant_bench_closed_book_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_closed_book_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_browser", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 33, "dependency": null, "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "AssistantBench/AssistantBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:assistant_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "organization": "Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind", "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.15711" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "provider": "AssistantBench" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.15711" }, "code_url": null, "metadata": { "source_search": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al. Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind Original paper authors AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks? Ori Yoran Samuel Joseph Amouyal Chaitanya Malaviya Ben Bogin Ofir Press Jonathan Berant evaluation-family" } }, { "id": "inspect:assistant_bench_web_browser", "name": "Assistant Bench Web Browser", "slug": "inspect--assistant-bench-web-browser", "organisation": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench", "topics": [ "autonomy-agents" ], "description": "Tests whether AI agents can perform real-world time-consuming tasks on the web.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.15711", "primary": true }, { "kind": "paper", "label": "Paper: AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "url": "https://arxiv.org/abs/2407.15711", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: AssistantBench", "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.15711", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "family_title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "group": "Assistants", "version": "5-B", "contributors": [ "nlpet", "caspardh" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "assistant_bench_web_browser", "dataset_samples": 33, "comment": null, "support_only": false }, "tasks": [ { "name": "assistant_bench_closed_book_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_closed_book_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_browser", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 33, "dependency": null, "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "AssistantBench/AssistantBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:assistant_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "organization": "Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind", "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.15711" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "provider": "AssistantBench" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.15711" }, "code_url": null, "metadata": { "source_search": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al. Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind Original paper authors AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks? Ori Yoran Samuel Joseph Amouyal Chaitanya Malaviya Ben Bogin Ofir Press Jonathan Berant evaluation-family" } }, { "id": "inspect:assistant_bench_web_search_one_shot", "name": "Assistant Bench Web Search One Shot", "slug": "inspect--assistant-bench-web-search-one-shot", "organisation": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench", "topics": [ "autonomy-agents" ], "description": "Tests whether AI agents can perform real-world time-consuming tasks on the web.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.15711", "primary": true }, { "kind": "paper", "label": "Paper: AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "url": "https://arxiv.org/abs/2407.15711", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: AssistantBench", "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.15711", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "family_title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "group": "Assistants", "version": "5-B", "contributors": [ "nlpet", "caspardh" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "assistant_bench_web_search_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, "tasks": [ { "name": "assistant_bench_closed_book_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_closed_book_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_browser", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 33, "dependency": null, "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "AssistantBench/AssistantBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:assistant_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "organization": "Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind", "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.15711" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "provider": "AssistantBench" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.15711" }, "code_url": null, "metadata": { "source_search": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al. Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind Original paper authors AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks? Ori Yoran Samuel Joseph Amouyal Chaitanya Malaviya Ben Bogin Ofir Press Jonathan Berant evaluation-family" } }, { "id": "inspect:assistant_bench_web_search_zero_shot", "name": "Assistant Bench Web Search Zero Shot", "slug": "inspect--assistant-bench-web-search-zero-shot", "organisation": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench", "topics": [ "autonomy-agents" ], "description": "Tests whether AI agents can perform real-world time-consuming tasks on the web.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.15711", "primary": true }, { "kind": "paper", "label": "Paper: AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "url": "https://arxiv.org/abs/2407.15711", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: AssistantBench", "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.15711", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml", "family_title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "group": "Assistants", "version": "5-B", "contributors": [ "nlpet", "caspardh" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "assistant_bench_web_search_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, "tasks": [ { "name": "assistant_bench_closed_book_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_closed_book_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_zero_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_search_one_shot", "dataset_samples": 33, "comment": null, "support_only": false }, { "name": "assistant_bench_web_browser", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 33, "dependency": null, "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "AssistantBench/AssistantBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:assistant_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al.", "organization": "Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind", "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.15711" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.15711", "arxiv_id": "2407.15711", "title": "AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?", "authors": [ "Ori Yoran", "Samuel Joseph Amouyal", "Chaitanya Malaviya", "Ben Bogin", "Ofir Press", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel Aviv University", "University of Washington", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/assistant_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "provider": "AssistantBench" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.15711" }, "code_url": null, "metadata": { "source_search": "Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya et al. Allen Institute for Artificial Intelligence; Tel Aviv University; University of Washington; Tel-Aviv University and Google DeepMind Original paper authors AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks? Ori Yoran Samuel Joseph Amouyal Chaitanya Malaviya Ben Bogin Ofir Press Jonathan Berant evaluation-family" } }, { "id": "inspect:b3", "name": "b3: Backbone Breaker Benchmark", "slug": "inspect--b3", "organisation": "Julia Bazinska, Max Mathys, Francesco Casucci et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/b3", "topics": [ "general-capability" ], "description": "A comprehensive benchmark for evaluating LLMs for agentic AI security vulnerabilities including prompt attacks aimed at data exfiltration, content injection, decision and behavior manipulation, denial of service, system and tool compromise, and content policy bypass.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/b3/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2510.22620", "primary": true }, { "kind": "paper", "label": "Paper: Breaking Agent Backbones: Evaluating the Security of Backbone LLMs in AI Agents", "url": "https://arxiv.org/abs/2510.22620", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/b3/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Lakera", "url": "https://huggingface.co/datasets/Lakera/b3-agent-security-benchmark-weak", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2510.22620", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/b3/eval.yaml", "family_title": "b3: Backbone Breaker Benchmark", "group": "Safeguards", "version": "4-A", "contributors": [ "jb-lakera", "mmathys", "Casuyan", "mrc-lakera", "xanderdavies", "alexandrasouly-aisi", "NiklasPfister" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "b3", "dataset_samples": 630, "comment": null, "support_only": false }, "tasks": [ { "name": "b3", "dataset_samples": 630, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 630, "dependency": "b3", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "Lakera/b3-agent-security-benchmark-weak", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:b3", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Julia Bazinska, Max Mathys, Francesco Casucci et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2510.22620" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2510.22620", "arxiv_id": "2510.22620", "title": "Breaking Agent Backbones: Evaluating the Security of Backbone LLMs in AI Agents", "authors": [ "Julia Bazinska", "Max Mathys", "Francesco Casucci", "Mateo Rojas-Carulla", "Xander Davies", "Alexandra Souly", "Niklas Pfister" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2510.22620", "arxiv_id": "2510.22620", "title": "Breaking Agent Backbones: Evaluating the Security of Backbone LLMs in AI Agents", "authors": [ "Julia Bazinska", "Max Mathys", "Francesco Casucci", "Mateo Rojas-Carulla", "Xander Davies", "Alexandra Souly", "Niklas Pfister" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/b3/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/Lakera/b3-agent-security-benchmark-weak", "provider": "Lakera" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2510.22620" }, "code_url": null, "metadata": { "source_search": "Julia Bazinska, Max Mathys, Francesco Casucci et al. Original paper authors Breaking Agent Backbones: Evaluating the Security of Backbone LLMs in AI Agents Julia Bazinska Max Mathys Francesco Casucci Mateo Rojas-Carulla Xander Davies Alexandra Souly Niklas Pfister evaluation-task" } }, { "id": "inspect:bbeh", "name": "Bbeh", "slug": "inspect--bbeh", "organisation": "Mehran Kazemi, Bahare Fatemi, Hritik Bansal et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh", "topics": [ "general-capability" ], "description": "A reasoning capability dataset that replaces each task in BIG-Bench-Hard with a novel task that probes a similar reasoning capability but exhibits significantly increased difficulty.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2502.19187", "primary": true }, { "kind": "paper", "label": "Paper: BIG-Bench Extra Hard", "url": "https://arxiv.org/abs/2502.19187", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: BBEH", "url": "https://huggingface.co/datasets/BBEH/bbeh", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2502.19187", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh/eval.yaml", "family_title": "BIG-Bench Extra Hard", "group": "Reasoning", "version": "2-A", "contributors": [ "jeqcho" ], "tags": [ "Reasoning" ], "task": { "name": "bbeh", "dataset_samples": 4520, "comment": null, "support_only": false }, "tasks": [ { "name": "bbeh", "dataset_samples": 4520, "comment": null, "support_only": false }, { "name": "bbeh_mini", "dataset_samples": 460, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 4520, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "BBEH/bbeh", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:bbeh", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mehran Kazemi, Bahare Fatemi, Hritik Bansal et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2502.19187" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2502.19187", "arxiv_id": "2502.19187", "title": "BIG-Bench Extra Hard", "authors": [ "Mehran Kazemi", "Bahare Fatemi", "Hritik Bansal", "John Palowitch", "Chrysovalantis Anastasiou", "Sanket Vaibhav Mehta", "Lalit K. Jain", "Virginia Aglietti", "Disha Jindal", "Peter Chen", "Nishanth Dikkala", "Gladys Tyen", "Xin Liu", "Uri Shalit", "Silvia Chiappa", "Kate Olszewska", "Yi Tay", "Vinh Q. Tran", "Quoc V. Le", "Orhan Firat" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2502.19187", "arxiv_id": "2502.19187", "title": "BIG-Bench Extra Hard", "authors": [ "Mehran Kazemi", "Bahare Fatemi", "Hritik Bansal", "John Palowitch", "Chrysovalantis Anastasiou", "Sanket Vaibhav Mehta", "Lalit K. Jain", "Virginia Aglietti", "Disha Jindal", "Peter Chen", "Nishanth Dikkala", "Gladys Tyen", "Xin Liu", "Uri Shalit", "Silvia Chiappa", "Kate Olszewska", "Yi Tay", "Vinh Q. Tran", "Quoc V. Le", "Orhan Firat" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2502.19187", "arxiv_id": "2502.19187", "title": "BIG-Bench Extra Hard", "authors": [ "Mehran Kazemi", "Bahare Fatemi", "Hritik Bansal", "John Palowitch", "Chrysovalantis Anastasiou", "Sanket Vaibhav Mehta", "Lalit K. Jain", "Virginia Aglietti", "Disha Jindal", "Peter Chen", "Nishanth Dikkala", "Gladys Tyen", "Xin Liu", "Uri Shalit", "Silvia Chiappa", "Kate Olszewska", "Yi Tay", "Vinh Q. Tran", "Quoc V. Le", "Orhan Firat" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/BBEH/bbeh", "provider": "BBEH" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2502.19187" }, "code_url": null, "metadata": { "source_search": "Mehran Kazemi, Bahare Fatemi, Hritik Bansal et al. Google DeepMind Original paper authors BIG-Bench Extra Hard Mehran Kazemi Bahare Fatemi Hritik Bansal John Palowitch Chrysovalantis Anastasiou Sanket Vaibhav Mehta Lalit K. Jain Virginia Aglietti Disha Jindal Peter Chen Nishanth Dikkala Gladys Tyen Xin Liu Uri Shalit Silvia Chiappa Kate Olszewska Yi Tay Vinh Q. Tran Quoc V. Le Orhan Firat evaluation-family" } }, { "id": "inspect:bbeh_mini", "name": "Bbeh Mini", "slug": "inspect--bbeh-mini", "organisation": "Mehran Kazemi, Bahare Fatemi, Hritik Bansal et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh", "topics": [ "general-capability" ], "description": "A reasoning capability dataset that replaces each task in BIG-Bench-Hard with a novel task that probes a similar reasoning capability but exhibits significantly increased difficulty.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2502.19187", "primary": true }, { "kind": "paper", "label": "Paper: BIG-Bench Extra Hard", "url": "https://arxiv.org/abs/2502.19187", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: BBEH", "url": "https://huggingface.co/datasets/BBEH/bbeh", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2502.19187", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh/eval.yaml", "family_title": "BIG-Bench Extra Hard", "group": "Reasoning", "version": "2-A", "contributors": [ "jeqcho" ], "tags": [ "Reasoning" ], "task": { "name": "bbeh_mini", "dataset_samples": 460, "comment": null, "support_only": false }, "tasks": [ { "name": "bbeh", "dataset_samples": 4520, "comment": null, "support_only": false }, { "name": "bbeh_mini", "dataset_samples": 460, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 460, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "BBEH/bbeh", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:bbeh", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mehran Kazemi, Bahare Fatemi, Hritik Bansal et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2502.19187" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2502.19187", "arxiv_id": "2502.19187", "title": "BIG-Bench Extra Hard", "authors": [ "Mehran Kazemi", "Bahare Fatemi", "Hritik Bansal", "John Palowitch", "Chrysovalantis Anastasiou", "Sanket Vaibhav Mehta", "Lalit K. Jain", "Virginia Aglietti", "Disha Jindal", "Peter Chen", "Nishanth Dikkala", "Gladys Tyen", "Xin Liu", "Uri Shalit", "Silvia Chiappa", "Kate Olszewska", "Yi Tay", "Vinh Q. Tran", "Quoc V. Le", "Orhan Firat" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2502.19187", "arxiv_id": "2502.19187", "title": "BIG-Bench Extra Hard", "authors": [ "Mehran Kazemi", "Bahare Fatemi", "Hritik Bansal", "John Palowitch", "Chrysovalantis Anastasiou", "Sanket Vaibhav Mehta", "Lalit K. Jain", "Virginia Aglietti", "Disha Jindal", "Peter Chen", "Nishanth Dikkala", "Gladys Tyen", "Xin Liu", "Uri Shalit", "Silvia Chiappa", "Kate Olszewska", "Yi Tay", "Vinh Q. Tran", "Quoc V. Le", "Orhan Firat" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2502.19187", "arxiv_id": "2502.19187", "title": "BIG-Bench Extra Hard", "authors": [ "Mehran Kazemi", "Bahare Fatemi", "Hritik Bansal", "John Palowitch", "Chrysovalantis Anastasiou", "Sanket Vaibhav Mehta", "Lalit K. Jain", "Virginia Aglietti", "Disha Jindal", "Peter Chen", "Nishanth Dikkala", "Gladys Tyen", "Xin Liu", "Uri Shalit", "Silvia Chiappa", "Kate Olszewska", "Yi Tay", "Vinh Q. Tran", "Quoc V. Le", "Orhan Firat" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbeh/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/BBEH/bbeh", "provider": "BBEH" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2502.19187" }, "code_url": null, "metadata": { "source_search": "Mehran Kazemi, Bahare Fatemi, Hritik Bansal et al. Google DeepMind Original paper authors BIG-Bench Extra Hard Mehran Kazemi Bahare Fatemi Hritik Bansal John Palowitch Chrysovalantis Anastasiou Sanket Vaibhav Mehta Lalit K. Jain Virginia Aglietti Disha Jindal Peter Chen Nishanth Dikkala Gladys Tyen Xin Liu Uri Shalit Silvia Chiappa Kate Olszewska Yi Tay Vinh Q. Tran Quoc V. Le Orhan Firat evaluation-family" } }, { "id": "inspect:bbh", "name": "BBH: Challenging BIG-Bench Tasks", "slug": "inspect--bbh", "organisation": "Mirac Suzgun, Nathan Scales, Nathanael Schärli et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbh", "topics": [ "general-capability" ], "description": "Tests AI models on a suite of 23 challenging BIG-Bench tasks that previously proved difficult even for advanced language models to solve.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbh/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2210.09261", "primary": true }, { "kind": "paper", "label": "Paper: Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them", "url": "https://arxiv.org/abs/2210.09261", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbh/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Joschka", "url": "https://huggingface.co/datasets/Joschka/big_bench_hard", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2210.09261", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbh/eval.yaml", "family_title": "BBH: Challenging BIG-Bench Tasks", "group": "Reasoning", "version": "2-A", "contributors": [ "JoschkaCBraun" ], "tags": [ "Reasoning" ], "task": { "name": "bbh", "dataset_samples": 250, "comment": null, "support_only": false }, "tasks": [ { "name": "bbh", "dataset_samples": 250, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 250, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "Joschka/big_bench_hard", "fetch_method": "hf_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "Joschka/big_bench_hard", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:bbh", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mirac Suzgun, Nathan Scales, Nathanael Schärli et al.", "organization": "Google; Google Research", "organizations": [ "Google", "Google Research" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2210.09261" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2210.09261", "arxiv_id": "2210.09261", "title": "Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them", "authors": [ "Mirac Suzgun", "Nathan Scales", "Nathanael Schärli", "Sebastian Gehrmann", "Yi Tay", "Hyung Won Chung", "Aakanksha Chowdhery", "Quoc V. Le", "Ed H. Chi", "Denny Zhou", "Jason Wei" ], "organizations": [ "Google", "Google Research" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2210.09261", "arxiv_id": "2210.09261", "title": "Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them", "authors": [ "Mirac Suzgun", "Nathan Scales", "Nathanael Schärli", "Sebastian Gehrmann", "Yi Tay", "Hyung Won Chung", "Aakanksha Chowdhery", "Quoc V. Le", "Ed H. Chi", "Denny Zhou", "Jason Wei" ], "organizations": [ "Google", "Google Research" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbh/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/Joschka/big_bench_hard", "provider": "Joschka" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2210.09261" }, "code_url": null, "metadata": { "source_search": "Mirac Suzgun, Nathan Scales, Nathanael Schärli et al. Google; Google Research Original paper authors Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them Mirac Suzgun Nathan Scales Nathanael Schärli Sebastian Gehrmann Yi Tay Hyung Won Chung Aakanksha Chowdhery Quoc V. Le Ed H. Chi Denny Zhou Jason Wei evaluation-task" } }, { "id": "inspect:bbq", "name": "BBQ: Bias Benchmark for Question Answering", "slug": "inspect--bbq", "organisation": "Alicia Parrish, Angelica Chen, Nikita Nangia et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbq", "topics": [ "general-capability" ], "description": "A dataset for evaluating bias in question answering models across multiple social dimensions.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbq/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2110.08193", "primary": true }, { "kind": "paper", "label": "Paper: BBQ: A hand-built bias benchmark for question answering", "url": "https://arxiv.org/abs/2110.08193", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbq/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Heegyu", "url": "https://huggingface.co/datasets/heegyu/bbq", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2110.08193", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbq/eval.yaml", "family_title": "BBQ: Bias Benchmark for Question Answering", "group": "Bias", "version": "2-A", "contributors": [ "harshraj172", "shubhobm" ], "tags": [ "Bias" ], "task": { "name": "bbq", "dataset_samples": 58492, "comment": null, "support_only": false }, "tasks": [ { "name": "bbq", "dataset_samples": 58492, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 58492, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "heegyu/bbq", "fetch_method": "snapshot_download", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:bbq", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Alicia Parrish, Angelica Chen, Nikita Nangia et al.", "organization": "New York University; NYU", "organizations": [ "New York University", "NYU" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2110.08193" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2110.08193", "arxiv_id": "2110.08193", "title": "BBQ: A hand-built bias benchmark for question answering", "authors": [ "Alicia Parrish", "Angelica Chen", "Nikita Nangia", "Vishakh Padmakumar", "Jason Phang", "Jana Thompson", "Phu Mon Htut", "Samuel R. Bowman" ], "organizations": [ "New York University", "NYU" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2110.08193", "arxiv_id": "2110.08193", "title": "BBQ: A hand-built bias benchmark for question answering", "authors": [ "Alicia Parrish", "Angelica Chen", "Nikita Nangia", "Vishakh Padmakumar", "Jason Phang", "Jana Thompson", "Phu Mon Htut", "Samuel R. Bowman" ], "organizations": [ "New York University", "NYU" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bbq/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/heegyu/bbq", "provider": "Heegyu" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2110.08193" }, "code_url": null, "metadata": { "source_search": "Alicia Parrish, Angelica Chen, Nikita Nangia et al. New York University; NYU Original paper authors BBQ: A hand-built bias benchmark for question answering Alicia Parrish Angelica Chen Nikita Nangia Vishakh Padmakumar Jason Phang Jana Thompson Phu Mon Htut Samuel R. Bowman evaluation-task" } }, { "id": "inspect:bfcl", "name": "BFCL", "slug": "inspect--bfcl", "organisation": "Berkeley Gorilla / UC Berkeley", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bfcl", "topics": [ "general-capability" ], "description": "Evaluates LLM function/tool-calling ability on a simplified split of the Berkeley Function-Calling Leaderboard (BFCL).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bfcl/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://gorilla.cs.berkeley.edu/leaderboard", "primary": true }, { "kind": "paper", "label": "Paper: The Berkeley Function Calling Leaderboard (BFCL): From Tool Use to Agentic Evaluation", "url": "https://openreview.net/forum?id=2GmDdhBdDk", "primary": false }, { "kind": "paper", "label": "Paper", "url": "https://openreview.net/pdf?id=2GmDdhBdDk", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ShishirPatil/gorilla", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bfcl/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://gorilla.cs.berkeley.edu/leaderboard", "primary": false } ], "resource_count": 6, "paper_url": "https://openreview.net/forum?id=2GmDdhBdDk", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bfcl/eval.yaml", "family_title": "BFCL: Berkeley Function-Calling Leaderboard", "group": "Assistants", "version": "7-B", "contributors": [ "alex-remedios-aisi", "rutvi02" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "bfcl", "dataset_samples": 4981, "comment": "V4 agentic categories are opt-in via -T categories=...: web_search adds 200 samples (requires SERPAPI_API_KEY) and memory adds 465 samples (requires a bfcl_prereqs snapshot referenced via -T snapshot_id=).", "support_only": false }, "tasks": [ { "name": "bfcl", "dataset_samples": 4981, "comment": "V4 agentic categories are opt-in via -T categories=...: web_search adds 200 samples (requires SERPAPI_API_KEY) and memory adds 465 samples (requires a bfcl_prereqs snapshot referenced via -T snapshot_id=).", "support_only": false }, { "name": "bfcl_prereqs", "dataset_samples": 111, "comment": "Setup task for V4 memory categories. Produces no scored metrics — its purpose is to drive the model through memory-building conversations and flush the resulting backend state to disk for `bfcl` to consume. Default sample count is 111 (37 prereq conversations per variant × 3 memory variants). Run before `bfcl` whenever memory categories are requested, passing the same snapshot_id to both tasks.", "support_only": true } ], "protocol": { "version": "7-B", "dataset_samples": 4981, "dependency": "bfcl", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/ShishirPatil/gorilla.git", "fetch_method": "git_clone", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:bfcl", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "BFCL is developed by the Berkeley Gorilla project. The Inspect task is a maintained implementation of a selected BFCL split." }, "origin": { "name": "Berkeley Gorilla / UC Berkeley", "organization": "University of California, Berkeley", "entity_type": "research_project", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://gorilla.cs.berkeley.edu/leaderboard", "organizations": [ "University of California, Berkeley" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://openreview.net/forum?id=2GmDdhBdDk", "arxiv_id": null, "title": "The Berkeley Function Calling Leaderboard (BFCL): From Tool Use to Agentic Evaluation", "authors": [], "organizations": [], "source": "openreview", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://openreview.net/forum?id=2GmDdhBdDk", "arxiv_id": null, "title": "The Berkeley Function Calling Leaderboard (BFCL): From Tool Use to Agentic Evaluation", "authors": [], "organizations": [], "source": "openreview", "metadata_status": "resolved" }, { "status": "present", "url": "https://openreview.net/pdf?id=2GmDdhBdDk", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ShishirPatil/gorilla", "owner": "Berkeley Gorilla", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bfcl/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://gorilla.cs.berkeley.edu/leaderboard" }, "code_url": "https://github.com/ShishirPatil/gorilla", "metadata": { "source_search": "Berkeley Gorilla / UC Berkeley University of California, Berkeley Evaluation developed by The Berkeley Function Calling Leaderboard (BFCL): From Tool Use to Agentic Evaluation Berkeley Gorilla evaluation-task" } }, { "id": "inspect:bigcodebench", "name": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "slug": "inspect--bigcodebench", "organisation": "Terry Yue Zhuo, Minh Chien Vu, Jenny Chim et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bigcodebench", "topics": [ "general-capability" ], "description": "Python coding benchmark with 1,140 diverse questions drawing on numerous python libraries.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bigcodebench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2406.15877", "primary": true }, { "kind": "paper", "label": "Paper: BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bigcodebench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: BigCode Project", "url": "https://huggingface.co/datasets/bigcode/bigcodebench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2406.15877", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bigcodebench/eval.yaml", "family_title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "group": "Coding", "version": "3-B", "contributors": [ "tim-hua-01" ], "tags": [ "Coding" ], "task": { "name": "bigcodebench", "dataset_samples": 1140, "comment": null, "support_only": false }, "tasks": [ { "name": "bigcodebench", "dataset_samples": 1140, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 1140, "dependency": null, "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "bigcode/bigcodebench", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:bigcodebench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Terry Yue Zhuo, Minh Chien Vu, Jenny Chim et al.", "organization": "University of Notre Dame; University of North Carolina at Chapel Hill; UC Berkeley; Microsoft Research; Indian Institute of Technology, Bombay", "organizations": [ "University of Notre Dame", "University of North Carolina at Chapel Hill", "UC Berkeley", "Microsoft Research", "Indian Institute of Technology, Bombay" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2406.15877" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2406.15877", "arxiv_id": "2406.15877", "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "authors": [ "Terry Yue Zhuo", "Minh Chien Vu", "Jenny Chim", "Han Hu", "Wenhao Yu", "Ratnadira Widyasari", "Imam Nur Bani Yusuf", "Haolan Zhan", "Junda He", "Indraneil Paul", "Simon Brunner", "Chen Gong", "Thong Hoang", "Armel Randy Zebaze", "Xiaoheng Hong", "Wen-Ding Li", "Jean Kaddour", "Ming Xu", "Zhihan Zhang", "Prateek Yadav", "Naman Jain", "Alex Gu", "Zhoujun Cheng", "Jiawei Liu", "Qian Liu", "Zijian Wang", "Binyuan Hui", "Niklas Muennighoff", "David Lo", "Daniel Fried", "Xiaoning Du", "Harm de Vries", "Leandro Von Werra" ], "organizations": [ "University of Notre Dame", "University of North Carolina at Chapel Hill", "UC Berkeley", "Microsoft Research", "Indian Institute of Technology, Bombay" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2406.15877", "arxiv_id": "2406.15877", "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "authors": [ "Terry Yue Zhuo", "Minh Chien Vu", "Jenny Chim", "Han Hu", "Wenhao Yu", "Ratnadira Widyasari", "Imam Nur Bani Yusuf", "Haolan Zhan", "Junda He", "Indraneil Paul", "Simon Brunner", "Chen Gong", "Thong Hoang", "Armel Randy Zebaze", "Xiaoheng Hong", "Wen-Ding Li", "Jean Kaddour", "Ming Xu", "Zhihan Zhang", "Prateek Yadav", "Naman Jain", "Alex Gu", "Zhoujun Cheng", "Jiawei Liu", "Qian Liu", "Zijian Wang", "Binyuan Hui", "Niklas Muennighoff", "David Lo", "Daniel Fried", "Xiaoning Du", "Harm de Vries", "Leandro Von Werra" ], "organizations": [ "University of Notre Dame", "University of North Carolina at Chapel Hill", "UC Berkeley", "Microsoft Research", "Indian Institute of Technology, Bombay" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bigcodebench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/bigcode/bigcodebench", "provider": "BigCode Project" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2406.15877" }, "code_url": null, "metadata": { "source_search": "Terry Yue Zhuo, Minh Chien Vu, Jenny Chim et al. University of Notre Dame; University of North Carolina at Chapel Hill; UC Berkeley; Microsoft Research; Indian Institute of Technology, Bombay Original paper authors BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions Terry Yue Zhuo Minh Chien Vu Jenny Chim Han Hu Wenhao Yu Ratnadira Widyasari Imam Nur Bani Yusuf Haolan Zhan Junda He Indraneil Paul Simon Brunner Chen Gong Thong Hoang Armel Randy Zebaze Xiaoheng Hong Wen-Ding Li Jean Kaddour Ming Xu Zhihan Zhang Prateek Yadav Naman Jain Alex Gu Zhoujun Cheng Jiawei Liu Qian Liu Zijian Wang Binyuan Hui Niklas Muennighoff David Lo Daniel Fried Xiaoning Du Harm de Vries Leandro Von Werra evaluation-task" } }, { "id": "register:bixbench", "name": "BixBench", "slug": "register--bixbench", "organisation": "Ludovico Mitchener, Jon M Laurent, Alex Andonian et al.", "source_type": "inspect-register", "source_url": "https://github.com/concordia-ai/concordia_evals", "topics": [ "bio-cbrn" ], "description": "BixBench evaluates LLM agents on open-ended bioinformatics data analysis tasks. Agents receive biological datasets and must produce Jupyter notebooks to analyze data and answer research questions. The eval uses a sandboxed IPython kernel environment where agents execute code interactively. Scoring is performed via an LLM judge comparing agent-generated answers against reference answers. The dataset is hosted on HuggingFace (futurehouse/BixBench) and tasks cover diverse bioinformatics analysis scenarios.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/bixbench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2503.00096", "primary": true }, { "kind": "paper", "label": "Paper: BixBench: a Comprehensive Benchmark for LLM-based Agents in Computational Biology", "url": "https://arxiv.org/abs/2503.00096", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/concordia-ai/concordia_evals", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/bixbench/eval.yaml", "primary": false } ], "paper_title": "BixBench a Comprehensive Benchmark for LLM-based Agents in Computational Biology", "paper_url": "https://arxiv.org/abs/2503.00096", "code_url": "https://github.com/concordia-ai/concordia_evals", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/bixbench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "jingedawang" ], "tasks": [ { "name": "bixbench", "path": "src/concordia_evals/bixbench/bixbench.py" } ], "protocol": { "implementation_commit": "e99597c8a5d68c85a5bbbb00020d7d1c813ad0e1", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:bixbench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Ludovico Mitchener, Jon M Laurent, Alex Andonian et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2503.00096" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2503.00096", "arxiv_id": "2503.00096", "title": "BixBench: a Comprehensive Benchmark for LLM-based Agents in Computational Biology", "authors": [ "Ludovico Mitchener", "Jon M Laurent", "Alex Andonian", "Benjamin Tenmann", "Siddharth Narayanan", "Geemi P Wellawatte", "Andrew White", "Lorenzo Sani", "Samuel G Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2503.00096", "arxiv_id": "2503.00096", "title": "BixBench: a Comprehensive Benchmark for LLM-based Agents in Computational Biology", "authors": [ "Ludovico Mitchener", "Jon M Laurent", "Alex Andonian", "Benjamin Tenmann", "Siddharth Narayanan", "Geemi P Wellawatte", "Andrew White", "Lorenzo Sani", "Samuel G Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/concordia-ai/concordia_evals", "owner": "Concordia Ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/bixbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2503.00096" }, "metadata": { "source_search": "Ludovico Mitchener, Jon M Laurent, Alex Andonian et al. Original paper authors BixBench: a Comprehensive Benchmark for LLM-based Agents in Computational Biology Ludovico Mitchener Jon M Laurent Alex Andonian Benjamin Tenmann Siddharth Narayanan Geemi P Wellawatte Andrew White Lorenzo Sani Samuel G Rodriques Concordia Ai evaluation-package" } }, { "id": "inspect:bold", "name": "BOLD: Bias in Open-ended Language Generation Dataset", "slug": "inspect--bold", "organisation": "Jwala Dhamala, Tony Sun, Varun Kumar et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bold", "topics": [ "general-capability" ], "description": "A dataset to measure fairness in open-ended text generation, covering five domains: profession, gender, race, religious ideologies, and political ideologies.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bold/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2101.11718", "primary": true }, { "kind": "paper", "label": "Paper: BOLD: Dataset and Metrics for Measuring Biases in Open-Ended Language Generation", "url": "https://arxiv.org/abs/2101.11718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bold/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: AmazonScience", "url": "https://huggingface.co/datasets/AmazonScience/bold", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2101.11718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bold/eval.yaml", "family_title": "BOLD: Bias in Open-ended Language Generation Dataset", "group": "Bias", "version": "2-A", "contributors": [ "harshraj172", "shubhobm" ], "tags": [ "Bias" ], "task": { "name": "bold", "dataset_samples": 7200, "comment": null, "support_only": false }, "tasks": [ { "name": "bold", "dataset_samples": 7200, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 7200, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "AmazonScience/bold", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:bold", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jwala Dhamala, Tony Sun, Varun Kumar et al.", "organization": "Amazon; Harvard University; UCLA", "organizations": [ "Amazon", "Harvard University", "UCLA" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2101.11718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2101.11718", "arxiv_id": "2101.11718", "title": "BOLD: Dataset and Metrics for Measuring Biases in Open-Ended Language Generation", "authors": [ "Jwala Dhamala", "Tony Sun", "Varun Kumar", "Satyapriya Krishna", "Yada Pruksachatkun", "Kai-Wei Chang", "Rahul Gupta" ], "organizations": [ "Amazon", "Harvard University", "UCLA" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2101.11718", "arxiv_id": "2101.11718", "title": "BOLD: Dataset and Metrics for Measuring Biases in Open-Ended Language Generation", "authors": [ "Jwala Dhamala", "Tony Sun", "Varun Kumar", "Satyapriya Krishna", "Yada Pruksachatkun", "Kai-Wei Chang", "Rahul Gupta" ], "organizations": [ "Amazon", "Harvard University", "UCLA" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/bold/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/AmazonScience/bold", "provider": "AmazonScience" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2101.11718" }, "code_url": null, "metadata": { "source_search": "Jwala Dhamala, Tony Sun, Varun Kumar et al. Amazon; Harvard University; UCLA Original paper authors BOLD: Dataset and Metrics for Measuring Biases in Open-Ended Language Generation Jwala Dhamala Tony Sun Varun Kumar Satyapriya Krishna Yada Pruksachatkun Kai-Wei Chang Rahul Gupta evaluation-task" } }, { "id": "inspect:boolq", "name": "BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions", "slug": "inspect--boolq", "organisation": "Christopher Clark, Kenton Lee, Ming-Wei Chang et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/boolq", "topics": [ "general-capability" ], "description": "Reading comprehension dataset that queries for complex, non-factoid information, and require difficult entailment-like inference to solve.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/boolq/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1905.10044", "primary": true }, { "kind": "paper", "label": "Paper: BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions", "url": "https://arxiv.org/abs/1905.10044", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/boolq/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Google Research", "url": "https://huggingface.co/datasets/google/boolq", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1905.10044", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/boolq/eval.yaml", "family_title": "BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions", "group": "Reasoning", "version": "2-A", "contributors": [ "seddy-aisi" ], "tags": [ "Reasoning" ], "task": { "name": "boolq", "dataset_samples": 3270, "comment": null, "support_only": false }, "tasks": [ { "name": "boolq", "dataset_samples": 3270, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 3270, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "google/boolq", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:boolq", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Christopher Clark, Kenton Lee, Ming-Wei Chang et al.", "organization": "Allen Institute for AI", "organizations": [ "Allen Institute for AI" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1905.10044" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1905.10044", "arxiv_id": "1905.10044", "title": "BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions", "authors": [ "Christopher Clark", "Kenton Lee", "Ming-Wei Chang", "Tom Kwiatkowski", "Michael Collins", "Kristina Toutanova" ], "organizations": [ "Allen Institute for AI" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1905.10044", "arxiv_id": "1905.10044", "title": "BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions", "authors": [ "Christopher Clark", "Kenton Lee", "Ming-Wei Chang", "Tom Kwiatkowski", "Michael Collins", "Kristina Toutanova" ], "organizations": [ "Allen Institute for AI" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/boolq/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/google/boolq", "provider": "Google Research" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1905.10044" }, "code_url": null, "metadata": { "source_search": "Christopher Clark, Kenton Lee, Ming-Wei Chang et al. Allen Institute for AI Original paper authors BoolQ: Exploring the Surprising Difficulty of Natural Yes/No Questions Christopher Clark Kenton Lee Ming-Wei Chang Tom Kwiatkowski Michael Collins Kristina Toutanova evaluation-task" } }, { "id": "register:brokenmath", "name": "BrokenMath", "slug": "register--brokenmath", "organisation": "Ivo Petrov, Jasper Dekoninck, Martin Vechev", "source_type": "inspect-register", "source_url": "https://github.com/Vedant-Agarwal/inspect-brokenmath", "topics": [ "general-capability" ], "description": "BrokenMath measures sycophancy in LLMs by presenting adversarially falsified recent olympiad theorems and asking models to prove them. This eval runs the dataset's `benchmark` split — 451 adversarial, proof-style problems (perturbed from 2025 competition sources and verified by an IMO medalist). A 4-way LLM-as-judge classifies each response as correct/detected/corrected/incorrect; the primary metric is the sycophancy rate (fraction judged incorrect — the model attempted a proof of the false statement without flagging the flawed premise).", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/brokenmath/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2510.04721", "primary": true }, { "kind": "paper", "label": "Paper: BrokenMath: A Benchmark for Sycophancy in Theorem Proving with LLMs", "url": "https://arxiv.org/abs/2510.04721", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Vedant-Agarwal/inspect-brokenmath", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/brokenmath/eval.yaml", "primary": false } ], "paper_title": "BrokenMath: A Benchmark for Sycophancy in Theorem Proving with LLMs", "paper_url": "https://arxiv.org/abs/2510.04721", "code_url": "https://github.com/Vedant-Agarwal/inspect-brokenmath", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/brokenmath/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "Vedant-Agarwal" ], "tasks": [ { "name": "brokenmath", "path": "src/brokenmath/brokenmath.py" } ], "protocol": { "implementation_commit": "d37b80bb16c9d98df990a7edd92323a1d73822bb", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:brokenmath", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Ivo Petrov, Jasper Dekoninck, Martin Vechev", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2510.04721" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2510.04721", "arxiv_id": "2510.04721", "title": "BrokenMath: A Benchmark for Sycophancy in Theorem Proving with LLMs", "authors": [ "Ivo Petrov", "Jasper Dekoninck", "Martin Vechev" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2510.04721", "arxiv_id": "2510.04721", "title": "BrokenMath: A Benchmark for Sycophancy in Theorem Proving with LLMs", "authors": [ "Ivo Petrov", "Jasper Dekoninck", "Martin Vechev" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2510.04721", "arxiv_id": "2510.04721", "title": "BrokenMath: A Benchmark for Sycophancy in Theorem Proving with LLMs", "authors": [ "Ivo Petrov", "Jasper Dekoninck", "Martin Vechev" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Vedant-Agarwal/inspect-brokenmath", "owner": "Vedant Agarwal", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/brokenmath/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2510.04721" }, "metadata": { "source_search": "Ivo Petrov, Jasper Dekoninck, Martin Vechev Original paper authors BrokenMath: A Benchmark for Sycophancy in Theorem Proving with LLMs Ivo Petrov Jasper Dekoninck Martin Vechev Vedant Agarwal evaluation-package" } }, { "id": "inspect:browse_comp", "name": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents", "slug": "inspect--browse-comp", "organisation": "OpenAI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/browse_comp", "topics": [ "autonomy-agents" ], "description": "A benchmark for evaluating agents' ability to browse the web. The dataset consists of challenging questions that generally require web-access to answer correctly.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/browse_comp/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://openai.com/index/browsecomp", "primary": true }, { "kind": "paper", "label": "Paper: BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents", "url": "https://arxiv.org/abs/2504.12516", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://openaipublic.blob.core.windows.net/simple-evals/browse_comp_test_set.csv", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/browse_comp/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://openai.com/index/browsecomp", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2504.12516", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/browse_comp/eval.yaml", "family_title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents", "group": "Assistants", "version": "3-B", "contributors": [ "AnselmC" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "browse_comp", "dataset_samples": 1266, "comment": null, "support_only": false }, "tasks": [ { "name": "browse_comp", "dataset_samples": 1266, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 1266, "dependency": null, "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "direct_url", "source": "https://openaipublic.blob.core.windows.net/simple-evals/browse_comp_test_set.csv", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:browse_comp", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://openai.com/index/browsecomp", "organizations": [ "OpenAI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2504.12516", "arxiv_id": "2504.12516", "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents", "authors": [ "Jason Wei", "Zhiqing Sun", "Spencer Papay", "Scott McKinney", "Jeffrey Han", "Isa Fulford", "Hyung Won Chung", "Alex Tachard Passos", "William Fedus", "Amelia Glaese" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2504.12516", "arxiv_id": "2504.12516", "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents", "authors": [ "Jason Wei", "Zhiqing Sun", "Spencer Papay", "Scott McKinney", "Jeffrey Han", "Isa Fulford", "Hyung Won Chung", "Alex Tachard Passos", "William Fedus", "Amelia Glaese" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2504.12516", "arxiv_id": "2504.12516", "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents", "authors": [ "Jason Wei", "Zhiqing Sun", "Spencer Papay", "Scott McKinney", "Jeffrey Han", "Isa Fulford", "Hyung Won Chung", "Alex Tachard Passos", "William Fedus", "Amelia Glaese" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://openaipublic.blob.core.windows.net/simple-evals/browse_comp_test_set.csv", "owner": "openaipublic.blob.core.windows.net", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/browse_comp/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://openai.com/index/browsecomp" }, "code_url": "https://openaipublic.blob.core.windows.net/simple-evals/browse_comp_test_set.csv", "metadata": { "source_search": "OpenAI OpenAI Evaluation developed by BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents Jason Wei Zhiqing Sun Spencer Papay Scott McKinney Jeffrey Han Isa Fulford Hyung Won Chung Alex Tachard Passos William Fedus Amelia Glaese openaipublic.blob.core.windows.net evaluation-task" } }, { "id": "register:castle", "name": "CASTLE", "slug": "register--castle", "organisation": "Richard A. Dubniczky, Krisztofer Zoltán Horvát, Tamás Bisztray et al.", "source_type": "inspect-register", "source_url": "https://github.com/AI-Sec-dev/inspect-eval-castle", "topics": [ "general-capability" ], "description": "CASTLE evaluates LLM vulnerability detection on 250 hand-crafted, compilable C programs covering 25 CWE types (6 vulnerable, 4 non-vulnerable per CWE). Models receive a system prompt requesting JSON output indicating vulnerability presence and CWE number. Scoring uses the CASTLE Score: +5 for correct vulnerability detection (minus 1 per extra false positive reported), +2 for correct true-negative identification, and -1 per false positive otherwise. TPR and FPR are also reported.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/castle/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2503.09433", "primary": true }, { "kind": "paper", "label": "Paper: CASTLE: Benchmarking Dataset for Static Code Analyzers and LLMs towards CWE Detection", "url": "https://arxiv.org/abs/2503.09433", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/AI-Sec-dev/inspect-eval-castle", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/castle/eval.yaml", "primary": false } ], "paper_title": "CASTLE: Benchmarking Dataset for Static Code Analyzers and LLMs towards CWE Detection", "paper_url": "https://arxiv.org/abs/2503.09433", "code_url": "https://github.com/AI-Sec-dev/inspect-eval-castle", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/castle/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "AI-Sec-dev" ], "tasks": [ { "name": "castle", "path": "src/castle/castle.py" } ], "protocol": { "implementation_commit": "dc4d5aa275120b6cc7943b54b67e2278c80f6f4c", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:castle", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Richard A. Dubniczky, Krisztofer Zoltán Horvát, Tamás Bisztray et al.", "organization": "Eötvös Loránd University; Technology Innovation Institute", "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2503.09433" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2503.09433", "arxiv_id": "2503.09433", "title": "CASTLE: Benchmarking Dataset for Static Code Analyzers and LLMs towards CWE Detection", "authors": [ "Richard A. Dubniczky", "Krisztofer Zoltán Horvát", "Tamás Bisztray", "Mohamed Amine Ferrag", "Lucas C. Cordeiro", "Norbert Tihanyi" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2503.09433", "arxiv_id": "2503.09433", "title": "CASTLE: Benchmarking Dataset for Static Code Analyzers and LLMs towards CWE Detection", "authors": [ "Richard A. Dubniczky", "Krisztofer Zoltán Horvát", "Tamás Bisztray", "Mohamed Amine Ferrag", "Lucas C. Cordeiro", "Norbert Tihanyi" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/AI-Sec-dev/inspect-eval-castle", "owner": "AI Sec Dev", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/castle/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2503.09433" }, "metadata": { "source_search": "Richard A. Dubniczky, Krisztofer Zoltán Horvát, Tamás Bisztray et al. Eötvös Loránd University; Technology Innovation Institute Original paper authors CASTLE: Benchmarking Dataset for Static Code Analyzers and LLMs towards CWE Detection Richard A. Dubniczky Krisztofer Zoltán Horvát Tamás Bisztray Mohamed Amine Ferrag Lucas C. Cordeiro Norbert Tihanyi AI Sec Dev evaluation-package" } }, { "id": "inspect:chembench", "name": "ChemBench: Are large language models superhuman chemists?", "slug": "inspect--chembench", "organisation": "Adrian Mirza, Nawaf Alampara, Sreekanth Kunchapu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/chembench", "topics": [ "bio-cbrn" ], "description": "ChemBench is designed to reveal limitations of current frontier models for use in the chemical sciences. It consists of 2786 question-answer pairs compiled from diverse sources. Our corpus measures reasoning, knowledge and intuition across a large fraction of the topics taught in undergraduate and graduate chemistry curricula. It can be used to evaluate any system that can return text (i.e., including tool-augmented systems).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/chembench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.01475", "primary": true }, { "kind": "paper", "label": "Paper: Are large language models superhuman chemists?", "url": "https://arxiv.org/abs/2404.01475", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/chembench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Jablonkagroup", "url": "https://huggingface.co/datasets/jablonkagroup/ChemBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.01475", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/chembench/eval.yaml", "family_title": "ChemBench: Are large language models superhuman chemists?", "group": "Knowledge", "version": "2-B", "contributors": [ "Esther-Guo", "MrtinoRG", "r-fedorov" ], "tags": [ "Knowledge" ], "task": { "name": "chembench", "dataset_samples": 2786, "comment": null, "support_only": false }, "tasks": [ { "name": "chembench", "dataset_samples": 2786, "comment": null, "support_only": false } ], "protocol": { "version": "2-B", "dataset_samples": 2786, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "jablonkagroup/ChemBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:chembench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Adrian Mirza, Nawaf Alampara, Sreekanth Kunchapu et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.01475" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.01475", "arxiv_id": "2404.01475", "title": "Are large language models superhuman chemists?", "authors": [ "Adrian Mirza", "Nawaf Alampara", "Sreekanth Kunchapu", "Martiño Ríos-García", "Benedict Emoekabu", "Aswanth Krishnan", "Tanya Gupta", "Mara Schilling-Wilhelmi", "Macjonathan Okereke", "Anagha Aneesh", "Amir Mohammad Elahi", "Mehrdad Asgari", "Juliane Eberhardt", "Hani M. Elbeheiry", "María Victoria Gil", "Maximilian Greiner", "Caroline T. Holick", "Christina Glaubitz", "Tim Hoffmann", "Abdelrahman Ibrahim", "Lea C. Klepsch", "Yannik Köster", "Fabian Alexander Kreth", "Jakob Meyer", "Santiago Miret", "Jan Matthias Peschel", "Michael Ringleb", "Nicole Roesner", "Johanna Schreiber", "Ulrich S. Schubert", "Leanne M. Stafast", "Dinga Wonanke", "Michael Pieler", "Philippe Schwaller", "Kevin Maik Jablonka" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.01475", "arxiv_id": "2404.01475", "title": "Are large language models superhuman chemists?", "authors": [ "Adrian Mirza", "Nawaf Alampara", "Sreekanth Kunchapu", "Martiño Ríos-García", "Benedict Emoekabu", "Aswanth Krishnan", "Tanya Gupta", "Mara Schilling-Wilhelmi", "Macjonathan Okereke", "Anagha Aneesh", "Amir Mohammad Elahi", "Mehrdad Asgari", "Juliane Eberhardt", "Hani M. Elbeheiry", "María Victoria Gil", "Maximilian Greiner", "Caroline T. Holick", "Christina Glaubitz", "Tim Hoffmann", "Abdelrahman Ibrahim", "Lea C. Klepsch", "Yannik Köster", "Fabian Alexander Kreth", "Jakob Meyer", "Santiago Miret", "Jan Matthias Peschel", "Michael Ringleb", "Nicole Roesner", "Johanna Schreiber", "Ulrich S. Schubert", "Leanne M. Stafast", "Dinga Wonanke", "Michael Pieler", "Philippe Schwaller", "Kevin Maik Jablonka" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2404.01475", "arxiv_id": "2404.01475", "title": "Are large language models superhuman chemists?", "authors": [ "Adrian Mirza", "Nawaf Alampara", "Sreekanth Kunchapu", "Martiño Ríos-García", "Benedict Emoekabu", "Aswanth Krishnan", "Tanya Gupta", "Mara Schilling-Wilhelmi", "Macjonathan Okereke", "Anagha Aneesh", "Amir Mohammad Elahi", "Mehrdad Asgari", "Juliane Eberhardt", "Hani M. Elbeheiry", "María Victoria Gil", "Maximilian Greiner", "Caroline T. Holick", "Christina Glaubitz", "Tim Hoffmann", "Abdelrahman Ibrahim", "Lea C. Klepsch", "Yannik Köster", "Fabian Alexander Kreth", "Jakob Meyer", "Santiago Miret", "Jan Matthias Peschel", "Michael Ringleb", "Nicole Roesner", "Johanna Schreiber", "Ulrich S. Schubert", "Leanne M. Stafast", "Dinga Wonanke", "Michael Pieler", "Philippe Schwaller", "Kevin Maik Jablonka" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/chembench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/jablonkagroup/ChemBench", "provider": "Jablonkagroup" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.01475" }, "code_url": null, "metadata": { "source_search": "Adrian Mirza, Nawaf Alampara, Sreekanth Kunchapu et al. Original paper authors Are large language models superhuman chemists? Adrian Mirza Nawaf Alampara Sreekanth Kunchapu Martiño Ríos-García Benedict Emoekabu Aswanth Krishnan Tanya Gupta Mara Schilling-Wilhelmi Macjonathan Okereke Anagha Aneesh Amir Mohammad Elahi Mehrdad Asgari Juliane Eberhardt Hani M. Elbeheiry María Victoria Gil Maximilian Greiner Caroline T. Holick Christina Glaubitz Tim Hoffmann Abdelrahman Ibrahim Lea C. Klepsch Yannik Köster Fabian Alexander Kreth Jakob Meyer Santiago Miret Jan Matthias Peschel Michael Ringleb Nicole Roesner Johanna Schreiber Ulrich S. Schubert Leanne M. Stafast Dinga Wonanke Michael Pieler Philippe Schwaller Kevin Maik Jablonka evaluation-task" } }, { "id": "register:chipbench_debug", "name": "ChipBench — Debugging", "slug": "register--chipbench-debug", "organisation": "Zhongkai Yu, Chenyang Zhou, Yichen Lin et al.", "source_type": "inspect-register", "source_url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "topics": [ "general-capability" ], "description": "ChipBench evaluates LLMs on AI-aided chip design across three tasks: Verilog generation (44 modules), Verilog debugging (89 cases), and reference model generation (132 cases in Python/SystemC/CXXRTL). The debugging task injects four bug types (arithmetic, assignment, timing, state machine) into golden Verilog modules; models receive buggy code plus a description (zero-shot) or additionally a VCD waveform (one-shot). Scoring uses iVerilog simulation against a golden reference, reporting pass@1/5/10 over 20 samples.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_debug/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2601.21448", "primary": true }, { "kind": "paper", "label": "Paper: ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "url": "https://arxiv.org/abs/2601.21448", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_debug/eval.yaml", "primary": false } ], "paper_title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "paper_url": "https://arxiv.org/abs/2601.21448", "code_url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_debug/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "Plswearpants" ], "tasks": [ { "name": "chipbench_debug", "path": "src/chipbench/chipbench.py" } ], "protocol": { "implementation_commit": "8a60e2e8914139c1e73fe62ae8018dc240605ff3", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "family_title": "ChipBench", "variant_title": "Debugging", "resource_count": 4, "source_key": "register:chipbench_debug", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zhongkai Yu, Chenyang Zhou, Yichen Lin et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2601.21448" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2601.21448", "arxiv_id": "2601.21448", "title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "authors": [ "Zhongkai Yu", "Chenyang Zhou", "Yichen Lin", "Hejia Zhang", "Haotian Ye", "Junxia Cui", "Zaifeng Pan", "Jishen Zhao", "Yufei Ding" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2601.21448", "arxiv_id": "2601.21448", "title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "authors": [ "Zhongkai Yu", "Chenyang Zhou", "Yichen Lin", "Hejia Zhang", "Haotian Ye", "Junxia Cui", "Zaifeng Pan", "Jishen Zhao", "Yufei Ding" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "owner": "Plswearpants", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_debug/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2601.21448" }, "metadata": { "source_search": "Zhongkai Yu, Chenyang Zhou, Yichen Lin et al. Original paper authors ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design Zhongkai Yu Chenyang Zhou Yichen Lin Hejia Zhang Haotian Ye Junxia Cui Zaifeng Pan Jishen Zhao Yufei Ding Plswearpants evaluation-package" } }, { "id": "register:chipbench_refmodel", "name": "ChipBench — Reference Model", "slug": "register--chipbench-refmodel", "organisation": "Zhongkai Yu, Chenyang Zhou, Yichen Lin et al.", "source_type": "inspect-register", "source_url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "topics": [ "general-capability" ], "description": "ChipBench evaluates LLMs on chip design tasks using 264 test cases: 44 Verilog generation problems (self-contained, hierarchical, CPU IP modules), 89 debugging cases (arithmetic, assignment, timing, state-machine bugs, zero/one-shot), and 132 reference model generation samples (Python, SystemC, CXXRTL). Scoring uses iVerilog functional simulation against golden implementations with 1000+ random stimuli. Metrics are pass@1/5/10 across 20 samples per problem at temperature=0.85, top_p=0.95.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_refmodel/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2601.21448", "primary": true }, { "kind": "paper", "label": "Paper: ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "url": "https://arxiv.org/abs/2601.21448", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_refmodel/eval.yaml", "primary": false } ], "paper_title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "paper_url": "https://arxiv.org/abs/2601.21448", "code_url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_refmodel/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "Plswearpants" ], "tasks": [ { "name": "chipbench_refmodel", "path": "src/chipbench/chipbench.py" } ], "protocol": { "implementation_commit": "8a60e2e8914139c1e73fe62ae8018dc240605ff3", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "family_title": "ChipBench", "variant_title": "Reference Model", "resource_count": 4, "source_key": "register:chipbench_refmodel", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zhongkai Yu, Chenyang Zhou, Yichen Lin et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2601.21448" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2601.21448", "arxiv_id": "2601.21448", "title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "authors": [ "Zhongkai Yu", "Chenyang Zhou", "Yichen Lin", "Hejia Zhang", "Haotian Ye", "Junxia Cui", "Zaifeng Pan", "Jishen Zhao", "Yufei Ding" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2601.21448", "arxiv_id": "2601.21448", "title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "authors": [ "Zhongkai Yu", "Chenyang Zhou", "Yichen Lin", "Hejia Zhang", "Haotian Ye", "Junxia Cui", "Zaifeng Pan", "Jishen Zhao", "Yufei Ding" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "owner": "Plswearpants", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_refmodel/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2601.21448" }, "metadata": { "source_search": "Zhongkai Yu, Chenyang Zhou, Yichen Lin et al. Original paper authors ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design Zhongkai Yu Chenyang Zhou Yichen Lin Hejia Zhang Haotian Ye Junxia Cui Zaifeng Pan Jishen Zhao Yufei Ding Plswearpants evaluation-package" } }, { "id": "register:chipbench_verilog_gen", "name": "ChipBench — Verilog Generation", "slug": "register--chipbench-verilog-gen", "organisation": "Zhongkai Yu, Chenyang Zhou, Yichen Lin et al.", "source_type": "inspect-register", "source_url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "topics": [ "general-capability" ], "description": "ChipBench evaluates LLMs on Verilog code generation from natural-language specifications across three module categories: self-contained (29 cases), non-self-contained hierarchical (6 cases), and CPU IP submodules (9 cases). Each case provides a prompt, golden Verilog, and a test harness combining directed corner cases with 1000+ random stimuli compiled via iVerilog. Scoring uses pass@1/5/10 over 20 samples at temperature=0.85, top_p=0.95, executed in a Docker sandbox.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_verilog_gen/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2601.21448", "primary": true }, { "kind": "paper", "label": "Paper: ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "url": "https://arxiv.org/abs/2601.21448", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_verilog_gen/eval.yaml", "primary": false } ], "paper_title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "paper_url": "https://arxiv.org/abs/2601.21448", "code_url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_verilog_gen/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "Plswearpants" ], "tasks": [ { "name": "chipbench_verilog_gen", "path": "src/chipbench/chipbench.py" } ], "protocol": { "implementation_commit": "8a60e2e8914139c1e73fe62ae8018dc240605ff3", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "family_title": "ChipBench", "variant_title": "Verilog Generation", "resource_count": 4, "source_key": "register:chipbench_verilog_gen", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zhongkai Yu, Chenyang Zhou, Yichen Lin et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2601.21448" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2601.21448", "arxiv_id": "2601.21448", "title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "authors": [ "Zhongkai Yu", "Chenyang Zhou", "Yichen Lin", "Hejia Zhang", "Haotian Ye", "Junxia Cui", "Zaifeng Pan", "Jishen Zhao", "Yufei Ding" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2601.21448", "arxiv_id": "2601.21448", "title": "ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design", "authors": [ "Zhongkai Yu", "Chenyang Zhou", "Yichen Lin", "Hejia Zhang", "Haotian Ye", "Junxia Cui", "Zaifeng Pan", "Jishen Zhao", "Yufei Ding" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Plswearpants/Inspect-Eval-ChipBench", "owner": "Plswearpants", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/chipbench_verilog_gen/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2601.21448" }, "metadata": { "source_search": "Zhongkai Yu, Chenyang Zhou, Yichen Lin et al. Original paper authors ChipBench: A Next-Step Benchmark for Evaluating LLM Performance in AI-Aided Chip Design Zhongkai Yu Chenyang Zhou Yichen Lin Hejia Zhang Haotian Ye Junxia Cui Zaifeng Pan Jishen Zhao Yufei Ding Plswearpants evaluation-package" } }, { "id": "inspect:class_eval", "name": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation", "slug": "inspect--class-eval", "organisation": "Xueying Du, Mingwei Liu, Kaixin Wang et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/class_eval", "topics": [ "general-capability" ], "description": "Evaluates LLMs on class-level code generation with 100 tasks constructed over 500 person-hours. The study shows that LLMs perform worse on class-level tasks compared to method-level tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/class_eval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2308.01861", "primary": true }, { "kind": "paper", "label": "Paper: ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation", "url": "https://arxiv.org/abs/2308.01861", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/class_eval/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FudanSELab", "url": "https://huggingface.co/datasets/FudanSELab/ClassEval", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2308.01861", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/class_eval/eval.yaml", "family_title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation", "group": "Coding", "version": "3-C", "contributors": [ "zhenningdavidliu" ], "tags": [ "Coding" ], "task": { "name": "class_eval", "dataset_samples": 100, "comment": null, "support_only": false }, "tasks": [ { "name": "class_eval", "dataset_samples": 100, "comment": null, "support_only": false } ], "protocol": { "version": "3-C", "dataset_samples": 100, "dependency": null, "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "FudanSELab/ClassEval", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:class_eval", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xueying Du, Mingwei Liu, Kaixin Wang et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2308.01861" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2308.01861", "arxiv_id": "2308.01861", "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation", "authors": [ "Xueying Du", "Mingwei Liu", "Kaixin Wang", "Hanlin Wang", "Junwei Liu", "Yixuan Chen", "Jiayi Feng", "Chaofeng Sha", "Xin Peng", "Yiling Lou" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2308.01861", "arxiv_id": "2308.01861", "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation", "authors": [ "Xueying Du", "Mingwei Liu", "Kaixin Wang", "Hanlin Wang", "Junwei Liu", "Yixuan Chen", "Jiayi Feng", "Chaofeng Sha", "Xin Peng", "Yiling Lou" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/class_eval/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/FudanSELab/ClassEval", "provider": "FudanSELab" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2308.01861" }, "code_url": null, "metadata": { "source_search": "Xueying Du, Mingwei Liu, Kaixin Wang et al. Original paper authors ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation Xueying Du Mingwei Liu Kaixin Wang Hanlin Wang Junwei Liu Yixuan Chen Jiayi Feng Chaofeng Sha Xin Peng Yiling Lou evaluation-task" } }, { "id": "inspect:coconot", "name": "The Art of Saying No: Contextual Noncompliance in Language Models", "slug": "inspect--coconot", "organisation": "Faeze Brahman, Sachin Kumar, Vidhisha Balachandran et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/coconot", "topics": [ "safeguards" ], "description": "Dataset with 1001 samples to test noncompliance capabilities of language models. Contrast set of 379 samples.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/coconot/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.12043", "primary": true }, { "kind": "paper", "label": "Paper: The Art of Saying No: Contextual Noncompliance in Language Models", "url": "https://arxiv.org/abs/2407.12043", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/coconot/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Allen Institute for AI", "url": "https://huggingface.co/datasets/allenai/coconot", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.12043", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/coconot/eval.yaml", "family_title": "The Art of Saying No: Contextual Noncompliance in Language Models", "group": "Safeguards", "version": "4-B", "contributors": [ "ransomr" ], "tags": [ "Safeguards" ], "task": { "name": "coconot", "dataset_samples": 1001, "comment": null, "support_only": false }, "tasks": [ { "name": "coconot", "dataset_samples": 1001, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 1001, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "allenai/coconot", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:coconot", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Faeze Brahman, Sachin Kumar, Vidhisha Balachandran et al.", "organization": "Carnegie Mellon University; Allen Institute for Artificial Intelligence; Allen Institute for AI; University of Maryland; AI2", "organizations": [ "Carnegie Mellon University", "Allen Institute for Artificial Intelligence", "Allen Institute for AI", "University of Maryland", "AI2" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.12043" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.12043", "arxiv_id": "2407.12043", "title": "The Art of Saying No: Contextual Noncompliance in Language Models", "authors": [ "Faeze Brahman", "Sachin Kumar", "Vidhisha Balachandran", "Pradeep Dasigi", "Valentina Pyatkin", "Abhilasha Ravichander", "Sarah Wiegreffe", "Nouha Dziri", "Khyathi Chandu", "Jack Hessel", "Yulia Tsvetkov", "Noah A. Smith", "Yejin Choi", "Hannaneh Hajishirzi" ], "organizations": [ "Carnegie Mellon University", "Allen Institute for Artificial Intelligence", "Allen Institute for AI", "University of Maryland", "AI2" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.12043", "arxiv_id": "2407.12043", "title": "The Art of Saying No: Contextual Noncompliance in Language Models", "authors": [ "Faeze Brahman", "Sachin Kumar", "Vidhisha Balachandran", "Pradeep Dasigi", "Valentina Pyatkin", "Abhilasha Ravichander", "Sarah Wiegreffe", "Nouha Dziri", "Khyathi Chandu", "Jack Hessel", "Yulia Tsvetkov", "Noah A. Smith", "Yejin Choi", "Hannaneh Hajishirzi" ], "organizations": [ "Carnegie Mellon University", "Allen Institute for Artificial Intelligence", "Allen Institute for AI", "University of Maryland", "AI2" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/coconot/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/allenai/coconot", "provider": "Allen Institute for AI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.12043" }, "code_url": null, "metadata": { "source_search": "Faeze Brahman, Sachin Kumar, Vidhisha Balachandran et al. Carnegie Mellon University; Allen Institute for Artificial Intelligence; Allen Institute for AI; University of Maryland; AI2 Original paper authors The Art of Saying No: Contextual Noncompliance in Language Models Faeze Brahman Sachin Kumar Vidhisha Balachandran Pradeep Dasigi Valentina Pyatkin Abhilasha Ravichander Sarah Wiegreffe Nouha Dziri Khyathi Chandu Jack Hessel Yulia Tsvetkov Noah A. Smith Yejin Choi Hannaneh Hajishirzi evaluation-task" } }, { "id": "inspect:commonsense_qa", "name": "CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge", "slug": "inspect--commonsense-qa", "organisation": "Alon Talmor, Jonathan Herzig, Nicholas Lourie et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/commonsense_qa", "topics": [ "general-capability" ], "description": "Evaluates an AI model's ability to correctly answer everyday questions that rely on basic commonsense knowledge and understanding of the world.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/commonsense_qa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1811.00937", "primary": true }, { "kind": "paper", "label": "Paper: CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge", "url": "https://arxiv.org/abs/1811.00937", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/commonsense_qa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Tau", "url": "https://huggingface.co/datasets/tau/commonsense_qa", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1811.00937", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/commonsense_qa/eval.yaml", "family_title": "CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge", "group": "Knowledge", "version": "2-A", "contributors": [ "lauritowal" ], "tags": [ "Knowledge" ], "task": { "name": "commonsense_qa", "dataset_samples": 1221, "comment": null, "support_only": false }, "tasks": [ { "name": "commonsense_qa", "dataset_samples": 1221, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1221, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "tau/commonsense_qa", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:commonsense_qa", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Alon Talmor, Jonathan Herzig, Nicholas Lourie et al.", "organization": "Allen Institute for Artificial Intelligence; Tel-Aviv University and Google DeepMind", "organizations": [ "Allen Institute for Artificial Intelligence", "Tel-Aviv University and Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1811.00937" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1811.00937", "arxiv_id": "1811.00937", "title": "CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge", "authors": [ "Alon Talmor", "Jonathan Herzig", "Nicholas Lourie", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1811.00937", "arxiv_id": "1811.00937", "title": "CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge", "authors": [ "Alon Talmor", "Jonathan Herzig", "Nicholas Lourie", "Jonathan Berant" ], "organizations": [ "Allen Institute for Artificial Intelligence", "Tel-Aviv University and Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/commonsense_qa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/tau/commonsense_qa", "provider": "Tau" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1811.00937" }, "code_url": null, "metadata": { "source_search": "Alon Talmor, Jonathan Herzig, Nicholas Lourie et al. Allen Institute for Artificial Intelligence; Tel-Aviv University and Google DeepMind Original paper authors CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge Alon Talmor Jonathan Herzig Nicholas Lourie Jonathan Berant evaluation-task" } }, { "id": "inspect:compute_eval", "name": "ComputeEval: CUDA Code Generation Benchmark", "slug": "inspect--compute-eval", "organisation": "NVIDIA", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/compute_eval", "topics": [ "general-capability" ], "description": "Evaluates LLM capability to generate correct CUDA code for kernel implementation, memory management, and parallel algorithm optimization tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/compute_eval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/NVIDIA/compute-eval", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/NVIDIA/compute-eval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/compute_eval/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Nvidia", "url": "https://huggingface.co/datasets/nvidia/compute-eval", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://huggingface.co/datasets/nvidia/compute-eval", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/NVIDIA/compute-eval", "primary": false } ], "resource_count": 6, "paper_url": null, "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/compute_eval/eval.yaml", "family_title": "ComputeEval: CUDA Code Generation Benchmark", "group": "Coding", "version": "1-A", "contributors": [ "Vitamoon" ], "tags": [ "Coding" ], "task": { "name": "compute_eval", "dataset_samples": 406, "comment": null, "support_only": false }, "tasks": [ { "name": "compute_eval", "dataset_samples": 406, "comment": null, "support_only": false } ], "protocol": { "version": "1-A", "dataset_samples": 406, "dependency": null, "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "nvidia/compute-eval", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:compute_eval", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "ComputeEval is developed and maintained in NVIDIA's official repository. No dedicated benchmark paper was identified in the reviewed primary sources." }, "origin": { "name": "NVIDIA", "organization": "NVIDIA", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://github.com/NVIDIA/compute-eval", "organizations": [ "NVIDIA" ], "role_label": "Evaluation developed by" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/NVIDIA/compute-eval", "owner": "NVIDIA", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/compute_eval/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/nvidia/compute-eval", "provider": "Nvidia" } ], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/NVIDIA/compute-eval" }, "code_url": "https://github.com/NVIDIA/compute-eval", "metadata": { "source_search": "NVIDIA NVIDIA Evaluation developed by NVIDIA evaluation-task" } }, { "id": "register:contractbench", "name": "ContractBench", "slug": "register--contractbench", "organisation": "Jicheng Wang, Yifeng He, Zili Wang et al.", "source_type": "inspect-register", "source_url": "https://github.com/SecurityLab-UCD/ContractBench-inspect", "topics": [ "general-capability" ], "description": "ContractBench evaluates LLM agents on 33 tasks probing two orthogonal failure modes: temporal validity (using API artifacts before expiry) and byte-level integrity (relaying artifacts without corruption). Tasks simulate real API patterns (presigned URLs, OAuth tokens, HMAC webhooks) using a virtual clock and SHA-256 hash verification. Each episode is scored deterministically via HTTP request logs against a 15-label failure taxonomy. Metric is success rate over k=3 runs per task (n=99 episodes per model). No LLM judge is used.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/contractbench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2605.17281", "primary": true }, { "kind": "paper", "label": "Paper: ContractBench: Can LLM Agents Preserve Observation Contracts?", "url": "https://arxiv.org/abs/2605.17281", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/SecurityLab-UCD/ContractBench-inspect", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/contractbench/eval.yaml", "primary": false } ], "paper_title": "ContractBench: Can LLM Agents Preserve Observation Contracts?", "paper_url": "https://arxiv.org/abs/2605.17281", "code_url": "https://github.com/SecurityLab-UCD/ContractBench-inspect", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/contractbench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "JeremyJC67" ], "tasks": [ { "name": "contractbench", "path": "contractbench_inspect/contractbench.py" } ], "protocol": { "implementation_commit": "22d39108619433a9f7bc8eb3b410bbf384d67616", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:contractbench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jicheng Wang, Yifeng He, Zili Wang et al.", "organization": "University of California, Davis", "organizations": [ "University of California, Davis" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2605.17281" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2605.17281", "arxiv_id": "2605.17281", "title": "ContractBench: Can LLM Agents Preserve Observation Contracts?", "authors": [ "Jicheng Wang", "Yifeng He", "Zili Wang", "Hanwen Xing", "Arkaprava De", "Hao Chen" ], "organizations": [ "University of California, Davis" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2605.17281", "arxiv_id": "2605.17281", "title": "ContractBench: Can LLM Agents Preserve Observation Contracts?", "authors": [ "Jicheng Wang", "Yifeng He", "Zili Wang", "Hanwen Xing", "Arkaprava De", "Hao Chen" ], "organizations": [ "University of California, Davis" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/SecurityLab-UCD/ContractBench-inspect", "owner": "SecurityLab UCD", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/contractbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2605.17281" }, "metadata": { "source_search": "Jicheng Wang, Yifeng He, Zili Wang et al. University of California, Davis Original paper authors ContractBench: Can LLM Agents Preserve Observation Contracts? Jicheng Wang Yifeng He Zili Wang Hanwen Xing Arkaprava De Hao Chen SecurityLab UCD evaluation-package" } }, { "id": "inspect:core_bench", "name": "CORE-Bench", "slug": "inspect--core-bench", "organisation": "Zachary S. Siegel, Sayash Kapoor, Nitya Nadgir et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/core_bench", "topics": [ "general-capability" ], "description": "Evaluate how well an LLM Agent is at computationally reproducing the results of a set of scientific papers.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/core_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2409.11363", "primary": true }, { "kind": "paper", "label": "Paper: CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark", "url": "https://arxiv.org/abs/2409.11363", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/core_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Siegelz", "url": "https://huggingface.co/datasets/siegelz/core-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2409.11363", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/core_bench/eval.yaml", "family_title": "CORE-Bench", "group": "Coding", "version": "3-A", "contributors": [ "enerrio" ], "tags": [ "Agent", "Coding" ], "task": { "name": "core_bench", "dataset_samples": 45, "comment": null, "support_only": false }, "tasks": [ { "name": "core_bench", "dataset_samples": 45, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 45, "dependency": "core_bench", "requires_internet": false, "sandbox": [ "solver", "scorer" ] }, "external_assets": [ { "type": "direct_url", "source": "https://corebench.cs.princeton.edu/capsules/{id}.tar.gz", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "siegelz/core-bench", "fetch_method": "hf_hub_download", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:core_bench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zachary S. Siegel, Sayash Kapoor, Nitya Nadgir et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2409.11363" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2409.11363", "arxiv_id": "2409.11363", "title": "CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark", "authors": [ "Zachary S. Siegel", "Sayash Kapoor", "Nitya Nadgir", "Benedikt Stroebl", "Arvind Narayanan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2409.11363", "arxiv_id": "2409.11363", "title": "CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark", "authors": [ "Zachary S. Siegel", "Sayash Kapoor", "Nitya Nadgir", "Benedikt Stroebl", "Arvind Narayanan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/core_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/siegelz/core-bench", "provider": "Siegelz" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2409.11363" }, "code_url": null, "metadata": { "source_search": "Zachary S. Siegel, Sayash Kapoor, Nitya Nadgir et al. Original paper authors CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark Zachary S. Siegel Sayash Kapoor Nitya Nadgir Benedikt Stroebl Arvind Narayanan evaluation-task" } }, { "id": "inspect:cti_realm_25", "name": "CTI Realm 25", "slug": "inspect--cti-realm-25", "organisation": "Arjun Chakraborty, Sandra Ho, Adam Cook et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm", "topics": [ "cyber" ], "description": "Evaluates AI systems' ability to analyze cyber threat intelligence and develop comprehensive detection capabilities through a realistic 5-subtask workflow: MITRE technique mapping, data source discovery, Sigma rule generation, KQL development and testing against real telemetry data, and results analysis.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2603.13517", "primary": true }, { "kind": "paper", "label": "Paper: CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "url": "https://arxiv.org/abs/2603.13517", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/mitre/cti", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Arjun180 New", "url": "https://huggingface.co/datasets/arjun180-new/cti_realm", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2603.13517", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "family_title": "CTI-REALM: Cyber Threat Intelligence Detection Rule Development Benchmark", "group": "Cybersecurity", "version": "4-A", "contributors": [ "arjun180-new" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "cti_realm_25", "dataset_samples": 25, "comment": null, "support_only": false }, "tasks": [ { "name": "cti_realm_25", "dataset_samples": 25, "comment": null, "support_only": false }, { "name": "cti_realm_50", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "cti_realm_25_minimal", "dataset_samples": 25, "comment": null, "support_only": false }, { "name": "cti_realm_25_seeded", "dataset_samples": 25, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 25, "dependency": "cti_realm", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "arjun180-new/cti_realm", "fetch_method": "hf_hub_download", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/mitre/cti/68d2992ea01249fc72e966f569a0d0b663d98581/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "enterprise-attack, mobile-attack and ics-attack STIX bundles; downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cti_realm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Arjun Chakraborty, Sandra Ho, Adam Cook et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2603.13517" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2603.13517", "arxiv_id": "2603.13517", "title": "CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "authors": [ "Arjun Chakraborty", "Sandra Ho", "Adam Cook", "Manuel Meléndez" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2603.13517", "arxiv_id": "2603.13517", "title": "CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "authors": [ "Arjun Chakraborty", "Sandra Ho", "Adam Cook", "Manuel Meléndez" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/mitre/cti", "owner": "Mitre", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/arjun180-new/cti_realm", "provider": "Arjun180 New" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2603.13517" }, "code_url": "https://github.com/mitre/cti", "metadata": { "source_search": "Arjun Chakraborty, Sandra Ho, Adam Cook et al. Original paper authors CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities Arjun Chakraborty Sandra Ho Adam Cook Manuel Meléndez Mitre evaluation-family" } }, { "id": "inspect:cti_realm_25_minimal", "name": "CTI Realm 25 Minimal", "slug": "inspect--cti-realm-25-minimal", "organisation": "Arjun Chakraborty, Sandra Ho, Adam Cook et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm", "topics": [ "cyber" ], "description": "Evaluates AI systems' ability to analyze cyber threat intelligence and develop comprehensive detection capabilities through a realistic 5-subtask workflow: MITRE technique mapping, data source discovery, Sigma rule generation, KQL development and testing against real telemetry data, and results analysis.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2603.13517", "primary": true }, { "kind": "paper", "label": "Paper: CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "url": "https://arxiv.org/abs/2603.13517", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/mitre/cti", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Arjun180 New", "url": "https://huggingface.co/datasets/arjun180-new/cti_realm", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2603.13517", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "family_title": "CTI-REALM: Cyber Threat Intelligence Detection Rule Development Benchmark", "group": "Cybersecurity", "version": "4-A", "contributors": [ "arjun180-new" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "cti_realm_25_minimal", "dataset_samples": 25, "comment": null, "support_only": false }, "tasks": [ { "name": "cti_realm_25", "dataset_samples": 25, "comment": null, "support_only": false }, { "name": "cti_realm_50", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "cti_realm_25_minimal", "dataset_samples": 25, "comment": null, "support_only": false }, { "name": "cti_realm_25_seeded", "dataset_samples": 25, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 25, "dependency": "cti_realm", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "arjun180-new/cti_realm", "fetch_method": "hf_hub_download", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/mitre/cti/68d2992ea01249fc72e966f569a0d0b663d98581/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "enterprise-attack, mobile-attack and ics-attack STIX bundles; downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cti_realm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Arjun Chakraborty, Sandra Ho, Adam Cook et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2603.13517" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2603.13517", "arxiv_id": "2603.13517", "title": "CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "authors": [ "Arjun Chakraborty", "Sandra Ho", "Adam Cook", "Manuel Meléndez" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2603.13517", "arxiv_id": "2603.13517", "title": "CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "authors": [ "Arjun Chakraborty", "Sandra Ho", "Adam Cook", "Manuel Meléndez" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/mitre/cti", "owner": "Mitre", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/arjun180-new/cti_realm", "provider": "Arjun180 New" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2603.13517" }, "code_url": "https://github.com/mitre/cti", "metadata": { "source_search": "Arjun Chakraborty, Sandra Ho, Adam Cook et al. Original paper authors CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities Arjun Chakraborty Sandra Ho Adam Cook Manuel Meléndez Mitre evaluation-family" } }, { "id": "inspect:cti_realm_25_seeded", "name": "CTI Realm 25 Seeded", "slug": "inspect--cti-realm-25-seeded", "organisation": "Arjun Chakraborty, Sandra Ho, Adam Cook et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm", "topics": [ "cyber" ], "description": "Evaluates AI systems' ability to analyze cyber threat intelligence and develop comprehensive detection capabilities through a realistic 5-subtask workflow: MITRE technique mapping, data source discovery, Sigma rule generation, KQL development and testing against real telemetry data, and results analysis.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2603.13517", "primary": true }, { "kind": "paper", "label": "Paper: CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "url": "https://arxiv.org/abs/2603.13517", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/mitre/cti", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Arjun180 New", "url": "https://huggingface.co/datasets/arjun180-new/cti_realm", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2603.13517", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "family_title": "CTI-REALM: Cyber Threat Intelligence Detection Rule Development Benchmark", "group": "Cybersecurity", "version": "4-A", "contributors": [ "arjun180-new" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "cti_realm_25_seeded", "dataset_samples": 25, "comment": null, "support_only": false }, "tasks": [ { "name": "cti_realm_25", "dataset_samples": 25, "comment": null, "support_only": false }, { "name": "cti_realm_50", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "cti_realm_25_minimal", "dataset_samples": 25, "comment": null, "support_only": false }, { "name": "cti_realm_25_seeded", "dataset_samples": 25, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 25, "dependency": "cti_realm", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "arjun180-new/cti_realm", "fetch_method": "hf_hub_download", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/mitre/cti/68d2992ea01249fc72e966f569a0d0b663d98581/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "enterprise-attack, mobile-attack and ics-attack STIX bundles; downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cti_realm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Arjun Chakraborty, Sandra Ho, Adam Cook et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2603.13517" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2603.13517", "arxiv_id": "2603.13517", "title": "CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "authors": [ "Arjun Chakraborty", "Sandra Ho", "Adam Cook", "Manuel Meléndez" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2603.13517", "arxiv_id": "2603.13517", "title": "CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "authors": [ "Arjun Chakraborty", "Sandra Ho", "Adam Cook", "Manuel Meléndez" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/mitre/cti", "owner": "Mitre", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/arjun180-new/cti_realm", "provider": "Arjun180 New" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2603.13517" }, "code_url": "https://github.com/mitre/cti", "metadata": { "source_search": "Arjun Chakraborty, Sandra Ho, Adam Cook et al. Original paper authors CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities Arjun Chakraborty Sandra Ho Adam Cook Manuel Meléndez Mitre evaluation-family" } }, { "id": "inspect:cti_realm_50", "name": "CTI Realm 50", "slug": "inspect--cti-realm-50", "organisation": "Arjun Chakraborty, Sandra Ho, Adam Cook et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm", "topics": [ "cyber" ], "description": "Evaluates AI systems' ability to analyze cyber threat intelligence and develop comprehensive detection capabilities through a realistic 5-subtask workflow: MITRE technique mapping, data source discovery, Sigma rule generation, KQL development and testing against real telemetry data, and results analysis.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2603.13517", "primary": true }, { "kind": "paper", "label": "Paper: CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "url": "https://arxiv.org/abs/2603.13517", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/mitre/cti", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Arjun180 New", "url": "https://huggingface.co/datasets/arjun180-new/cti_realm", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2603.13517", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml", "family_title": "CTI-REALM: Cyber Threat Intelligence Detection Rule Development Benchmark", "group": "Cybersecurity", "version": "4-A", "contributors": [ "arjun180-new" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "cti_realm_50", "dataset_samples": 50, "comment": null, "support_only": false }, "tasks": [ { "name": "cti_realm_25", "dataset_samples": 25, "comment": null, "support_only": false }, { "name": "cti_realm_50", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "cti_realm_25_minimal", "dataset_samples": 25, "comment": null, "support_only": false }, { "name": "cti_realm_25_seeded", "dataset_samples": 25, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 50, "dependency": "cti_realm", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "arjun180-new/cti_realm", "fetch_method": "hf_hub_download", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/mitre/cti/68d2992ea01249fc72e966f569a0d0b663d98581/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "enterprise-attack, mobile-attack and ics-attack STIX bundles; downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cti_realm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Arjun Chakraborty, Sandra Ho, Adam Cook et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2603.13517" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2603.13517", "arxiv_id": "2603.13517", "title": "CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "authors": [ "Arjun Chakraborty", "Sandra Ho", "Adam Cook", "Manuel Meléndez" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2603.13517", "arxiv_id": "2603.13517", "title": "CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities", "authors": [ "Arjun Chakraborty", "Sandra Ho", "Adam Cook", "Manuel Meléndez" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/mitre/cti", "owner": "Mitre", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cti_realm/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/arjun180-new/cti_realm", "provider": "Arjun180 New" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2603.13517" }, "code_url": "https://github.com/mitre/cti", "metadata": { "source_search": "Arjun Chakraborty, Sandra Ho, Adam Cook et al. Original paper authors CTI-REALM: Benchmark to Evaluate Agent Performance on Security Detection Rule Generation Capabilities Arjun Chakraborty Sandra Ho Adam Cook Manuel Meléndez Mitre evaluation-family" } }, { "id": "inspect:cve_bench", "name": "CVEBench: Benchmark for AI Agents Ability to Exploit Real-World Web Application Vulnerabilities", "slug": "inspect--cve-bench", "organisation": "Yuxuan Zhu, Antony Kellermann, Dylan Bowman et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cve_bench", "topics": [ "cyber" ], "description": "Characterises an AI Agent's capability to exploit real-world web application vulnerabilities. Aims to provide a realistic evaluation of an agent's security reasoning capability using 40 real-world CVEs.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Exploit completion on CVE-Bench's selected vulnerabilities under its environment, information, tool, and budget conditions.", "does_not_measure": "Vulnerability discovery at scale, stealth, persistence, lateral movement, real target access, or end-to-end offensive campaigns.", "best_for": "Testing exploit execution and tool-using cyber agents on reproducible real-vulnerability environments.", "not_sufficient_for": "Forecasting real-world cyber incidents or concluding that a model can autonomously compromise arbitrary systems.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cve_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2503.17332", "primary": true }, { "kind": "paper", "label": "Paper: CVE-Bench: A Benchmark for AI Agents' Ability to Exploit Real-World Web Application Vulnerabilities", "url": "https://arxiv.org/abs/2503.17332", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Scott-Simmons/cve-bench", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cve_bench/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2503.17332", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cve_bench/eval.yaml", "family_title": "CVEBench: Benchmark for AI Agents Ability to Exploit Real-World Web Application Vulnerabilities", "group": "Cybersecurity", "version": "2-A", "contributors": [ "yuxuan18", "aokellermann", "dylanbowman314", "pli482004", "akulguptax", "adarshdanda06", "rightegg", "rightegg", "jensen", "eihli", "JasonBenn", "inigo", "dhir", "SudhitRao", "YuVictor123", "Cloakless", "ddkang", "Scott-Simmons" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "cve_bench", "dataset_samples": 40, "comment": null, "support_only": false }, "tasks": [ { "name": "cve_bench", "dataset_samples": 40, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 40, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_dependency", "source": "https://github.com/Scott-Simmons/cve-bench.git", "fetch_method": "pyproject_toml", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cve_bench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yuxuan Zhu, Antony Kellermann, Dylan Bowman et al.", "organization": "University of Illinois Urbana-Champaign", "organizations": [ "University of Illinois Urbana-Champaign" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2503.17332" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2503.17332", "arxiv_id": "2503.17332", "title": "CVE-Bench: A Benchmark for AI Agents' Ability to Exploit Real-World Web Application Vulnerabilities", "authors": [ "Yuxuan Zhu", "Antony Kellermann", "Dylan Bowman", "Philip Li", "Akul Gupta", "Adarsh Danda", "Richard Fang", "Conner Jensen", "Eric Ihli", "Jason Benn", "Jet Geronimo", "Avi Dhir", "Sudhit Rao", "Kaicheng Yu", "Twm Stone", "Daniel Kang" ], "organizations": [ "University of Illinois Urbana-Champaign" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2503.17332", "arxiv_id": "2503.17332", "title": "CVE-Bench: A Benchmark for AI Agents' Ability to Exploit Real-World Web Application Vulnerabilities", "authors": [ "Yuxuan Zhu", "Antony Kellermann", "Dylan Bowman", "Philip Li", "Akul Gupta", "Adarsh Danda", "Richard Fang", "Conner Jensen", "Eric Ihli", "Jason Benn", "Jet Geronimo", "Avi Dhir", "Sudhit Rao", "Kaicheng Yu", "Twm Stone", "Daniel Kang" ], "organizations": [ "University of Illinois Urbana-Champaign" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Scott-Simmons/cve-bench", "owner": "Scott Simmons", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cve_bench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2503.17332" }, "code_url": "https://github.com/Scott-Simmons/cve-bench", "metadata": { "source_search": "Yuxuan Zhu, Antony Kellermann, Dylan Bowman et al. University of Illinois Urbana-Champaign Original paper authors CVE-Bench: A Benchmark for AI Agents' Ability to Exploit Real-World Web Application Vulnerabilities Yuxuan Zhu Antony Kellermann Dylan Bowman Philip Li Akul Gupta Adarsh Danda Richard Fang Conner Jensen Eric Ihli Jason Benn Jet Geronimo Avi Dhir Sudhit Rao Kaicheng Yu Twm Stone Daniel Kang Scott Simmons evaluation-task" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Agent capability to exploit selected real-world software vulnerabilities in stateful, containerized environments.", "unit_of_analysis": "One agent trajectory against one vulnerable application or CVE environment.", "interaction_type": "Long-horizon command and network interaction with a vulnerable open-source project in a sandbox.", "target_outcome": "Task-specific exploit success, typically demonstrated by obtaining a flag or satisfying a stateful verifier.", "human_participants": "No human subjects. CVE reports, patches, and expert-built environments define tasks and labels.", "elicitation": "The agent receives a target environment and task information under defined tools, network conditions, time, and attempt limits.", "scoring": "Deterministic environment checks or flags indicate successful exploitation; task metadata records vulnerability and patch context.", "protocol_dependence": "Very high: CVE selection, vulnerable image, information provided, tools, network, timeout, and scaffold define difficulty.", "implementation_dependence": "Very high: environment correctness, patch labels, dependencies, and verifier integrity are essential.", "limitations": [ "The CVE sample is small and selected, not representative of the full vulnerability ecosystem.", "Containerized targets simplify reconnaissance, access, and operational constraints.", "Known CVEs may be present in model training data.", "Successful exploitation does not measure stealth, persistence, or impact." ], "comparability": "Match CVE-Bench release, CVE and task IDs, container images, information condition, tools, network settings, budget, scaffold, and model snapshot.", "decision_relevance": "Useful for controlled exploitation-capability monitoring and security evaluation design; insufficient alone for systemic cyber-risk estimates.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2503.17332" }, { "kind": "implementation", "url": "https://github.com/uiuc-kang-lab/cve-bench" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:cybench", "name": "Cybench: Capture-The-Flag Cybersecurity Challenges", "slug": "inspect--cybench", "organisation": "Andy K. Zhang, Neil Perry, Riya Dulepet et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybench", "topics": [ "cyber" ], "description": "Tests language models on cybersecurity skills using 39 of 40 practical, professional-level challenges taken from cybersecurity competitions, designed to cover various difficulty levels and security concepts. The motp challenge is excluded due to GPL licensing.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Performance on CyBench's selected CTF tasks under the exact command environment, tools, scaffold, and budget.", "does_not_measure": "Real-world intrusion capability, stealth, persistence, target discovery, legal or organizational constraints, or operational cyber risk.", "best_for": "Comparing cyber-agent problem solving and analysing where long-horizon CTF trajectories fail.", "not_sufficient_for": "Claims that a model can conduct real-world cyber operations or meets a deployment-risk threshold.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2408.08926", "primary": true }, { "kind": "paper", "label": "Paper: Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risks of Language Models", "url": "https://arxiv.org/abs/2408.08926", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://www.haproxy.org/download/2.8/src/haproxy-2.8.1.tar.gz", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybench/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2408.08926", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybench/eval.yaml", "family_title": "Cybench: Capture-The-Flag Cybersecurity Challenges", "group": "Cybersecurity", "version": "4-C", "contributors": [ "sinman-aisi", "sam-deverett-dsit", "kola-aisi", "pgiav" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "cybench", "dataset_samples": 39, "comment": null, "support_only": false }, "tasks": [ { "name": "cybench", "dataset_samples": 39, "comment": null, "support_only": false } ], "protocol": { "version": "4-C", "dataset_samples": 39, "dependency": "cybench", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "direct_url", "source": "https://www.haproxy.org/download/2.8/src/haproxy-2.8.1.tar.gz", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/andyzorigin/cybench/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "challenge files, including binaries and malware-like artifacts, fetched instead of vendored" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/LosFuzzys/GlacierCTF2023_writeups/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "challenge files for the Glacier challenges, which cybench references but does not commit" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cybench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Andy K. Zhang, Neil Perry, Riya Dulepet et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2408.08926" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2408.08926", "arxiv_id": "2408.08926", "title": "Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risks of Language Models", "authors": [ "Andy K. Zhang", "Neil Perry", "Riya Dulepet", "Joey Ji", "Celeste Menders", "Justin W. Lin", "Eliot Jones", "Gashon Hussein", "Samantha Liu", "Donovan Jasper", "Pura Peetathawatchai", "Ari Glenn", "Vikram Sivashankar", "Daniel Zamoshchin", "Leo Glikbarg", "Derek Askaryar", "Mike Yang", "Teddy Zhang", "Rishi Alluri", "Nathan Tran", "Rinnara Sangpisit", "Polycarpos Yiorkadjis", "Kenny Osele", "Gautham Raghupathi", "Dan Boneh", "Daniel E. Ho", "Percy Liang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2408.08926", "arxiv_id": "2408.08926", "title": "Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risks of Language Models", "authors": [ "Andy K. Zhang", "Neil Perry", "Riya Dulepet", "Joey Ji", "Celeste Menders", "Justin W. Lin", "Eliot Jones", "Gashon Hussein", "Samantha Liu", "Donovan Jasper", "Pura Peetathawatchai", "Ari Glenn", "Vikram Sivashankar", "Daniel Zamoshchin", "Leo Glikbarg", "Derek Askaryar", "Mike Yang", "Teddy Zhang", "Rishi Alluri", "Nathan Tran", "Rinnara Sangpisit", "Polycarpos Yiorkadjis", "Kenny Osele", "Gautham Raghupathi", "Dan Boneh", "Daniel E. Ho", "Percy Liang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://www.haproxy.org/download/2.8/src/haproxy-2.8.1.tar.gz", "owner": "haproxy.org", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2408.08926" }, "code_url": "https://www.haproxy.org/download/2.8/src/haproxy-2.8.1.tar.gz", "metadata": { "source_search": "Andy K. Zhang, Neil Perry, Riya Dulepet et al. Original paper authors Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risks of Language Models Andy K. Zhang Neil Perry Riya Dulepet Joey Ji Celeste Menders Justin W. Lin Eliot Jones Gashon Hussein Samantha Liu Donovan Jasper Pura Peetathawatchai Ari Glenn Vikram Sivashankar Daniel Zamoshchin Leo Glikbarg Derek Askaryar Mike Yang Teddy Zhang Rishi Alluri Nathan Tran Rinnara Sangpisit Polycarpos Yiorkadjis Kenny Osele Gautham Raghupathi Dan Boneh Daniel E. Ho Percy Liang haproxy.org evaluation-task" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Autonomous cybersecurity problem-solving on selected professional capture-the-flag challenges.", "unit_of_analysis": "One agent trajectory attempting one CTF task or subtask in a command-line environment.", "interaction_type": "Long-horizon tool use, code execution, file inspection, and iterative exploitation in sandboxed challenges.", "target_outcome": "Recovery of the correct flag, with subtask progress measures where provided.", "human_participants": "No human subjects. Professional CTF tasks and expert decompositions provide challenge and reference structure.", "elicitation": "The agent receives a challenge description, environment, command tools, and a bounded interaction budget; optional subtasks can scaffold progress.", "scoring": "Binary flag capture plus subtask completion and pass-rate aggregation across challenge categories.", "protocol_dependence": "Very high: challenge selection, environment, tool access, subtask hints, attempt budget, and scaffold matter.", "implementation_dependence": "High: container reliability, flag validators, package versions, and command-tool behavior affect results.", "limitations": [ "CTF challenges are stylized and differ from live networks and organizational systems.", "Public task familiarity and leakage can inflate apparent capability.", "Binary flags conceal unsafe or inefficient intermediate behavior.", "Subtask scaffolding changes the capability being measured." ], "comparability": "Use the same CyBench release, task subset, environment images, hint or subtask condition, tool access, budget, scaffold, and model version.", "decision_relevance": "Useful for controlled cyber capability monitoring and targeted red teaming; operational risk assessment requires realistic targets and deployment assumptions.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2408.08926" }, { "kind": "implementation", "url": "https://github.com/andyzorigin/cybench" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:cybergym", "name": "CyberGym: Evaluating AI Agents' Real-World Cybersecurity Capabilities at Scale", "slug": "inspect--cybergym", "organisation": "Zhun Wang, Tianneng Shi, Jingxuan He et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybergym", "topics": [ "cyber" ], "description": "A large-scale, high-quality cybersecurity evaluation framework designed to rigorously assess the capabilities of AI agents on real-world vulnerability analysis tasks. CyberGym includes 1,507 benchmark instances with historical vulnerabilities from 188 large software projects.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybergym/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2506.02548", "primary": true }, { "kind": "paper", "label": "Paper: CyberGym: Evaluating AI Agents'Real-World Cybersecurity Capabilities at Scale", "url": "https://arxiv.org/abs/2506.02548", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybergym/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Sunblaze Ucb", "url": "https://huggingface.co/datasets/sunblaze-ucb/cybergym", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2506.02548", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybergym/eval.yaml", "family_title": "CyberGym: Evaluating AI Agents' Real-World Cybersecurity Capabilities at Scale", "group": "Cybersecurity", "version": "3-B", "contributors": [ "wzunknown", "stneng", "LostBenjamin", "pro-wh" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "cybergym", "dataset_samples": 6028, "comment": null, "support_only": false }, "tasks": [ { "name": "cybergym", "dataset_samples": 6028, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 6028, "dependency": "cybergym", "requires_internet": false, "sandbox": [ "solver", "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "sunblaze-ucb/cybergym", "fetch_method": "snapshot_download", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cybergym", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zhun Wang, Tianneng Shi, Jingxuan He et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2506.02548" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.02548", "arxiv_id": "2506.02548", "title": "CyberGym: Evaluating AI Agents'Real-World Cybersecurity Capabilities at Scale", "authors": [ "Zhun Wang", "Tianneng Shi", "Jingxuan He", "Matthew Cai", "Jialin Zhang", "Dawn Song" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.02548", "arxiv_id": "2506.02548", "title": "CyberGym: Evaluating AI Agents'Real-World Cybersecurity Capabilities at Scale", "authors": [ "Zhun Wang", "Tianneng Shi", "Jingxuan He", "Matthew Cai", "Jialin Zhang", "Dawn Song" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybergym/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/sunblaze-ucb/cybergym", "provider": "Sunblaze Ucb" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2506.02548" }, "code_url": null, "metadata": { "source_search": "Zhun Wang, Tianneng Shi, Jingxuan He et al. Original paper authors CyberGym: Evaluating AI Agents'Real-World Cybersecurity Capabilities at Scale Zhun Wang Tianneng Shi Jingxuan He Matthew Cai Jialin Zhang Dawn Song evaluation-task" } }, { "id": "inspect:cybermetric_10000", "name": "Cybermetric 10000", "slug": "inspect--cybermetric-10000", "organisation": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric", "topics": [ "cyber" ], "description": "Datasets containing 80, 500, 2000 and 10000 multiple-choice questions, designed to evaluate understanding across nine domains within cybersecurity", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.07688", "primary": true }, { "kind": "paper", "label": "Paper: CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "url": "https://arxiv.org/abs/2402.07688", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/cybermetric/CyberMetric", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.07688", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "family_title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "group": "Cybersecurity", "version": "2-A", "contributors": [ "neilshaabi" ], "tags": [ "Cybersecurity" ], "task": { "name": "cybermetric_10000", "dataset_samples": 10000, "comment": null, "support_only": false }, "tasks": [ { "name": "cybermetric_80", "dataset_samples": 80, "comment": null, "support_only": false }, { "name": "cybermetric_500", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "cybermetric_2000", "dataset_samples": 2000, "comment": null, "support_only": false }, { "name": "cybermetric_10000", "dataset_samples": 10000, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 10000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/cybermetric/CyberMetric/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cybermetric", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al.", "organization": "Eötvös Loránd University; Technology Innovation Institute", "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.07688" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.07688", "arxiv_id": "2402.07688", "title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "authors": [ "Norbert Tihanyi", "Mohamed Amine Ferrag", "Ridhi Jain", "Tamas Bisztray", "Merouane Debbah" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.07688", "arxiv_id": "2402.07688", "title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "authors": [ "Norbert Tihanyi", "Mohamed Amine Ferrag", "Ridhi Jain", "Tamas Bisztray", "Merouane Debbah" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/cybermetric/CyberMetric", "owner": "Cybermetric", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.07688" }, "code_url": "https://github.com/cybermetric/CyberMetric", "metadata": { "source_search": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al. Eötvös Loránd University; Technology Innovation Institute Original paper authors CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge Norbert Tihanyi Mohamed Amine Ferrag Ridhi Jain Tamas Bisztray Merouane Debbah Cybermetric evaluation-family" } }, { "id": "inspect:cybermetric_2000", "name": "Cybermetric 2000", "slug": "inspect--cybermetric-2000", "organisation": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric", "topics": [ "cyber" ], "description": "Datasets containing 80, 500, 2000 and 10000 multiple-choice questions, designed to evaluate understanding across nine domains within cybersecurity", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.07688", "primary": true }, { "kind": "paper", "label": "Paper: CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "url": "https://arxiv.org/abs/2402.07688", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/cybermetric/CyberMetric", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.07688", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "family_title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "group": "Cybersecurity", "version": "2-A", "contributors": [ "neilshaabi" ], "tags": [ "Cybersecurity" ], "task": { "name": "cybermetric_2000", "dataset_samples": 2000, "comment": null, "support_only": false }, "tasks": [ { "name": "cybermetric_80", "dataset_samples": 80, "comment": null, "support_only": false }, { "name": "cybermetric_500", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "cybermetric_2000", "dataset_samples": 2000, "comment": null, "support_only": false }, { "name": "cybermetric_10000", "dataset_samples": 10000, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 2000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/cybermetric/CyberMetric/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cybermetric", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al.", "organization": "Eötvös Loránd University; Technology Innovation Institute", "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.07688" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.07688", "arxiv_id": "2402.07688", "title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "authors": [ "Norbert Tihanyi", "Mohamed Amine Ferrag", "Ridhi Jain", "Tamas Bisztray", "Merouane Debbah" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.07688", "arxiv_id": "2402.07688", "title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "authors": [ "Norbert Tihanyi", "Mohamed Amine Ferrag", "Ridhi Jain", "Tamas Bisztray", "Merouane Debbah" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/cybermetric/CyberMetric", "owner": "Cybermetric", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.07688" }, "code_url": "https://github.com/cybermetric/CyberMetric", "metadata": { "source_search": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al. Eötvös Loránd University; Technology Innovation Institute Original paper authors CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge Norbert Tihanyi Mohamed Amine Ferrag Ridhi Jain Tamas Bisztray Merouane Debbah Cybermetric evaluation-family" } }, { "id": "inspect:cybermetric_500", "name": "Cybermetric 500", "slug": "inspect--cybermetric-500", "organisation": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric", "topics": [ "cyber" ], "description": "Datasets containing 80, 500, 2000 and 10000 multiple-choice questions, designed to evaluate understanding across nine domains within cybersecurity", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.07688", "primary": true }, { "kind": "paper", "label": "Paper: CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "url": "https://arxiv.org/abs/2402.07688", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/cybermetric/CyberMetric", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.07688", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "family_title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "group": "Cybersecurity", "version": "2-A", "contributors": [ "neilshaabi" ], "tags": [ "Cybersecurity" ], "task": { "name": "cybermetric_500", "dataset_samples": 500, "comment": null, "support_only": false }, "tasks": [ { "name": "cybermetric_80", "dataset_samples": 80, "comment": null, "support_only": false }, { "name": "cybermetric_500", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "cybermetric_2000", "dataset_samples": 2000, "comment": null, "support_only": false }, { "name": "cybermetric_10000", "dataset_samples": 10000, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 500, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/cybermetric/CyberMetric/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cybermetric", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al.", "organization": "Eötvös Loránd University; Technology Innovation Institute", "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.07688" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.07688", "arxiv_id": "2402.07688", "title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "authors": [ "Norbert Tihanyi", "Mohamed Amine Ferrag", "Ridhi Jain", "Tamas Bisztray", "Merouane Debbah" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.07688", "arxiv_id": "2402.07688", "title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "authors": [ "Norbert Tihanyi", "Mohamed Amine Ferrag", "Ridhi Jain", "Tamas Bisztray", "Merouane Debbah" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/cybermetric/CyberMetric", "owner": "Cybermetric", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.07688" }, "code_url": "https://github.com/cybermetric/CyberMetric", "metadata": { "source_search": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al. Eötvös Loránd University; Technology Innovation Institute Original paper authors CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge Norbert Tihanyi Mohamed Amine Ferrag Ridhi Jain Tamas Bisztray Merouane Debbah Cybermetric evaluation-family" } }, { "id": "inspect:cybermetric_80", "name": "Cybermetric 80", "slug": "inspect--cybermetric-80", "organisation": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric", "topics": [ "cyber" ], "description": "Datasets containing 80, 500, 2000 and 10000 multiple-choice questions, designed to evaluate understanding across nine domains within cybersecurity", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.07688", "primary": true }, { "kind": "paper", "label": "Paper: CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "url": "https://arxiv.org/abs/2402.07688", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/cybermetric/CyberMetric", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.07688", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml", "family_title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "group": "Cybersecurity", "version": "2-A", "contributors": [ "neilshaabi" ], "tags": [ "Cybersecurity" ], "task": { "name": "cybermetric_80", "dataset_samples": 80, "comment": null, "support_only": false }, "tasks": [ { "name": "cybermetric_80", "dataset_samples": 80, "comment": null, "support_only": false }, { "name": "cybermetric_500", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "cybermetric_2000", "dataset_samples": 2000, "comment": null, "support_only": false }, { "name": "cybermetric_10000", "dataset_samples": 10000, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 80, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/cybermetric/CyberMetric/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cybermetric", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al.", "organization": "Eötvös Loránd University; Technology Innovation Institute", "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.07688" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.07688", "arxiv_id": "2402.07688", "title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "authors": [ "Norbert Tihanyi", "Mohamed Amine Ferrag", "Ridhi Jain", "Tamas Bisztray", "Merouane Debbah" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.07688", "arxiv_id": "2402.07688", "title": "CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge", "authors": [ "Norbert Tihanyi", "Mohamed Amine Ferrag", "Ridhi Jain", "Tamas Bisztray", "Merouane Debbah" ], "organizations": [ "Eötvös Loránd University", "Technology Innovation Institute" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/cybermetric/CyberMetric", "owner": "Cybermetric", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cybermetric/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.07688" }, "code_url": "https://github.com/cybermetric/CyberMetric", "metadata": { "source_search": "Norbert Tihanyi, Mohamed Amine Ferrag, Ridhi Jain et al. Eötvös Loránd University; Technology Innovation Institute Original paper authors CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge Norbert Tihanyi Mohamed Amine Ferrag Ridhi Jain Tamas Bisztray Merouane Debbah Cybermetric evaluation-family" } }, { "id": "canonical:cyberseceval", "name": "CyberSecEval", "slug": "canonical--cyberseceval", "organisation": "Meta", "source_type": "canonical-source", "source_url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks", "topics": [ "cyber", "safeguards" ], "description": "Cybersecurity suite covering insecure code, exploitation, prompt injection and related risks.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks", "primary": false } ], "resource_count": 3, "source_key": "canonical:cyberseceval", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Meta", "organization": "Meta", "organizations": [ "Meta" ], "entity_type": "organization", "role": "evaluation_developer", "role_label": "Evaluation developed by", "confidence": "curated", "evidence_url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks", "owner": "Meta", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks" }, "paper_url": null, "code_url": "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks", "metadata": { "source_search": "Meta Meta Evaluation developed by Meta evaluation-resource" } }, { "id": "inspect:cyse2_interpreter_abuse", "name": "Cyse2 Interpreter Abuse", "slug": "inspect--cyse2-interpreter-abuse", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2", "topics": [ "general-capability" ], "description": "Assesses language models for cybersecurity risks, specifically testing their potential to misuse programming interpreters, vulnerability to malicious prompt injections, and capability to exploit known software vulnerabilities.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml", "family_title": "CyberSecEval_2: Cybersecurity Risk and Vulnerability Evaluation", "group": "Cybersecurity", "version": "4-A", "contributors": [ "its-emile" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse2_interpreter_abuse", "dataset_samples": 500, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse2_interpreter_abuse", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "cyse2_prompt_injection", "dataset_samples": 251, "comment": null, "support_only": false }, { "name": "cyse2_vulnerability_exploit", "dataset_samples": 585, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 500, "dependency": null, "requires_internet": false, "sandbox": [ "scorer", "solver" ] }, "external_assets": [ { "type": "direct_url", "source": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "fetch_method": "curl", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/{SHA}/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "interpreter.json and prompt_injection.json; fetched via fsspec" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_2", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "owner": "nodejs.org", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe nodejs.org evaluation-family" } }, { "id": "inspect:cyse2_prompt_injection", "name": "Cyse2 Prompt Injection", "slug": "inspect--cyse2-prompt-injection", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2", "topics": [ "cyber" ], "description": "Assesses language models for cybersecurity risks, specifically testing their potential to misuse programming interpreters, vulnerability to malicious prompt injections, and capability to exploit known software vulnerabilities.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml", "family_title": "CyberSecEval_2: Cybersecurity Risk and Vulnerability Evaluation", "group": "Cybersecurity", "version": "4-A", "contributors": [ "its-emile" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse2_prompt_injection", "dataset_samples": 251, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse2_interpreter_abuse", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "cyse2_prompt_injection", "dataset_samples": 251, "comment": null, "support_only": false }, { "name": "cyse2_vulnerability_exploit", "dataset_samples": 585, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 251, "dependency": null, "requires_internet": false, "sandbox": [ "scorer", "solver" ] }, "external_assets": [ { "type": "direct_url", "source": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "fetch_method": "curl", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/{SHA}/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "interpreter.json and prompt_injection.json; fetched via fsspec" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_2", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "owner": "nodejs.org", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe nodejs.org evaluation-family" } }, { "id": "inspect:cyse2_vulnerability_exploit", "name": "Cyse2 Vulnerability Exploit", "slug": "inspect--cyse2-vulnerability-exploit", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2", "topics": [ "cyber" ], "description": "Assesses language models for cybersecurity risks, specifically testing their potential to misuse programming interpreters, vulnerability to malicious prompt injections, and capability to exploit known software vulnerabilities.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml", "family_title": "CyberSecEval_2: Cybersecurity Risk and Vulnerability Evaluation", "group": "Cybersecurity", "version": "4-A", "contributors": [ "its-emile" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse2_vulnerability_exploit", "dataset_samples": 585, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse2_interpreter_abuse", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "cyse2_prompt_injection", "dataset_samples": 251, "comment": null, "support_only": false }, { "name": "cyse2_vulnerability_exploit", "dataset_samples": 585, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 585, "dependency": null, "requires_internet": false, "sandbox": [ "scorer", "solver" ] }, "external_assets": [ { "type": "direct_url", "source": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "fetch_method": "curl", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/{SHA}/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "interpreter.json and prompt_injection.json; fetched via fsspec" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_2", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "owner": "nodejs.org", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_2/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://nodejs.org/dist/v20.18.3/node-v20.18.3-linux-x64.tar.xz", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe nodejs.org evaluation-family" } }, { "id": "inspect:cyse3_visual_prompt_injection", "name": "CYBERSECEVAL 3: Advancing the Evaluation of Cybersecurity Risks and Capabilities in Large Language Models", "slug": "inspect--cyse3-visual-prompt-injection", "organisation": "Manish Bhatt, Sahana Chennabasappa, Cyrus Nikolaidis et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_3", "topics": [ "cyber", "multimodal" ], "description": "Evaluates Large Language Models for cybersecurity risk to third parties, application developers and end users.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_3/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2312.04724", "primary": true }, { "kind": "paper", "label": "Paper: Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models", "url": "https://arxiv.org/abs/2312.04724", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_3/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Facebook", "url": "https://huggingface.co/datasets/facebook/cyberseceval3-visual-prompt-injection", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2312.04724", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_3/eval.yaml", "family_title": "CYBERSECEVAL 3: Advancing the Evaluation of Cybersecurity Risks and Capabilities in Large Language Models", "group": "Cybersecurity", "version": "3-A", "contributors": [ "onionymous" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse3_visual_prompt_injection", "dataset_samples": 1000, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse3_visual_prompt_injection", "dataset_samples": 1000, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 1000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "facebook/cyberseceval3-visual-prompt-injection", "fetch_method": "snapshot_download", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "facebook/cyberseceval3-visual-prompt-injection", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_3", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Cyrus Nikolaidis et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2312.04724" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2312.04724", "arxiv_id": "2312.04724", "title": "Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Cyrus Nikolaidis", "Shengye Wan", "Ivan Evtimov", "Dominik Gabi", "Daniel Song", "Faizan Ahmad", "Cornelius Aschermann", "Lorenzo Fontana", "Sasha Frolov", "Ravi Prakash Giri", "Dhaval Kapil", "Yiannis Kozyrakis", "David LeBlanc", "James Milazzo", "Aleksandar Straumann", "Gabriel Synnaeve", "Varun Vontimitta", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2312.04724", "arxiv_id": "2312.04724", "title": "Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Cyrus Nikolaidis", "Shengye Wan", "Ivan Evtimov", "Dominik Gabi", "Daniel Song", "Faizan Ahmad", "Cornelius Aschermann", "Lorenzo Fontana", "Sasha Frolov", "Ravi Prakash Giri", "Dhaval Kapil", "Yiannis Kozyrakis", "David LeBlanc", "James Milazzo", "Aleksandar Straumann", "Gabriel Synnaeve", "Varun Vontimitta", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_3/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/facebook/cyberseceval3-visual-prompt-injection", "provider": "Facebook" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2312.04724" }, "code_url": null, "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Cyrus Nikolaidis et al. Original paper authors Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models Manish Bhatt Sahana Chennabasappa Cyrus Nikolaidis Shengye Wan Ivan Evtimov Dominik Gabi Daniel Song Faizan Ahmad Cornelius Aschermann Lorenzo Fontana Sasha Frolov Ravi Prakash Giri Dhaval Kapil Yiannis Kozyrakis David LeBlanc James Milazzo Aleksandar Straumann Gabriel Synnaeve Varun Vontimitta Spencer Whitman Joshua Saxe evaluation-task" } }, { "id": "inspect:cyse4_autocomplete", "name": "Cyse4 Autocomplete", "slug": "inspect--cyse4-autocomplete", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4", "topics": [ "general-capability" ], "description": "A suite of cybersecurity evaluation benchmarks adapted from Meta's PurpleLlama CybersecurityBenchmarks. Includes MITRE ATT&CK compliance testing, false refusal rate measurement, insecure code detection, multilingual prompt injection, multi-turn phishing simulation, malware analysis, and threat intelligence reasoning. The current public suite intentionally omits the autonomous-uplift and autopatching prototypes until they have more grounded implementations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meta-llama/PurpleLlama", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "family_title": "CyberSecEval 4: Advanced Cybersecurity Evaluation Benchmarks", "group": "Cybersecurity", "version": "5-B", "contributors": [ "ckane" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse4_autocomplete", "dataset_samples": 1916, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse4_mitre", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "cyse4_mitre_frr", "dataset_samples": 750, "comment": null, "support_only": false }, { "name": "cyse4_instruct", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_autocomplete", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_multiturn_phishing", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "cyse4_multilingual_prompt_injection", "dataset_samples": 1004, "comment": null, "support_only": false }, { "name": "cyse4_malware_analysis", "dataset_samples": 609, "comment": null, "support_only": false }, { "name": "cyse4_threat_intelligence", "dataset_samples": 588, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 1916, "dependency": "cyberseceval_4", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "instruct.json, autocomplete.json, mitre_benchmark_100_per_category_with_augmentation.json, mitre_frr.json, and multiturn_phishing_challenges.json; fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/prompt_injection/prompt_injection_multilingual_machine_translated.json", "fetch_method": "other", "state": "pinned", "comment": "fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autonomous_uplift/in/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "system_prompt.txt and cyber_range_pairs_sample.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autopatch/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "autopatch_bench.json and autopatch_lite.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://github.com/CrowdStrike/CyberSOCEval_data/archive/ce7daa5bc7da51559ca97476d2277be02631783e.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": "malware-analysis and threat-intelligence report data; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://web.archive.org/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "non-CrowdStrike threat-intelligence report PDFs; each report's pinned web.archive.org snapshot (url_source in threat_intel_reasoning/report_questions.json) is downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_4", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/meta-llama/PurpleLlama", "owner": "Meta", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://github.com/meta-llama/PurpleLlama", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe Meta evaluation-family" } }, { "id": "inspect:cyse4_instruct", "name": "Cyse4 Instruct", "slug": "inspect--cyse4-instruct", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4", "topics": [ "general-capability" ], "description": "A suite of cybersecurity evaluation benchmarks adapted from Meta's PurpleLlama CybersecurityBenchmarks. Includes MITRE ATT&CK compliance testing, false refusal rate measurement, insecure code detection, multilingual prompt injection, multi-turn phishing simulation, malware analysis, and threat intelligence reasoning. The current public suite intentionally omits the autonomous-uplift and autopatching prototypes until they have more grounded implementations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meta-llama/PurpleLlama", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "family_title": "CyberSecEval 4: Advanced Cybersecurity Evaluation Benchmarks", "group": "Cybersecurity", "version": "5-B", "contributors": [ "ckane" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse4_instruct", "dataset_samples": 1916, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse4_mitre", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "cyse4_mitre_frr", "dataset_samples": 750, "comment": null, "support_only": false }, { "name": "cyse4_instruct", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_autocomplete", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_multiturn_phishing", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "cyse4_multilingual_prompt_injection", "dataset_samples": 1004, "comment": null, "support_only": false }, { "name": "cyse4_malware_analysis", "dataset_samples": 609, "comment": null, "support_only": false }, { "name": "cyse4_threat_intelligence", "dataset_samples": 588, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 1916, "dependency": "cyberseceval_4", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "instruct.json, autocomplete.json, mitre_benchmark_100_per_category_with_augmentation.json, mitre_frr.json, and multiturn_phishing_challenges.json; fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/prompt_injection/prompt_injection_multilingual_machine_translated.json", "fetch_method": "other", "state": "pinned", "comment": "fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autonomous_uplift/in/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "system_prompt.txt and cyber_range_pairs_sample.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autopatch/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "autopatch_bench.json and autopatch_lite.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://github.com/CrowdStrike/CyberSOCEval_data/archive/ce7daa5bc7da51559ca97476d2277be02631783e.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": "malware-analysis and threat-intelligence report data; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://web.archive.org/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "non-CrowdStrike threat-intelligence report PDFs; each report's pinned web.archive.org snapshot (url_source in threat_intel_reasoning/report_questions.json) is downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_4", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/meta-llama/PurpleLlama", "owner": "Meta", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://github.com/meta-llama/PurpleLlama", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe Meta evaluation-family" } }, { "id": "inspect:cyse4_malware_analysis", "name": "Cyse4 Malware Analysis", "slug": "inspect--cyse4-malware-analysis", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4", "topics": [ "cyber" ], "description": "A suite of cybersecurity evaluation benchmarks adapted from Meta's PurpleLlama CybersecurityBenchmarks. Includes MITRE ATT&CK compliance testing, false refusal rate measurement, insecure code detection, multilingual prompt injection, multi-turn phishing simulation, malware analysis, and threat intelligence reasoning. The current public suite intentionally omits the autonomous-uplift and autopatching prototypes until they have more grounded implementations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meta-llama/PurpleLlama", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "family_title": "CyberSecEval 4: Advanced Cybersecurity Evaluation Benchmarks", "group": "Cybersecurity", "version": "5-B", "contributors": [ "ckane" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse4_malware_analysis", "dataset_samples": 609, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse4_mitre", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "cyse4_mitre_frr", "dataset_samples": 750, "comment": null, "support_only": false }, { "name": "cyse4_instruct", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_autocomplete", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_multiturn_phishing", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "cyse4_multilingual_prompt_injection", "dataset_samples": 1004, "comment": null, "support_only": false }, { "name": "cyse4_malware_analysis", "dataset_samples": 609, "comment": null, "support_only": false }, { "name": "cyse4_threat_intelligence", "dataset_samples": 588, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 609, "dependency": "cyberseceval_4", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "instruct.json, autocomplete.json, mitre_benchmark_100_per_category_with_augmentation.json, mitre_frr.json, and multiturn_phishing_challenges.json; fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/prompt_injection/prompt_injection_multilingual_machine_translated.json", "fetch_method": "other", "state": "pinned", "comment": "fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autonomous_uplift/in/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "system_prompt.txt and cyber_range_pairs_sample.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autopatch/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "autopatch_bench.json and autopatch_lite.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://github.com/CrowdStrike/CyberSOCEval_data/archive/ce7daa5bc7da51559ca97476d2277be02631783e.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": "malware-analysis and threat-intelligence report data; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://web.archive.org/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "non-CrowdStrike threat-intelligence report PDFs; each report's pinned web.archive.org snapshot (url_source in threat_intel_reasoning/report_questions.json) is downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_4", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/meta-llama/PurpleLlama", "owner": "Meta", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://github.com/meta-llama/PurpleLlama", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe Meta evaluation-family" } }, { "id": "inspect:cyse4_mitre", "name": "Cyse4 Mitre", "slug": "inspect--cyse4-mitre", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4", "topics": [ "cyber" ], "description": "A suite of cybersecurity evaluation benchmarks adapted from Meta's PurpleLlama CybersecurityBenchmarks. Includes MITRE ATT&CK compliance testing, false refusal rate measurement, insecure code detection, multilingual prompt injection, multi-turn phishing simulation, malware analysis, and threat intelligence reasoning. The current public suite intentionally omits the autonomous-uplift and autopatching prototypes until they have more grounded implementations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meta-llama/PurpleLlama", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "family_title": "CyberSecEval 4: Advanced Cybersecurity Evaluation Benchmarks", "group": "Cybersecurity", "version": "5-B", "contributors": [ "ckane" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse4_mitre", "dataset_samples": 1000, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse4_mitre", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "cyse4_mitre_frr", "dataset_samples": 750, "comment": null, "support_only": false }, { "name": "cyse4_instruct", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_autocomplete", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_multiturn_phishing", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "cyse4_multilingual_prompt_injection", "dataset_samples": 1004, "comment": null, "support_only": false }, { "name": "cyse4_malware_analysis", "dataset_samples": 609, "comment": null, "support_only": false }, { "name": "cyse4_threat_intelligence", "dataset_samples": 588, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 1000, "dependency": "cyberseceval_4", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "instruct.json, autocomplete.json, mitre_benchmark_100_per_category_with_augmentation.json, mitre_frr.json, and multiturn_phishing_challenges.json; fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/prompt_injection/prompt_injection_multilingual_machine_translated.json", "fetch_method": "other", "state": "pinned", "comment": "fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autonomous_uplift/in/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "system_prompt.txt and cyber_range_pairs_sample.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autopatch/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "autopatch_bench.json and autopatch_lite.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://github.com/CrowdStrike/CyberSOCEval_data/archive/ce7daa5bc7da51559ca97476d2277be02631783e.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": "malware-analysis and threat-intelligence report data; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://web.archive.org/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "non-CrowdStrike threat-intelligence report PDFs; each report's pinned web.archive.org snapshot (url_source in threat_intel_reasoning/report_questions.json) is downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_4", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/meta-llama/PurpleLlama", "owner": "Meta", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://github.com/meta-llama/PurpleLlama", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe Meta evaluation-family" } }, { "id": "inspect:cyse4_mitre_frr", "name": "Cyse4 Mitre Frr", "slug": "inspect--cyse4-mitre-frr", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4", "topics": [ "cyber" ], "description": "A suite of cybersecurity evaluation benchmarks adapted from Meta's PurpleLlama CybersecurityBenchmarks. Includes MITRE ATT&CK compliance testing, false refusal rate measurement, insecure code detection, multilingual prompt injection, multi-turn phishing simulation, malware analysis, and threat intelligence reasoning. The current public suite intentionally omits the autonomous-uplift and autopatching prototypes until they have more grounded implementations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meta-llama/PurpleLlama", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "family_title": "CyberSecEval 4: Advanced Cybersecurity Evaluation Benchmarks", "group": "Cybersecurity", "version": "5-B", "contributors": [ "ckane" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse4_mitre_frr", "dataset_samples": 750, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse4_mitre", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "cyse4_mitre_frr", "dataset_samples": 750, "comment": null, "support_only": false }, { "name": "cyse4_instruct", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_autocomplete", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_multiturn_phishing", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "cyse4_multilingual_prompt_injection", "dataset_samples": 1004, "comment": null, "support_only": false }, { "name": "cyse4_malware_analysis", "dataset_samples": 609, "comment": null, "support_only": false }, { "name": "cyse4_threat_intelligence", "dataset_samples": 588, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 750, "dependency": "cyberseceval_4", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "instruct.json, autocomplete.json, mitre_benchmark_100_per_category_with_augmentation.json, mitre_frr.json, and multiturn_phishing_challenges.json; fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/prompt_injection/prompt_injection_multilingual_machine_translated.json", "fetch_method": "other", "state": "pinned", "comment": "fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autonomous_uplift/in/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "system_prompt.txt and cyber_range_pairs_sample.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autopatch/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "autopatch_bench.json and autopatch_lite.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://github.com/CrowdStrike/CyberSOCEval_data/archive/ce7daa5bc7da51559ca97476d2277be02631783e.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": "malware-analysis and threat-intelligence report data; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://web.archive.org/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "non-CrowdStrike threat-intelligence report PDFs; each report's pinned web.archive.org snapshot (url_source in threat_intel_reasoning/report_questions.json) is downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_4", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/meta-llama/PurpleLlama", "owner": "Meta", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://github.com/meta-llama/PurpleLlama", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe Meta evaluation-family" } }, { "id": "inspect:cyse4_multilingual_prompt_injection", "name": "Cyse4 Multilingual Prompt Injection", "slug": "inspect--cyse4-multilingual-prompt-injection", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4", "topics": [ "cyber" ], "description": "A suite of cybersecurity evaluation benchmarks adapted from Meta's PurpleLlama CybersecurityBenchmarks. Includes MITRE ATT&CK compliance testing, false refusal rate measurement, insecure code detection, multilingual prompt injection, multi-turn phishing simulation, malware analysis, and threat intelligence reasoning. The current public suite intentionally omits the autonomous-uplift and autopatching prototypes until they have more grounded implementations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meta-llama/PurpleLlama", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "family_title": "CyberSecEval 4: Advanced Cybersecurity Evaluation Benchmarks", "group": "Cybersecurity", "version": "5-B", "contributors": [ "ckane" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse4_multilingual_prompt_injection", "dataset_samples": 1004, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse4_mitre", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "cyse4_mitre_frr", "dataset_samples": 750, "comment": null, "support_only": false }, { "name": "cyse4_instruct", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_autocomplete", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_multiturn_phishing", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "cyse4_multilingual_prompt_injection", "dataset_samples": 1004, "comment": null, "support_only": false }, { "name": "cyse4_malware_analysis", "dataset_samples": 609, "comment": null, "support_only": false }, { "name": "cyse4_threat_intelligence", "dataset_samples": 588, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 1004, "dependency": "cyberseceval_4", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "instruct.json, autocomplete.json, mitre_benchmark_100_per_category_with_augmentation.json, mitre_frr.json, and multiturn_phishing_challenges.json; fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/prompt_injection/prompt_injection_multilingual_machine_translated.json", "fetch_method": "other", "state": "pinned", "comment": "fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autonomous_uplift/in/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "system_prompt.txt and cyber_range_pairs_sample.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autopatch/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "autopatch_bench.json and autopatch_lite.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://github.com/CrowdStrike/CyberSOCEval_data/archive/ce7daa5bc7da51559ca97476d2277be02631783e.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": "malware-analysis and threat-intelligence report data; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://web.archive.org/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "non-CrowdStrike threat-intelligence report PDFs; each report's pinned web.archive.org snapshot (url_source in threat_intel_reasoning/report_questions.json) is downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_4", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/meta-llama/PurpleLlama", "owner": "Meta", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://github.com/meta-llama/PurpleLlama", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe Meta evaluation-family" } }, { "id": "inspect:cyse4_multiturn_phishing", "name": "Cyse4 Multiturn Phishing", "slug": "inspect--cyse4-multiturn-phishing", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4", "topics": [ "cyber" ], "description": "A suite of cybersecurity evaluation benchmarks adapted from Meta's PurpleLlama CybersecurityBenchmarks. Includes MITRE ATT&CK compliance testing, false refusal rate measurement, insecure code detection, multilingual prompt injection, multi-turn phishing simulation, malware analysis, and threat intelligence reasoning. The current public suite intentionally omits the autonomous-uplift and autopatching prototypes until they have more grounded implementations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meta-llama/PurpleLlama", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "family_title": "CyberSecEval 4: Advanced Cybersecurity Evaluation Benchmarks", "group": "Cybersecurity", "version": "5-B", "contributors": [ "ckane" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse4_multiturn_phishing", "dataset_samples": 100, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse4_mitre", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "cyse4_mitre_frr", "dataset_samples": 750, "comment": null, "support_only": false }, { "name": "cyse4_instruct", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_autocomplete", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_multiturn_phishing", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "cyse4_multilingual_prompt_injection", "dataset_samples": 1004, "comment": null, "support_only": false }, { "name": "cyse4_malware_analysis", "dataset_samples": 609, "comment": null, "support_only": false }, { "name": "cyse4_threat_intelligence", "dataset_samples": 588, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 100, "dependency": "cyberseceval_4", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "instruct.json, autocomplete.json, mitre_benchmark_100_per_category_with_augmentation.json, mitre_frr.json, and multiturn_phishing_challenges.json; fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/prompt_injection/prompt_injection_multilingual_machine_translated.json", "fetch_method": "other", "state": "pinned", "comment": "fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autonomous_uplift/in/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "system_prompt.txt and cyber_range_pairs_sample.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autopatch/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "autopatch_bench.json and autopatch_lite.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://github.com/CrowdStrike/CyberSOCEval_data/archive/ce7daa5bc7da51559ca97476d2277be02631783e.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": "malware-analysis and threat-intelligence report data; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://web.archive.org/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "non-CrowdStrike threat-intelligence report PDFs; each report's pinned web.archive.org snapshot (url_source in threat_intel_reasoning/report_questions.json) is downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_4", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/meta-llama/PurpleLlama", "owner": "Meta", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://github.com/meta-llama/PurpleLlama", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe Meta evaluation-family" } }, { "id": "inspect:cyse4_threat_intelligence", "name": "Cyse4 Threat Intelligence", "slug": "inspect--cyse4-threat-intelligence", "organisation": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4", "topics": [ "general-capability" ], "description": "A suite of cybersecurity evaluation benchmarks adapted from Meta's PurpleLlama CybersecurityBenchmarks. Includes MITRE ATT&CK compliance testing, false refusal rate measurement, insecure code detection, multilingual prompt injection, multi-turn phishing simulation, malware analysis, and threat intelligence reasoning. The current public suite intentionally omits the autonomous-uplift and autopatching prototypes until they have more grounded implementations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.13161", "primary": true }, { "kind": "paper", "label": "Paper: CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "url": "https://arxiv.org/abs/2404.13161", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meta-llama/PurpleLlama", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.13161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml", "family_title": "CyberSecEval 4: Advanced Cybersecurity Evaluation Benchmarks", "group": "Cybersecurity", "version": "5-B", "contributors": [ "ckane" ], "tags": [ "Cybersecurity" ], "task": { "name": "cyse4_threat_intelligence", "dataset_samples": 588, "comment": null, "support_only": false }, "tasks": [ { "name": "cyse4_mitre", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "cyse4_mitre_frr", "dataset_samples": 750, "comment": null, "support_only": false }, { "name": "cyse4_instruct", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_autocomplete", "dataset_samples": 1916, "comment": null, "support_only": false }, { "name": "cyse4_multiturn_phishing", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "cyse4_multilingual_prompt_injection", "dataset_samples": 1004, "comment": null, "support_only": false }, { "name": "cyse4_malware_analysis", "dataset_samples": 609, "comment": null, "support_only": false }, { "name": "cyse4_threat_intelligence", "dataset_samples": 588, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 588, "dependency": "cyberseceval_4", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/", "fetch_method": "other", "state": "pinned", "comment": "instruct.json, autocomplete.json, mitre_benchmark_100_per_category_with_augmentation.json, mitre_frr.json, and multiturn_phishing_challenges.json; fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/prompt_injection/prompt_injection_multilingual_machine_translated.json", "fetch_method": "other", "state": "pinned", "comment": "fetched via fsspec (load_json_dataset)" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autonomous_uplift/in/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "system_prompt.txt and cyber_range_pairs_sample.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/meta-llama/PurpleLlama/fe05293b610dabc3967443f2dd4dc35c4e8971b6/CybersecurityBenchmarks/datasets/autopatch/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "autopatch_bench.json and autopatch_lite.json; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://github.com/CrowdStrike/CyberSOCEval_data/archive/ce7daa5bc7da51559ca97476d2277be02631783e.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": "malware-analysis and threat-intelligence report data; downloaded via download_and_verify" }, { "type": "direct_url", "source": "https://web.archive.org/", "fetch_method": "download_and_verify", "state": "pinned", "comment": "non-CrowdStrike threat-intelligence report PDFs; each report's pinned web.archive.org snapshot (url_source in threat_intel_reasoning/report_questions.json) is downloaded via download_and_verify" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:cyberseceval_4", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.13161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.13161", "arxiv_id": "2404.13161", "title": "CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models", "authors": [ "Manish Bhatt", "Sahana Chennabasappa", "Yue Li", "Cyrus Nikolaidis", "Daniel Song", "Shengye Wan", "Faizan Ahmad", "Cornelius Aschermann", "Yaohui Chen", "Dhaval Kapil", "David Molnar", "Spencer Whitman", "Joshua Saxe" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/meta-llama/PurpleLlama", "owner": "Meta", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/cyberseceval_4/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.13161" }, "code_url": "https://github.com/meta-llama/PurpleLlama", "metadata": { "source_search": "Manish Bhatt, Sahana Chennabasappa, Yue Li et al. Original paper authors CyberSecEval 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models Manish Bhatt Sahana Chennabasappa Yue Li Cyrus Nikolaidis Daniel Song Shengye Wan Faizan Ahmad Cornelius Aschermann Yaohui Chen Dhaval Kapil David Molnar Spencer Whitman Joshua Saxe Meta evaluation-family" } }, { "id": "canonical:deepmind-dangerous-capabilities", "name": "Dangerous Capability Evaluations", "slug": "canonical--deepmind-dangerous-capabilities", "organisation": "Google DeepMind", "source_type": "canonical-source", "source_url": "https://deepmind.google/frontier-safety/", "topics": [ "bio-cbrn", "cyber", "autonomy-agents" ], "description": "Evaluation suite spanning dangerous capabilities including autonomy, cyber and CBRN-related domains.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "suite_developer", "label": "Suite developed by", "url": "https://deepmind.google/frontier-safety", "primary": true }, { "kind": "official", "label": "Official or project page", "url": "https://deepmind.google/frontier-safety", "primary": false } ], "resource_count": 2, "source_key": "canonical:deepmind-dangerous-capabilities", "record_type": "evaluation-suite", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "organization", "role": "suite_developer", "role_label": "Suite developed by", "confidence": "curated", "evidence_url": "https://deepmind.google/frontier-safety" }, "paper": { "status": "not-applicable", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-applicable", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Suite developed by", "attribution_status": "verified", "preferred_source": { "label": "Suite developed by", "kind": "suite_developer", "url": "https://deepmind.google/frontier-safety" }, "paper_url": null, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Suite developed by evaluation-suite" } }, { "id": "register:deceptionbench", "name": "DeceptionBench", "slug": "register--deceptionbench", "organisation": "PKU-Alignment and the DeceptionBench paper authors", "source_type": "inspect-register", "source_url": "https://github.com/WatchTree-19/inspect-deceptionbench", "topics": [ "human-influence", "deception-misalignment" ], "description": "Measures belief-vs-behaviour deception in LLMs. For each of 180 scenarios the eval elicits the model's internal belief from a neutral prompt, then places the model under scenario pressure and captures its chain-of-thought (``) and final answer (``). Three verbatim GPT-4o judges check whether the reasoning tracks the belief (R~U_m) while the answer diverges from it (Y!~U_m). The headline metric is the Deception Tendency Rate (DTR) — the fraction of scenarios showing that deceptive pattern — reported overall and across five categories (sycophancy, strategic deception, honesty evasion, alignment faking, sandbagging).", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Deceptive response patterns under DeceptionBench's scenarios and operational definitions, conditional on the validity of its belief and reasoning proxies.", "does_not_measure": "A universal deception trait, private intent, spontaneous scheming, real-world deception prevalence, human susceptibility, or downstream harm.", "best_for": "Comparing protocol-defined deceptive behavior across scenarios and identifying sensitivity to incentives, prompts, and model conditions.", "not_sufficient_for": "Claims that a model knowingly deceives in general or will deceive users in deployment.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/deceptionbench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://arxiv.org/abs/2505.18807", "primary": true }, { "kind": "paper", "label": "Paper: Mitigating Deceptive Alignment via Self-Monitoring", "url": "https://arxiv.org/abs/2505.18807", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/WatchTree-19/inspect-deceptionbench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/deceptionbench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.18807", "primary": false } ], "paper_title": "Mitigating Deceptive Alignment via Self-Monitoring", "paper_url": "https://arxiv.org/abs/2505.18807", "code_url": "https://github.com/WatchTree-19/inspect-deceptionbench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/deceptionbench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "Safety", "Deception" ], "contributors": [ "WatchTree-19" ], "tasks": [ { "name": "deceptionbench", "path": "src/inspect_deceptionbench/deceptionbench.py" } ], "protocol": { "implementation_commit": "8683bf5a480f938b812aa5078fb0ee7d12a74483", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": "Dataset PKU-Alignment/DeceptionBench is associated with arXiv:2505.18807 (not the same-named arXiv:2510.15501, a different benchmark)." }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 5, "source_key": "register:deceptionbench", "record_type": "evaluation-package", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "PKU-Alignment and the DeceptionBench paper authors", "organization": "PKU-Alignment", "entity_type": "research_group", "role": "original_research", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.18807", "organizations": [ "PKU-Alignment" ], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.18807", "arxiv_id": "2505.18807", "title": "Mitigating Deceptive Alignment via Self-Monitoring", "authors": [ "Jiaming Ji", "Wenqi Chen", "Kaile Wang", "Donghai Hong", "Sitong Fang", "Boyuan Chen", "Jiayi Zhou", "Juntao Dai", "Sirui Han", "Yike Guo", "Yaodong Yang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.18807", "arxiv_id": "2505.18807", "title": "Mitigating Deceptive Alignment via Self-Monitoring", "authors": [ "Jiaming Ji", "Wenqi Chen", "Kaile Wang", "Donghai Hong", "Sitong Fang", "Boyuan Chen", "Jiayi Zhou", "Juntao Dai", "Sirui Han", "Yike Guo", "Yaodong Yang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/WatchTree-19/inspect-deceptionbench", "owner": "WatchTree-19", "relation": "inspect_compatible_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/deceptionbench/eval.yaml" }, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://arxiv.org/abs/2505.18807" }, "metadata": { "source_search": "PKU-Alignment and the DeceptionBench paper authors PKU-Alignment Original research by Mitigating Deceptive Alignment via Self-Monitoring Jiaming Ji Wenqi Chen Kaile Wang Donghai Hong Sitong Fang Boyuan Chen Jiayi Zhou Juntao Dai Sirui Han Yike Guo Yaodong Yang WatchTree-19 evaluation-package" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "moderate", "construct": "Protocol-defined deceptive behavior in controlled scenarios where a model's generated statement can diverge from an elicited belief proxy or known task state.", "unit_of_analysis": "One scenario response, often paired with a belief-elicitation or reasoning trace used by the benchmark to classify deception.", "interaction_type": "Controlled question answering or role-based generation across deception scenarios and conditions.", "target_outcome": "Benchmark-specific deception classification or deception-tendency rate derived from response, task state, and elicited belief or reasoning proxies.", "human_participants": "No human subjects. Researchers construct scenarios, labels, and evaluation rules.", "elicitation": "The model is placed in scenarios with opportunities or incentives to provide a misleading answer; some tasks separately elicit belief or reasoning proxies.", "scoring": "Rule-based or model-graded classification combines truth conditions, model responses, and protocol-specific belief or reasoning evidence.", "protocol_dependence": "Very high: scenario assumptions, belief elicitation, incentive framing, chain-of-thought access, and deception definition determine classification.", "implementation_dependence": "High: judge models, hidden-state assumptions, answer parsers, and model endpoint behavior affect reported rates.", "limitations": [ "Elicited beliefs and generated reasoning are behavioral proxies, not direct observations of internal intent.", "The benchmark combines heterogeneous deception scenarios that may not share one construct.", "Reasoning traces can be incomplete, post hoc, or strategically generated.", "Scenario-level rates should not be extrapolated to deployment base rates." ], "comparability": "Require the same DeceptionBench version, scenario subsets, incentive conditions, belief and reasoning elicitation, judge, model snapshot, and generation settings.", "decision_relevance": "Useful for targeted deception research and stress testing; deployment decisions require agentic, longitudinal, and independently validated evidence.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2505.18807" }, { "kind": "implementation", "url": "https://github.com/WatchTree-19/inspect-deceptionbench" }, { "kind": "registry", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/register/deceptionbench" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "register:do_not_answer", "name": "Do-Not-Answer", "slug": "register--do-not-answer", "organisation": "Yuxia Wang, Haonan Li, Xudong Han et al.", "source_type": "inspect-register", "source_url": "https://github.com/mkzung/inspect-evals-do-not-answer", "topics": [ "safeguards" ], "description": "Evaluates LLM safety by presenting 939 prompts spanning five risk areas (information hazards, malicious uses, discrimination/toxicity, misinformation, human-chatbot harms) that responsible models should not directly answer. An LLM judge classifies each response into seven action categories (0-6), where category 6 indicates directly following the risky instruction. Metrics include refusal rate, safe rate (not category 6), and follow rate (category 6), broken down by risk area. Dataset was GPT-4-generated and human-filtered/annotated. An Inspect AI implementation of the original codebase publicly available at .", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/do_not_answer/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2308.13387", "primary": true }, { "kind": "paper", "label": "Paper: Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "url": "https://arxiv.org/abs/2308.13387", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/mkzung/inspect-evals-do-not-answer", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/do_not_answer/eval.yaml", "primary": false } ], "paper_title": "Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "paper_url": "https://arxiv.org/abs/2308.13387", "code_url": "https://github.com/mkzung/inspect-evals-do-not-answer", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/do_not_answer/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "mkzung" ], "tasks": [ { "name": "do_not_answer", "path": "src/do_not_answer/do_not_answer.py" } ], "protocol": { "implementation_commit": "b386689ab2d469ea1ff9eb423ac048549f7542c5", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:do_not_answer", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yuxia Wang, Haonan Li, Xudong Han et al.", "organization": "MBZUAI; LibrAI", "organizations": [ "MBZUAI", "LibrAI" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2308.13387" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2308.13387", "arxiv_id": "2308.13387", "title": "Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "authors": [ "Yuxia Wang", "Haonan Li", "Xudong Han", "Preslav Nakov", "Timothy Baldwin" ], "organizations": [ "MBZUAI", "LibrAI" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2308.13387", "arxiv_id": "2308.13387", "title": "Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "authors": [ "Yuxia Wang", "Haonan Li", "Xudong Han", "Preslav Nakov", "Timothy Baldwin" ], "organizations": [ "MBZUAI", "LibrAI" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2308.13387", "arxiv_id": "2308.13387", "title": "Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "authors": [ "Yuxia Wang", "Haonan Li", "Xudong Han", "Preslav Nakov", "Timothy Baldwin" ], "organizations": [ "MBZUAI", "LibrAI" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/mkzung/inspect-evals-do-not-answer", "owner": "Mkzung", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/do_not_answer/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2308.13387" }, "metadata": { "source_search": "Yuxia Wang, Haonan Li, Xudong Han et al. MBZUAI; LibrAI Original paper authors Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs Yuxia Wang Haonan Li Xudong Han Preslav Nakov Timothy Baldwin Mkzung evaluation-package" } }, { "id": "register:do_not_answer_adversarial", "name": "Do-Not-Answer Adversarial", "slug": "register--do-not-answer-adversarial", "organisation": "Yuxia Wang, Haonan Li, Xudong Han et al.", "source_type": "inspect-register", "source_url": "https://github.com/mkzung/inspect-evals-do-not-answer", "topics": [ "safeguards" ], "description": "An adversarial extension of the Do-Not-Answer benchmark that wraps each of the 939 harmful prompts (spanning five risk areas) in one of three jailbreak transforms -- prefix injection, refusal suppression, or their combination -- drawn from Wei et al. 2023. The model's response is scored against the original instruction using a seven-action behavioural taxonomy. The primary metric is follow_rate per risk area, with the delta versus the clean baseline indicating safeguard degradation under attack.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/do_not_answer_adversarial/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2308.13387", "primary": true }, { "kind": "paper", "label": "Paper: Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "url": "https://arxiv.org/abs/2308.13387", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/mkzung/inspect-evals-do-not-answer", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/do_not_answer_adversarial/eval.yaml", "primary": false } ], "paper_title": "Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "paper_url": "https://arxiv.org/abs/2308.13387", "code_url": "https://github.com/mkzung/inspect-evals-do-not-answer", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/do_not_answer_adversarial/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "mkzung" ], "tasks": [ { "name": "do_not_answer_adversarial", "path": "src/do_not_answer/adversarial.py" } ], "protocol": { "implementation_commit": "b386689ab2d469ea1ff9eb423ac048549f7542c5", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:do_not_answer_adversarial", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yuxia Wang, Haonan Li, Xudong Han et al.", "organization": "MBZUAI; LibrAI", "organizations": [ "MBZUAI", "LibrAI" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2308.13387" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2308.13387", "arxiv_id": "2308.13387", "title": "Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "authors": [ "Yuxia Wang", "Haonan Li", "Xudong Han", "Preslav Nakov", "Timothy Baldwin" ], "organizations": [ "MBZUAI", "LibrAI" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2308.13387", "arxiv_id": "2308.13387", "title": "Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "authors": [ "Yuxia Wang", "Haonan Li", "Xudong Han", "Preslav Nakov", "Timothy Baldwin" ], "organizations": [ "MBZUAI", "LibrAI" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2308.13387", "arxiv_id": "2308.13387", "title": "Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs", "authors": [ "Yuxia Wang", "Haonan Li", "Xudong Han", "Preslav Nakov", "Timothy Baldwin" ], "organizations": [ "MBZUAI", "LibrAI" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/mkzung/inspect-evals-do-not-answer", "owner": "Mkzung", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/do_not_answer_adversarial/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2308.13387" }, "metadata": { "source_search": "Yuxia Wang, Haonan Li, Xudong Han et al. MBZUAI; LibrAI Original paper authors Do-Not-Answer: A Dataset for Evaluating Safeguards in LLMs Yuxia Wang Haonan Li Xudong Han Preslav Nakov Timothy Baldwin Mkzung evaluation-package" } }, { "id": "inspect:docvqa", "name": "DocVQA: A Dataset for VQA on Document Images", "slug": "inspect--docvqa", "organisation": "Minesh Mathew, Dimosthenis Karatzas, C. V. Jawahar", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/docvqa", "topics": [ "multimodal" ], "description": "DocVQA is a Visual Question Answering benchmark that consists of 50,000 questions covering 12,000+ document images. This implementation solves and scores the \"validation\" split.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/docvqa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2007.00398", "primary": true }, { "kind": "paper", "label": "Paper: DocVQA: A Dataset for VQA on Document Images", "url": "https://arxiv.org/abs/2007.00398", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/docvqa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Lmms Lab Encoder", "url": "https://huggingface.co/datasets/lmms-lab-encoder/DocVQA", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2007.00398", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/docvqa/eval.yaml", "family_title": "DocVQA: A Dataset for VQA on Document Images", "group": "Multimodal", "version": "2-A", "contributors": [ "evanmiller-anthropic" ], "tags": [ "Multimodal" ], "task": { "name": "docvqa", "dataset_samples": 5349, "comment": null, "support_only": false }, "tasks": [ { "name": "docvqa", "dataset_samples": 5349, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 5349, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "lmms-lab-encoder/DocVQA", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:docvqa", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Minesh Mathew, Dimosthenis Karatzas, C. V. Jawahar", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2007.00398" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2007.00398", "arxiv_id": "2007.00398", "title": "DocVQA: A Dataset for VQA on Document Images", "authors": [ "Minesh Mathew", "Dimosthenis Karatzas", "C. V. Jawahar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2007.00398", "arxiv_id": "2007.00398", "title": "DocVQA: A Dataset for VQA on Document Images", "authors": [ "Minesh Mathew", "Dimosthenis Karatzas", "C. V. Jawahar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/docvqa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/lmms-lab-encoder/DocVQA", "provider": "Lmms Lab Encoder" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2007.00398" }, "code_url": null, "metadata": { "source_search": "Minesh Mathew, Dimosthenis Karatzas, C. V. Jawahar Original paper authors DocVQA: A Dataset for VQA on Document Images Minesh Mathew Dimosthenis Karatzas C. V. Jawahar evaluation-task" } }, { "id": "inspect:drop", "name": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs", "slug": "inspect--drop", "organisation": "Dheeru Dua, Yizhong Wang, Pradeep Dasigi et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/drop", "topics": [ "general-capability" ], "description": "Evaluates reading comprehension where models must resolve references in a question, perhaps to multiple input positions, and perform discrete operations over them (such as addition, counting, or sorting).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/drop/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1903.00161", "primary": true }, { "kind": "paper", "label": "Paper: DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs", "url": "https://arxiv.org/abs/1903.00161", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/drop/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: EleutherAI", "url": "https://huggingface.co/datasets/EleutherAI/drop", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1903.00161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/drop/eval.yaml", "family_title": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs", "group": "Reasoning", "version": "2-A", "contributors": [ "xeon27" ], "tags": [ "Reasoning" ], "task": { "name": "drop", "dataset_samples": 9535, "comment": null, "support_only": false }, "tasks": [ { "name": "drop", "dataset_samples": 9535, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 9535, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "EleutherAI/drop", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:drop", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Dheeru Dua, Yizhong Wang, Pradeep Dasigi et al.", "organization": "University of Washington; Allen Institute for Artificial Intelligence; Hebrew University of Jerusalem; University of California, Irvine; Envive AI (fka Spiffy AI)", "organizations": [ "University of Washington", "Allen Institute for Artificial Intelligence", "Hebrew University of Jerusalem", "University of California, Irvine", "Envive AI (fka Spiffy AI)" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1903.00161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1903.00161", "arxiv_id": "1903.00161", "title": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs", "authors": [ "Dheeru Dua", "Yizhong Wang", "Pradeep Dasigi", "Gabriel Stanovsky", "Sameer Singh", "Matt Gardner" ], "organizations": [ "University of Washington", "Allen Institute for Artificial Intelligence", "Hebrew University of Jerusalem", "University of California, Irvine", "Envive AI (fka Spiffy AI)" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1903.00161", "arxiv_id": "1903.00161", "title": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs", "authors": [ "Dheeru Dua", "Yizhong Wang", "Pradeep Dasigi", "Gabriel Stanovsky", "Sameer Singh", "Matt Gardner" ], "organizations": [ "University of Washington", "Allen Institute for Artificial Intelligence", "Hebrew University of Jerusalem", "University of California, Irvine", "Envive AI (fka Spiffy AI)" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/drop/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/EleutherAI/drop", "provider": "EleutherAI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1903.00161" }, "code_url": null, "metadata": { "source_search": "Dheeru Dua, Yizhong Wang, Pradeep Dasigi et al. University of Washington; Allen Institute for Artificial Intelligence; Hebrew University of Jerusalem; University of California, Irvine; Envive AI (fka Spiffy AI) Original paper authors DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs Dheeru Dua Yizhong Wang Pradeep Dasigi Gabriel Stanovsky Sameer Singh Matt Gardner evaluation-task" } }, { "id": "inspect:ds1000", "name": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation", "slug": "inspect--ds1000", "organisation": "Yuhang Lai, Chengxi Li, Yiming Wang et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ds1000", "topics": [ "general-capability" ], "description": "Code generation benchmark with a thousand data science problems spanning seven Python libraries.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ds1000/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2211.11501", "primary": true }, { "kind": "paper", "label": "Paper: DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation", "url": "https://arxiv.org/abs/2211.11501", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ds1000/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xlangai", "url": "https://huggingface.co/datasets/xlangai/DS-1000", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2211.11501", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ds1000/eval.yaml", "family_title": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation", "group": "Coding", "version": "3-B", "contributors": [ "bienehito" ], "tags": [ "Coding" ], "task": { "name": "ds1000", "dataset_samples": 1000, "comment": null, "support_only": false }, "tasks": [ { "name": "ds1000", "dataset_samples": 1000, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 1000, "dependency": null, "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "xlangai/DS-1000", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:ds1000", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yuhang Lai, Chengxi Li, Yiming Wang et al.", "organization": "University of Washington; Facebook; Carnegie Mellon University; Yale University", "organizations": [ "University of Washington", "Facebook", "Carnegie Mellon University", "Yale University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2211.11501" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2211.11501", "arxiv_id": "2211.11501", "title": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation", "authors": [ "Yuhang Lai", "Chengxi Li", "Yiming Wang", "Tianyi Zhang", "Ruiqi Zhong", "Luke Zettlemoyer", "Scott Wen-tau Yih", "Daniel Fried", "Sida Wang", "Tao Yu" ], "organizations": [ "University of Washington", "Facebook", "Carnegie Mellon University", "Yale University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2211.11501", "arxiv_id": "2211.11501", "title": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation", "authors": [ "Yuhang Lai", "Chengxi Li", "Yiming Wang", "Tianyi Zhang", "Ruiqi Zhong", "Luke Zettlemoyer", "Scott Wen-tau Yih", "Daniel Fried", "Sida Wang", "Tao Yu" ], "organizations": [ "University of Washington", "Facebook", "Carnegie Mellon University", "Yale University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ds1000/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xlangai/DS-1000", "provider": "Xlangai" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2211.11501" }, "code_url": null, "metadata": { "source_search": "Yuhang Lai, Chengxi Li, Yiming Wang et al. University of Washington; Facebook; Carnegie Mellon University; Yale University Original paper authors DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation Yuhang Lai Chengxi Li Yiming Wang Tianyi Zhang Ruiqi Zhong Luke Zettlemoyer Scott Wen-tau Yih Daniel Fried Sida Wang Tao Yu evaluation-task" } }, { "id": "register:exploitbench", "name": "ExploitBench", "slug": "register--exploitbench", "organisation": "Seunghyun Lee, David Brumley", "source_type": "inspect-register", "source_url": "https://github.com/ChaoticCooties/exploitbench-eval", "topics": [ "cyber" ], "description": "ExploitBench evaluates LLM agents on developing memory-corruption exploits against 41 real V8 JavaScript engine N-day vulnerabilities with all production mitigations enabled. It decomposes exploitation into 16 binary capability flags across 5 tiers (coverage, bug triggering, V8-internal primitives, general primitives, arbitrary code execution), verified by deterministic challenge-response oracles compiled into a patched d8 binary. Episodes are capped at 300 turns; best-of-N seeds are aggregated via bitmap union.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/exploitbench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2605.14153", "primary": true }, { "kind": "paper", "label": "Paper: ExploitBench: A Capability Ladder Benchmark for LLM Cybersecurity Agents", "url": "https://arxiv.org/abs/2605.14153", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ChaoticCooties/exploitbench-eval", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/exploitbench/eval.yaml", "primary": false } ], "paper_title": "ExploitBench: A Capability Ladder Benchmark for LLM Cybersecurity Agents", "paper_url": "https://arxiv.org/abs/2605.14153", "code_url": "https://github.com/ChaoticCooties/exploitbench-eval", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/exploitbench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "ChaoticCooties" ], "tasks": [ { "name": "eb_v8", "path": "src/exploitbench/exploitbench.py" } ], "protocol": { "implementation_commit": "ccdd13a128cf4dd25c720e8bf1f13d2867b6ba23", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:exploitbench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Seunghyun Lee, David Brumley", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2605.14153" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2605.14153", "arxiv_id": "2605.14153", "title": "ExploitBench: A Capability Ladder Benchmark for LLM Cybersecurity Agents", "authors": [ "Seunghyun Lee", "David Brumley" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2605.14153", "arxiv_id": "2605.14153", "title": "ExploitBench: A Capability Ladder Benchmark for LLM Cybersecurity Agents", "authors": [ "Seunghyun Lee", "David Brumley" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2605.14153", "arxiv_id": "2605.14153", "title": "ExploitBench: A Capability Ladder Benchmark for LLM Cybersecurity Agents", "authors": [ "Seunghyun Lee", "David Brumley" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ChaoticCooties/exploitbench-eval", "owner": "ChaoticCooties", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/exploitbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2605.14153" }, "metadata": { "source_search": "Seunghyun Lee, David Brumley Original paper authors ExploitBench: A Capability Ladder Benchmark for LLM Cybersecurity Agents Seunghyun Lee David Brumley ChaoticCooties evaluation-package" } }, { "id": "inspect:fortress_adversarial", "name": "Fortress Adversarial", "slug": "inspect--fortress-adversarial", "organisation": "Christina Q. Knight, Kaustubh Deshpande, Ved Sirdeshmukh et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress", "topics": [ "safeguards" ], "description": "A dataset of 500 expert-crafted adversarial prompts with instance-based rubrics of 4-7 binary questions for automated evaluation across 3 domains relevant to national security and public safety (NSPS).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2506.14922", "primary": true }, { "kind": "paper", "label": "Paper: FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "url": "https://arxiv.org/abs/2506.14922", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Scale AI", "url": "https://huggingface.co/datasets/ScaleAI/fortress_public", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2506.14922", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress/eval.yaml", "family_title": "FORTRESS", "group": "Safeguards", "version": "3-A", "contributors": [ "jeqcho" ], "tags": [ "Safeguards" ], "task": { "name": "fortress_adversarial", "dataset_samples": 500, "comment": null, "support_only": false }, "tasks": [ { "name": "fortress_adversarial", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "fortress_benign", "dataset_samples": 500, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 500, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "ScaleAI/fortress_public", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:fortress", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Christina Q. Knight, Kaustubh Deshpande, Ved Sirdeshmukh et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2506.14922" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.14922", "arxiv_id": "2506.14922", "title": "FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "authors": [ "Christina Q. Knight", "Kaustubh Deshpande", "Ved Sirdeshmukh", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "Julian Michael" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.14922", "arxiv_id": "2506.14922", "title": "FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "authors": [ "Christina Q. Knight", "Kaustubh Deshpande", "Ved Sirdeshmukh", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "Julian Michael" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/ScaleAI/fortress_public", "provider": "Scale AI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2506.14922" }, "code_url": null, "metadata": { "source_search": "Christina Q. Knight, Kaustubh Deshpande, Ved Sirdeshmukh et al. Original paper authors FORTRESS: Frontier Risk Evaluation for National Security and Public Safety Christina Q. Knight Kaustubh Deshpande Ved Sirdeshmukh Meher Mankikar Scale Red Team SEAL Research Team Julian Michael evaluation-family" } }, { "id": "inspect:fortress_benign", "name": "Fortress Benign", "slug": "inspect--fortress-benign", "organisation": "Christina Q. Knight, Kaustubh Deshpande, Ved Sirdeshmukh et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress", "topics": [ "safeguards" ], "description": "A dataset of 500 expert-crafted adversarial prompts with instance-based rubrics of 4-7 binary questions for automated evaluation across 3 domains relevant to national security and public safety (NSPS).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2506.14922", "primary": true }, { "kind": "paper", "label": "Paper: FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "url": "https://arxiv.org/abs/2506.14922", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Scale AI", "url": "https://huggingface.co/datasets/ScaleAI/fortress_public", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2506.14922", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress/eval.yaml", "family_title": "FORTRESS", "group": "Safeguards", "version": "3-A", "contributors": [ "jeqcho" ], "tags": [ "Safeguards" ], "task": { "name": "fortress_benign", "dataset_samples": 500, "comment": null, "support_only": false }, "tasks": [ { "name": "fortress_adversarial", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "fortress_benign", "dataset_samples": 500, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 500, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "ScaleAI/fortress_public", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:fortress", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Christina Q. Knight, Kaustubh Deshpande, Ved Sirdeshmukh et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2506.14922" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.14922", "arxiv_id": "2506.14922", "title": "FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "authors": [ "Christina Q. Knight", "Kaustubh Deshpande", "Ved Sirdeshmukh", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "Julian Michael" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.14922", "arxiv_id": "2506.14922", "title": "FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "authors": [ "Christina Q. Knight", "Kaustubh Deshpande", "Ved Sirdeshmukh", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "Julian Michael" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/fortress/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/ScaleAI/fortress_public", "provider": "Scale AI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2506.14922" }, "code_url": null, "metadata": { "source_search": "Christina Q. Knight, Kaustubh Deshpande, Ved Sirdeshmukh et al. Original paper authors FORTRESS: Frontier Risk Evaluation for National Security and Public Safety Christina Q. Knight Kaustubh Deshpande Ved Sirdeshmukh Meher Mankikar Scale Red Team SEAL Research Team Julian Michael evaluation-family" } }, { "id": "register:frames", "name": "FRAMES", "slug": "register--frames", "organisation": "Satyapriya Krishna, Kalpesh Krishna, Anhad Mohananey et al.", "source_type": "inspect-register", "source_url": "https://github.com/sahil350/frames-eval", "topics": [ "general-capability" ], "description": "FRAMES evaluates multi-hop reasoning by requiring models to answer questions that can only be resolved by chaining information across multiple Wikipedia documents. Two task variants are provided: - **frames_baseline**: all source documents are provided upfront; tests whether a model can reason across a full document set. - **frames_socrates**: documents are withheld and the model must iteratively request them via a `request_document` tool. Retrieval is constrained to a per-sample allowlist of ground-truth source documents, preventing eval-awareness contamination. Scored with a decaying reward `accuracy × min(optimal_hops / actual_hops, 1)` that penalises unnecessary hops and hallucinated document requests.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/frames/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2409.12941", "primary": true }, { "kind": "paper", "label": "Paper: Fact, Fetch, and Reason: A Unified Evaluation of Retrieval-Augmented Generation", "url": "https://arxiv.org/abs/2409.12941", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/sahil350/frames-eval", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/frames/eval.yaml", "primary": false } ], "paper_title": "Fact, Fetch, and Reason: A Unified Evaluation of Retrieval-Augmented Generation", "paper_url": "https://arxiv.org/abs/2409.12941", "code_url": "https://github.com/sahil350/frames-eval", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/frames/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "reasoning", "retrieval", "multi-hop", "Agent" ], "contributors": [ "sahil350" ], "tasks": [ { "name": "frames_baseline", "path": "src/frames_eval/frames.py" }, { "name": "frames_socrates", "path": "src/frames_eval/frames.py" } ], "protocol": { "implementation_commit": "2fbb1a123431d559180547e325f0765d97ab2a39", "report_commit": "2fbb1a123431d559180547e325f0765d97ab2a39", "report_timestamp": "May 2026", "command": null, "source_comment": null }, "reported_results": [ { "model": "ollama/llama3.2", "provider": null, "metrics": [ { "key": "accuracy", "value": 0.06 } ] }, { "model": "ollama/llama3.2", "provider": null, "metrics": [ { "key": "mean_score", "value": 0.108 }, { "key": "accuracy", "value": 0.12 }, { "key": "avg_hops", "value": 2 }, { "key": "avg_hallucinations", "value": 0.58 } ] }, { "model": "ollama/llama3.1:8b", "provider": null, "metrics": [ { "key": "accuracy", "value": 0.08 } ] }, { "model": "ollama/llama3.1:8b", "provider": null, "metrics": [ { "key": "mean_score", "value": 0.27 }, { "key": "stderr", "value": 0.07 }, { "key": "accuracy", "value": 0.275 }, { "key": "avg_hops", "value": 2 }, { "key": "avg_hallucinations", "value": 0.47 } ] } ], "reported_result_count": 4, "reported_results_truncated": false, "report_notes": [ "frames_socrates score = accuracy × min(5 / actual_hops, 1) − 0.2 × hallucinations", "Retrieval is constrained to a per-sample allowlist of ground-truth Wikipedia source documents.", "Wikipedia content fetched live via the MediaWiki API at eval time." ], "models_reported": [ "ollama/llama3.2", "ollama/llama3.2", "ollama/llama3.1:8b", "ollama/llama3.1:8b" ], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:frames", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Satyapriya Krishna, Kalpesh Krishna, Anhad Mohananey et al.", "organization": "Harvard University; Staff Research Scientist, Google DeepMind", "organizations": [ "Harvard University", "Staff Research Scientist, Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2409.12941" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2409.12941", "arxiv_id": "2409.12941", "title": "Fact, Fetch, and Reason: A Unified Evaluation of Retrieval-Augmented Generation", "authors": [ "Satyapriya Krishna", "Kalpesh Krishna", "Anhad Mohananey", "Steven Schwarcz", "Adam Stambler", "Shyam Upadhyay", "Manaal Faruqui" ], "organizations": [ "Harvard University", "Staff Research Scientist, Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2409.12941", "arxiv_id": "2409.12941", "title": "Fact, Fetch, and Reason: A Unified Evaluation of Retrieval-Augmented Generation", "authors": [ "Satyapriya Krishna", "Kalpesh Krishna", "Anhad Mohananey", "Steven Schwarcz", "Adam Stambler", "Shyam Upadhyay", "Manaal Faruqui" ], "organizations": [ "Harvard University", "Staff Research Scientist, Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/sahil350/frames-eval", "owner": "Sahil350", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/frames/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2409.12941" }, "metadata": { "source_search": "Satyapriya Krishna, Kalpesh Krishna, Anhad Mohananey et al. Harvard University; Staff Research Scientist, Google DeepMind Original paper authors Fact, Fetch, and Reason: A Unified Evaluation of Retrieval-Augmented Generation Satyapriya Krishna Kalpesh Krishna Anhad Mohananey Steven Schwarcz Adam Stambler Shyam Upadhyay Manaal Faruqui Sahil350 evaluation-package" } }, { "id": "inspect:frontier_cs", "name": "Frontier Cs", "slug": "inspect--frontier-cs", "organisation": "Qiuyang Mang, Wenhao Chai, Zhifei Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs", "topics": [ "general-capability" ], "description": "238 open-ended computer science problems spanning algorithmic (172) and research (66) tracks. Problems feature continuous partial scoring, with algorithmic solutions evaluated via compilation and test-case checking, and research solutions evaluated via custom evaluator scripts. Current frontier models score well below human expert baselines, making this a challenging, unsaturated benchmark.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2512.15699", "primary": true }, { "kind": "paper", "label": "Paper: FrontierCS: Evolving Challenges for Evolving Intelligence", "url": "https://arxiv.org/abs/2512.15699", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/FrontierCS/Frontier-CS", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FrontierCS", "url": "https://huggingface.co/datasets/FrontierCS/Frontier-CS", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2512.15699", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml", "family_title": "Frontier-CS: Benchmarking LLMs on Computer Science Problems", "group": "Coding", "version": "2-A", "contributors": [ "JayBaileyCS" ], "tags": [ "Agent", "Coding" ], "task": { "name": "frontier_cs", "dataset_samples": 238, "comment": null, "support_only": false }, "tasks": [ { "name": "frontier_cs", "dataset_samples": 238, "comment": null, "support_only": false }, { "name": "frontier_cs_algorithmic", "dataset_samples": 172, "comment": null, "support_only": false }, { "name": "frontier_cs_research", "dataset_samples": 66, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 238, "dependency": null, "requires_internet": false, "sandbox": [ "solver", "scorer" ] }, "external_assets": [ { "type": "direct_url", "source": "https://github.com/FrontierCS/Frontier-CS/archive/{SHA}.tar.gz", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/MikeMirzayanov/testlib/{SHA}/testlib.h", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://julialang-s3.julialang.org/bin/linux/x64/1.11/julia-1.11.3-linux-x86_64.tar.gz", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "FrontierCS/Frontier-CS", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:frontier_cs", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Qiuyang Mang, Wenhao Chai, Zhifei Li et al.", "organization": "Stanford University", "organizations": [ "Stanford University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2512.15699" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2512.15699", "arxiv_id": "2512.15699", "title": "FrontierCS: Evolving Challenges for Evolving Intelligence", "authors": [ "Qiuyang Mang", "Wenhao Chai", "Zhifei Li", "Huanzhi Mao", "Shang Zhou", "Alexander Du", "Hanchen Li", "Shu Liu", "Edwin Chen", "Yichuan Wang", "Xieting Chu", "Zerui Cheng", "Yuan Xu", "Tian Xia", "Zirui Wang", "Tianneng Shi", "Jianzhu Yao", "Yilong Zhao", "Qizheng Zhang", "Charlie Ruan", "Zeyu Shen", "Kaiyuan Liu", "Runyuan He", "Dong Xing", "Zerui Li", "Zirong Zeng", "Yige Jiang", "Lufeng Cheng", "Ziyi Zhao", "Youran Sun", "Wesley Zheng", "Meiyuwang Zhang", "Ruyi Ji", "Xuechang Tu", "Zihan Zheng", "Zexing Chen", "Kangyang Zhou", "Zhaozi Wang", "Jingbang Chen", "Aleksandra Korolova", "Peter Henderson", "Pramod Viswanath", "Vijay Ganesh", "Saining Xie", "Zhuang Liu", "Dawn Song", "Sewon Min", "Ion Stoica", "Joseph E. Gonzalez", "Jingbo Shang", "Alvin Cheung" ], "organizations": [ "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2512.15699", "arxiv_id": "2512.15699", "title": "FrontierCS: Evolving Challenges for Evolving Intelligence", "authors": [ "Qiuyang Mang", "Wenhao Chai", "Zhifei Li", "Huanzhi Mao", "Shang Zhou", "Alexander Du", "Hanchen Li", "Shu Liu", "Edwin Chen", "Yichuan Wang", "Xieting Chu", "Zerui Cheng", "Yuan Xu", "Tian Xia", "Zirui Wang", "Tianneng Shi", "Jianzhu Yao", "Yilong Zhao", "Qizheng Zhang", "Charlie Ruan", "Zeyu Shen", "Kaiyuan Liu", "Runyuan He", "Dong Xing", "Zerui Li", "Zirong Zeng", "Yige Jiang", "Lufeng Cheng", "Ziyi Zhao", "Youran Sun", "Wesley Zheng", "Meiyuwang Zhang", "Ruyi Ji", "Xuechang Tu", "Zihan Zheng", "Zexing Chen", "Kangyang Zhou", "Zhaozi Wang", "Jingbang Chen", "Aleksandra Korolova", "Peter Henderson", "Pramod Viswanath", "Vijay Ganesh", "Saining Xie", "Zhuang Liu", "Dawn Song", "Sewon Min", "Ion Stoica", "Joseph E. Gonzalez", "Jingbo Shang", "Alvin Cheung" ], "organizations": [ "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/FrontierCS/Frontier-CS", "owner": "Frontier-CS", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/FrontierCS/Frontier-CS", "provider": "FrontierCS" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2512.15699" }, "code_url": "https://github.com/FrontierCS/Frontier-CS", "metadata": { "source_search": "Qiuyang Mang, Wenhao Chai, Zhifei Li et al. Stanford University Original paper authors FrontierCS: Evolving Challenges for Evolving Intelligence Qiuyang Mang Wenhao Chai Zhifei Li Huanzhi Mao Shang Zhou Alexander Du Hanchen Li Shu Liu Edwin Chen Yichuan Wang Xieting Chu Zerui Cheng Yuan Xu Tian Xia Zirui Wang Tianneng Shi Jianzhu Yao Yilong Zhao Qizheng Zhang Charlie Ruan Zeyu Shen Kaiyuan Liu Runyuan He Dong Xing Zerui Li Zirong Zeng Yige Jiang Lufeng Cheng Ziyi Zhao Youran Sun Wesley Zheng Meiyuwang Zhang Ruyi Ji Xuechang Tu Zihan Zheng Zexing Chen Kangyang Zhou Zhaozi Wang Jingbang Chen Aleksandra Korolova Peter Henderson Pramod Viswanath Vijay Ganesh Saining Xie Zhuang Liu Dawn Song Sewon Min Ion Stoica Joseph E. Gonzalez Jingbo Shang Alvin Cheung Frontier-CS evaluation-family" } }, { "id": "inspect:frontier_cs_algorithmic", "name": "Frontier Cs Algorithmic", "slug": "inspect--frontier-cs-algorithmic", "organisation": "Qiuyang Mang, Wenhao Chai, Zhifei Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs", "topics": [ "general-capability" ], "description": "238 open-ended computer science problems spanning algorithmic (172) and research (66) tracks. Problems feature continuous partial scoring, with algorithmic solutions evaluated via compilation and test-case checking, and research solutions evaluated via custom evaluator scripts. Current frontier models score well below human expert baselines, making this a challenging, unsaturated benchmark.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2512.15699", "primary": true }, { "kind": "paper", "label": "Paper: FrontierCS: Evolving Challenges for Evolving Intelligence", "url": "https://arxiv.org/abs/2512.15699", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/FrontierCS/Frontier-CS", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FrontierCS", "url": "https://huggingface.co/datasets/FrontierCS/Frontier-CS", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2512.15699", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml", "family_title": "Frontier-CS: Benchmarking LLMs on Computer Science Problems", "group": "Coding", "version": "2-A", "contributors": [ "JayBaileyCS" ], "tags": [ "Agent", "Coding" ], "task": { "name": "frontier_cs_algorithmic", "dataset_samples": 172, "comment": null, "support_only": false }, "tasks": [ { "name": "frontier_cs", "dataset_samples": 238, "comment": null, "support_only": false }, { "name": "frontier_cs_algorithmic", "dataset_samples": 172, "comment": null, "support_only": false }, { "name": "frontier_cs_research", "dataset_samples": 66, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 172, "dependency": null, "requires_internet": false, "sandbox": [ "solver", "scorer" ] }, "external_assets": [ { "type": "direct_url", "source": "https://github.com/FrontierCS/Frontier-CS/archive/{SHA}.tar.gz", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/MikeMirzayanov/testlib/{SHA}/testlib.h", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://julialang-s3.julialang.org/bin/linux/x64/1.11/julia-1.11.3-linux-x86_64.tar.gz", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "FrontierCS/Frontier-CS", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:frontier_cs", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Qiuyang Mang, Wenhao Chai, Zhifei Li et al.", "organization": "Stanford University", "organizations": [ "Stanford University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2512.15699" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2512.15699", "arxiv_id": "2512.15699", "title": "FrontierCS: Evolving Challenges for Evolving Intelligence", "authors": [ "Qiuyang Mang", "Wenhao Chai", "Zhifei Li", "Huanzhi Mao", "Shang Zhou", "Alexander Du", "Hanchen Li", "Shu Liu", "Edwin Chen", "Yichuan Wang", "Xieting Chu", "Zerui Cheng", "Yuan Xu", "Tian Xia", "Zirui Wang", "Tianneng Shi", "Jianzhu Yao", "Yilong Zhao", "Qizheng Zhang", "Charlie Ruan", "Zeyu Shen", "Kaiyuan Liu", "Runyuan He", "Dong Xing", "Zerui Li", "Zirong Zeng", "Yige Jiang", "Lufeng Cheng", "Ziyi Zhao", "Youran Sun", "Wesley Zheng", "Meiyuwang Zhang", "Ruyi Ji", "Xuechang Tu", "Zihan Zheng", "Zexing Chen", "Kangyang Zhou", "Zhaozi Wang", "Jingbang Chen", "Aleksandra Korolova", "Peter Henderson", "Pramod Viswanath", "Vijay Ganesh", "Saining Xie", "Zhuang Liu", "Dawn Song", "Sewon Min", "Ion Stoica", "Joseph E. Gonzalez", "Jingbo Shang", "Alvin Cheung" ], "organizations": [ "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2512.15699", "arxiv_id": "2512.15699", "title": "FrontierCS: Evolving Challenges for Evolving Intelligence", "authors": [ "Qiuyang Mang", "Wenhao Chai", "Zhifei Li", "Huanzhi Mao", "Shang Zhou", "Alexander Du", "Hanchen Li", "Shu Liu", "Edwin Chen", "Yichuan Wang", "Xieting Chu", "Zerui Cheng", "Yuan Xu", "Tian Xia", "Zirui Wang", "Tianneng Shi", "Jianzhu Yao", "Yilong Zhao", "Qizheng Zhang", "Charlie Ruan", "Zeyu Shen", "Kaiyuan Liu", "Runyuan He", "Dong Xing", "Zerui Li", "Zirong Zeng", "Yige Jiang", "Lufeng Cheng", "Ziyi Zhao", "Youran Sun", "Wesley Zheng", "Meiyuwang Zhang", "Ruyi Ji", "Xuechang Tu", "Zihan Zheng", "Zexing Chen", "Kangyang Zhou", "Zhaozi Wang", "Jingbang Chen", "Aleksandra Korolova", "Peter Henderson", "Pramod Viswanath", "Vijay Ganesh", "Saining Xie", "Zhuang Liu", "Dawn Song", "Sewon Min", "Ion Stoica", "Joseph E. Gonzalez", "Jingbo Shang", "Alvin Cheung" ], "organizations": [ "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/FrontierCS/Frontier-CS", "owner": "Frontier-CS", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/FrontierCS/Frontier-CS", "provider": "FrontierCS" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2512.15699" }, "code_url": "https://github.com/FrontierCS/Frontier-CS", "metadata": { "source_search": "Qiuyang Mang, Wenhao Chai, Zhifei Li et al. Stanford University Original paper authors FrontierCS: Evolving Challenges for Evolving Intelligence Qiuyang Mang Wenhao Chai Zhifei Li Huanzhi Mao Shang Zhou Alexander Du Hanchen Li Shu Liu Edwin Chen Yichuan Wang Xieting Chu Zerui Cheng Yuan Xu Tian Xia Zirui Wang Tianneng Shi Jianzhu Yao Yilong Zhao Qizheng Zhang Charlie Ruan Zeyu Shen Kaiyuan Liu Runyuan He Dong Xing Zerui Li Zirong Zeng Yige Jiang Lufeng Cheng Ziyi Zhao Youran Sun Wesley Zheng Meiyuwang Zhang Ruyi Ji Xuechang Tu Zihan Zheng Zexing Chen Kangyang Zhou Zhaozi Wang Jingbang Chen Aleksandra Korolova Peter Henderson Pramod Viswanath Vijay Ganesh Saining Xie Zhuang Liu Dawn Song Sewon Min Ion Stoica Joseph E. Gonzalez Jingbo Shang Alvin Cheung Frontier-CS evaluation-family" } }, { "id": "inspect:frontier_cs_research", "name": "Frontier Cs Research", "slug": "inspect--frontier-cs-research", "organisation": "Qiuyang Mang, Wenhao Chai, Zhifei Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs", "topics": [ "ai-rd" ], "description": "238 open-ended computer science problems spanning algorithmic (172) and research (66) tracks. Problems feature continuous partial scoring, with algorithmic solutions evaluated via compilation and test-case checking, and research solutions evaluated via custom evaluator scripts. Current frontier models score well below human expert baselines, making this a challenging, unsaturated benchmark.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Performance on the benchmark's selected frontier computer-science research questions under its elicitation and grading protocol.", "does_not_measure": "End-to-end research execution, empirical validation, research taste, durable autonomy, or general scientific productivity.", "best_for": "Probing advanced research reasoning and identifying problem-specific capability gaps.", "not_sufficient_for": "Claims that a model can independently conduct original computer-science research or accelerate an entire R&D pipeline.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2512.15699", "primary": true }, { "kind": "paper", "label": "Paper: FrontierCS: Evolving Challenges for Evolving Intelligence", "url": "https://arxiv.org/abs/2512.15699", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/FrontierCS/Frontier-CS", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FrontierCS", "url": "https://huggingface.co/datasets/FrontierCS/Frontier-CS", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2512.15699", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml", "family_title": "Frontier-CS: Benchmarking LLMs on Computer Science Problems", "group": "Coding", "version": "2-A", "contributors": [ "JayBaileyCS" ], "tags": [ "Agent", "Coding" ], "task": { "name": "frontier_cs_research", "dataset_samples": 66, "comment": null, "support_only": false }, "tasks": [ { "name": "frontier_cs", "dataset_samples": 238, "comment": null, "support_only": false }, { "name": "frontier_cs_algorithmic", "dataset_samples": 172, "comment": null, "support_only": false }, { "name": "frontier_cs_research", "dataset_samples": 66, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 66, "dependency": null, "requires_internet": false, "sandbox": [ "solver", "scorer" ] }, "external_assets": [ { "type": "direct_url", "source": "https://github.com/FrontierCS/Frontier-CS/archive/{SHA}.tar.gz", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/MikeMirzayanov/testlib/{SHA}/testlib.h", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://julialang-s3.julialang.org/bin/linux/x64/1.11/julia-1.11.3-linux-x86_64.tar.gz", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "FrontierCS/Frontier-CS", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:frontier_cs", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Qiuyang Mang, Wenhao Chai, Zhifei Li et al.", "organization": "Stanford University", "organizations": [ "Stanford University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2512.15699" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2512.15699", "arxiv_id": "2512.15699", "title": "FrontierCS: Evolving Challenges for Evolving Intelligence", "authors": [ "Qiuyang Mang", "Wenhao Chai", "Zhifei Li", "Huanzhi Mao", "Shang Zhou", "Alexander Du", "Hanchen Li", "Shu Liu", "Edwin Chen", "Yichuan Wang", "Xieting Chu", "Zerui Cheng", "Yuan Xu", "Tian Xia", "Zirui Wang", "Tianneng Shi", "Jianzhu Yao", "Yilong Zhao", "Qizheng Zhang", "Charlie Ruan", "Zeyu Shen", "Kaiyuan Liu", "Runyuan He", "Dong Xing", "Zerui Li", "Zirong Zeng", "Yige Jiang", "Lufeng Cheng", "Ziyi Zhao", "Youran Sun", "Wesley Zheng", "Meiyuwang Zhang", "Ruyi Ji", "Xuechang Tu", "Zihan Zheng", "Zexing Chen", "Kangyang Zhou", "Zhaozi Wang", "Jingbang Chen", "Aleksandra Korolova", "Peter Henderson", "Pramod Viswanath", "Vijay Ganesh", "Saining Xie", "Zhuang Liu", "Dawn Song", "Sewon Min", "Ion Stoica", "Joseph E. Gonzalez", "Jingbo Shang", "Alvin Cheung" ], "organizations": [ "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2512.15699", "arxiv_id": "2512.15699", "title": "FrontierCS: Evolving Challenges for Evolving Intelligence", "authors": [ "Qiuyang Mang", "Wenhao Chai", "Zhifei Li", "Huanzhi Mao", "Shang Zhou", "Alexander Du", "Hanchen Li", "Shu Liu", "Edwin Chen", "Yichuan Wang", "Xieting Chu", "Zerui Cheng", "Yuan Xu", "Tian Xia", "Zirui Wang", "Tianneng Shi", "Jianzhu Yao", "Yilong Zhao", "Qizheng Zhang", "Charlie Ruan", "Zeyu Shen", "Kaiyuan Liu", "Runyuan He", "Dong Xing", "Zerui Li", "Zirong Zeng", "Yige Jiang", "Lufeng Cheng", "Ziyi Zhao", "Youran Sun", "Wesley Zheng", "Meiyuwang Zhang", "Ruyi Ji", "Xuechang Tu", "Zihan Zheng", "Zexing Chen", "Kangyang Zhou", "Zhaozi Wang", "Jingbang Chen", "Aleksandra Korolova", "Peter Henderson", "Pramod Viswanath", "Vijay Ganesh", "Saining Xie", "Zhuang Liu", "Dawn Song", "Sewon Min", "Ion Stoica", "Joseph E. Gonzalez", "Jingbo Shang", "Alvin Cheung" ], "organizations": [ "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/FrontierCS/Frontier-CS", "owner": "Frontier-CS", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontier_cs/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/FrontierCS/Frontier-CS", "provider": "FrontierCS" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2512.15699" }, "code_url": "https://github.com/FrontierCS/Frontier-CS", "metadata": { "source_search": "Qiuyang Mang, Wenhao Chai, Zhifei Li et al. Stanford University Original paper authors FrontierCS: Evolving Challenges for Evolving Intelligence Qiuyang Mang Wenhao Chai Zhifei Li Huanzhi Mao Shang Zhou Alexander Du Hanchen Li Shu Liu Edwin Chen Yichuan Wang Xieting Chu Zerui Cheng Yuan Xu Tian Xia Zirui Wang Tianneng Shi Jianzhu Yao Yilong Zhao Qizheng Zhang Charlie Ruan Zeyu Shen Kaiyuan Liu Runyuan He Dong Xing Zerui Li Zirong Zeng Yige Jiang Lufeng Cheng Ziyi Zhao Youran Sun Wesley Zheng Meiyuwang Zhang Ruyi Ji Xuechang Tu Zihan Zheng Zexing Chen Kangyang Zhou Zhaozi Wang Jingbang Chen Aleksandra Korolova Peter Henderson Pramod Viswanath Vijay Ganesh Saining Xie Zhuang Liu Dawn Song Sewon Min Ion Stoica Joseph E. Gonzalez Jingbo Shang Alvin Cheung Frontier-CS evaluation-family" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Open-ended reasoning on expert-authored frontier computer-science research problems.", "unit_of_analysis": "A proposed solution to one research problem, assessed against problem-specific criteria.", "interaction_type": "Long-form research reasoning, sometimes supported by code or formal artifacts depending on the task.", "target_outcome": "Problem-specific partial-credit score from custom evaluators and/or judge rubrics.", "human_participants": "No human subjects. Domain experts create problems, reference expectations, and scoring criteria.", "elicitation": "The model receives an unpublished research-style problem and produces a solution under the benchmark's prompting and tool conditions.", "scoring": "Continuous partial credit using problem-specific evaluators or rubrics rather than one uniform metric.", "protocol_dependence": "High: problem selection, unpublished status, allowed tools, answer format, and grader design affect scores.", "implementation_dependence": "Moderate to high: custom evaluators and judge models can vary in reliability across subfields.", "limitations": [ "The expert-selected problem set is not a representative sample of computer-science research.", "Some tasks may reward written solution quality without requiring empirical execution.", "Custom graders reduce metric uniformity and complicate aggregation.", "Confidential tasks improve contamination resistance but limit external scrutiny." ], "comparability": "Compare only within the same task release, evaluator versions, tool access, prompt format, and model-system settings; do not treat the aggregate as interchangeable with replication benchmarks.", "decision_relevance": "Useful for frontier research-reasoning monitoring and targeted follow-up; weak evidence for operational research autonomy.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2512.15699" }, { "kind": "metadata", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/frontier_cs" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "canonical:frontiermath", "name": "FrontierMath", "slug": "canonical--frontiermath", "organisation": "Epoch AI", "source_type": "canonical-source", "source_url": "https://epoch.ai/frontiermath", "topics": [ "general-capability", "ai-rd" ], "description": "Expert-written advanced mathematics benchmark intended to remain difficult for frontier models.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Success on the evaluated FrontierMath problem set under the stated elicitation, tool, sampling, and grading conditions.", "does_not_measure": "General intelligence, mathematical research autonomy, theorem-proving reliability across the field, AI safety, or AI R&D capability by itself.", "best_for": "Tracking high-end mathematical problem solving on a difficult, expert-authored and contamination-conscious task set.", "not_sufficient_for": "Safety conclusions, claims of autonomous mathematical research, or broad comparisons that mix different FrontierMath sets or elicitation budgets.", "evidence_reach": [ "artifact-production" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://epoch.ai/frontiermath", "primary": true }, { "kind": "official", "label": "Official or project page", "url": "https://epoch.ai/frontiermath", "primary": false } ], "resource_count": 2, "source_key": "canonical:frontiermath", "record_type": "evaluation-resource", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Epoch AI", "organization": "Epoch AI", "organizations": [ "Epoch AI" ], "entity_type": "organization", "role": "evaluation_developer", "role_label": "Evaluation developed by", "confidence": "curated", "evidence_url": "https://epoch.ai/frontiermath" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://epoch.ai/frontiermath" }, "paper_url": null, "code_url": null, "metadata": { "source_search": "Epoch AI Epoch AI Evaluation developed by evaluation-resource" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "high", "construct": "Performance on expert-authored advanced mathematics problems designed to remain difficult for frontier models and resistant to simple memorization.", "unit_of_analysis": "One model solution or final answer to one advanced mathematics problem.", "interaction_type": "Long-form mathematical reasoning, sometimes with code or tools depending on the evaluation configuration.", "target_outcome": "Problem correctness under expert-authored answers, exact or symbolic checks, and problem-specific verification procedures.", "human_participants": "No human subjects. Professional mathematicians author, review, and validate problems and solutions.", "elicitation": "The model receives selected held-out or public problems under a specified prompt, token budget, sampling strategy, and tool condition.", "scoring": "Problem-level correctness is established through answer keys and verification procedures, then aggregated over explicitly identified problem sets or tiers.", "protocol_dependence": "High: problem set and tier, secrecy status, prompt, reasoning and token budget, tool access, sampling, and verification method shape performance.", "implementation_dependence": "Moderate to high: answer extraction, symbolic equivalence, judge procedures, and model-system tool integration affect scores.", "limitations": [ "A selected problem set cannot represent all of advanced mathematics or mathematical research.", "Hidden problems improve contamination resistance but limit independent inspection and reproduction.", "Pass rates depend strongly on sampling and compute budgets.", "Correct final answers do not necessarily establish valid, understandable, or reusable proofs." ], "comparability": "Match the exact FrontierMath set and tier, problem-version date, prompt, tool access, sampling count, token and compute budget, answer-verification procedure, and model snapshot.", "decision_relevance": "Useful as high-end capability context and for monitoring mathematical reasoning. It is not a direct dangerous-capability or safety evaluation.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2411.04872" }, { "kind": "official", "url": "https://epoch.ai/frontiermath" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:frontierscience", "name": "FrontierScience: Expert-Level Scientific Reasoning", "slug": "inspect--frontierscience", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontierscience", "topics": [ "general-capability" ], "description": "Evaluates AI capabilities for expert-level scientific reasoning across physics, chemistry, and biology. Contains 160 problems with two evaluation formats: Olympic (100 samples with reference answers) and Research (60 samples with rubrics).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontierscience/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://openai.com/index/frontierscience", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://openai.com/index/frontierscience", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontierscience/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: OpenAI", "url": "https://huggingface.co/datasets/openai/frontierscience", "primary": false } ], "resource_count": 4, "paper_url": "https://openai.com/index/frontierscience", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontierscience/eval.yaml", "family_title": "FrontierScience: Expert-Level Scientific Reasoning", "group": "Knowledge", "version": "2-A", "contributors": [ "tommyly201", "mnarayan" ], "tags": [ "Knowledge" ], "task": { "name": "frontierscience", "dataset_samples": 160, "comment": null, "support_only": false }, "tasks": [ { "name": "frontierscience", "dataset_samples": 160, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 160, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "openai/frontierscience", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:frontierscience", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://openai.com/index/frontierscience" }, "paper": { "status": "present", "url": "https://openai.com/index/frontierscience", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://openai.com/index/frontierscience", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/frontierscience/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/openai/frontierscience", "provider": "OpenAI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://openai.com/index/frontierscience" }, "code_url": null, "metadata": { "source_search": "Associated paper authors Original paper authors evaluation-task" } }, { "id": "inspect:gaia", "name": "Gaia", "slug": "inspect--gaia", "organisation": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia", "topics": [ "autonomy-agents" ], "description": "Proposes real-world questions that require a set of fundamental abilities such as reasoning, multi-modality handling, web browsing, and generally tool-use proficiency. GAIA questions are conceptually simple for humans yet challenging for most advanced AIs.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2311.12983", "primary": true }, { "kind": "paper", "label": "Paper: GAIA: a benchmark for General AI Assistants", "url": "https://arxiv.org/abs/2311.12983", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Gaia Benchmark", "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2311.12983", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "family_title": "GAIA: A Benchmark for General AI Assistants", "group": "Assistants", "version": "3-B", "contributors": [ "max-kaufmann" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "gaia", "dataset_samples": 165, "comment": null, "support_only": false }, "tasks": [ { "name": "gaia", "dataset_samples": 165, "comment": null, "support_only": false }, { "name": "gaia_level1", "dataset_samples": 53, "comment": null, "support_only": false }, { "name": "gaia_level2", "dataset_samples": 86, "comment": null, "support_only": false }, { "name": "gaia_level3", "dataset_samples": 26, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 165, "dependency": "gaia", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "gaia-benchmark/GAIA", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gaia", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2311.12983" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.12983", "arxiv_id": "2311.12983", "title": "GAIA: a benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.12983", "arxiv_id": "2311.12983", "title": "GAIA: a benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "provider": "Gaia Benchmark" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2311.12983" }, "code_url": null, "metadata": { "source_search": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al. Original paper authors GAIA: a benchmark for General AI Assistants Grégoire Mialon Clémentine Fourrier Craig Swift Thomas Wolf Yann LeCun Thomas Scialom evaluation-family" } }, { "id": "inspect:gaia_level1", "name": "Gaia Level1", "slug": "inspect--gaia-level1", "organisation": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia", "topics": [ "autonomy-agents" ], "description": "Proposes real-world questions that require a set of fundamental abilities such as reasoning, multi-modality handling, web browsing, and generally tool-use proficiency. GAIA questions are conceptually simple for humans yet challenging for most advanced AIs.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2311.12983", "primary": true }, { "kind": "paper", "label": "Paper: GAIA: a benchmark for General AI Assistants", "url": "https://arxiv.org/abs/2311.12983", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Gaia Benchmark", "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2311.12983", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "family_title": "GAIA: A Benchmark for General AI Assistants", "group": "Assistants", "version": "3-B", "contributors": [ "max-kaufmann" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "gaia_level1", "dataset_samples": 53, "comment": null, "support_only": false }, "tasks": [ { "name": "gaia", "dataset_samples": 165, "comment": null, "support_only": false }, { "name": "gaia_level1", "dataset_samples": 53, "comment": null, "support_only": false }, { "name": "gaia_level2", "dataset_samples": 86, "comment": null, "support_only": false }, { "name": "gaia_level3", "dataset_samples": 26, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 53, "dependency": "gaia", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "gaia-benchmark/GAIA", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gaia", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2311.12983" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.12983", "arxiv_id": "2311.12983", "title": "GAIA: a benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.12983", "arxiv_id": "2311.12983", "title": "GAIA: a benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "provider": "Gaia Benchmark" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2311.12983" }, "code_url": null, "metadata": { "source_search": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al. Original paper authors GAIA: a benchmark for General AI Assistants Grégoire Mialon Clémentine Fourrier Craig Swift Thomas Wolf Yann LeCun Thomas Scialom evaluation-family" } }, { "id": "inspect:gaia_level2", "name": "Gaia Level2", "slug": "inspect--gaia-level2", "organisation": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia", "topics": [ "autonomy-agents" ], "description": "Proposes real-world questions that require a set of fundamental abilities such as reasoning, multi-modality handling, web browsing, and generally tool-use proficiency. GAIA questions are conceptually simple for humans yet challenging for most advanced AIs.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2311.12983", "primary": true }, { "kind": "paper", "label": "Paper: GAIA: a benchmark for General AI Assistants", "url": "https://arxiv.org/abs/2311.12983", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Gaia Benchmark", "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2311.12983", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "family_title": "GAIA: A Benchmark for General AI Assistants", "group": "Assistants", "version": "3-B", "contributors": [ "max-kaufmann" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "gaia_level2", "dataset_samples": 86, "comment": null, "support_only": false }, "tasks": [ { "name": "gaia", "dataset_samples": 165, "comment": null, "support_only": false }, { "name": "gaia_level1", "dataset_samples": 53, "comment": null, "support_only": false }, { "name": "gaia_level2", "dataset_samples": 86, "comment": null, "support_only": false }, { "name": "gaia_level3", "dataset_samples": 26, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 86, "dependency": "gaia", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "gaia-benchmark/GAIA", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gaia", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2311.12983" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.12983", "arxiv_id": "2311.12983", "title": "GAIA: a benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.12983", "arxiv_id": "2311.12983", "title": "GAIA: a benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "provider": "Gaia Benchmark" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2311.12983" }, "code_url": null, "metadata": { "source_search": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al. Original paper authors GAIA: a benchmark for General AI Assistants Grégoire Mialon Clémentine Fourrier Craig Swift Thomas Wolf Yann LeCun Thomas Scialom evaluation-family" } }, { "id": "inspect:gaia_level3", "name": "Gaia Level3", "slug": "inspect--gaia-level3", "organisation": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia", "topics": [ "autonomy-agents" ], "description": "Proposes real-world questions that require a set of fundamental abilities such as reasoning, multi-modality handling, web browsing, and generally tool-use proficiency. GAIA questions are conceptually simple for humans yet challenging for most advanced AIs.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2311.12983", "primary": true }, { "kind": "paper", "label": "Paper: GAIA: a benchmark for General AI Assistants", "url": "https://arxiv.org/abs/2311.12983", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Gaia Benchmark", "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2311.12983", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml", "family_title": "GAIA: A Benchmark for General AI Assistants", "group": "Assistants", "version": "3-B", "contributors": [ "max-kaufmann" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "gaia_level3", "dataset_samples": 26, "comment": null, "support_only": false }, "tasks": [ { "name": "gaia", "dataset_samples": 165, "comment": null, "support_only": false }, { "name": "gaia_level1", "dataset_samples": 53, "comment": null, "support_only": false }, { "name": "gaia_level2", "dataset_samples": 86, "comment": null, "support_only": false }, { "name": "gaia_level3", "dataset_samples": 26, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 26, "dependency": "gaia", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "gaia-benchmark/GAIA", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gaia", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2311.12983" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.12983", "arxiv_id": "2311.12983", "title": "GAIA: a benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.12983", "arxiv_id": "2311.12983", "title": "GAIA: a benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gaia/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "provider": "Gaia Benchmark" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2311.12983" }, "code_url": null, "metadata": { "source_search": "Grégoire Mialon, Clémentine Fourrier, Craig Swift et al. Original paper authors GAIA: a benchmark for General AI Assistants Grégoire Mialon Clémentine Fourrier Craig Swift Thomas Wolf Yann LeCun Thomas Scialom evaluation-family" } }, { "id": "inspect:gdm_classifier_evasion", "name": "GDM Classifier Evasion", "slug": "inspect--gdm-classifier-evasion", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth", "topics": [ "general-capability" ], "description": "Test AI's ability to reason about and circumvent oversight.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "family_title": "GDM Dangerous Capabilities: Stealth", "group": "Scheming", "version": "4-A", "contributors": [ "ZiyueWang25" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_classifier_evasion", "dataset_samples": 9, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_classifier_evasion", "dataset_samples": 9, "comment": null, "support_only": false }, { "name": "gdm_cover_your_tracks", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_oversight_pattern", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_strategic_rule_breaking", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 9, "dependency": "gdm_stealth", "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_stealth", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_cover_your_tracks", "name": "GDM Cover Your Tracks", "slug": "inspect--gdm-cover-your-tracks", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about and circumvent oversight.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "family_title": "GDM Dangerous Capabilities: Stealth", "group": "Scheming", "version": "4-A", "contributors": [ "ZiyueWang25" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_cover_your_tracks", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_classifier_evasion", "dataset_samples": 9, "comment": null, "support_only": false }, { "name": "gdm_cover_your_tracks", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_oversight_pattern", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_strategic_rule_breaking", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 2, "dependency": "gdm_stealth", "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_stealth", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_in_house_ctf", "name": "GDM Dangerous Capabilities: Capture the Flag", "slug": "inspect--gdm-in-house-ctf", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_in_house_ctf", "topics": [ "cyber" ], "description": "CTF challenges covering web app vulnerabilities, off-the-shelf exploits, databases, Linux privilege escalation, password cracking and spraying. Demonstrates tool use and sandboxing untrusted model code.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_in_house_ctf/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://hub.docker.com/u/marshw", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_in_house_ctf/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_in_house_ctf/eval.yaml", "family_title": "GDM Dangerous Capabilities: Capture the Flag", "group": "Cybersecurity", "version": "6-A", "contributors": [ "XkunW" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "gdm_in_house_ctf", "dataset_samples": 13, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_in_house_ctf", "dataset_samples": 13, "comment": null, "support_only": false } ], "protocol": { "version": "6-A", "dataset_samples": 13, "dependency": null, "requires_internet": false, "sandbox": [ "solver", "scorer" ] }, "external_assets": [ { "type": "direct_url", "source": "https://hub.docker.com/u/marshw", "fetch_method": "other", "state": "floating", "comment": "Per-challenge Docker images (marshw/cmd_injection, marshw/db_3, marshw/grafana, etc.) pulled at eval runtime" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_in_house_ctf", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://hub.docker.com/u/marshw", "owner": "hub.docker.com", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_in_house_ctf/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://hub.docker.com/u/marshw", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane hub.docker.com evaluation-task" } }, { "id": "inspect:gdm_intercode_ctf", "name": "InterCode: Security and Coding Capture-the-Flag Challenges", "slug": "inspect--gdm-intercode-ctf", "organisation": "John Yang, Akshara Prabhakar, Karthik Narasimhan et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_intercode_ctf", "topics": [ "cyber" ], "description": "Tests AI's ability in coding, cryptography, reverse engineering, and vulnerability identification through practical capture-the-flag (CTF) cybersecurity scenarios.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_intercode_ctf/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2306.14898", "primary": true }, { "kind": "paper", "label": "Paper: InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback", "url": "https://arxiv.org/abs/2306.14898", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/princeton-nlp/intercode", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_intercode_ctf/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2306.14898", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_intercode_ctf/eval.yaml", "family_title": "InterCode: Security and Coding Capture-the-Flag Challenges", "group": "Cybersecurity", "version": "4-B", "contributors": [ "jjallaire" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "gdm_intercode_ctf", "dataset_samples": 78, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_intercode_ctf", "dataset_samples": 78, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 78, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "direct_url", "source": "https://github.com/princeton-nlp/intercode/archive/{SHA}.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_intercode_ctf", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "John Yang, Akshara Prabhakar, Karthik Narasimhan et al.", "organization": "Princeton U", "organizations": [ "Princeton U" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2306.14898" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2306.14898", "arxiv_id": "2306.14898", "title": "InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback", "authors": [ "John Yang", "Akshara Prabhakar", "Karthik Narasimhan", "Shunyu Yao" ], "organizations": [ "Princeton U" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2306.14898", "arxiv_id": "2306.14898", "title": "InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback", "authors": [ "John Yang", "Akshara Prabhakar", "Karthik Narasimhan", "Shunyu Yao" ], "organizations": [ "Princeton U" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/princeton-nlp/intercode", "owner": "Princeton NLP", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_intercode_ctf/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2306.14898" }, "code_url": "https://github.com/princeton-nlp/intercode", "metadata": { "source_search": "John Yang, Akshara Prabhakar, Karthik Narasimhan et al. Princeton U Original paper authors InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback John Yang Akshara Prabhakar Karthik Narasimhan Shunyu Yao Princeton NLP evaluation-task" } }, { "id": "inspect:gdm_oversight_pattern", "name": "GDM Oversight Pattern", "slug": "inspect--gdm-oversight-pattern", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about and circumvent oversight.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "family_title": "GDM Dangerous Capabilities: Stealth", "group": "Scheming", "version": "4-A", "contributors": [ "ZiyueWang25" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_oversight_pattern", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_classifier_evasion", "dataset_samples": 9, "comment": null, "support_only": false }, { "name": "gdm_cover_your_tracks", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_oversight_pattern", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_strategic_rule_breaking", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 2, "dependency": "gdm_stealth", "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_stealth", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_approved_directories", "name": "GDM Self Reasoning Approved Directories", "slug": "inspect--gdm-self-reasoning-approved-directories", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_calculator_improvement", "name": "GDM Self Reasoning Calculator Improvement", "slug": "inspect--gdm-self-reasoning-calculator-improvement", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_context_length_mod_instrumental_only", "name": "GDM Self Reasoning Context Length Mod Instrumental Only", "slug": "inspect--gdm-self-reasoning-context-length-mod-instrumental-only", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_context_length_mod_irreversibility_only", "name": "GDM Self Reasoning Context Length Mod Irreversibility Only", "slug": "inspect--gdm-self-reasoning-context-length-mod-irreversibility-only", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_database_tool", "name": "GDM Self Reasoning Database Tool", "slug": "inspect--gdm-self-reasoning-database-tool", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_latency_calculator", "name": "GDM Self Reasoning Latency Calculator", "slug": "inspect--gdm-self-reasoning-latency-calculator", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_max_messages_calculator", "name": "GDM Self Reasoning Max Messages Calculator", "slug": "inspect--gdm-self-reasoning-max-messages-calculator", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_max_tokens", "name": "GDM Self Reasoning Max Tokens", "slug": "inspect--gdm-self-reasoning-max-tokens", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_oversight_frequency", "name": "GDM Self Reasoning Oversight Frequency", "slug": "inspect--gdm-self-reasoning-oversight-frequency", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_read_logs", "name": "GDM Self Reasoning Read Logs", "slug": "inspect--gdm-self-reasoning-read-logs", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_self_reasoning_turn_off_filters", "name": "GDM Self Reasoning Turn Off Filters", "slug": "inspect--gdm-self-reasoning-turn-off-filters", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about its environment.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-reasoning", "group": "Scheming", "version": "4-B", "contributors": [ "ZiyueWang25", "XkunW" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_self_reasoning_approved_directories", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_calculator_improvement", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_instrumental_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_context_length_mod_irreversibility_only", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_database_tool", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_latency_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_messages_calculator", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_max_tokens", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_oversight_frequency", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_read_logs", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_self_reasoning_turn_off_filters", "dataset_samples": 2, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 2, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_reasoning", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_reasoning/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdm_sp01_e2e", "name": "GDM Sp01 E2e", "slug": "inspect--gdm-sp01-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp01_milestones", "name": "GDM Sp01 Milestones", "slug": "inspect--gdm-sp01-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp01_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp02_e2e", "name": "GDM Sp02 E2e", "slug": "inspect--gdm-sp02-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp02_milestones", "name": "GDM Sp02 Milestones", "slug": "inspect--gdm-sp02-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp02_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp03_e2e", "name": "GDM Sp03 E2e", "slug": "inspect--gdm-sp03-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp03_milestones", "name": "GDM Sp03 Milestones", "slug": "inspect--gdm-sp03-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp03_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp04_e2e", "name": "GDM Sp04 E2e", "slug": "inspect--gdm-sp04-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp04_milestones", "name": "GDM Sp04 Milestones", "slug": "inspect--gdm-sp04-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp04_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp05_e2e", "name": "GDM Sp05 E2e", "slug": "inspect--gdm-sp05-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp05_milestones", "name": "GDM Sp05 Milestones", "slug": "inspect--gdm-sp05-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp05_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp07_e2e", "name": "GDM Sp07 E2e", "slug": "inspect--gdm-sp07-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp07_milestones", "name": "GDM Sp07 Milestones", "slug": "inspect--gdm-sp07-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp07_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp08_e2e", "name": "GDM Sp08 E2e", "slug": "inspect--gdm-sp08-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp08_milestones", "name": "GDM Sp08 Milestones", "slug": "inspect--gdm-sp08-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp08_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp09_e2e", "name": "GDM Sp09 E2e", "slug": "inspect--gdm-sp09-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp09_milestones", "name": "GDM Sp09 Milestones", "slug": "inspect--gdm-sp09-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp09_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp10_e2e", "name": "GDM Sp10 E2e", "slug": "inspect--gdm-sp10-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp10_milestones", "name": "GDM Sp10 Milestones", "slug": "inspect--gdm-sp10-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp10_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp12_e2e", "name": "GDM Sp12 E2e", "slug": "inspect--gdm-sp12-e2e", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": 1, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_sp12_milestones", "name": "GDM Sp12 Milestones", "slug": "inspect--gdm-sp12-milestones", "organisation": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation", "topics": [ "general-capability" ], "description": "Ten real-world–inspired tasks from Google DeepMind's Dangerous Capabilities Evaluations assessing self-proliferation behaviors (e.g., email setup, model installation, web agent setup, wallet operations). Supports end-to-end, milestones, and expert best-of-N modes.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.13793", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.13793", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml", "family_title": "GDM Dangerous Capabilities: Self-proliferation", "group": "Scheming", "version": "6-B", "contributors": [ "XkunW", "MariaIzobava", "kohankhaki" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_sp12_milestones", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_sp01_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp02_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp03_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp04_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp05_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp07_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp08_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp09_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp10_e2e", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "gdm_sp12_e2e", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "6-B", "dataset_samples": null, "dependency": "gdm_self_proliferation", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/mistralai/mistral-src.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2557e12d0e1878a1562ece30434820f80aa3e12a in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "pip install git+https://github.com/huggingface/{transformers,peft,accelerate}.git", "fetch_method": "pip_install_git", "state": "pinned", "comment": "Pinned to commit SHAs in mistral.Dockerfile" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/UKGovernmentBEIS/inspect_evals/{version}/...secrets.zip", "fetch_method": "requests", "state": "controlled", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_self_proliferation", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mary Phuong, Matthew Aitchison, Elliot Catt et al.", "organization": "Google DeepMind", "organizations": [ "Google DeepMind" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.13793" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2403.13793", "arxiv_id": "2403.13793", "title": "Evaluating Frontier Models for Dangerous Capabilities", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "organizations": [ "Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "owner": "files.mistral-7b-v0-1.mistral.ai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_self_proliferation/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.13793" }, "code_url": "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar", "metadata": { "source_search": "Mary Phuong, Matthew Aitchison, Elliot Catt et al. Google DeepMind Original paper authors Evaluating Frontier Models for Dangerous Capabilities Mary Phuong Matthew Aitchison Elliot Catt Sarah Cogan Alexandre Kaskasoli Victoria Krakovna David Lindner Matthew Rahtz Yannis Assael Sarah Hodkinson Heidi Howard Tom Lieberum Ramana Kumar Maria Abi Raad Albert Webson Lewis Ho Sharon Lin Sebastian Farquhar Marcus Hutter Gregoire Deletang Anian Ruoss Seliem El-Sayed Sasha Brown Anca Dragan Rohin Shah Allan Dafoe Toby Shevlane files.mistral-7b-v0-1.mistral.ai evaluation-family" } }, { "id": "inspect:gdm_strategic_rule_breaking", "name": "GDM Strategic Rule Breaking", "slug": "inspect--gdm-strategic-rule-breaking", "organisation": "Google DeepMind", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth", "topics": [ "deception-misalignment" ], "description": "Test AI's ability to reason about and circumvent oversight.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://arxiv.org/abs/2505.01420", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2505.01420", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.01420", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml", "family_title": "GDM Dangerous Capabilities: Stealth", "group": "Scheming", "version": "4-A", "contributors": [ "ZiyueWang25" ], "tags": [ "Agent", "Scheming" ], "task": { "name": "gdm_strategic_rule_breaking", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "gdm_classifier_evasion", "dataset_samples": 9, "comment": null, "support_only": false }, { "name": "gdm_cover_your_tracks", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_oversight_pattern", "dataset_samples": 2, "comment": null, "support_only": false }, { "name": "gdm_strategic_rule_breaking", "dataset_samples": 1, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 1, "dependency": "gdm_stealth", "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdm_stealth", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Google DeepMind", "organization": "Google DeepMind", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2505.01420", "organizations": [ "Google DeepMind" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.01420", "arxiv_id": "2505.01420", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdm_stealth/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://arxiv.org/abs/2505.01420" }, "code_url": null, "metadata": { "source_search": "Google DeepMind Google DeepMind Evaluation developed by Evaluating Frontier Models for Stealth and Situational Awareness Mary Phuong Roland S. Zimmermann Ziyue Wang David Lindner Victoria Krakovna Sarah Cogan Allan Dafoe Lewis Ho Rohin Shah evaluation-family" } }, { "id": "inspect:gdpval", "name": "GDPval", "slug": "inspect--gdpval", "organisation": "Tejal Patwardhan, Rachel Dias, Elizabeth Proehl et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdpval", "topics": [ "general-capability" ], "description": "GDPval measures model performance on economically valuable, real-world tasks across 44 occupations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdpval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2510.04374", "primary": true }, { "kind": "paper", "label": "Paper: GDPval: Evaluating AI Model Performance on Real-World Economically Valuable Tasks", "url": "https://arxiv.org/abs/2510.04374", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdpval/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: OpenAI", "url": "https://huggingface.co/datasets/openai/gdpval", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2510.04374", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdpval/eval.yaml", "family_title": "GDPval", "group": "Assistants", "version": "2-A", "contributors": [ "jeqcho" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "gdpval", "dataset_samples": 220, "comment": null, "support_only": false }, "tasks": [ { "name": "gdpval", "dataset_samples": 220, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 220, "dependency": "gdpval", "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "huggingface", "source": "openai/gdpval", "fetch_method": "hf_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "openai/gdpval", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gdpval", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Tejal Patwardhan, Rachel Dias, Elizabeth Proehl et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2510.04374" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2510.04374", "arxiv_id": "2510.04374", "title": "GDPval: Evaluating AI Model Performance on Real-World Economically Valuable Tasks", "authors": [ "Tejal Patwardhan", "Rachel Dias", "Elizabeth Proehl", "Grace Kim", "Michele Wang", "Olivia Watkins", "Simón Posada Fishman", "Marwan Aljubeh", "Phoebe Thacker", "Laurance Fauconnet", "Natalie S. Kim", "Patrick Chao", "Samuel Miserendino", "Gildas Chabot", "David Li", "Michael Sharman", "Alexandra Barr", "Amelia Glaese", "Jerry Tworek" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2510.04374", "arxiv_id": "2510.04374", "title": "GDPval: Evaluating AI Model Performance on Real-World Economically Valuable Tasks", "authors": [ "Tejal Patwardhan", "Rachel Dias", "Elizabeth Proehl", "Grace Kim", "Michele Wang", "Olivia Watkins", "Simón Posada Fishman", "Marwan Aljubeh", "Phoebe Thacker", "Laurance Fauconnet", "Natalie S. Kim", "Patrick Chao", "Samuel Miserendino", "Gildas Chabot", "David Li", "Michael Sharman", "Alexandra Barr", "Amelia Glaese", "Jerry Tworek" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gdpval/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/openai/gdpval", "provider": "OpenAI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2510.04374" }, "code_url": null, "metadata": { "source_search": "Tejal Patwardhan, Rachel Dias, Elizabeth Proehl et al. Original paper authors GDPval: Evaluating AI Model Performance on Real-World Economically Valuable Tasks Tejal Patwardhan Rachel Dias Elizabeth Proehl Grace Kim Michele Wang Olivia Watkins Simón Posada Fishman Marwan Aljubeh Phoebe Thacker Laurance Fauconnet Natalie S. Kim Patrick Chao Samuel Miserendino Gildas Chabot David Li Michael Sharman Alexandra Barr Amelia Glaese Jerry Tworek evaluation-task" } }, { "id": "inspect:gpqa_diamond", "name": "GPQA: Graduate-Level STEM Knowledge Challenge", "slug": "inspect--gpqa-diamond", "organisation": "David Rein, Betty Li Hou, Asa Cooper Stickland et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gpqa", "topics": [ "general-capability" ], "description": "Contains challenging multiple-choice questions created by domain experts in biology, physics, and chemistry, designed to test advanced scientific understanding beyond basic internet searches. Experts at PhD level in the corresponding domains reach 65% accuracy.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gpqa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2311.12022", "primary": true }, { "kind": "paper", "label": "Paper: GPQA: A Graduate-Level Google-Proof Q&A Benchmark", "url": "https://arxiv.org/abs/2311.12022", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gpqa/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2311.12022", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gpqa/eval.yaml", "family_title": "GPQA: Graduate-Level STEM Knowledge Challenge", "group": "Knowledge", "version": "2-B", "contributors": [ "jjallaire" ], "tags": [ "Knowledge" ], "task": { "name": "gpqa_diamond", "dataset_samples": 198, "comment": null, "support_only": false }, "tasks": [ { "name": "gpqa_diamond", "dataset_samples": 198, "comment": null, "support_only": false } ], "protocol": { "version": "2-B", "dataset_samples": 198, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gpqa", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "David Rein, Betty Li Hou, Asa Cooper Stickland et al.", "organization": "METR (Model Evaluation and Threat Research); UK AI Security Institute; New York University; Yale University", "organizations": [ "METR (Model Evaluation and Threat Research)", "UK AI Security Institute", "New York University", "Yale University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2311.12022" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.12022", "arxiv_id": "2311.12022", "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark", "authors": [ "David Rein", "Betty Li Hou", "Asa Cooper Stickland", "Jackson Petty", "Richard Yuanzhe Pang", "Julien Dirani", "Julian Michael", "Samuel R. Bowman" ], "organizations": [ "METR (Model Evaluation and Threat Research)", "UK AI Security Institute", "New York University", "Yale University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.12022", "arxiv_id": "2311.12022", "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark", "authors": [ "David Rein", "Betty Li Hou", "Asa Cooper Stickland", "Jackson Petty", "Richard Yuanzhe Pang", "Julien Dirani", "Julian Michael", "Samuel R. Bowman" ], "organizations": [ "METR (Model Evaluation and Threat Research)", "UK AI Security Institute", "New York University", "Yale University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv", "owner": "openaipublic.blob.core.windows.net", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gpqa/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2311.12022" }, "code_url": "https://openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv", "metadata": { "source_search": "David Rein, Betty Li Hou, Asa Cooper Stickland et al. METR (Model Evaluation and Threat Research); UK AI Security Institute; New York University; Yale University Original paper authors GPQA: A Graduate-Level Google-Proof Q&A Benchmark David Rein Betty Li Hou Asa Cooper Stickland Jackson Petty Richard Yuanzhe Pang Julien Dirani Julian Michael Samuel R. Bowman openaipublic.blob.core.windows.net evaluation-task" } }, { "id": "inspect:gsm8k", "name": "GSM8K: Grade School Math Word Problems", "slug": "inspect--gsm8k", "organisation": "Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gsm8k", "topics": [ "general-capability" ], "description": "Measures how effectively language models solve realistic, linguistically rich math word problems suitable for grade-school-level mathematics.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gsm8k/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2110.14168", "primary": true }, { "kind": "paper", "label": "Paper: Training Verifiers to Solve Math Word Problems", "url": "https://arxiv.org/abs/2110.14168", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gsm8k/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: OpenAI", "url": "https://huggingface.co/datasets/openai/gsm8k", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2110.14168", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gsm8k/eval.yaml", "family_title": "GSM8K: Grade School Math Word Problems", "group": "Mathematics", "version": "2-A", "contributors": [ "jjallaire" ], "tags": [ "Mathematics" ], "task": { "name": "gsm8k", "dataset_samples": 1319, "comment": null, "support_only": false }, "tasks": [ { "name": "gsm8k", "dataset_samples": 1319, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1319, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "openai/gsm8k", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:gsm8k", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2110.14168" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2110.14168", "arxiv_id": "2110.14168", "title": "Training Verifiers to Solve Math Word Problems", "authors": [ "Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian", "Mark Chen", "Heewoo Jun", "Lukasz Kaiser", "Matthias Plappert", "Jerry Tworek", "Jacob Hilton", "Reiichiro Nakano", "Christopher Hesse", "John Schulman" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2110.14168", "arxiv_id": "2110.14168", "title": "Training Verifiers to Solve Math Word Problems", "authors": [ "Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian", "Mark Chen", "Heewoo Jun", "Lukasz Kaiser", "Matthias Plappert", "Jerry Tworek", "Jacob Hilton", "Reiichiro Nakano", "Christopher Hesse", "John Schulman" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/gsm8k/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/openai/gsm8k", "provider": "OpenAI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2110.14168" }, "code_url": null, "metadata": { "source_search": "Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian et al. Original paper authors Training Verifiers to Solve Math Word Problems Karl Cobbe Vineet Kosaraju Mohammad Bavarian Mark Chen Heewoo Jun Lukasz Kaiser Matthias Plappert Jerry Tworek Jacob Hilton Reiichiro Nakano Christopher Hesse John Schulman evaluation-task" } }, { "id": "register:hangman-bench", "name": "Hangman Bench", "slug": "register--hangman-bench", "organisation": "Matt Fisher", "source_type": "inspect-register", "source_url": "https://github.com/MattFisher/hangman-bench", "topics": [ "general-capability" ], "description": "A benchmark for testing AI models' ability to play the classic game of Hangman. Built as a demonstration of how to enable models to play games using tools within the Inspect framework.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/hangman-bench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/MattFisher/hangman-bench", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/MattFisher/hangman-bench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/hangman-bench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/MattFisher/hangman-bench", "primary": false } ], "paper_title": null, "paper_url": null, "code_url": "https://github.com/MattFisher/hangman-bench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/hangman-bench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "games", "tools" ], "contributors": [ "Scott-Simmons" ], "tasks": [ { "name": "hangman", "path": "src/hangman_bench/hangman.py" } ], "protocol": { "implementation_commit": "9f1f396e68191eb5ec99dbffbc6fea63609bd0b6", "report_commit": "9f1f396e68191eb5ec99dbffbc6fea63609bd0b6", "report_timestamp": "April 2026", "command": "uv run inspect eval src/hangman_bench/hangman.py@hangman --model openai/gpt-5-nano --limit 100", "source_comment": null }, "reported_results": [ { "model": "openai/gpt-5-nano", "provider": "OpenAI", "metrics": [ { "key": "accuracy", "value": 0.93 }, { "key": "stderr", "value": 0.026 } ] } ], "reported_result_count": 1, "reported_results_truncated": false, "report_notes": [ "Run on 100 samples, 1 epoch.", "The headline `accuracy` is the aggregate score from the eval's grouped game scorer. Per-difficulty breakdown: `v_easy`/`easy`/`medium`/`hard` = 0.95, `v_hard` = 0.85." ], "models_reported": [ "openai/gpt-5-nano" ], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:hangman-bench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "The repository describes Hangman Bench as an Inspect demonstration benchmark. No dedicated paper was identified." }, "origin": { "name": "Matt Fisher", "organization": null, "entity_type": "individual_developer", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://github.com/MattFisher/hangman-bench", "organizations": [], "role_label": "Evaluation developed by" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/MattFisher/hangman-bench", "owner": "Matt Fisher", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/hangman-bench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/MattFisher/hangman-bench" }, "metadata": { "source_search": "Matt Fisher Evaluation developed by Matt Fisher evaluation-package" } }, { "id": "canonical:harmbench", "name": "HarmBench", "slug": "canonical--harmbench", "organisation": "Center for AI Safety and the HarmBench paper authors", "source_type": "canonical-source", "source_url": "https://www.harmbench.org/", "topics": [ "safeguards" ], "description": "Framework for automated red teaming and harmful-behaviour robustness.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Elicitability of HarmBench's harmful behaviors under the selected red-team methods and target configuration.", "does_not_measure": "Real-world harm, user intent, deployment prevalence, universal safeguard robustness, or agentic end-to-end misuse.", "best_for": "Comparing red-teaming methods, target-model vulnerability, and defense regressions on a standardized behavior set.", "not_sufficient_for": "A comprehensive safety certification or estimate of incident probability.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/centerforaisafety/HarmBench", "primary": true }, { "kind": "paper", "label": "Paper: HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/centerforaisafety/HarmBench", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.harmbench.org", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/centerforaisafety/HarmBench", "primary": false } ], "resource_count": 5, "source_key": "canonical:harmbench", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Center for AI Safety and the HarmBench paper authors", "organization": "Center for AI Safety", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://github.com/centerforaisafety/HarmBench", "organizations": [ "Center for AI Safety" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.04249", "arxiv_id": "2402.04249", "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "authors": [ "Mantas Mazeika", "Long Phan", "Xuwang Yin", "Andy Zou", "Zifan Wang", "Norman Mu", "Elham Sakhaee", "Nathaniel Li", "Steven Basart", "Bo Li", "David Forsyth", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.04249", "arxiv_id": "2402.04249", "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "authors": [ "Mantas Mazeika", "Long Phan", "Xuwang Yin", "Andy Zou", "Zifan Wang", "Norman Mu", "Elham Sakhaee", "Nathaniel Li", "Steven Basart", "Bo Li", "David Forsyth", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/centerforaisafety/HarmBench", "owner": "Center for AI Safety", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/centerforaisafety/HarmBench" }, "paper_url": "https://arxiv.org/abs/2402.04249", "code_url": "https://github.com/centerforaisafety/HarmBench", "metadata": { "source_search": "Center for AI Safety and the HarmBench paper authors Center for AI Safety Evaluation developed by HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal Mantas Mazeika Long Phan Xuwang Yin Andy Zou Zifan Wang Norman Mu Elham Sakhaee Nathaniel Li Steven Basart Bo Li David Forsyth Dan Hendrycks Center for AI Safety evaluation-resource" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Standardized red-team elicitation of a broad set of harmful text behaviors.", "unit_of_analysis": "One harmful behavior attempted against one target model using one red-teaming method.", "interaction_type": "Adversarial prompt generation and target-model interaction under benchmark-defined methods and behavior categories.", "target_outcome": "Whether the response is classified as successfully exhibiting the specified harmful behavior.", "human_participants": "No human subjects. Human curation and validation inform behaviors and classifiers.", "elicitation": "Multiple red-teaming methods are run with defined budgets against target models and, in some cases, defenses.", "scoring": "Automated behavior classifiers and aggregate attack-success metrics across behaviors, methods, and targets.", "protocol_dependence": "Very high: behavior taxonomy, attack method and budget, target setup, generation settings, and classifier all matter.", "implementation_dependence": "High: classifier calibration, prompt templates, target API changes, and attack-code versions affect results.", "limitations": [ "The behavior set cannot cover the full space of harmful requests.", "Automated classifiers introduce false positives and false negatives.", "Static attacks may underestimate adaptive adversaries.", "Text-only elicitation is narrower than tool-using or multimodal misuse." ], "comparability": "Use the same HarmBench release, behavior subset, attack implementations and budgets, target snapshot, generation settings, and classifier.", "decision_relevance": "Useful for standardized red-team coverage and regression tracking; it should feed into, not substitute for, a broader safety case.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2402.04249" }, { "kind": "official", "url": "https://www.harmbench.org/" }, { "kind": "implementation", "url": "https://github.com/centerforaisafety/HarmBench" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:healthbench", "name": "Healthbench", "slug": "inspect--healthbench", "organisation": "OpenAI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench", "topics": [ "general-capability" ], "description": "A comprehensive evaluation benchmark designed to assess language models' medical capabilities across a wide range of healthcare scenarios.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://openai.com/index/healthbench", "primary": true }, { "kind": "paper", "label": "Paper: HealthBench: Evaluating Large Language Models Towards Improved Human Health", "url": "https://arxiv.org/abs/2505.08775", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://openai.com/index/healthbench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.08775", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "family_title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "group": "Knowledge", "version": "2-B", "contributors": [ "retroam" ], "tags": [ "Medical", "Healthcare", "Knowledge" ], "task": { "name": "healthbench", "dataset_samples": 5000, "comment": null, "support_only": false }, "tasks": [ { "name": "healthbench", "dataset_samples": 5000, "comment": null, "support_only": false }, { "name": "healthbench_hard", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "healthbench_consensus", "dataset_samples": 3671, "comment": null, "support_only": false }, { "name": "healthbench_meta_eval", "dataset_samples": 29511, "comment": null, "support_only": false } ], "protocol": { "version": "2-B", "dataset_samples": 5000, "dependency": "healthbench", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://openaipublic.blob.core.windows.net/simple-evals/healthbench/{file}.jsonl", "fetch_method": "other", "state": "pinned", "comment": "4 files with timestamp-encoded filenames; fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:healthbench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://openai.com/index/healthbench", "organizations": [ "OpenAI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.08775", "arxiv_id": "2505.08775", "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "authors": [ "Rahul K. Arora", "Jason Wei", "Rebecca Soskin Hicks", "Preston Bowman", "Joaquin Quiñonero-Candela", "Foivos Tsimpourlas", "Michael Sharman", "Meghan Shah", "Andrea Vallone", "Alex Beutel", "Johannes Heidecke", "Karan Singhal" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.08775", "arxiv_id": "2505.08775", "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "authors": [ "Rahul K. Arora", "Jason Wei", "Rebecca Soskin Hicks", "Preston Bowman", "Joaquin Quiñonero-Candela", "Foivos Tsimpourlas", "Michael Sharman", "Meghan Shah", "Andrea Vallone", "Alex Beutel", "Johannes Heidecke", "Karan Singhal" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://openai.com/index/healthbench" }, "code_url": null, "metadata": { "source_search": "OpenAI OpenAI Evaluation developed by HealthBench: Evaluating Large Language Models Towards Improved Human Health Rahul K. Arora Jason Wei Rebecca Soskin Hicks Preston Bowman Joaquin Quiñonero-Candela Foivos Tsimpourlas Michael Sharman Meghan Shah Andrea Vallone Alex Beutel Johannes Heidecke Karan Singhal evaluation-family" } }, { "id": "inspect:healthbench_consensus", "name": "Healthbench Consensus", "slug": "inspect--healthbench-consensus", "organisation": "OpenAI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench", "topics": [ "general-capability" ], "description": "A comprehensive evaluation benchmark designed to assess language models' medical capabilities across a wide range of healthcare scenarios.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://openai.com/index/healthbench", "primary": true }, { "kind": "paper", "label": "Paper: HealthBench: Evaluating Large Language Models Towards Improved Human Health", "url": "https://arxiv.org/abs/2505.08775", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://openai.com/index/healthbench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.08775", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "family_title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "group": "Knowledge", "version": "2-B", "contributors": [ "retroam" ], "tags": [ "Medical", "Healthcare", "Knowledge" ], "task": { "name": "healthbench_consensus", "dataset_samples": 3671, "comment": null, "support_only": false }, "tasks": [ { "name": "healthbench", "dataset_samples": 5000, "comment": null, "support_only": false }, { "name": "healthbench_hard", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "healthbench_consensus", "dataset_samples": 3671, "comment": null, "support_only": false }, { "name": "healthbench_meta_eval", "dataset_samples": 29511, "comment": null, "support_only": false } ], "protocol": { "version": "2-B", "dataset_samples": 3671, "dependency": "healthbench", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://openaipublic.blob.core.windows.net/simple-evals/healthbench/{file}.jsonl", "fetch_method": "other", "state": "pinned", "comment": "4 files with timestamp-encoded filenames; fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:healthbench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://openai.com/index/healthbench", "organizations": [ "OpenAI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.08775", "arxiv_id": "2505.08775", "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "authors": [ "Rahul K. Arora", "Jason Wei", "Rebecca Soskin Hicks", "Preston Bowman", "Joaquin Quiñonero-Candela", "Foivos Tsimpourlas", "Michael Sharman", "Meghan Shah", "Andrea Vallone", "Alex Beutel", "Johannes Heidecke", "Karan Singhal" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.08775", "arxiv_id": "2505.08775", "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "authors": [ "Rahul K. Arora", "Jason Wei", "Rebecca Soskin Hicks", "Preston Bowman", "Joaquin Quiñonero-Candela", "Foivos Tsimpourlas", "Michael Sharman", "Meghan Shah", "Andrea Vallone", "Alex Beutel", "Johannes Heidecke", "Karan Singhal" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://openai.com/index/healthbench" }, "code_url": null, "metadata": { "source_search": "OpenAI OpenAI Evaluation developed by HealthBench: Evaluating Large Language Models Towards Improved Human Health Rahul K. Arora Jason Wei Rebecca Soskin Hicks Preston Bowman Joaquin Quiñonero-Candela Foivos Tsimpourlas Michael Sharman Meghan Shah Andrea Vallone Alex Beutel Johannes Heidecke Karan Singhal evaluation-family" } }, { "id": "inspect:healthbench_hard", "name": "Healthbench Hard", "slug": "inspect--healthbench-hard", "organisation": "OpenAI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench", "topics": [ "general-capability" ], "description": "A comprehensive evaluation benchmark designed to assess language models' medical capabilities across a wide range of healthcare scenarios.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://openai.com/index/healthbench", "primary": true }, { "kind": "paper", "label": "Paper: HealthBench: Evaluating Large Language Models Towards Improved Human Health", "url": "https://arxiv.org/abs/2505.08775", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://openai.com/index/healthbench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.08775", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "family_title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "group": "Knowledge", "version": "2-B", "contributors": [ "retroam" ], "tags": [ "Medical", "Healthcare", "Knowledge" ], "task": { "name": "healthbench_hard", "dataset_samples": 1000, "comment": null, "support_only": false }, "tasks": [ { "name": "healthbench", "dataset_samples": 5000, "comment": null, "support_only": false }, { "name": "healthbench_hard", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "healthbench_consensus", "dataset_samples": 3671, "comment": null, "support_only": false }, { "name": "healthbench_meta_eval", "dataset_samples": 29511, "comment": null, "support_only": false } ], "protocol": { "version": "2-B", "dataset_samples": 1000, "dependency": "healthbench", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://openaipublic.blob.core.windows.net/simple-evals/healthbench/{file}.jsonl", "fetch_method": "other", "state": "pinned", "comment": "4 files with timestamp-encoded filenames; fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:healthbench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://openai.com/index/healthbench", "organizations": [ "OpenAI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.08775", "arxiv_id": "2505.08775", "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "authors": [ "Rahul K. Arora", "Jason Wei", "Rebecca Soskin Hicks", "Preston Bowman", "Joaquin Quiñonero-Candela", "Foivos Tsimpourlas", "Michael Sharman", "Meghan Shah", "Andrea Vallone", "Alex Beutel", "Johannes Heidecke", "Karan Singhal" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.08775", "arxiv_id": "2505.08775", "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "authors": [ "Rahul K. Arora", "Jason Wei", "Rebecca Soskin Hicks", "Preston Bowman", "Joaquin Quiñonero-Candela", "Foivos Tsimpourlas", "Michael Sharman", "Meghan Shah", "Andrea Vallone", "Alex Beutel", "Johannes Heidecke", "Karan Singhal" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://openai.com/index/healthbench" }, "code_url": null, "metadata": { "source_search": "OpenAI OpenAI Evaluation developed by HealthBench: Evaluating Large Language Models Towards Improved Human Health Rahul K. Arora Jason Wei Rebecca Soskin Hicks Preston Bowman Joaquin Quiñonero-Candela Foivos Tsimpourlas Michael Sharman Meghan Shah Andrea Vallone Alex Beutel Johannes Heidecke Karan Singhal evaluation-family" } }, { "id": "inspect:healthbench_meta_eval", "name": "Healthbench Meta Eval", "slug": "inspect--healthbench-meta-eval", "organisation": "OpenAI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench", "topics": [ "general-capability" ], "description": "A comprehensive evaluation benchmark designed to assess language models' medical capabilities across a wide range of healthcare scenarios.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://openai.com/index/healthbench", "primary": true }, { "kind": "paper", "label": "Paper: HealthBench: Evaluating Large Language Models Towards Improved Human Health", "url": "https://arxiv.org/abs/2505.08775", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://openai.com/index/healthbench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.08775", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml", "family_title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "group": "Knowledge", "version": "2-B", "contributors": [ "retroam" ], "tags": [ "Medical", "Healthcare", "Knowledge" ], "task": { "name": "healthbench_meta_eval", "dataset_samples": 29511, "comment": null, "support_only": false }, "tasks": [ { "name": "healthbench", "dataset_samples": 5000, "comment": null, "support_only": false }, { "name": "healthbench_hard", "dataset_samples": 1000, "comment": null, "support_only": false }, { "name": "healthbench_consensus", "dataset_samples": 3671, "comment": null, "support_only": false }, { "name": "healthbench_meta_eval", "dataset_samples": 29511, "comment": null, "support_only": false } ], "protocol": { "version": "2-B", "dataset_samples": 29511, "dependency": "healthbench", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://openaipublic.blob.core.windows.net/simple-evals/healthbench/{file}.jsonl", "fetch_method": "other", "state": "pinned", "comment": "4 files with timestamp-encoded filenames; fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:healthbench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://openai.com/index/healthbench", "organizations": [ "OpenAI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.08775", "arxiv_id": "2505.08775", "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "authors": [ "Rahul K. Arora", "Jason Wei", "Rebecca Soskin Hicks", "Preston Bowman", "Joaquin Quiñonero-Candela", "Foivos Tsimpourlas", "Michael Sharman", "Meghan Shah", "Andrea Vallone", "Alex Beutel", "Johannes Heidecke", "Karan Singhal" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.08775", "arxiv_id": "2505.08775", "title": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "authors": [ "Rahul K. Arora", "Jason Wei", "Rebecca Soskin Hicks", "Preston Bowman", "Joaquin Quiñonero-Candela", "Foivos Tsimpourlas", "Michael Sharman", "Meghan Shah", "Andrea Vallone", "Alex Beutel", "Johannes Heidecke", "Karan Singhal" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/healthbench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://openai.com/index/healthbench" }, "code_url": null, "metadata": { "source_search": "OpenAI OpenAI Evaluation developed by HealthBench: Evaluating Large Language Models Towards Improved Human Health Rahul K. Arora Jason Wei Rebecca Soskin Hicks Preston Bowman Joaquin Quiñonero-Candela Foivos Tsimpourlas Michael Sharman Meghan Shah Andrea Vallone Alex Beutel Johannes Heidecke Karan Singhal evaluation-family" } }, { "id": "inspect:hellaswag", "name": "HellaSwag: Commonsense Event Continuation", "slug": "inspect--hellaswag", "organisation": "Rowan Zellers, Ari Holtzman, Yonatan Bisk et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hellaswag", "topics": [ "general-capability" ], "description": "Tests models' commonsense reasoning abilities by asking them to select the most likely next step or continuation for a given everyday situation.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hellaswag/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1905.07830", "primary": true }, { "kind": "paper", "label": "Paper: HellaSwag: Can a Machine Really Finish Your Sentence?", "url": "https://arxiv.org/abs/1905.07830", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hellaswag/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Rowan", "url": "https://huggingface.co/datasets/Rowan/hellaswag", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1905.07830", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hellaswag/eval.yaml", "family_title": "HellaSwag: Commonsense Event Continuation", "group": "Reasoning", "version": "2-A", "contributors": [ "jjallaire" ], "tags": [ "Reasoning" ], "task": { "name": "hellaswag", "dataset_samples": 10042, "comment": null, "support_only": false }, "tasks": [ { "name": "hellaswag", "dataset_samples": 10042, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 10042, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "Rowan/hellaswag", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:hellaswag", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Rowan Zellers, Ari Holtzman, Yonatan Bisk et al.", "organization": "University of Washington; Carnegie Mellon University", "organizations": [ "University of Washington", "Carnegie Mellon University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1905.07830" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1905.07830", "arxiv_id": "1905.07830", "title": "HellaSwag: Can a Machine Really Finish Your Sentence?", "authors": [ "Rowan Zellers", "Ari Holtzman", "Yonatan Bisk", "Ali Farhadi", "Yejin Choi" ], "organizations": [ "University of Washington", "Carnegie Mellon University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1905.07830", "arxiv_id": "1905.07830", "title": "HellaSwag: Can a Machine Really Finish Your Sentence?", "authors": [ "Rowan Zellers", "Ari Holtzman", "Yonatan Bisk", "Ali Farhadi", "Yejin Choi" ], "organizations": [ "University of Washington", "Carnegie Mellon University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hellaswag/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/Rowan/hellaswag", "provider": "Rowan" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1905.07830" }, "code_url": null, "metadata": { "source_search": "Rowan Zellers, Ari Holtzman, Yonatan Bisk et al. University of Washington; Carnegie Mellon University Original paper authors HellaSwag: Can a Machine Really Finish Your Sentence? Rowan Zellers Ari Holtzman Yonatan Bisk Ali Farhadi Yejin Choi evaluation-task" } }, { "id": "canonical:helm-safety", "name": "HELM Safety", "slug": "canonical--helm-safety", "organisation": "Stanford CRFM", "source_type": "canonical-source", "source_url": "https://crfm.stanford.edu/helm/", "topics": [ "safeguards", "evaluation-integrity" ], "description": "Scenario-based safety evaluation within the HELM framework.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "suite_developer", "label": "Suite developed by", "url": "https://crfm.stanford.edu/helm", "primary": true }, { "kind": "official", "label": "Official or project page", "url": "https://crfm.stanford.edu/helm", "primary": false } ], "resource_count": 2, "source_key": "canonical:helm-safety", "record_type": "evaluation-suite", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Stanford CRFM", "organization": "Stanford CRFM", "organizations": [ "Stanford CRFM" ], "entity_type": "organization", "role": "suite_developer", "role_label": "Suite developed by", "confidence": "curated", "evidence_url": "https://crfm.stanford.edu/helm" }, "paper": { "status": "not-applicable", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-applicable", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Suite developed by", "attribution_status": "verified", "preferred_source": { "label": "Suite developed by", "kind": "suite_developer", "url": "https://crfm.stanford.edu/helm" }, "paper_url": null, "code_url": null, "metadata": { "source_search": "Stanford CRFM Stanford CRFM Suite developed by evaluation-suite" } }, { "id": "inspect:humaneval", "name": "HumanEval: Python Function Generation from Instructions", "slug": "inspect--humaneval", "organisation": "Mark Chen, Jerry Tworek, Heewoo Jun et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/humaneval", "topics": [ "general-capability" ], "description": "Assesses how accurately language models can write correct Python functions based solely on natural-language instructions provided as docstrings.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/humaneval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2107.03374", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating Large Language Models Trained on Code", "url": "https://arxiv.org/abs/2107.03374", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/humaneval/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: OpenAI", "url": "https://huggingface.co/datasets/openai/openai_humaneval", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2107.03374", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/humaneval/eval.yaml", "family_title": "HumanEval: Python Function Generation from Instructions", "group": "Coding", "version": "2-A", "contributors": [ "adil-a" ], "tags": [ "Coding" ], "task": { "name": "humaneval", "dataset_samples": 164, "comment": null, "support_only": false }, "tasks": [ { "name": "humaneval", "dataset_samples": 164, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 164, "dependency": null, "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "openai/openai_humaneval", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:humaneval", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Mark Chen, Jerry Tworek, Heewoo Jun et al.", "organization": "OpenAI", "organizations": [ "OpenAI" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2107.03374" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2107.03374", "arxiv_id": "2107.03374", "title": "Evaluating Large Language Models Trained on Code", "authors": [ "Mark Chen", "Jerry Tworek", "Heewoo Jun", "Qiming Yuan", "Henrique Ponde de Oliveira Pinto", "Jared Kaplan", "Harri Edwards", "Yuri Burda", "Nicholas Joseph", "Greg Brockman", "Alex Ray", "Raul Puri", "Gretchen Krueger", "Michael Petrov", "Heidy Khlaaf", "Girish Sastry", "Pamela Mishkin", "Brooke Chan", "Scott Gray", "Nick Ryder", "Mikhail Pavlov", "Alethea Power", "Lukasz Kaiser", "Mohammad Bavarian", "Clemens Winter", "Philippe Tillet", "Felipe Petroski Such", "Dave Cummings", "Matthias Plappert", "Fotios Chantzis", "Elizabeth Barnes", "Ariel Herbert-Voss", "William Hebgen Guss", "Alex Nichol", "Alex Paino", "Nikolas Tezak", "Jie Tang", "Igor Babuschkin", "Suchir Balaji", "Shantanu Jain", "William Saunders", "Christopher Hesse", "Andrew N. Carr", "Jan Leike", "Josh Achiam", "Vedant Misra", "Evan Morikawa", "Alec Radford", "Matthew Knight", "Miles Brundage", "Mira Murati", "Katie Mayer", "Peter Welinder", "Bob McGrew", "Dario Amodei", "Sam McCandlish", "Ilya Sutskever", "Wojciech Zaremba" ], "organizations": [ "OpenAI" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2107.03374", "arxiv_id": "2107.03374", "title": "Evaluating Large Language Models Trained on Code", "authors": [ "Mark Chen", "Jerry Tworek", "Heewoo Jun", "Qiming Yuan", "Henrique Ponde de Oliveira Pinto", "Jared Kaplan", "Harri Edwards", "Yuri Burda", "Nicholas Joseph", "Greg Brockman", "Alex Ray", "Raul Puri", "Gretchen Krueger", "Michael Petrov", "Heidy Khlaaf", "Girish Sastry", "Pamela Mishkin", "Brooke Chan", "Scott Gray", "Nick Ryder", "Mikhail Pavlov", "Alethea Power", "Lukasz Kaiser", "Mohammad Bavarian", "Clemens Winter", "Philippe Tillet", "Felipe Petroski Such", "Dave Cummings", "Matthias Plappert", "Fotios Chantzis", "Elizabeth Barnes", "Ariel Herbert-Voss", "William Hebgen Guss", "Alex Nichol", "Alex Paino", "Nikolas Tezak", "Jie Tang", "Igor Babuschkin", "Suchir Balaji", "Shantanu Jain", "William Saunders", "Christopher Hesse", "Andrew N. Carr", "Jan Leike", "Josh Achiam", "Vedant Misra", "Evan Morikawa", "Alec Radford", "Matthew Knight", "Miles Brundage", "Mira Murati", "Katie Mayer", "Peter Welinder", "Bob McGrew", "Dario Amodei", "Sam McCandlish", "Ilya Sutskever", "Wojciech Zaremba" ], "organizations": [ "OpenAI" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/humaneval/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/openai/openai_humaneval", "provider": "OpenAI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2107.03374" }, "code_url": null, "metadata": { "source_search": "Mark Chen, Jerry Tworek, Heewoo Jun et al. OpenAI Original paper authors Evaluating Large Language Models Trained on Code Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de Oliveira Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever Wojciech Zaremba evaluation-task" } }, { "id": "inspect:hle", "name": "Humanity's Last Exam", "slug": "inspect--hle", "organisation": "Long Phan, Alice Gatti, Ziwen Han et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hle", "topics": [ "general-capability" ], "description": "Humanity's Last Exam (HLE) is a multi-modal benchmark at the frontier of human knowledge, designed to be the final closed-ended academic benchmark of its kind with broad subject coverage. Humanity's Last Exam consists of 2,500 questions across dozens of subjects, including mathematics, humanities, and the natural sciences. HLE is developed globally by subject-matter experts and consists of multiple-choice and short-answer questions suitable for automated grading.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hle/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2501.14249", "primary": true }, { "kind": "paper", "label": "Paper: A benchmark of expert-level academic questions to assess AI capabilities", "url": "https://arxiv.org/abs/2501.14249", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hle/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Center for AI Safety", "url": "https://huggingface.co/datasets/cais/hle", "primary": false }, { "kind": "dataset", "label": "Dataset: Skylenage Ai", "url": "https://huggingface.co/datasets/skylenage-ai/HLE-Verified", "primary": false }, { "kind": "dataset", "label": "Dataset: Center for AI Safety", "url": "https://huggingface.co/datasets/cais/hle-rolling", "primary": false } ], "resource_count": 6, "paper_url": "https://arxiv.org/abs/2501.14249", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hle/eval.yaml", "family_title": "Humanity's Last Exam", "group": "Knowledge", "version": "5-C", "contributors": [ "SasankYadati" ], "tags": [ "Knowledge" ], "task": { "name": "hle", "dataset_samples": 2500, "comment": null, "support_only": false }, "tasks": [ { "name": "hle", "dataset_samples": 2500, "comment": null, "support_only": false } ], "protocol": { "version": "5-C", "dataset_samples": 2500, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "cais/hle", "fetch_method": "hf_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "skylenage-ai/HLE-Verified", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "cais/hle-rolling", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:hle", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Long Phan, Alice Gatti, Ziwen Han et al.", "organization": "University of Pennsylvania; Center for AI Safety; University of Waterloo; Amazon AGI; CMU LTI; ASAPP; ASAPP Inc; Carnegie Mellon University; National University; De La Salle University; Sapienza, University of Rome; Yonsei University; University of Michigan; Washington University in St. Louis; Mila, McGill; Mozilla.ai; University of California, Berkeley; EleutherAI; UC Berkeley; Georgia Institute of Technology; Brown University; Minerva University; Johns Hopkins University; Stanford University; Nanyang Technological University; ETH Zurich; CMU; ASU, BITS-Pilani; AIM Intelligence; Apple; University of Oxford; Beni Suef University; Language Techonology Institute; University of Washington; SEACrowd; University of Copenhagen; University of Minnesota; McGill University; Mila; KAIST; Universidad Nacional de Educación a Distancia (UNED); Northeastern University; Mathematics Department; Faculty of Science; Kafrelsheikh University; Methods of Plasticity Research, Department of Psychology, University of Zurich, Zurich, Switzerland; Neuroscience Center Zurich (ZNZ), University of Zurich & ETH Zurich, Zurich, Switzerland; Stony Brook University", "organizations": [ "University of Pennsylvania", "Center for AI Safety", "University of Waterloo", "Amazon AGI", "CMU LTI", "ASAPP", "ASAPP Inc", "Carnegie Mellon University", "National University", "De La Salle University", "Sapienza, University of Rome", "Yonsei University", "University of Michigan", "Washington University in St. Louis", "Mila, McGill", "Mozilla.ai", "University of California, Berkeley", "EleutherAI", "UC Berkeley", "Georgia Institute of Technology", "Brown University", "Minerva University", "Johns Hopkins University", "Stanford University", "Nanyang Technological University", "ETH Zurich", "CMU", "ASU, BITS-Pilani", "AIM Intelligence", "Apple", "University of Oxford", "Beni Suef University", "Language Techonology Institute", "University of Washington", "SEACrowd", "University of Copenhagen", "University of Minnesota", "McGill University", "Mila", "KAIST", "Universidad Nacional de Educación a Distancia (UNED)", "Northeastern University", "Mathematics Department", "Faculty of Science", "Kafrelsheikh University", "Methods of Plasticity Research, Department of Psychology, University of Zurich, Zurich, Switzerland", "Neuroscience Center Zurich (ZNZ), University of Zurich & ETH Zurich, Zurich, Switzerland", "Stony Brook University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2501.14249" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2501.14249", "arxiv_id": "2501.14249", "title": "A benchmark of expert-level academic questions to assess AI capabilities", "authors": [ "Long Phan", "Alice Gatti", "Ziwen Han", "Nathaniel Li", "Josephina Hu", "Hugh Zhang", "Chen Bo Calvin Zhang", "Mohamed Shaaban", "John Ling", "Sean Shi", "Michael Choi", "Anish Agrawal", "Arnav Chopra", "Adam Khoja", "Ryan Kim", "Richard Ren", "Jason Hausenloy", "Oliver Zhang", "Mantas Mazeika", "Dmitry Dodonov", "Tung Nguyen", "Jaeho Lee", "Daron Anderson", "Mikhail Doroshenko", "Alun Cennyth Stokes", "Mobeen Mahmood", "Oleksandr Pokutnyi", "Oleg Iskra", "Jessica P. Wang", "John-Clark Levin", "Mstyslav Kazakov", "Fiona Feng", "Steven Y. Feng", "Haoran Zhao", "Michael Yu", "Varun Gangal", "Chelsea Zou", "Zihan Wang", "Serguei Popov", "Robert Gerbicz", "Geoff Galgon", "Johannes Schmitt", "Will Yeadon", "Yongki Lee", "Scott Sauers", "Alvaro Sanchez", "Fabian Giska", "Marc Roth", "Søren Riis", "Saiteja Utpala", "Noah Burns", "Gashaw M. Goshu", "Mohinder Maheshbhai Naiya", "Chidozie Agu", "Zachary Giboney", "Antrell Cheatom", "Francesco Fournier-Facio", "Sarah-Jane Crowson", "Lennart Finke", "Zerui Cheng", "Jennifer Zampese", "Ryan G. Hoerr", "Mark Nandor", "Hyunwoo Park", "Tim Gehrunger", "Jiaqi Cai", "Ben McCarty", "Alexis C Garretson", "Edwin Taylor", "Damien Sileo", "Qiuyu Ren", "Usman Qazi", "Lianghui Li", "Jungbae Nam", "John B. Wydallis", "Pavel Arkhipov", "Jack Wei Lun Shi", "Aras Bacho", "Chris G. Willcocks", "Hangrui Cao", "Sumeet Motwani", "Emily de Oliveira Santos", "Johannes Veith", "Edward Vendrow", "Doru Cojoc", "Kengo Zenitani", "Joshua Robinson", "Longke Tang", "Yuqi Li", "Joshua Vendrow", "Natanael Wildner Fraga", "Vladyslav Kuchkin", "Andrey Pupasov Maksimov", "Pierre Marion", "Denis Efremov", "Jayson Lynch", "Kaiqu Liang", "Aleksandar Mikov", "Andrew Gritsevskiy", "Julien Guillod", "Gözdenur Demir", "Dakotah Martinez", "Ben Pageler", "Kevin Zhou", "Saeed Soori", "Ori Press", "Henry Tang", "Paolo Rissone", "Sean R. Green", "Lina Brüssel", "Moon Twayana", "Aymeric Dieuleveut", "Joseph Marvin Imperial", "Ameya Prabhu", "Jinzhou Yang", "Nick Crispino", "Arun Rao", "Dimitri Zvonkine", "Gabriel Loiseau", "Mikhail Kalinin", "Marco Lukas", "Ciprian Manolescu", "Nate Stambaugh", "Subrata Mishra", "Tad Hogg", "Carlo Bosio", "Brian P Coppola", "Julian Salazar", "Jaehyeok Jin", "Rafael Sayous", "Stefan Ivanov", "Philippe Schwaller", "Shaipranesh Senthilkuma", "Andres M Bran", "Andres Algaba", "Kelsey Van den Houte", "Lynn Van Der Sypt", "Brecht Verbeken", "David Noever", "Alexei Kopylov", "Benjamin Myklebust", "Bikun Li", "Lisa Schut", "Evgenii Zheltonozhskii", "Qiaochu Yuan", "Derek Lim", "Richard Stanley", "Tong Yang", "John Maar", "Julian Wykowski", "Martí Oller", "Anmol Sahu", "Cesare Giulio Ardito", "Yuzheng Hu", "Ariel Ghislain Kemogne Kamdoum", "Alvin Jin", "Tobias Garcia Vilchis", "Yuexuan Zu", "Martin Lackner", "James Koppel", "Gongbo Sun", "Daniil S. Antonenko", "Steffi Chern", "Bingchen Zhao", "Pierrot Arsene", "Joseph M Cavanagh", "Daofeng Li", "Jiawei Shen", "Donato Crisostomi", "Wenjin Zhang", "Ali Dehghan", "Sergey Ivanov", "David Perrella", "Nurdin Kaparov", "Allen Zang", "Ilia Sucholutsky", "Arina Kharlamova", "Daniil Orel", "Vladislav Poritski", "Shalev Ben-David", "Zachary Berger", "Parker Whitfill", "Michael Foster", "Daniel Munro", "Linh Ho", "Shankar Sivarajan", "Dan Bar Hava", "Aleksey Kuchkin", "David Holmes", "Alexandra Rodriguez-Romero", "Frank Sommerhage", "Anji Zhang", "Richard Moat", "Keith Schneider", "Zakayo Kazibwe", "Don Clarke", "Dae Hyun Kim", "Felipe Meneguitti Dias", "Sara Fish", "Veit Elser", "Tobias Kreiman", "Victor Efren Guadarrama Vilchis", "Immo Klose", "Ujjwala Anantheswaran", "Adam Zweiger", "Kaivalya Rawal", "Jeffery Li", "Jeremy Nguyen", "Nicolas Daans", "Haline Heidinger", "Maksim Radionov", "Václav Rozhoň", "Vincent Ginis", "Christian Stump", "Niv Cohen", "Rafał Poświata", "Josef Tkadlec", "Alan Goldfarb", "Chenguang Wang", "Piotr Padlewski", "Stanislaw Barzowski", "Kyle Montgomery", "Ryan Stendall", "Jamie Tucker-Foltz", "Jack Stade", "T. Ryan Rogers", "Tom Goertzen", "Declan Grabb", "Abhishek Shukla", "Alan Givré", "John Arnold Ambay", "Archan Sen", "Muhammad Fayez Aziz", "Mark H Inlow", "Hao He", "Ling Zhang", "Younesse Kaddar", "Ivar Ängquist", "Yanxu Chen", "Harrison K Wang", "Kalyan Ramakrishnan", "Elliott Thornley", "Antonio Terpin", "Hailey Schoelkopf", "Eric Zheng", "Avishy Carmi", "Ethan D. L. Brown", "Kelin Zhu", "Max Bartolo", "Richard Wheeler", "Martin Stehberger", "Peter Bradshaw", "JP Heimonen", "Kaustubh Sridhar", "Ido Akov", "Jennifer Sandlin", "Yury Makarychev", "Joanna Tam", "Hieu Hoang", "David M. Cunningham", "Vladimir Goryachev", "Demosthenes Patramanis", "Michael Krause", "Andrew Redenti", "David Aldous", "Jesyin Lai", "Shannon Coleman", "Jiangnan Xu", "Sangwon Lee", "Ilias Magoulas", "Sandy Zhao", "Ning Tang", "Michael K. Cohen", "Orr Paradise", "Jan Hendrik Kirchner", "Maksym Ovchynnikov", "Jason O. Matos", "Adithya Shenoy", "Michael Wang", "Yuzhou Nie", "Anna Sztyber-Betley", "Paolo Faraboschi", "Robin Riblet", "Jonathan Crozier", "Shiv Halasyamani", "Shreyas Verma", "Prashant Joshi", "Eli Meril", "Ziqiao Ma", "Jérémy Andréoletti", "Raghav Singhal", "Jacob Platnick", "Volodymyr Nevirkovets", "Luke Basler", "Alexander Ivanov", "Seri Khoury", "Nils Gustafsson", "Marco Piccardo", "Hamid Mostaghimi", "Qijia Chen", "Virendra Singh", "Tran Quoc Khánh", "Paul Rosu", "Hannah Szlyk", "Zachary Brown", "Himanshu Narayan", "Aline Menezes", "Jonathan Roberts", "William Alley", "Kunyang Sun", "Arkil Patel", "Max Lamparth", "Anka Reuel", "Linwei Xin", "Hanmeng Xu", "Jacob Loader", "Freddie Martin", "Zixuan Wang", "Andrea Achilleos", "Thomas Preu", "Tomek Korbak", "Ida Bosio", "Fereshteh Kazemi", "Ziye Chen", "Biró Bálint", "Eve J. Y. Lo", "Jiaqi Wang", "Maria Inês S. Nunes", "Jeremiah Milbauer", "M Saiful Bari", "Zihao Wang", "Behzad Ansarinejad", "Yewen Sun", "Stephane Durand", "Hossam Elgnainy", "Guillaume Douville", "Daniel Tordera", "George Balabanian", "Hew Wolff", "Lynna Kvistad", "Hsiaoyun Milliron", "Ahmad Sakor", "Murat Eron", "Andrew Favre D. O.", "Shailesh Shah", "Xiaoxiang Zhou", "Firuz Kamalov", "Sherwin Abdoli", "Tim Santens", "Shaul Barkan", "Allison Tee", "Robin Zhang", "Alessandro Tomasiello", "G. Bruno De Luca", "Shi-Zhuo Looi", "Vinh-Kha Le", "Noam Kolt", "Jiayi Pan", "Emma Rodman", "Jacob Drori", "Carl J Fossum", "Niklas Muennighoff", "Milind Jagota", "Ronak Pradeep", "Honglu Fan", "Jonathan Eicher", "Michael Chen", "Kushal Thaman", "William Merrill", "Moritz Firsching", "Carter Harris", "Stefan Ciobâcă", "Jason Gross", "Rohan Pandey", "Ilya Gusev", "Adam Jones", "Shashank Agnihotri", "Pavel Zhelnov", "Mohammadreza Mofayezi", "Alexander Piperski", "David K. Zhang", "Kostiantyn Dobarskyi", "Roman Leventov", "Ignat Soroko", "Joshua Duersch", "Vage Taamazyan", "Andrew Ho", "Wenjie Ma", "William Held", "Ruicheng Xian", "Armel Randy Zebaze", "Mohanad Mohamed", "Julian Noah Leser", "Michelle X Yuan", "Laila Yacar", "Johannes Lengler", "Katarzyna Olszewska", "Claudio Di Fratta", "Edson Oliveira", "Joseph W. Jackson", "Andy Zou", "Muthu Chidambaram", "Timothy Manik", "Hector Haffenden", "Dashiell Stander", "Ali Dasouqi", "Alexander Shen", "Bita Golshani", "David Stap", "Egor Kretov", "Mikalai Uzhou", "Alina Borisovna Zhidkovskaya", "Nick Winter", "Miguel Orbegozo Rodriguez", "Robert Lauff", "Dustin Wehr", "Colin Tang", "Zaki Hossain", "Shaun Phillips", "Fortuna Samuele", "Fredrik Ekström", "Angela Hammon", "Oam Patel", "Faraz Farhidi", "George Medley", "Forough Mohammadzadeh", "Madellene Peñaflor", "Haile Kassahun", "Alena Friedrich", "Rayner Hernandez Perez", "Daniel Pyda", "Taom Sakal", "Omkar Dhamane", "Ali Khajegili Mirabadi", "Eric Hallman", "Kenchi Okutsu", "Mike Battaglia", "Mohammad Maghsoudimehrabani", "Alon Amit", "Dave Hulbert", "Roberto Pereira", "Simon Weber", "Handoko", "Anton Peristyy", "Stephen Malina", "Mustafa Mehkary", "Rami Aly", "Frank Reidegeld", "Anna-Katharina Dick", "Cary Friday", "Mukhwinder Singh", "Hassan Shapourian", "Wanyoung Kim", "Mariana Costa", "Hubeyb Gurdogan", "Harsh Kumar", "Chiara Ceconello", "Chao Zhuang", "Haon Park", "Micah Carroll", "Andrew R. Tawfeek", "Stefan Steinerberger", "Daattavya Aggarwal", "Michael Kirchhof", "Linjie Dai", "Evan Kim", "Johan Ferret", "Jainam Shah", "Yuzhou Wang", "Minghao Yan", "Krzysztof Burdzy", "Lixin Zhang", "Antonio Franca", "Diana T. Pham", "Kang Yong Loh", "Abram Jackson", "Paolo Giordano", "Philipp Petersen", "Adrian Cosma", "Jesus Colino", "Colin White", "Jacob Votava", "Vladimir Vinnikov", "Ethan Delaney", "Petr Spelda", "Vit Stritecky", "Syed M. Shahid", "Jean-Christophe Mourrat", "Lavr Vetoshkin", "Koen Sponselee", "Renas Bacho", "Zheng-Xin Yong", "Florencia de la Rosa", "Nathan Cho", "Xiuyu Li", "Guillaume Malod", "Orion Weller", "Guglielmo Albani", "Leon Lang", "Julien Laurendeau", "Dmitry Kazakov", "Fatimah Adesanya", "Julien Portier", "Lawrence Hollom", "Victor Souza", "Yuchen Anna Zhou", "Julien Degorre", "Yiğit Yalın", "Gbenga Daniel Obikoya", "Rai", "Filippo Bigi", "M. C. Boscá", "Oleg Shumar", "Kaniuar Bacho", "Gabriel Recchia", "Mara Popescu", "Nikita Shulga", "Ngefor Mildred Tanwie", "Thomas C. H. Lux", "Ben Rank", "Colin Ni", "Matthew Brooks", "Alesia Yakimchyk", "Huanxu", "Liu", "Stefano Cavalleri", "Olle Häggström", "Emil Verkama", "Joshua Newbould", "Hans Gundlach", "Leonor Brito-Santana", "Brian Amaro", "Vivek Vajipey", "Rynaa Grover", "Ting Wang", "Yosi Kratish", "Wen-Ding Li", "Sivakanth Gopi", "Andrea Caciolai", "Christian Schroeder de Witt", "Pablo Hernández-Cámara", "Emanuele Rodolà", "Jules Robins", "Dominic Williamson", "Vincent Cheng", "Brad Raynor", "Hao Qi", "Ben Segev", "Jingxuan Fan", "Sarah Martinson", "Erik Y. Wang", "Kaylie Hausknecht", "Michael P. Brenner", "Mao Mao", "Christoph Demian", "Peyman Kassani", "Xinyu Zhang", "David Avagian", "Eshawn Jessica Scipio", "Alon Ragoler", "Justin Tan", "Blake Sims", "Rebeka Plecnik", "Aaron Kirtland", "Omer Faruk Bodur", "D. P. Shinde", "Yan Carlos Leyva Labrador", "Zahra Adoul", "Mohamed Zekry", "Ali Karakoc", "Tania C. B. Santos", "Samir Shamseldeen", "Loukmane Karim", "Anna Liakhovitskaia", "Nate Resman", "Nicholas Farina", "Juan Carlos Gonzalez", "Gabe Maayan", "Earth Anderson", "Rodrigo De Oliveira Pena", "Elizabeth Kelley", "Hodjat Mariji", "Rasoul Pouriamanesh", "Wentao Wu", "Ross Finocchio", "Ismail Alarab", "Joshua Cole", "Danyelle Ferreira", "Bryan Johnson", "Mohammad Safdari", "Liangti Dai", "Siriphan Arthornthurasuk", "Isaac C. McAlister", "Alejandro José Moyano", "Alexey Pronin", "Jing Fan", "Angel Ramirez-Trinidad", "Yana Malysheva", "Daphiny Pottmaier", "Omid Taheri", "Stanley Stepanic", "Samuel Perry", "Luke Askew", "Raúl Adrián Huerta Rodríguez", "Ali M. R. Minissi", "Ricardo Lorena", "Krishnamurthy Iyer", "Arshad Anil Fasiludeen", "Ronald Clark", "Josh Ducey", "Matheus Piza", "Maja Somrak", "Eric Vergo", "Juehang Qin", "Benjámin Borbás", "Eric Chu", "Jack Lindsey", "Antoine Jallon", "I. M. J. McInnis", "Evan Chen", "Avi Semler", "Luk Gloor", "Tej Shah", "Marc Carauleanu", "Pascal Lauer", "Tran Đuc Huy", "Hossein Shahrtash", "Emilien Duc", "Lukas Lewark", "Assaf Brown", "Samuel Albanie", "Brian Weber", "Warren S. Vaz", "Pierre Clavier", "Yiyang Fan", "Gabriel Poesia Reis e Silva", "Long", "Lian", "Marcus Abramovitch", "Xi Jiang", "Sandra Mendoza", "Murat Islam", "Juan Gonzalez", "Vasilios Mavroudis", "Justin Xu", "Pawan Kumar", "Laxman Prasad Goswami", "Daniel Bugas", "Nasser Heydari", "Ferenc Jeanplong", "Thorben Jansen", "Antonella Pinto", "Archimedes Apronti", "Abdallah Galal", "Ng Ze-An", "Ankit Singh", "Tong Jiang", "Joan of Arc Xavier", "Kanu Priya Agarwal", "Mohammed Berkani", "Gang Zhang", "Zhehang Du", "Benedito Alves de Oliveira Junior", "Dmitry Malishev", "Nicolas Remy", "Taylor D. Hartman", "Tim Tarver", "Stephen Mensah", "Gautier Abou Loume", "Wiktor Morak", "Farzad Habibi", "Sarah Hoback", "Will Cai", "Javier Gimenez", "Roselynn Grace Montecillo", "Jakub Łucki", "Russell Campbell", "Asankhaya Sharma", "Khalida Meer", "Shreen Gul", "Daniel Espinosa Gonzalez", "Xavier Alapont", "Alex Hoover", "Gunjan Chhablani", "Freddie Vargus", "Arunim Agarwal", "Yibo Jiang", "Deepakkumar Patil", "David Outevsky", "Kevin Joseph Scaria", "Rajat Maheshwari", "Abdelkader Dendane", "Priti Shukla", "Ashley Cartwright", "Sergei Bogdanov", "Niels Mündler", "Sören Möller", "Luca Arnaboldi", "Kunvar Thaman", "Muhammad Rehan Siddiqi", "Prajvi Saxena", "Himanshu Gupta", "Tony Fruhauff", "Glen Sherman", "Mátyás Vincze", "Siranut Usawasutsakorn", "Dylan Ler", "Anil Radhakrishnan", "Innocent Enyekwe", "Sk Md Salauddin", "Jiang Muzhen", "Aleksandr Maksapetyan", "Vivien Rossbach", "Chris Harjadi", "Mohsen Bahaloohoreh", "Claire Sparrow", "Jasdeep Sidhu", "Sam Ali", "Song Bian", "John Lai", "Eric Singer", "Justine Leon Uro", "Greg Bateman", "Mohamed Sayed", "Ahmed Menshawy", "Darling Duclosel", "Dario Bezzi", "Yashaswini Jain", "Ashley Aaron", "Murat Tiryakioglu", "Sheeshram Siddh", "Keith Krenek", "Imad Ali Shah", "Jun Jin", "Scott Creighton", "Denis Peskoff", "Zienab EL-Wasif", "Ragavendran P", "Michael Richmond", "Joseph McGowan", "Tejal Patwardhan", "Hao-Yu Sun", "Ting Sun", "Nikola Zubić", "Samuele Sala", "Stephen Ebert", "Jean Kaddour", "Manuel Schottdorf", "Dianzhuo Wang", "Gerol Petruzella", "Alex Meiburg", "Tilen Medved", "Ali ElSheikh", "S Ashwin Hebbar", "Lorenzo Vaquero", "Xianjun Yang", "Jason Poulos", "Vilém Zouhar", "Sergey Bogdanik", "Mingfang Zhang", "Jorge Sanz-Ros", "David Anugraha", "Yinwei Dai", "Anh N. Nhu", "Xue Wang", "Ali Anil Demircali", "Zhibai Jia", "Yuyin Zhou", "Juncheng Wu", "Mike He", "Nitin Chandok", "Aarush Sinha", "Gaoxiang Luo", "Long Le", "Mickaël Noyé", "Michał Perełkiewicz", "Ioannis Pantidis", "Tianbo Qi", "Soham Sachin Purohit", "Letitia Parcalabescu", "Thai-Hoa Nguyen", "Genta Indra Winata", "Edoardo M. Ponti", "Hanchen Li", "Kaustubh Dhole", "Jongee Park", "Dario Abbondanza", "Yuanli Wang", "Anupam Nayak", "Diogo M. Caetano", "Antonio A. W. L. Wong", "Maria del Rio-Chanona", "Dániel Kondor", "Pieter Francois", "Ed Chalstrey", "Jakob Zsambok", "Dan Hoyer", "Jenny Reddish", "Jakob Hauser", "Francisco-Javier Rodrigo-Ginés", "Suchandra Datta", "Maxwell Shepherd", "Thom Kamphuis", "Qizheng Zhang", "Hyunjun Kim", "Ruiji Sun", "Jianzhu Yao", "Franck Dernoncourt", "Satyapriya Krishna", "Sina Rismanchian", "Bonan Pu", "Francesco Pinto", "Yingheng Wang", "Kumar Shridhar", "Kalon J. Overholt", "Glib Briia", "Hieu Nguyen", "David", "Soler Bartomeu", "Tony CY Pang", "Adam Wecker", "Yifan Xiong", "Fanfei Li", "Lukas S. Huber", "Joshua Jaeger", "Romano De Maddalena", "Xing Han Lù", "Yuhui Zhang", "Claas Beger", "Patrick Tser Jern Kon", "Sean Li", "Vivek Sanker", "Ming Yin", "Yihao Liang", "Xinlu Zhang", "Ankit Agrawal", "Li S. Yifei", "Zechen Zhang", "Mu Cai", "Yasin Sonmez", "Costin Cozianu", "Changhao Li", "Alex Slen", "Shoubin Yu", "Hyun Kyu Park", "Gabriele Sarti", "Marcin Briański", "Alessandro Stolfo", "Truong An Nguyen", "Mike Zhang", "Yotam Perlitz", "Jose Hernandez-Orallo", "Runjia Li", "Amin Shabani", "Felix Juefei-Xu", "Shikhar Dhingra", "Orr Zohar", "My Chiffon Nguyen", "Alexander Pondaven", "Abdurrahim Yilmaz", "Xuandong Zhao", "Chuanyang Jin", "Muyan Jiang", "Stefan Todoran", "Xinyao Han", "Jules Kreuer", "Brian Rabern", "Anna Plassart", "Martino Maggetti", "Luther Yap", "Robert Geirhos", "Jonathon Kean", "Dingsu Wang", "Sina Mollaei", "Chenkai Sun", "Yifan Yin", "Shiqi Wang", "Rui Li", "Yaowen Chang", "Anjiang Wei", "Alice Bizeul", "Xiaohan Wang", "Alexandre Oliveira Arrais", "Kushin Mukherjee", "Jorge Chamorro-Padial", "Jiachen Liu", "Xingyu Qu", "Junyi Guan", "Adam Bouyamourn", "Shuyu Wu", "Martyna Plomecka", "Junda Chen", "Mengze Tang", "Jiaqi Deng", "Shreyas Subramanian", "Haocheng Xi", "Haoxuan Chen", "Weizhi Zhang", "Yinuo Ren", "Haoqin Tu", "Sejong Kim", "Yushun Chen", "Sara Vera Marjanović", "Junwoo Ha", "Grzegorz Luczyna", "Jeff J. Ma", "Zewen Shen", "Dawn Song", "Cedegao E. Zhang", "Zhun Wang", "Gaël Gendron", "Yunze Xiao", "Leo Smucker", "Erica Weng", "Kwok Hao Lee", "Zhe Ye", "Stefano Ermon", "Ignacio D. Lopez-Miguel", "Theo Knights", "Anthony Gitter", "Namkyu Park", "Boyi Wei", "Hongzheng Chen", "Kunal Pai", "Ahmed Elkhanany", "Han Lin", "Philipp D. Siedler", "Jichao Fang", "Ritwik Mishra", "Károly Zsolnai-Fehér", "Xilin Jiang", "Shadab Khan", "Jun Yuan", "Rishab Kumar Jain", "Xi Lin", "Mike Peterson", "Zhe Wang", "Aditya Malusare", "Maosen Tang", "Isha Gupta", "Ivan Fosin", "Timothy Kang", "Barbara Dworakowska", "Kazuki Matsumoto", "Guangyao Zheng", "Gerben Sewuster", "Jorge Pretel Villanueva", "Ivan Rannev", "Igor Chernyavsky", "Jiale Chen", "Deepayan Banik", "Ben Racz", "Wenchao Dong", "Jianxin Wang", "Laila Bashmal", "Duarte V. Gonçalves", "Wei Hu", "Kaushik Bar", "Ondrej Bohdal", "Atharv Singh Patlan", "Shehzaad Dhuliawala", "Caroline Geirhos", "Julien Wist", "Yuval Kansal", "Bingsen Chen", "Kutay Tire", "Atak Talay Yücel", "Brandon Christof", "Veerupaksh Singla", "Zijian Song", "Sanxing Chen", "Jiaxin Ge", "Kaustubh Ponkshe", "Isaac Park", "Tianneng Shi", "Martin Q. Ma", "Joshua Mak", "Sherwin Lai", "Antoine Moulin", "Zhuo Cheng", "Zhanda Zhu", "Ziyi Zhang", "Vaidehi Patil", "Ketan Jha", "Qiutong Men", "Jiaxuan Wu", "Tianchi Zhang", "Bruno Hebling Vieira", "Alham Fikri Aji", "Jae-Won Chung", "Mohammed Mahfoud", "Ha Thi Hoang", "Marc Sperzel", "Wei Hao", "Kristof Meding", "Sihan Xu", "Vassilis Kostakos", "Davide Manini", "Yueying Liu", "Christopher Toukmaji", "Jay Paek", "Eunmi Yu", "Arif Engin Demircali", "Zhiyi Sun", "Ivan Dewerpe", "Hongsen Qin", "Roman Pflugfelder", "James Bailey", "Johnathan Morris", "Ville Heilala", "Sybille Rosset", "Zishun Yu", "Peter E. Chen", "Woongyeong Yeo", "Eeshaan Jain", "Ryan Yang", "Sreekar Chigurupati", "Julia Chernyavsky", "Sai Prajwal Reddy", "Subhashini Venugopalan", "Hunar Batra", "Core Francisco Park", "Hieu Tran", "Guilherme Maximiano", "Genghan Zhang", "Yizhuo Liang", "Hu Shiyu", "Rongwu Xu", "Rui Pan", "Siddharth Suresh", "Ziqi Liu", "Samaksh Gulati", "Songyang Zhang", "Peter Turchin", "Christopher W. Bartlett", "Christopher R. Scotese", "Phuong M. Cao", "Ben Wu", "Jacek Karwowski", "Davide Scaramuzza", "Aakaash Nattanmai", "Gordon McKellips", "Anish Cheraku", "Asim Suhail", "Ethan Luo", "Marvin Deng", "Jason Luo", "Ashley Zhang", "Kavin Jindel", "Kasper Halevy", "Allen Baranov", "Michael Liu", "Advaith Avadhanam", "David Zhang", "Brad Ma", "Evan Fu", "Liam Do", "Joshua Lass", "Hubert Yang", "Surya Sunkari", "Vishruth Bharath", "Violet Ai", "James Leung", "Rishit Agrawal", "Alan Zhou", "Kevin Chen", "Tejas Kalpathi", "Ziqi Xu", "Gavin Wang", "Tyler Xiao", "Erik Maung", "Sam Lee", "Roy Yue", "Ben Zhao", "Julia Yoon", "Sunny Sun", "Aryan Singh", "Clark Peng", "Tyler Osbey", "Taozhi Wang", "Daryl Echeazu", "Timothy Wu", "Spandan Patel", "Vidhi Kulkarni", "Vijaykaarti Sundarapandiyan", "Andrew Le", "Zafir Nasim", "Srikar Yalam", "Ritesh Kasamsetty", "Soham Samal", "David Sun", "Nihar Shah", "Abhijeet Saha", "Alex Zhang", "Leon Nguyen", "Laasya Nagumalli", "Kaixin Wang", "Aidan Wu", "Anwith Telluri", "Steven Dillmann", "Zhengxiang Wang", "Junyu Luo", "Hugo Lunn", "Artem Gazizov", "Haitz Sáez de Ocáriz Borde", "Ivan Trus", "Morgan Hervault", "Zheyu Zhang", "Bo Chen", "Yuchen Wu", "Christopher J. Cordier", "Gün Kaynar", "Cansin Ayvaz", "Polina Avdiunina", "Johannes Brust", "Xingjian Diao", "K. D. Meaney", "Yifan Gu", "Chenyu Wang", "Chenzhuo Dong", "William Wright", "Simon Brave", "Owen Root", "Jiayuan Liu", "Chow Chun Lok", "Tianqin Li", "Shiyi Du", "Dailan He", "Lufeiya Liu", "Sina Jamalzadegan", "Anil Ramakrishna", "Xuanqing Xu", "Xin Qing", "Xin Luo", "Wenkai Li", "Shi Bo", "Filipp Gusev", "Maximos Skandalis", "Desheng Ma", "Chunhui Zhang", "Haoran Qiu", "Allen G Hart", "Rickard Brüel Gabrielsson", "Artem Lukoianov", "Summer Yue", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "University of Pennsylvania", "Center for AI Safety", "University of Waterloo", "Amazon AGI", "CMU LTI", "ASAPP", "ASAPP Inc", "Carnegie Mellon University", "National University", "De La Salle University", "Sapienza, University of Rome", "Yonsei University", "University of Michigan", "Washington University in St. Louis", "Mila, McGill", "Mozilla.ai", "University of California, Berkeley", "EleutherAI", "UC Berkeley", "Georgia Institute of Technology", "Brown University", "Minerva University", "Johns Hopkins University", "Stanford University", "Nanyang Technological University", "ETH Zurich", "CMU", "ASU, BITS-Pilani", "AIM Intelligence", "Apple", "University of Oxford", "Beni Suef University", "Language Techonology Institute", "University of Washington", "SEACrowd", "University of Copenhagen", "University of Minnesota", "McGill University", "Mila", "KAIST", "Universidad Nacional de Educación a Distancia (UNED)", "Northeastern University", "Mathematics Department", "Faculty of Science", "Kafrelsheikh University", "Methods of Plasticity Research, Department of Psychology, University of Zurich, Zurich, Switzerland", "Neuroscience Center Zurich (ZNZ), University of Zurich & ETH Zurich, Zurich, Switzerland", "Stony Brook University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2501.14249", "arxiv_id": "2501.14249", "title": "A benchmark of expert-level academic questions to assess AI capabilities", "authors": [ "Long Phan", "Alice Gatti", "Ziwen Han", "Nathaniel Li", "Josephina Hu", "Hugh Zhang", "Chen Bo Calvin Zhang", "Mohamed Shaaban", "John Ling", "Sean Shi", "Michael Choi", "Anish Agrawal", "Arnav Chopra", "Adam Khoja", "Ryan Kim", "Richard Ren", "Jason Hausenloy", "Oliver Zhang", "Mantas Mazeika", "Dmitry Dodonov", "Tung Nguyen", "Jaeho Lee", "Daron Anderson", "Mikhail Doroshenko", "Alun Cennyth Stokes", "Mobeen Mahmood", "Oleksandr Pokutnyi", "Oleg Iskra", "Jessica P. Wang", "John-Clark Levin", "Mstyslav Kazakov", "Fiona Feng", "Steven Y. Feng", "Haoran Zhao", "Michael Yu", "Varun Gangal", "Chelsea Zou", "Zihan Wang", "Serguei Popov", "Robert Gerbicz", "Geoff Galgon", "Johannes Schmitt", "Will Yeadon", "Yongki Lee", "Scott Sauers", "Alvaro Sanchez", "Fabian Giska", "Marc Roth", "Søren Riis", "Saiteja Utpala", "Noah Burns", "Gashaw M. Goshu", "Mohinder Maheshbhai Naiya", "Chidozie Agu", "Zachary Giboney", "Antrell Cheatom", "Francesco Fournier-Facio", "Sarah-Jane Crowson", "Lennart Finke", "Zerui Cheng", "Jennifer Zampese", "Ryan G. Hoerr", "Mark Nandor", "Hyunwoo Park", "Tim Gehrunger", "Jiaqi Cai", "Ben McCarty", "Alexis C Garretson", "Edwin Taylor", "Damien Sileo", "Qiuyu Ren", "Usman Qazi", "Lianghui Li", "Jungbae Nam", "John B. Wydallis", "Pavel Arkhipov", "Jack Wei Lun Shi", "Aras Bacho", "Chris G. Willcocks", "Hangrui Cao", "Sumeet Motwani", "Emily de Oliveira Santos", "Johannes Veith", "Edward Vendrow", "Doru Cojoc", "Kengo Zenitani", "Joshua Robinson", "Longke Tang", "Yuqi Li", "Joshua Vendrow", "Natanael Wildner Fraga", "Vladyslav Kuchkin", "Andrey Pupasov Maksimov", "Pierre Marion", "Denis Efremov", "Jayson Lynch", "Kaiqu Liang", "Aleksandar Mikov", "Andrew Gritsevskiy", "Julien Guillod", "Gözdenur Demir", "Dakotah Martinez", "Ben Pageler", "Kevin Zhou", "Saeed Soori", "Ori Press", "Henry Tang", "Paolo Rissone", "Sean R. Green", "Lina Brüssel", "Moon Twayana", "Aymeric Dieuleveut", "Joseph Marvin Imperial", "Ameya Prabhu", "Jinzhou Yang", "Nick Crispino", "Arun Rao", "Dimitri Zvonkine", "Gabriel Loiseau", "Mikhail Kalinin", "Marco Lukas", "Ciprian Manolescu", "Nate Stambaugh", "Subrata Mishra", "Tad Hogg", "Carlo Bosio", "Brian P Coppola", "Julian Salazar", "Jaehyeok Jin", "Rafael Sayous", "Stefan Ivanov", "Philippe Schwaller", "Shaipranesh Senthilkuma", "Andres M Bran", "Andres Algaba", "Kelsey Van den Houte", "Lynn Van Der Sypt", "Brecht Verbeken", "David Noever", "Alexei Kopylov", "Benjamin Myklebust", "Bikun Li", "Lisa Schut", "Evgenii Zheltonozhskii", "Qiaochu Yuan", "Derek Lim", "Richard Stanley", "Tong Yang", "John Maar", "Julian Wykowski", "Martí Oller", "Anmol Sahu", "Cesare Giulio Ardito", "Yuzheng Hu", "Ariel Ghislain Kemogne Kamdoum", "Alvin Jin", "Tobias Garcia Vilchis", "Yuexuan Zu", "Martin Lackner", "James Koppel", "Gongbo Sun", "Daniil S. Antonenko", "Steffi Chern", "Bingchen Zhao", "Pierrot Arsene", "Joseph M Cavanagh", "Daofeng Li", "Jiawei Shen", "Donato Crisostomi", "Wenjin Zhang", "Ali Dehghan", "Sergey Ivanov", "David Perrella", "Nurdin Kaparov", "Allen Zang", "Ilia Sucholutsky", "Arina Kharlamova", "Daniil Orel", "Vladislav Poritski", "Shalev Ben-David", "Zachary Berger", "Parker Whitfill", "Michael Foster", "Daniel Munro", "Linh Ho", "Shankar Sivarajan", "Dan Bar Hava", "Aleksey Kuchkin", "David Holmes", "Alexandra Rodriguez-Romero", "Frank Sommerhage", "Anji Zhang", "Richard Moat", "Keith Schneider", "Zakayo Kazibwe", "Don Clarke", "Dae Hyun Kim", "Felipe Meneguitti Dias", "Sara Fish", "Veit Elser", "Tobias Kreiman", "Victor Efren Guadarrama Vilchis", "Immo Klose", "Ujjwala Anantheswaran", "Adam Zweiger", "Kaivalya Rawal", "Jeffery Li", "Jeremy Nguyen", "Nicolas Daans", "Haline Heidinger", "Maksim Radionov", "Václav Rozhoň", "Vincent Ginis", "Christian Stump", "Niv Cohen", "Rafał Poświata", "Josef Tkadlec", "Alan Goldfarb", "Chenguang Wang", "Piotr Padlewski", "Stanislaw Barzowski", "Kyle Montgomery", "Ryan Stendall", "Jamie Tucker-Foltz", "Jack Stade", "T. Ryan Rogers", "Tom Goertzen", "Declan Grabb", "Abhishek Shukla", "Alan Givré", "John Arnold Ambay", "Archan Sen", "Muhammad Fayez Aziz", "Mark H Inlow", "Hao He", "Ling Zhang", "Younesse Kaddar", "Ivar Ängquist", "Yanxu Chen", "Harrison K Wang", "Kalyan Ramakrishnan", "Elliott Thornley", "Antonio Terpin", "Hailey Schoelkopf", "Eric Zheng", "Avishy Carmi", "Ethan D. L. Brown", "Kelin Zhu", "Max Bartolo", "Richard Wheeler", "Martin Stehberger", "Peter Bradshaw", "JP Heimonen", "Kaustubh Sridhar", "Ido Akov", "Jennifer Sandlin", "Yury Makarychev", "Joanna Tam", "Hieu Hoang", "David M. Cunningham", "Vladimir Goryachev", "Demosthenes Patramanis", "Michael Krause", "Andrew Redenti", "David Aldous", "Jesyin Lai", "Shannon Coleman", "Jiangnan Xu", "Sangwon Lee", "Ilias Magoulas", "Sandy Zhao", "Ning Tang", "Michael K. Cohen", "Orr Paradise", "Jan Hendrik Kirchner", "Maksym Ovchynnikov", "Jason O. Matos", "Adithya Shenoy", "Michael Wang", "Yuzhou Nie", "Anna Sztyber-Betley", "Paolo Faraboschi", "Robin Riblet", "Jonathan Crozier", "Shiv Halasyamani", "Shreyas Verma", "Prashant Joshi", "Eli Meril", "Ziqiao Ma", "Jérémy Andréoletti", "Raghav Singhal", "Jacob Platnick", "Volodymyr Nevirkovets", "Luke Basler", "Alexander Ivanov", "Seri Khoury", "Nils Gustafsson", "Marco Piccardo", "Hamid Mostaghimi", "Qijia Chen", "Virendra Singh", "Tran Quoc Khánh", "Paul Rosu", "Hannah Szlyk", "Zachary Brown", "Himanshu Narayan", "Aline Menezes", "Jonathan Roberts", "William Alley", "Kunyang Sun", "Arkil Patel", "Max Lamparth", "Anka Reuel", "Linwei Xin", "Hanmeng Xu", "Jacob Loader", "Freddie Martin", "Zixuan Wang", "Andrea Achilleos", "Thomas Preu", "Tomek Korbak", "Ida Bosio", "Fereshteh Kazemi", "Ziye Chen", "Biró Bálint", "Eve J. Y. Lo", "Jiaqi Wang", "Maria Inês S. Nunes", "Jeremiah Milbauer", "M Saiful Bari", "Zihao Wang", "Behzad Ansarinejad", "Yewen Sun", "Stephane Durand", "Hossam Elgnainy", "Guillaume Douville", "Daniel Tordera", "George Balabanian", "Hew Wolff", "Lynna Kvistad", "Hsiaoyun Milliron", "Ahmad Sakor", "Murat Eron", "Andrew Favre D. O.", "Shailesh Shah", "Xiaoxiang Zhou", "Firuz Kamalov", "Sherwin Abdoli", "Tim Santens", "Shaul Barkan", "Allison Tee", "Robin Zhang", "Alessandro Tomasiello", "G. Bruno De Luca", "Shi-Zhuo Looi", "Vinh-Kha Le", "Noam Kolt", "Jiayi Pan", "Emma Rodman", "Jacob Drori", "Carl J Fossum", "Niklas Muennighoff", "Milind Jagota", "Ronak Pradeep", "Honglu Fan", "Jonathan Eicher", "Michael Chen", "Kushal Thaman", "William Merrill", "Moritz Firsching", "Carter Harris", "Stefan Ciobâcă", "Jason Gross", "Rohan Pandey", "Ilya Gusev", "Adam Jones", "Shashank Agnihotri", "Pavel Zhelnov", "Mohammadreza Mofayezi", "Alexander Piperski", "David K. Zhang", "Kostiantyn Dobarskyi", "Roman Leventov", "Ignat Soroko", "Joshua Duersch", "Vage Taamazyan", "Andrew Ho", "Wenjie Ma", "William Held", "Ruicheng Xian", "Armel Randy Zebaze", "Mohanad Mohamed", "Julian Noah Leser", "Michelle X Yuan", "Laila Yacar", "Johannes Lengler", "Katarzyna Olszewska", "Claudio Di Fratta", "Edson Oliveira", "Joseph W. Jackson", "Andy Zou", "Muthu Chidambaram", "Timothy Manik", "Hector Haffenden", "Dashiell Stander", "Ali Dasouqi", "Alexander Shen", "Bita Golshani", "David Stap", "Egor Kretov", "Mikalai Uzhou", "Alina Borisovna Zhidkovskaya", "Nick Winter", "Miguel Orbegozo Rodriguez", "Robert Lauff", "Dustin Wehr", "Colin Tang", "Zaki Hossain", "Shaun Phillips", "Fortuna Samuele", "Fredrik Ekström", "Angela Hammon", "Oam Patel", "Faraz Farhidi", "George Medley", "Forough Mohammadzadeh", "Madellene Peñaflor", "Haile Kassahun", "Alena Friedrich", "Rayner Hernandez Perez", "Daniel Pyda", "Taom Sakal", "Omkar Dhamane", "Ali Khajegili Mirabadi", "Eric Hallman", "Kenchi Okutsu", "Mike Battaglia", "Mohammad Maghsoudimehrabani", "Alon Amit", "Dave Hulbert", "Roberto Pereira", "Simon Weber", "Handoko", "Anton Peristyy", "Stephen Malina", "Mustafa Mehkary", "Rami Aly", "Frank Reidegeld", "Anna-Katharina Dick", "Cary Friday", "Mukhwinder Singh", "Hassan Shapourian", "Wanyoung Kim", "Mariana Costa", "Hubeyb Gurdogan", "Harsh Kumar", "Chiara Ceconello", "Chao Zhuang", "Haon Park", "Micah Carroll", "Andrew R. Tawfeek", "Stefan Steinerberger", "Daattavya Aggarwal", "Michael Kirchhof", "Linjie Dai", "Evan Kim", "Johan Ferret", "Jainam Shah", "Yuzhou Wang", "Minghao Yan", "Krzysztof Burdzy", "Lixin Zhang", "Antonio Franca", "Diana T. Pham", "Kang Yong Loh", "Abram Jackson", "Paolo Giordano", "Philipp Petersen", "Adrian Cosma", "Jesus Colino", "Colin White", "Jacob Votava", "Vladimir Vinnikov", "Ethan Delaney", "Petr Spelda", "Vit Stritecky", "Syed M. Shahid", "Jean-Christophe Mourrat", "Lavr Vetoshkin", "Koen Sponselee", "Renas Bacho", "Zheng-Xin Yong", "Florencia de la Rosa", "Nathan Cho", "Xiuyu Li", "Guillaume Malod", "Orion Weller", "Guglielmo Albani", "Leon Lang", "Julien Laurendeau", "Dmitry Kazakov", "Fatimah Adesanya", "Julien Portier", "Lawrence Hollom", "Victor Souza", "Yuchen Anna Zhou", "Julien Degorre", "Yiğit Yalın", "Gbenga Daniel Obikoya", "Rai", "Filippo Bigi", "M. C. Boscá", "Oleg Shumar", "Kaniuar Bacho", "Gabriel Recchia", "Mara Popescu", "Nikita Shulga", "Ngefor Mildred Tanwie", "Thomas C. H. Lux", "Ben Rank", "Colin Ni", "Matthew Brooks", "Alesia Yakimchyk", "Huanxu", "Liu", "Stefano Cavalleri", "Olle Häggström", "Emil Verkama", "Joshua Newbould", "Hans Gundlach", "Leonor Brito-Santana", "Brian Amaro", "Vivek Vajipey", "Rynaa Grover", "Ting Wang", "Yosi Kratish", "Wen-Ding Li", "Sivakanth Gopi", "Andrea Caciolai", "Christian Schroeder de Witt", "Pablo Hernández-Cámara", "Emanuele Rodolà", "Jules Robins", "Dominic Williamson", "Vincent Cheng", "Brad Raynor", "Hao Qi", "Ben Segev", "Jingxuan Fan", "Sarah Martinson", "Erik Y. Wang", "Kaylie Hausknecht", "Michael P. Brenner", "Mao Mao", "Christoph Demian", "Peyman Kassani", "Xinyu Zhang", "David Avagian", "Eshawn Jessica Scipio", "Alon Ragoler", "Justin Tan", "Blake Sims", "Rebeka Plecnik", "Aaron Kirtland", "Omer Faruk Bodur", "D. P. Shinde", "Yan Carlos Leyva Labrador", "Zahra Adoul", "Mohamed Zekry", "Ali Karakoc", "Tania C. B. Santos", "Samir Shamseldeen", "Loukmane Karim", "Anna Liakhovitskaia", "Nate Resman", "Nicholas Farina", "Juan Carlos Gonzalez", "Gabe Maayan", "Earth Anderson", "Rodrigo De Oliveira Pena", "Elizabeth Kelley", "Hodjat Mariji", "Rasoul Pouriamanesh", "Wentao Wu", "Ross Finocchio", "Ismail Alarab", "Joshua Cole", "Danyelle Ferreira", "Bryan Johnson", "Mohammad Safdari", "Liangti Dai", "Siriphan Arthornthurasuk", "Isaac C. McAlister", "Alejandro José Moyano", "Alexey Pronin", "Jing Fan", "Angel Ramirez-Trinidad", "Yana Malysheva", "Daphiny Pottmaier", "Omid Taheri", "Stanley Stepanic", "Samuel Perry", "Luke Askew", "Raúl Adrián Huerta Rodríguez", "Ali M. R. Minissi", "Ricardo Lorena", "Krishnamurthy Iyer", "Arshad Anil Fasiludeen", "Ronald Clark", "Josh Ducey", "Matheus Piza", "Maja Somrak", "Eric Vergo", "Juehang Qin", "Benjámin Borbás", "Eric Chu", "Jack Lindsey", "Antoine Jallon", "I. M. J. McInnis", "Evan Chen", "Avi Semler", "Luk Gloor", "Tej Shah", "Marc Carauleanu", "Pascal Lauer", "Tran Đuc Huy", "Hossein Shahrtash", "Emilien Duc", "Lukas Lewark", "Assaf Brown", "Samuel Albanie", "Brian Weber", "Warren S. Vaz", "Pierre Clavier", "Yiyang Fan", "Gabriel Poesia Reis e Silva", "Long", "Lian", "Marcus Abramovitch", "Xi Jiang", "Sandra Mendoza", "Murat Islam", "Juan Gonzalez", "Vasilios Mavroudis", "Justin Xu", "Pawan Kumar", "Laxman Prasad Goswami", "Daniel Bugas", "Nasser Heydari", "Ferenc Jeanplong", "Thorben Jansen", "Antonella Pinto", "Archimedes Apronti", "Abdallah Galal", "Ng Ze-An", "Ankit Singh", "Tong Jiang", "Joan of Arc Xavier", "Kanu Priya Agarwal", "Mohammed Berkani", "Gang Zhang", "Zhehang Du", "Benedito Alves de Oliveira Junior", "Dmitry Malishev", "Nicolas Remy", "Taylor D. Hartman", "Tim Tarver", "Stephen Mensah", "Gautier Abou Loume", "Wiktor Morak", "Farzad Habibi", "Sarah Hoback", "Will Cai", "Javier Gimenez", "Roselynn Grace Montecillo", "Jakub Łucki", "Russell Campbell", "Asankhaya Sharma", "Khalida Meer", "Shreen Gul", "Daniel Espinosa Gonzalez", "Xavier Alapont", "Alex Hoover", "Gunjan Chhablani", "Freddie Vargus", "Arunim Agarwal", "Yibo Jiang", "Deepakkumar Patil", "David Outevsky", "Kevin Joseph Scaria", "Rajat Maheshwari", "Abdelkader Dendane", "Priti Shukla", "Ashley Cartwright", "Sergei Bogdanov", "Niels Mündler", "Sören Möller", "Luca Arnaboldi", "Kunvar Thaman", "Muhammad Rehan Siddiqi", "Prajvi Saxena", "Himanshu Gupta", "Tony Fruhauff", "Glen Sherman", "Mátyás Vincze", "Siranut Usawasutsakorn", "Dylan Ler", "Anil Radhakrishnan", "Innocent Enyekwe", "Sk Md Salauddin", "Jiang Muzhen", "Aleksandr Maksapetyan", "Vivien Rossbach", "Chris Harjadi", "Mohsen Bahaloohoreh", "Claire Sparrow", "Jasdeep Sidhu", "Sam Ali", "Song Bian", "John Lai", "Eric Singer", "Justine Leon Uro", "Greg Bateman", "Mohamed Sayed", "Ahmed Menshawy", "Darling Duclosel", "Dario Bezzi", "Yashaswini Jain", "Ashley Aaron", "Murat Tiryakioglu", "Sheeshram Siddh", "Keith Krenek", "Imad Ali Shah", "Jun Jin", "Scott Creighton", "Denis Peskoff", "Zienab EL-Wasif", "Ragavendran P", "Michael Richmond", "Joseph McGowan", "Tejal Patwardhan", "Hao-Yu Sun", "Ting Sun", "Nikola Zubić", "Samuele Sala", "Stephen Ebert", "Jean Kaddour", "Manuel Schottdorf", "Dianzhuo Wang", "Gerol Petruzella", "Alex Meiburg", "Tilen Medved", "Ali ElSheikh", "S Ashwin Hebbar", "Lorenzo Vaquero", "Xianjun Yang", "Jason Poulos", "Vilém Zouhar", "Sergey Bogdanik", "Mingfang Zhang", "Jorge Sanz-Ros", "David Anugraha", "Yinwei Dai", "Anh N. Nhu", "Xue Wang", "Ali Anil Demircali", "Zhibai Jia", "Yuyin Zhou", "Juncheng Wu", "Mike He", "Nitin Chandok", "Aarush Sinha", "Gaoxiang Luo", "Long Le", "Mickaël Noyé", "Michał Perełkiewicz", "Ioannis Pantidis", "Tianbo Qi", "Soham Sachin Purohit", "Letitia Parcalabescu", "Thai-Hoa Nguyen", "Genta Indra Winata", "Edoardo M. Ponti", "Hanchen Li", "Kaustubh Dhole", "Jongee Park", "Dario Abbondanza", "Yuanli Wang", "Anupam Nayak", "Diogo M. Caetano", "Antonio A. W. L. Wong", "Maria del Rio-Chanona", "Dániel Kondor", "Pieter Francois", "Ed Chalstrey", "Jakob Zsambok", "Dan Hoyer", "Jenny Reddish", "Jakob Hauser", "Francisco-Javier Rodrigo-Ginés", "Suchandra Datta", "Maxwell Shepherd", "Thom Kamphuis", "Qizheng Zhang", "Hyunjun Kim", "Ruiji Sun", "Jianzhu Yao", "Franck Dernoncourt", "Satyapriya Krishna", "Sina Rismanchian", "Bonan Pu", "Francesco Pinto", "Yingheng Wang", "Kumar Shridhar", "Kalon J. Overholt", "Glib Briia", "Hieu Nguyen", "David", "Soler Bartomeu", "Tony CY Pang", "Adam Wecker", "Yifan Xiong", "Fanfei Li", "Lukas S. Huber", "Joshua Jaeger", "Romano De Maddalena", "Xing Han Lù", "Yuhui Zhang", "Claas Beger", "Patrick Tser Jern Kon", "Sean Li", "Vivek Sanker", "Ming Yin", "Yihao Liang", "Xinlu Zhang", "Ankit Agrawal", "Li S. Yifei", "Zechen Zhang", "Mu Cai", "Yasin Sonmez", "Costin Cozianu", "Changhao Li", "Alex Slen", "Shoubin Yu", "Hyun Kyu Park", "Gabriele Sarti", "Marcin Briański", "Alessandro Stolfo", "Truong An Nguyen", "Mike Zhang", "Yotam Perlitz", "Jose Hernandez-Orallo", "Runjia Li", "Amin Shabani", "Felix Juefei-Xu", "Shikhar Dhingra", "Orr Zohar", "My Chiffon Nguyen", "Alexander Pondaven", "Abdurrahim Yilmaz", "Xuandong Zhao", "Chuanyang Jin", "Muyan Jiang", "Stefan Todoran", "Xinyao Han", "Jules Kreuer", "Brian Rabern", "Anna Plassart", "Martino Maggetti", "Luther Yap", "Robert Geirhos", "Jonathon Kean", "Dingsu Wang", "Sina Mollaei", "Chenkai Sun", "Yifan Yin", "Shiqi Wang", "Rui Li", "Yaowen Chang", "Anjiang Wei", "Alice Bizeul", "Xiaohan Wang", "Alexandre Oliveira Arrais", "Kushin Mukherjee", "Jorge Chamorro-Padial", "Jiachen Liu", "Xingyu Qu", "Junyi Guan", "Adam Bouyamourn", "Shuyu Wu", "Martyna Plomecka", "Junda Chen", "Mengze Tang", "Jiaqi Deng", "Shreyas Subramanian", "Haocheng Xi", "Haoxuan Chen", "Weizhi Zhang", "Yinuo Ren", "Haoqin Tu", "Sejong Kim", "Yushun Chen", "Sara Vera Marjanović", "Junwoo Ha", "Grzegorz Luczyna", "Jeff J. Ma", "Zewen Shen", "Dawn Song", "Cedegao E. Zhang", "Zhun Wang", "Gaël Gendron", "Yunze Xiao", "Leo Smucker", "Erica Weng", "Kwok Hao Lee", "Zhe Ye", "Stefano Ermon", "Ignacio D. Lopez-Miguel", "Theo Knights", "Anthony Gitter", "Namkyu Park", "Boyi Wei", "Hongzheng Chen", "Kunal Pai", "Ahmed Elkhanany", "Han Lin", "Philipp D. Siedler", "Jichao Fang", "Ritwik Mishra", "Károly Zsolnai-Fehér", "Xilin Jiang", "Shadab Khan", "Jun Yuan", "Rishab Kumar Jain", "Xi Lin", "Mike Peterson", "Zhe Wang", "Aditya Malusare", "Maosen Tang", "Isha Gupta", "Ivan Fosin", "Timothy Kang", "Barbara Dworakowska", "Kazuki Matsumoto", "Guangyao Zheng", "Gerben Sewuster", "Jorge Pretel Villanueva", "Ivan Rannev", "Igor Chernyavsky", "Jiale Chen", "Deepayan Banik", "Ben Racz", "Wenchao Dong", "Jianxin Wang", "Laila Bashmal", "Duarte V. Gonçalves", "Wei Hu", "Kaushik Bar", "Ondrej Bohdal", "Atharv Singh Patlan", "Shehzaad Dhuliawala", "Caroline Geirhos", "Julien Wist", "Yuval Kansal", "Bingsen Chen", "Kutay Tire", "Atak Talay Yücel", "Brandon Christof", "Veerupaksh Singla", "Zijian Song", "Sanxing Chen", "Jiaxin Ge", "Kaustubh Ponkshe", "Isaac Park", "Tianneng Shi", "Martin Q. Ma", "Joshua Mak", "Sherwin Lai", "Antoine Moulin", "Zhuo Cheng", "Zhanda Zhu", "Ziyi Zhang", "Vaidehi Patil", "Ketan Jha", "Qiutong Men", "Jiaxuan Wu", "Tianchi Zhang", "Bruno Hebling Vieira", "Alham Fikri Aji", "Jae-Won Chung", "Mohammed Mahfoud", "Ha Thi Hoang", "Marc Sperzel", "Wei Hao", "Kristof Meding", "Sihan Xu", "Vassilis Kostakos", "Davide Manini", "Yueying Liu", "Christopher Toukmaji", "Jay Paek", "Eunmi Yu", "Arif Engin Demircali", "Zhiyi Sun", "Ivan Dewerpe", "Hongsen Qin", "Roman Pflugfelder", "James Bailey", "Johnathan Morris", "Ville Heilala", "Sybille Rosset", "Zishun Yu", "Peter E. Chen", "Woongyeong Yeo", "Eeshaan Jain", "Ryan Yang", "Sreekar Chigurupati", "Julia Chernyavsky", "Sai Prajwal Reddy", "Subhashini Venugopalan", "Hunar Batra", "Core Francisco Park", "Hieu Tran", "Guilherme Maximiano", "Genghan Zhang", "Yizhuo Liang", "Hu Shiyu", "Rongwu Xu", "Rui Pan", "Siddharth Suresh", "Ziqi Liu", "Samaksh Gulati", "Songyang Zhang", "Peter Turchin", "Christopher W. Bartlett", "Christopher R. Scotese", "Phuong M. Cao", "Ben Wu", "Jacek Karwowski", "Davide Scaramuzza", "Aakaash Nattanmai", "Gordon McKellips", "Anish Cheraku", "Asim Suhail", "Ethan Luo", "Marvin Deng", "Jason Luo", "Ashley Zhang", "Kavin Jindel", "Kasper Halevy", "Allen Baranov", "Michael Liu", "Advaith Avadhanam", "David Zhang", "Brad Ma", "Evan Fu", "Liam Do", "Joshua Lass", "Hubert Yang", "Surya Sunkari", "Vishruth Bharath", "Violet Ai", "James Leung", "Rishit Agrawal", "Alan Zhou", "Kevin Chen", "Tejas Kalpathi", "Ziqi Xu", "Gavin Wang", "Tyler Xiao", "Erik Maung", "Sam Lee", "Roy Yue", "Ben Zhao", "Julia Yoon", "Sunny Sun", "Aryan Singh", "Clark Peng", "Tyler Osbey", "Taozhi Wang", "Daryl Echeazu", "Timothy Wu", "Spandan Patel", "Vidhi Kulkarni", "Vijaykaarti Sundarapandiyan", "Andrew Le", "Zafir Nasim", "Srikar Yalam", "Ritesh Kasamsetty", "Soham Samal", "David Sun", "Nihar Shah", "Abhijeet Saha", "Alex Zhang", "Leon Nguyen", "Laasya Nagumalli", "Kaixin Wang", "Aidan Wu", "Anwith Telluri", "Steven Dillmann", "Zhengxiang Wang", "Junyu Luo", "Hugo Lunn", "Artem Gazizov", "Haitz Sáez de Ocáriz Borde", "Ivan Trus", "Morgan Hervault", "Zheyu Zhang", "Bo Chen", "Yuchen Wu", "Christopher J. Cordier", "Gün Kaynar", "Cansin Ayvaz", "Polina Avdiunina", "Johannes Brust", "Xingjian Diao", "K. D. Meaney", "Yifan Gu", "Chenyu Wang", "Chenzhuo Dong", "William Wright", "Simon Brave", "Owen Root", "Jiayuan Liu", "Chow Chun Lok", "Tianqin Li", "Shiyi Du", "Dailan He", "Lufeiya Liu", "Sina Jamalzadegan", "Anil Ramakrishna", "Xuanqing Xu", "Xin Qing", "Xin Luo", "Wenkai Li", "Shi Bo", "Filipp Gusev", "Maximos Skandalis", "Desheng Ma", "Chunhui Zhang", "Haoran Qiu", "Allen G Hart", "Rickard Brüel Gabrielsson", "Artem Lukoianov", "Summer Yue", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "University of Pennsylvania", "Center for AI Safety", "University of Waterloo", "Amazon AGI", "CMU LTI", "ASAPP", "ASAPP Inc", "Carnegie Mellon University", "National University", "De La Salle University", "Sapienza, University of Rome", "Yonsei University", "University of Michigan", "Washington University in St. Louis", "Mila, McGill", "Mozilla.ai", "University of California, Berkeley", "EleutherAI", "UC Berkeley", "Georgia Institute of Technology", "Brown University", "Minerva University", "Johns Hopkins University", "Stanford University", "Nanyang Technological University", "ETH Zurich", "CMU", "ASU, BITS-Pilani", "AIM Intelligence", "Apple", "University of Oxford", "Beni Suef University", "Language Techonology Institute", "University of Washington", "SEACrowd", "University of Copenhagen", "University of Minnesota", "McGill University", "Mila", "KAIST", "Universidad Nacional de Educación a Distancia (UNED)", "Northeastern University", "Mathematics Department", "Faculty of Science", "Kafrelsheikh University", "Methods of Plasticity Research, Department of Psychology, University of Zurich, Zurich, Switzerland", "Neuroscience Center Zurich (ZNZ), University of Zurich & ETH Zurich, Zurich, Switzerland", "Stony Brook University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/hle/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/cais/hle", "provider": "Center for AI Safety" }, { "url": "https://huggingface.co/datasets/skylenage-ai/HLE-Verified", "provider": "Skylenage Ai" }, { "url": "https://huggingface.co/datasets/cais/hle-rolling", "provider": "Center for AI Safety" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2501.14249" }, "code_url": null, "metadata": { "source_search": "Long Phan, Alice Gatti, Ziwen Han et al. University of Pennsylvania; Center for AI Safety; University of Waterloo; Amazon AGI; CMU LTI; ASAPP; ASAPP Inc; Carnegie Mellon University; National University; De La Salle University; Sapienza, University of Rome; Yonsei University; University of Michigan; Washington University in St. Louis; Mila, McGill; Mozilla.ai; University of California, Berkeley; EleutherAI; UC Berkeley; Georgia Institute of Technology; Brown University; Minerva University; Johns Hopkins University; Stanford University; Nanyang Technological University; ETH Zurich; CMU; ASU, BITS-Pilani; AIM Intelligence; Apple; University of Oxford; Beni Suef University; Language Techonology Institute; University of Washington; SEACrowd; University of Copenhagen; University of Minnesota; McGill University; Mila; KAIST; Universidad Nacional de Educación a Distancia (UNED); Northeastern University; Mathematics Department; Faculty of Science; Kafrelsheikh University; Methods of Plasticity Research, Department of Psychology, University of Zurich, Zurich, Switzerland; Neuroscience Center Zurich (ZNZ), University of Zurich & ETH Zurich, Zurich, Switzerland; Stony Brook University Original paper authors A benchmark of expert-level academic questions to assess AI capabilities Long Phan Alice Gatti Ziwen Han Nathaniel Li Josephina Hu Hugh Zhang Chen Bo Calvin Zhang Mohamed Shaaban John Ling Sean Shi Michael Choi Anish Agrawal Arnav Chopra Adam Khoja Ryan Kim Richard Ren Jason Hausenloy Oliver Zhang Mantas Mazeika Dmitry Dodonov Tung Nguyen Jaeho Lee Daron Anderson Mikhail Doroshenko Alun Cennyth Stokes Mobeen Mahmood Oleksandr Pokutnyi Oleg Iskra Jessica P. Wang John-Clark Levin Mstyslav Kazakov Fiona Feng Steven Y. Feng Haoran Zhao Michael Yu Varun Gangal Chelsea Zou Zihan Wang Serguei Popov Robert Gerbicz Geoff Galgon Johannes Schmitt Will Yeadon Yongki Lee Scott Sauers Alvaro Sanchez Fabian Giska Marc Roth Søren Riis Saiteja Utpala Noah Burns Gashaw M. Goshu Mohinder Maheshbhai Naiya Chidozie Agu Zachary Giboney Antrell Cheatom Francesco Fournier-Facio Sarah-Jane Crowson Lennart Finke Zerui Cheng Jennifer Zampese Ryan G. Hoerr Mark Nandor Hyunwoo Park Tim Gehrunger Jiaqi Cai Ben McCarty Alexis C Garretson Edwin Taylor Damien Sileo Qiuyu Ren Usman Qazi Lianghui Li Jungbae Nam John B. Wydallis Pavel Arkhipov Jack Wei Lun Shi Aras Bacho Chris G. Willcocks Hangrui Cao Sumeet Motwani Emily de Oliveira Santos Johannes Veith Edward Vendrow Doru Cojoc Kengo Zenitani Joshua Robinson Longke Tang Yuqi Li Joshua Vendrow Natanael Wildner Fraga Vladyslav Kuchkin Andrey Pupasov Maksimov Pierre Marion Denis Efremov Jayson Lynch Kaiqu Liang Aleksandar Mikov Andrew Gritsevskiy Julien Guillod Gözdenur Demir Dakotah Martinez Ben Pageler Kevin Zhou Saeed Soori Ori Press Henry Tang Paolo Rissone Sean R. Green Lina Brüssel Moon Twayana Aymeric Dieuleveut Joseph Marvin Imperial Ameya Prabhu Jinzhou Yang Nick Crispino Arun Rao Dimitri Zvonkine Gabriel Loiseau Mikhail Kalinin Marco Lukas Ciprian Manolescu Nate Stambaugh Subrata Mishra Tad Hogg Carlo Bosio Brian P Coppola Julian Salazar Jaehyeok Jin Rafael Sayous Stefan Ivanov Philippe Schwaller Shaipranesh Senthilkuma Andres M Bran Andres Algaba Kelsey Van den Houte Lynn Van Der Sypt Brecht Verbeken David Noever Alexei Kopylov Benjamin Myklebust Bikun Li Lisa Schut Evgenii Zheltonozhskii Qiaochu Yuan Derek Lim Richard Stanley Tong Yang John Maar Julian Wykowski Martí Oller Anmol Sahu Cesare Giulio Ardito Yuzheng Hu Ariel Ghislain Kemogne Kamdoum Alvin Jin Tobias Garcia Vilchis Yuexuan Zu Martin Lackner James Koppel Gongbo Sun Daniil S. Antonenko Steffi Chern Bingchen Zhao Pierrot Arsene Joseph M Cavanagh Daofeng Li Jiawei Shen Donato Crisostomi Wenjin Zhang Ali Dehghan Sergey Ivanov David Perrella Nurdin Kaparov Allen Zang Ilia Sucholutsky Arina Kharlamova Daniil Orel Vladislav Poritski Shalev Ben-David Zachary Berger Parker Whitfill Michael Foster Daniel Munro Linh Ho Shankar Sivarajan Dan Bar Hava Aleksey Kuchkin David Holmes Alexandra Rodriguez-Romero Frank Sommerhage Anji Zhang Richard Moat Keith Schneider Zakayo Kazibwe Don Clarke Dae Hyun Kim Felipe Meneguitti Dias Sara Fish Veit Elser Tobias Kreiman Victor Efren Guadarrama Vilchis Immo Klose Ujjwala Anantheswaran Adam Zweiger Kaivalya Rawal Jeffery Li Jeremy Nguyen Nicolas Daans Haline Heidinger Maksim Radionov Václav Rozhoň Vincent Ginis Christian Stump Niv Cohen Rafał Poświata Josef Tkadlec Alan Goldfarb Chenguang Wang Piotr Padlewski Stanislaw Barzowski Kyle Montgomery Ryan Stendall Jamie Tucker-Foltz Jack Stade T. Ryan Rogers Tom Goertzen Declan Grabb Abhishek Shukla Alan Givré John Arnold Ambay Archan Sen Muhammad Fayez Aziz Mark H Inlow Hao He Ling Zhang Younesse Kaddar Ivar Ängquist Yanxu Chen Harrison K Wang Kalyan Ramakrishnan Elliott Thornley Antonio Terpin Hailey Schoelkopf Eric Zheng Avishy Carmi Ethan D. L. Brown Kelin Zhu Max Bartolo Richard Wheeler Martin Stehberger Peter Bradshaw JP Heimonen Kaustubh Sridhar Ido Akov Jennifer Sandlin Yury Makarychev Joanna Tam Hieu Hoang David M. Cunningham Vladimir Goryachev Demosthenes Patramanis Michael Krause Andrew Redenti David Aldous Jesyin Lai Shannon Coleman Jiangnan Xu Sangwon Lee Ilias Magoulas Sandy Zhao Ning Tang Michael K. Cohen Orr Paradise Jan Hendrik Kirchner Maksym Ovchynnikov Jason O. Matos Adithya Shenoy Michael Wang Yuzhou Nie Anna Sztyber-Betley Paolo Faraboschi Robin Riblet Jonathan Crozier Shiv Halasyamani Shreyas Verma Prashant Joshi Eli Meril Ziqiao Ma Jérémy Andréoletti Raghav Singhal Jacob Platnick Volodymyr Nevirkovets Luke Basler Alexander Ivanov Seri Khoury Nils Gustafsson Marco Piccardo Hamid Mostaghimi Qijia Chen Virendra Singh Tran Quoc Khánh Paul Rosu Hannah Szlyk Zachary Brown Himanshu Narayan Aline Menezes Jonathan Roberts William Alley Kunyang Sun Arkil Patel Max Lamparth Anka Reuel Linwei Xin Hanmeng Xu Jacob Loader Freddie Martin Zixuan Wang Andrea Achilleos Thomas Preu Tomek Korbak Ida Bosio Fereshteh Kazemi Ziye Chen Biró Bálint Eve J. Y. Lo Jiaqi Wang Maria Inês S. Nunes Jeremiah Milbauer M Saiful Bari Zihao Wang Behzad Ansarinejad Yewen Sun Stephane Durand Hossam Elgnainy Guillaume Douville Daniel Tordera George Balabanian Hew Wolff Lynna Kvistad Hsiaoyun Milliron Ahmad Sakor Murat Eron Andrew Favre D. O. Shailesh Shah Xiaoxiang Zhou Firuz Kamalov Sherwin Abdoli Tim Santens Shaul Barkan Allison Tee Robin Zhang Alessandro Tomasiello G. Bruno De Luca Shi-Zhuo Looi Vinh-Kha Le Noam Kolt Jiayi Pan Emma Rodman Jacob Drori Carl J Fossum Niklas Muennighoff Milind Jagota Ronak Pradeep Honglu Fan Jonathan Eicher Michael Chen Kushal Thaman William Merrill Moritz Firsching Carter Harris Stefan Ciobâcă Jason Gross Rohan Pandey Ilya Gusev Adam Jones Shashank Agnihotri Pavel Zhelnov Mohammadreza Mofayezi Alexander Piperski David K. Zhang Kostiantyn Dobarskyi Roman Leventov Ignat Soroko Joshua Duersch Vage Taamazyan Andrew Ho Wenjie Ma William Held Ruicheng Xian Armel Randy Zebaze Mohanad Mohamed Julian Noah Leser Michelle X Yuan Laila Yacar Johannes Lengler Katarzyna Olszewska Claudio Di Fratta Edson Oliveira Joseph W. Jackson Andy Zou Muthu Chidambaram Timothy Manik Hector Haffenden Dashiell Stander Ali Dasouqi Alexander Shen Bita Golshani David Stap Egor Kretov Mikalai Uzhou Alina Borisovna Zhidkovskaya Nick Winter Miguel Orbegozo Rodriguez Robert Lauff Dustin Wehr Colin Tang Zaki Hossain Shaun Phillips Fortuna Samuele Fredrik Ekström Angela Hammon Oam Patel Faraz Farhidi George Medley Forough Mohammadzadeh Madellene Peñaflor Haile Kassahun Alena Friedrich Rayner Hernandez Perez Daniel Pyda Taom Sakal Omkar Dhamane Ali Khajegili Mirabadi Eric Hallman Kenchi Okutsu Mike Battaglia Mohammad Maghsoudimehrabani Alon Amit Dave Hulbert Roberto Pereira Simon Weber Handoko Anton Peristyy Stephen Malina Mustafa Mehkary Rami Aly Frank Reidegeld Anna-Katharina Dick Cary Friday Mukhwinder Singh Hassan Shapourian Wanyoung Kim Mariana Costa Hubeyb Gurdogan Harsh Kumar Chiara Ceconello Chao Zhuang Haon Park Micah Carroll Andrew R. Tawfeek Stefan Steinerberger Daattavya Aggarwal Michael Kirchhof Linjie Dai Evan Kim Johan Ferret Jainam Shah Yuzhou Wang Minghao Yan Krzysztof Burdzy Lixin Zhang Antonio Franca Diana T. Pham Kang Yong Loh Abram Jackson Paolo Giordano Philipp Petersen Adrian Cosma Jesus Colino Colin White Jacob Votava Vladimir Vinnikov Ethan Delaney Petr Spelda Vit Stritecky Syed M. Shahid Jean-Christophe Mourrat Lavr Vetoshkin Koen Sponselee Renas Bacho Zheng-Xin Yong Florencia de la Rosa Nathan Cho Xiuyu Li Guillaume Malod Orion Weller Guglielmo Albani Leon Lang Julien Laurendeau Dmitry Kazakov Fatimah Adesanya Julien Portier Lawrence Hollom Victor Souza Yuchen Anna Zhou Julien Degorre Yiğit Yalın Gbenga Daniel Obikoya Rai Filippo Bigi M. C. Boscá Oleg Shumar Kaniuar Bacho Gabriel Recchia Mara Popescu Nikita Shulga Ngefor Mildred Tanwie Thomas C. H. Lux Ben Rank Colin Ni Matthew Brooks Alesia Yakimchyk Huanxu Liu Stefano Cavalleri Olle Häggström Emil Verkama Joshua Newbould Hans Gundlach Leonor Brito-Santana Brian Amaro Vivek Vajipey Rynaa Grover Ting Wang Yosi Kratish Wen-Ding Li Sivakanth Gopi Andrea Caciolai Christian Schroeder de Witt Pablo Hernández-Cámara Emanuele Rodolà Jules Robins Dominic Williamson Vincent Cheng Brad Raynor Hao Qi Ben Segev Jingxuan Fan Sarah Martinson Erik Y. Wang Kaylie Hausknecht Michael P. Brenner Mao Mao Christoph Demian Peyman Kassani Xinyu Zhang David Avagian Eshawn Jessica Scipio Alon Ragoler Justin Tan Blake Sims Rebeka Plecnik Aaron Kirtland Omer Faruk Bodur D. P. Shinde Yan Carlos Leyva Labrador Zahra Adoul Mohamed Zekry Ali Karakoc Tania C. B. Santos Samir Shamseldeen Loukmane Karim Anna Liakhovitskaia Nate Resman Nicholas Farina Juan Carlos Gonzalez Gabe Maayan Earth Anderson Rodrigo De Oliveira Pena Elizabeth Kelley Hodjat Mariji Rasoul Pouriamanesh Wentao Wu Ross Finocchio Ismail Alarab Joshua Cole Danyelle Ferreira Bryan Johnson Mohammad Safdari Liangti Dai Siriphan Arthornthurasuk Isaac C. McAlister Alejandro José Moyano Alexey Pronin Jing Fan Angel Ramirez-Trinidad Yana Malysheva Daphiny Pottmaier Omid Taheri Stanley Stepanic Samuel Perry Luke Askew Raúl Adrián Huerta Rodríguez Ali M. R. Minissi Ricardo Lorena Krishnamurthy Iyer Arshad Anil Fasiludeen Ronald Clark Josh Ducey Matheus Piza Maja Somrak Eric Vergo Juehang Qin Benjámin Borbás Eric Chu Jack Lindsey Antoine Jallon I. M. J. McInnis Evan Chen Avi Semler Luk Gloor Tej Shah Marc Carauleanu Pascal Lauer Tran Đuc Huy Hossein Shahrtash Emilien Duc Lukas Lewark Assaf Brown Samuel Albanie Brian Weber Warren S. Vaz Pierre Clavier Yiyang Fan Gabriel Poesia Reis e Silva Long Lian Marcus Abramovitch Xi Jiang Sandra Mendoza Murat Islam Juan Gonzalez Vasilios Mavroudis Justin Xu Pawan Kumar Laxman Prasad Goswami Daniel Bugas Nasser Heydari Ferenc Jeanplong Thorben Jansen Antonella Pinto Archimedes Apronti Abdallah Galal Ng Ze-An Ankit Singh Tong Jiang Joan of Arc Xavier Kanu Priya Agarwal Mohammed Berkani Gang Zhang Zhehang Du Benedito Alves de Oliveira Junior Dmitry Malishev Nicolas Remy Taylor D. Hartman Tim Tarver Stephen Mensah Gautier Abou Loume Wiktor Morak Farzad Habibi Sarah Hoback Will Cai Javier Gimenez Roselynn Grace Montecillo Jakub Łucki Russell Campbell Asankhaya Sharma Khalida Meer Shreen Gul Daniel Espinosa Gonzalez Xavier Alapont Alex Hoover Gunjan Chhablani Freddie Vargus Arunim Agarwal Yibo Jiang Deepakkumar Patil David Outevsky Kevin Joseph Scaria Rajat Maheshwari Abdelkader Dendane Priti Shukla Ashley Cartwright Sergei Bogdanov Niels Mündler Sören Möller Luca Arnaboldi Kunvar Thaman Muhammad Rehan Siddiqi Prajvi Saxena Himanshu Gupta Tony Fruhauff Glen Sherman Mátyás Vincze Siranut Usawasutsakorn Dylan Ler Anil Radhakrishnan Innocent Enyekwe Sk Md Salauddin Jiang Muzhen Aleksandr Maksapetyan Vivien Rossbach Chris Harjadi Mohsen Bahaloohoreh Claire Sparrow Jasdeep Sidhu Sam Ali Song Bian John Lai Eric Singer Justine Leon Uro Greg Bateman Mohamed Sayed Ahmed Menshawy Darling Duclosel Dario Bezzi Yashaswini Jain Ashley Aaron Murat Tiryakioglu Sheeshram Siddh Keith Krenek Imad Ali Shah Jun Jin Scott Creighton Denis Peskoff Zienab EL-Wasif Ragavendran P Michael Richmond Joseph McGowan Tejal Patwardhan Hao-Yu Sun Ting Sun Nikola Zubić Samuele Sala Stephen Ebert Jean Kaddour Manuel Schottdorf Dianzhuo Wang Gerol Petruzella Alex Meiburg Tilen Medved Ali ElSheikh S Ashwin Hebbar Lorenzo Vaquero Xianjun Yang Jason Poulos Vilém Zouhar Sergey Bogdanik Mingfang Zhang Jorge Sanz-Ros David Anugraha Yinwei Dai Anh N. Nhu Xue Wang Ali Anil Demircali Zhibai Jia Yuyin Zhou Juncheng Wu Mike He Nitin Chandok Aarush Sinha Gaoxiang Luo Long Le Mickaël Noyé Michał Perełkiewicz Ioannis Pantidis Tianbo Qi Soham Sachin Purohit Letitia Parcalabescu Thai-Hoa Nguyen Genta Indra Winata Edoardo M. Ponti Hanchen Li Kaustubh Dhole Jongee Park Dario Abbondanza Yuanli Wang Anupam Nayak Diogo M. Caetano Antonio A. W. L. Wong Maria del Rio-Chanona Dániel Kondor Pieter Francois Ed Chalstrey Jakob Zsambok Dan Hoyer Jenny Reddish Jakob Hauser Francisco-Javier Rodrigo-Ginés Suchandra Datta Maxwell Shepherd Thom Kamphuis Qizheng Zhang Hyunjun Kim Ruiji Sun Jianzhu Yao Franck Dernoncourt Satyapriya Krishna Sina Rismanchian Bonan Pu Francesco Pinto Yingheng Wang Kumar Shridhar Kalon J. Overholt Glib Briia Hieu Nguyen David Soler Bartomeu Tony CY Pang Adam Wecker Yifan Xiong Fanfei Li Lukas S. Huber Joshua Jaeger Romano De Maddalena Xing Han Lù Yuhui Zhang Claas Beger Patrick Tser Jern Kon Sean Li Vivek Sanker Ming Yin Yihao Liang Xinlu Zhang Ankit Agrawal Li S. Yifei Zechen Zhang Mu Cai Yasin Sonmez Costin Cozianu Changhao Li Alex Slen Shoubin Yu Hyun Kyu Park Gabriele Sarti Marcin Briański Alessandro Stolfo Truong An Nguyen Mike Zhang Yotam Perlitz Jose Hernandez-Orallo Runjia Li Amin Shabani Felix Juefei-Xu Shikhar Dhingra Orr Zohar My Chiffon Nguyen Alexander Pondaven Abdurrahim Yilmaz Xuandong Zhao Chuanyang Jin Muyan Jiang Stefan Todoran Xinyao Han Jules Kreuer Brian Rabern Anna Plassart Martino Maggetti Luther Yap Robert Geirhos Jonathon Kean Dingsu Wang Sina Mollaei Chenkai Sun Yifan Yin Shiqi Wang Rui Li Yaowen Chang Anjiang Wei Alice Bizeul Xiaohan Wang Alexandre Oliveira Arrais Kushin Mukherjee Jorge Chamorro-Padial Jiachen Liu Xingyu Qu Junyi Guan Adam Bouyamourn Shuyu Wu Martyna Plomecka Junda Chen Mengze Tang Jiaqi Deng Shreyas Subramanian Haocheng Xi Haoxuan Chen Weizhi Zhang Yinuo Ren Haoqin Tu Sejong Kim Yushun Chen Sara Vera Marjanović Junwoo Ha Grzegorz Luczyna Jeff J. Ma Zewen Shen Dawn Song Cedegao E. Zhang Zhun Wang Gaël Gendron Yunze Xiao Leo Smucker Erica Weng Kwok Hao Lee Zhe Ye Stefano Ermon Ignacio D. Lopez-Miguel Theo Knights Anthony Gitter Namkyu Park Boyi Wei Hongzheng Chen Kunal Pai Ahmed Elkhanany Han Lin Philipp D. Siedler Jichao Fang Ritwik Mishra Károly Zsolnai-Fehér Xilin Jiang Shadab Khan Jun Yuan Rishab Kumar Jain Xi Lin Mike Peterson Zhe Wang Aditya Malusare Maosen Tang Isha Gupta Ivan Fosin Timothy Kang Barbara Dworakowska Kazuki Matsumoto Guangyao Zheng Gerben Sewuster Jorge Pretel Villanueva Ivan Rannev Igor Chernyavsky Jiale Chen Deepayan Banik Ben Racz Wenchao Dong Jianxin Wang Laila Bashmal Duarte V. Gonçalves Wei Hu Kaushik Bar Ondrej Bohdal Atharv Singh Patlan Shehzaad Dhuliawala Caroline Geirhos Julien Wist Yuval Kansal Bingsen Chen Kutay Tire Atak Talay Yücel Brandon Christof Veerupaksh Singla Zijian Song Sanxing Chen Jiaxin Ge Kaustubh Ponkshe Isaac Park Tianneng Shi Martin Q. Ma Joshua Mak Sherwin Lai Antoine Moulin Zhuo Cheng Zhanda Zhu Ziyi Zhang Vaidehi Patil Ketan Jha Qiutong Men Jiaxuan Wu Tianchi Zhang Bruno Hebling Vieira Alham Fikri Aji Jae-Won Chung Mohammed Mahfoud Ha Thi Hoang Marc Sperzel Wei Hao Kristof Meding Sihan Xu Vassilis Kostakos Davide Manini Yueying Liu Christopher Toukmaji Jay Paek Eunmi Yu Arif Engin Demircali Zhiyi Sun Ivan Dewerpe Hongsen Qin Roman Pflugfelder James Bailey Johnathan Morris Ville Heilala Sybille Rosset Zishun Yu Peter E. Chen Woongyeong Yeo Eeshaan Jain Ryan Yang Sreekar Chigurupati Julia Chernyavsky Sai Prajwal Reddy Subhashini Venugopalan Hunar Batra Core Francisco Park Hieu Tran Guilherme Maximiano Genghan Zhang Yizhuo Liang Hu Shiyu Rongwu Xu Rui Pan Siddharth Suresh Ziqi Liu Samaksh Gulati Songyang Zhang Peter Turchin Christopher W. Bartlett Christopher R. Scotese Phuong M. Cao Ben Wu Jacek Karwowski Davide Scaramuzza Aakaash Nattanmai Gordon McKellips Anish Cheraku Asim Suhail Ethan Luo Marvin Deng Jason Luo Ashley Zhang Kavin Jindel Kasper Halevy Allen Baranov Michael Liu Advaith Avadhanam David Zhang Brad Ma Evan Fu Liam Do Joshua Lass Hubert Yang Surya Sunkari Vishruth Bharath Violet Ai James Leung Rishit Agrawal Alan Zhou Kevin Chen Tejas Kalpathi Ziqi Xu Gavin Wang Tyler Xiao Erik Maung Sam Lee Roy Yue Ben Zhao Julia Yoon Sunny Sun Aryan Singh Clark Peng Tyler Osbey Taozhi Wang Daryl Echeazu Timothy Wu Spandan Patel Vidhi Kulkarni Vijaykaarti Sundarapandiyan Andrew Le Zafir Nasim Srikar Yalam Ritesh Kasamsetty Soham Samal David Sun Nihar Shah Abhijeet Saha Alex Zhang Leon Nguyen Laasya Nagumalli Kaixin Wang Aidan Wu Anwith Telluri Steven Dillmann Zhengxiang Wang Junyu Luo Hugo Lunn Artem Gazizov Haitz Sáez de Ocáriz Borde Ivan Trus Morgan Hervault Zheyu Zhang Bo Chen Yuchen Wu Christopher J. Cordier Gün Kaynar Cansin Ayvaz Polina Avdiunina Johannes Brust Xingjian Diao K. D. Meaney Yifan Gu Chenyu Wang Chenzhuo Dong William Wright Simon Brave Owen Root Jiayuan Liu Chow Chun Lok Tianqin Li Shiyi Du Dailan He Lufeiya Liu Sina Jamalzadegan Anil Ramakrishna Xuanqing Xu Xin Qing Xin Luo Wenkai Li Shi Bo Filipp Gusev Maximos Skandalis Desheng Ma Chunhui Zhang Haoran Qiu Allen G Hart Rickard Brüel Gabrielsson Artem Lukoianov Summer Yue Alexandr Wang Dan Hendrycks evaluation-task" } }, { "id": "inspect:ifeval", "name": "IFEval: Instruction-Following Evaluation", "slug": "inspect--ifeval", "organisation": "Jeffrey Zhou, Tianjian Lu, Swaroop Mishra et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifeval", "topics": [ "general-capability" ], "description": "Evaluates how well language models can strictly follow detailed instructions, such as writing responses with specific word counts or including required keywords.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifeval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2311.07911", "primary": true }, { "kind": "paper", "label": "Paper: Instruction-Following Evaluation for Large Language Models", "url": "https://arxiv.org/abs/2311.07911", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/josejg/instruction_following_eval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifeval/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Google Research", "url": "https://huggingface.co/datasets/google/IFEval", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2311.07911", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifeval/eval.yaml", "family_title": "IFEval: Instruction-Following Evaluation", "group": "Reasoning", "version": "2-A", "contributors": [ "adil-a" ], "tags": [ "Reasoning" ], "task": { "name": "ifeval", "dataset_samples": 541, "comment": null, "support_only": false }, "tasks": [ { "name": "ifeval", "dataset_samples": 541, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 541, "dependency": "ifeval", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "git_dependency", "source": "https://github.com/josejg/instruction_following_eval", "fetch_method": "pyproject_toml", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "google/IFEval", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:ifeval", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jeffrey Zhou, Tianjian Lu, Swaroop Mishra et al.", "organization": "Arizona State University; Google Research", "organizations": [ "Arizona State University", "Google Research" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2311.07911" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.07911", "arxiv_id": "2311.07911", "title": "Instruction-Following Evaluation for Large Language Models", "authors": [ "Jeffrey Zhou", "Tianjian Lu", "Swaroop Mishra", "Siddhartha Brahma", "Sujoy Basu", "Yi Luan", "Denny Zhou", "Le Hou" ], "organizations": [ "Arizona State University", "Google Research" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.07911", "arxiv_id": "2311.07911", "title": "Instruction-Following Evaluation for Large Language Models", "authors": [ "Jeffrey Zhou", "Tianjian Lu", "Swaroop Mishra", "Siddhartha Brahma", "Sujoy Basu", "Yi Luan", "Denny Zhou", "Le Hou" ], "organizations": [ "Arizona State University", "Google Research" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/josejg/instruction_following_eval", "owner": "Josejg", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifeval/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/google/IFEval", "provider": "Google Research" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2311.07911" }, "code_url": "https://github.com/josejg/instruction_following_eval", "metadata": { "source_search": "Jeffrey Zhou, Tianjian Lu, Swaroop Mishra et al. Arizona State University; Google Research Original paper authors Instruction-Following Evaluation for Large Language Models Jeffrey Zhou Tianjian Lu Swaroop Mishra Siddhartha Brahma Sujoy Basu Yi Luan Denny Zhou Le Hou Josejg evaluation-task" } }, { "id": "inspect:ifevalcode", "name": "IFEvalCode: Controlled Code Generation", "slug": "inspect--ifevalcode", "organisation": "Jian Yang, Wei Zhang, Shukai Liu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifevalcode", "topics": [ "general-capability" ], "description": "Evaluates code generation models on their ability to produce correct code while adhering to specific instruction constraints across 8 programming languages.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifevalcode/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2507.22462", "primary": true }, { "kind": "paper", "label": "Paper: IFEvalCode: Controlled Code Generation", "url": "https://arxiv.org/abs/2507.22462", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifevalcode/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Multilingual Multimodal NLP", "url": "https://huggingface.co/datasets/Multilingual-Multimodal-NLP/IfEvalCode-testset", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2507.22462", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifevalcode/eval.yaml", "family_title": "IFEvalCode: Controlled Code Generation", "group": "Coding", "version": "2-A", "contributors": [ "PranshuSrivastava" ], "tags": [ "Coding" ], "task": { "name": "ifevalcode", "dataset_samples": 810, "comment": null, "support_only": false }, "tasks": [ { "name": "ifevalcode", "dataset_samples": 810, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 810, "dependency": "ifevalcode", "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "Multilingual-Multimodal-NLP/IfEvalCode-testset", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:ifevalcode", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jian Yang, Wei Zhang, Shukai Liu et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2507.22462" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2507.22462", "arxiv_id": "2507.22462", "title": "IFEvalCode: Controlled Code Generation", "authors": [ "Jian Yang", "Wei Zhang", "Shukai Liu", "Linzheng Chai", "Yingshui Tan", "Jiaheng Liu", "Ge Zhang", "Wangchunshu Zhou", "Guanglin Niu", "Zhoujun Li", "Binyuan Hui", "Junyang Lin" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2507.22462", "arxiv_id": "2507.22462", "title": "IFEvalCode: Controlled Code Generation", "authors": [ "Jian Yang", "Wei Zhang", "Shukai Liu", "Linzheng Chai", "Yingshui Tan", "Jiaheng Liu", "Ge Zhang", "Wangchunshu Zhou", "Guanglin Niu", "Zhoujun Li", "Binyuan Hui", "Junyang Lin" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ifevalcode/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/Multilingual-Multimodal-NLP/IfEvalCode-testset", "provider": "Multilingual Multimodal NLP" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2507.22462" }, "code_url": null, "metadata": { "source_search": "Jian Yang, Wei Zhang, Shukai Liu et al. Original paper authors IFEvalCode: Controlled Code Generation Jian Yang Wei Zhang Shukai Liu Linzheng Chai Yingshui Tan Jiaheng Liu Ge Zhang Wangchunshu Zhou Guanglin Niu Zhoujun Li Binyuan Hui Junyang Lin evaluation-task" } }, { "id": "inspect:infinite_bench_code_debug", "name": "Infinite Bench Code Debug", "slug": "inspect--infinite-bench-code-debug", "organisation": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench", "topics": [ "general-capability" ], "description": "LLM benchmark featuring an average data length surpassing 100K tokens. Comprises synthetic and realistic tasks spanning diverse domains in English and Chinese.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.13718", "primary": true }, { "kind": "paper", "label": "Paper: ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xinrongzhang2022", "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.13718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "family_title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "group": "Reasoning", "version": "2-A", "contributors": [ "celiawaggoner" ], "tags": [ "Reasoning" ], "task": { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, "tasks": [ { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 394, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "xinrongzhang2022/InfiniteBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:infinite_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "organization": "Tsinghua University", "organizations": [ "Tsinghua University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.13718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "provider": "Xinrongzhang2022" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.13718" }, "code_url": null, "metadata": { "source_search": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al. Tsinghua University Original paper authors ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens Xinrong Zhang Yingfa Chen Shengding Hu Zihang Xu Junhao Chen Moo Khai Hao Xu Han Zhen Leng Thai Shuo Wang Zhiyuan Liu Maosong Sun evaluation-family" } }, { "id": "inspect:infinite_bench_code_run", "name": "Infinite Bench Code Run", "slug": "inspect--infinite-bench-code-run", "organisation": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench", "topics": [ "general-capability" ], "description": "LLM benchmark featuring an average data length surpassing 100K tokens. Comprises synthetic and realistic tasks spanning diverse domains in English and Chinese.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.13718", "primary": true }, { "kind": "paper", "label": "Paper: ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xinrongzhang2022", "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.13718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "family_title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "group": "Reasoning", "version": "2-A", "contributors": [ "celiawaggoner" ], "tags": [ "Reasoning" ], "task": { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, "tasks": [ { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 400, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "xinrongzhang2022/InfiniteBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:infinite_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "organization": "Tsinghua University", "organizations": [ "Tsinghua University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.13718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "provider": "Xinrongzhang2022" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.13718" }, "code_url": null, "metadata": { "source_search": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al. Tsinghua University Original paper authors ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens Xinrong Zhang Yingfa Chen Shengding Hu Zihang Xu Junhao Chen Moo Khai Hao Xu Han Zhen Leng Thai Shuo Wang Zhiyuan Liu Maosong Sun evaluation-family" } }, { "id": "inspect:infinite_bench_kv_retrieval", "name": "Infinite Bench Kv Retrieval", "slug": "inspect--infinite-bench-kv-retrieval", "organisation": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench", "topics": [ "general-capability" ], "description": "LLM benchmark featuring an average data length surpassing 100K tokens. Comprises synthetic and realistic tasks spanning diverse domains in English and Chinese.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.13718", "primary": true }, { "kind": "paper", "label": "Paper: ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xinrongzhang2022", "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.13718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "family_title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "group": "Reasoning", "version": "2-A", "contributors": [ "celiawaggoner" ], "tags": [ "Reasoning" ], "task": { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, "tasks": [ { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 500, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "xinrongzhang2022/InfiniteBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:infinite_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "organization": "Tsinghua University", "organizations": [ "Tsinghua University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.13718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "provider": "Xinrongzhang2022" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.13718" }, "code_url": null, "metadata": { "source_search": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al. Tsinghua University Original paper authors ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens Xinrong Zhang Yingfa Chen Shengding Hu Zihang Xu Junhao Chen Moo Khai Hao Xu Han Zhen Leng Thai Shuo Wang Zhiyuan Liu Maosong Sun evaluation-family" } }, { "id": "inspect:infinite_bench_longbook_choice_eng", "name": "Infinite Bench Longbook Choice Eng", "slug": "inspect--infinite-bench-longbook-choice-eng", "organisation": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench", "topics": [ "general-capability" ], "description": "LLM benchmark featuring an average data length surpassing 100K tokens. Comprises synthetic and realistic tasks spanning diverse domains in English and Chinese.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.13718", "primary": true }, { "kind": "paper", "label": "Paper: ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xinrongzhang2022", "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.13718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "family_title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "group": "Reasoning", "version": "2-A", "contributors": [ "celiawaggoner" ], "tags": [ "Reasoning" ], "task": { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, "tasks": [ { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 229, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "xinrongzhang2022/InfiniteBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:infinite_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "organization": "Tsinghua University", "organizations": [ "Tsinghua University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.13718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "provider": "Xinrongzhang2022" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.13718" }, "code_url": null, "metadata": { "source_search": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al. Tsinghua University Original paper authors ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens Xinrong Zhang Yingfa Chen Shengding Hu Zihang Xu Junhao Chen Moo Khai Hao Xu Han Zhen Leng Thai Shuo Wang Zhiyuan Liu Maosong Sun evaluation-family" } }, { "id": "inspect:infinite_bench_longdialogue_qa_eng", "name": "Infinite Bench Longdialogue QA Eng", "slug": "inspect--infinite-bench-longdialogue-qa-eng", "organisation": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench", "topics": [ "general-capability" ], "description": "LLM benchmark featuring an average data length surpassing 100K tokens. Comprises synthetic and realistic tasks spanning diverse domains in English and Chinese.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.13718", "primary": true }, { "kind": "paper", "label": "Paper: ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xinrongzhang2022", "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.13718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "family_title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "group": "Reasoning", "version": "2-A", "contributors": [ "celiawaggoner" ], "tags": [ "Reasoning" ], "task": { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, "tasks": [ { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 200, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "xinrongzhang2022/InfiniteBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:infinite_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "organization": "Tsinghua University", "organizations": [ "Tsinghua University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.13718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "provider": "Xinrongzhang2022" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.13718" }, "code_url": null, "metadata": { "source_search": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al. Tsinghua University Original paper authors ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens Xinrong Zhang Yingfa Chen Shengding Hu Zihang Xu Junhao Chen Moo Khai Hao Xu Han Zhen Leng Thai Shuo Wang Zhiyuan Liu Maosong Sun evaluation-family" } }, { "id": "inspect:infinite_bench_math_calc", "name": "Infinite Bench Math Calc", "slug": "inspect--infinite-bench-math-calc", "organisation": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench", "topics": [ "general-capability" ], "description": "LLM benchmark featuring an average data length surpassing 100K tokens. Comprises synthetic and realistic tasks spanning diverse domains in English and Chinese.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.13718", "primary": true }, { "kind": "paper", "label": "Paper: ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xinrongzhang2022", "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.13718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "family_title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "group": "Reasoning", "version": "2-A", "contributors": [ "celiawaggoner" ], "tags": [ "Reasoning" ], "task": { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, "tasks": [ { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 50, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "xinrongzhang2022/InfiniteBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:infinite_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "organization": "Tsinghua University", "organizations": [ "Tsinghua University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.13718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "provider": "Xinrongzhang2022" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.13718" }, "code_url": null, "metadata": { "source_search": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al. Tsinghua University Original paper authors ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens Xinrong Zhang Yingfa Chen Shengding Hu Zihang Xu Junhao Chen Moo Khai Hao Xu Han Zhen Leng Thai Shuo Wang Zhiyuan Liu Maosong Sun evaluation-family" } }, { "id": "inspect:infinite_bench_math_find", "name": "Infinite Bench Math Find", "slug": "inspect--infinite-bench-math-find", "organisation": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench", "topics": [ "general-capability" ], "description": "LLM benchmark featuring an average data length surpassing 100K tokens. Comprises synthetic and realistic tasks spanning diverse domains in English and Chinese.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.13718", "primary": true }, { "kind": "paper", "label": "Paper: ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xinrongzhang2022", "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.13718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "family_title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "group": "Reasoning", "version": "2-A", "contributors": [ "celiawaggoner" ], "tags": [ "Reasoning" ], "task": { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, "tasks": [ { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 350, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "xinrongzhang2022/InfiniteBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:infinite_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "organization": "Tsinghua University", "organizations": [ "Tsinghua University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.13718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "provider": "Xinrongzhang2022" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.13718" }, "code_url": null, "metadata": { "source_search": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al. Tsinghua University Original paper authors ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens Xinrong Zhang Yingfa Chen Shengding Hu Zihang Xu Junhao Chen Moo Khai Hao Xu Han Zhen Leng Thai Shuo Wang Zhiyuan Liu Maosong Sun evaluation-family" } }, { "id": "inspect:infinite_bench_number_string", "name": "Infinite Bench Number String", "slug": "inspect--infinite-bench-number-string", "organisation": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench", "topics": [ "general-capability" ], "description": "LLM benchmark featuring an average data length surpassing 100K tokens. Comprises synthetic and realistic tasks spanning diverse domains in English and Chinese.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.13718", "primary": true }, { "kind": "paper", "label": "Paper: ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xinrongzhang2022", "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.13718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "family_title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "group": "Reasoning", "version": "2-A", "contributors": [ "celiawaggoner" ], "tags": [ "Reasoning" ], "task": { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, "tasks": [ { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 590, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "xinrongzhang2022/InfiniteBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:infinite_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "organization": "Tsinghua University", "organizations": [ "Tsinghua University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.13718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "provider": "Xinrongzhang2022" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.13718" }, "code_url": null, "metadata": { "source_search": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al. Tsinghua University Original paper authors ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens Xinrong Zhang Yingfa Chen Shengding Hu Zihang Xu Junhao Chen Moo Khai Hao Xu Han Zhen Leng Thai Shuo Wang Zhiyuan Liu Maosong Sun evaluation-family" } }, { "id": "inspect:infinite_bench_passkey", "name": "Infinite Bench Passkey", "slug": "inspect--infinite-bench-passkey", "organisation": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench", "topics": [ "general-capability" ], "description": "LLM benchmark featuring an average data length surpassing 100K tokens. Comprises synthetic and realistic tasks spanning diverse domains in English and Chinese.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.13718", "primary": true }, { "kind": "paper", "label": "Paper: ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "url": "https://arxiv.org/abs/2402.13718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Xinrongzhang2022", "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.13718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml", "family_title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "group": "Reasoning", "version": "2-A", "contributors": [ "celiawaggoner" ], "tags": [ "Reasoning" ], "task": { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false }, "tasks": [ { "name": "infinite_bench_code_debug", "dataset_samples": 394, "comment": null, "support_only": false }, { "name": "infinite_bench_code_run", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "infinite_bench_kv_retrieval", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "infinite_bench_longbook_choice_eng", "dataset_samples": 229, "comment": null, "support_only": false }, { "name": "infinite_bench_longdialogue_qa_eng", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "infinite_bench_math_calc", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "infinite_bench_math_find", "dataset_samples": 350, "comment": null, "support_only": false }, { "name": "infinite_bench_number_string", "dataset_samples": 590, "comment": null, "support_only": false }, { "name": "infinite_bench_passkey", "dataset_samples": 590, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 590, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "xinrongzhang2022/InfiniteBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:infinite_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al.", "organization": "Tsinghua University", "organizations": [ "Tsinghua University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.13718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.13718", "arxiv_id": "2402.13718", "title": "∞Bench: Extending Long Context Evaluation Beyond 100K Tokens", "authors": [ "Xinrong Zhang", "Yingfa Chen", "Shengding Hu", "Zihang Xu", "Junhao Chen", "Moo Khai Hao", "Xu Han", "Zhen Leng Thai", "Shuo Wang", "Zhiyuan Liu", "Maosong Sun" ], "organizations": [ "Tsinghua University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/infinite_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench", "provider": "Xinrongzhang2022" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.13718" }, "code_url": null, "metadata": { "source_search": "Xinrong Zhang, Yingfa Chen, Shengding Hu et al. Tsinghua University Original paper authors ∞Bench: Extending Long Context Evaluation Beyond 100K Tokens Xinrong Zhang Yingfa Chen Shengding Hu Zihang Xu Junhao Chen Moo Khai Hao Xu Han Zhen Leng Thai Shuo Wang Zhiyuan Liu Maosong Sun evaluation-family" } }, { "id": "canonical:inspect-ai", "name": "Inspect AI", "slug": "canonical--inspect-ai", "organisation": "UK AI Security Institute", "source_type": "canonical-source", "source_url": "https://inspect.aisi.org.uk/", "topics": [ "evaluation-integrity" ], "description": "Open-source framework for model evaluations including agents, tools and sandboxes.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "framework_developer", "label": "Framework developed by", "url": "https://inspect.aisi.org.uk", "primary": true }, { "kind": "official", "label": "Official or project page", "url": "https://inspect.aisi.org.uk", "primary": false } ], "resource_count": 2, "source_key": "canonical:inspect-ai", "record_type": "evaluation-framework", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "UK AI Security Institute", "organization": "UK AI Security Institute", "organizations": [ "UK AI Security Institute" ], "entity_type": "organization", "role": "framework_developer", "role_label": "Framework developed by", "confidence": "curated", "evidence_url": "https://inspect.aisi.org.uk" }, "paper": { "status": "not-applicable", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-applicable", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Framework developed by", "attribution_status": "verified", "preferred_source": { "label": "Framework developed by", "kind": "framework_developer", "url": "https://inspect.aisi.org.uk" }, "paper_url": null, "code_url": null, "metadata": { "source_search": "UK AI Security Institute UK AI Security Institute Framework developed by evaluation-framework" } }, { "id": "register:inspect-india-bharatbbq", "name": "BharatBBQ", "slug": "register--inspect-india-bharatbbq", "organisation": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "source_type": "inspect-register", "source_url": "https://github.com/MetaFazer/inspect-india-evals", "topics": [ "general-capability" ], "description": "Evaluates social bias in LLM responses within the Indian context, part of the Inspect India Evals suite for assessing model performance in the Indian context.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-bharatbbq/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2607.25375", "primary": true }, { "kind": "paper", "label": "Paper: Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "url": "https://arxiv.org/abs/2607.25375", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/MetaFazer/inspect-india-evals", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-bharatbbq/eval.yaml", "primary": false } ], "paper_title": "Inspect India Evals: A Benchmark for Indian Language Safety, DPI, and Cultural Knowledge", "paper_url": "https://arxiv.org/abs/2607.25375", "code_url": "https://github.com/MetaFazer/inspect-india-evals", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-bharatbbq/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "MetaFazer", "ShreyNag" ], "tasks": [ { "name": "bharatbbq", "path": "india_evals/bias/task.py" } ], "protocol": { "implementation_commit": "9c2e0bd9d8089444751ede080367dcc4983b9dc2", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:inspect-india-bharatbbq", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2607.25375" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/MetaFazer/inspect-india-evals", "owner": "MetaFazer", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-bharatbbq/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2607.25375" }, "metadata": { "source_search": "Abhishek Kumar Singh, Shrey Nag, Sachita et al. Original paper authors Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context Abhishek Kumar Singh Shrey Nag Sachita Lipi Goel Rajeshwar Singh Janwar MetaFazer evaluation-package" } }, { "id": "register:inspect-india-cultural_knowledge", "name": "India Cultural Knowledge", "slug": "register--inspect-india-cultural-knowledge", "organisation": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "source_type": "inspect-register", "source_url": "https://github.com/MetaFazer/inspect-india-evals", "topics": [ "general-capability" ], "description": "Evaluates LLM knowledge of Indian culture, history, and context, part of the Inspect India Evals suite for assessing model performance in the Indian context.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-cultural_knowledge/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2607.25375", "primary": true }, { "kind": "paper", "label": "Paper: Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "url": "https://arxiv.org/abs/2607.25375", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/MetaFazer/inspect-india-evals", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-cultural_knowledge/eval.yaml", "primary": false } ], "paper_title": "Inspect India Evals: A Benchmark for Indian Language Safety, DPI, and Cultural Knowledge", "paper_url": "https://arxiv.org/abs/2607.25375", "code_url": "https://github.com/MetaFazer/inspect-india-evals", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-cultural_knowledge/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "MetaFazer", "ShreyNag" ], "tasks": [ { "name": "cultural_knowledge", "path": "india_evals/cultural_knowledge/task.py" } ], "protocol": { "implementation_commit": "9c2e0bd9d8089444751ede080367dcc4983b9dc2", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:inspect-india-cultural_knowledge", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2607.25375" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/MetaFazer/inspect-india-evals", "owner": "MetaFazer", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-cultural_knowledge/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2607.25375" }, "metadata": { "source_search": "Abhishek Kumar Singh, Shrey Nag, Sachita et al. Original paper authors Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context Abhishek Kumar Singh Shrey Nag Sachita Lipi Goel Rajeshwar Singh Janwar MetaFazer evaluation-package" } }, { "id": "register:inspect-india-dpi_safety", "name": "India DPI Safety", "slug": "register--inspect-india-dpi-safety", "organisation": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "source_type": "inspect-register", "source_url": "https://github.com/MetaFazer/inspect-india-evals", "topics": [ "safeguards" ], "description": "Evaluates LLM compliance with Digital Public Infrastructure (DPI) safety expectations in the Indian context, part of the Inspect India Evals suite for assessing model performance in the Indian context.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-dpi_safety/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2607.25375", "primary": true }, { "kind": "paper", "label": "Paper: Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "url": "https://arxiv.org/abs/2607.25375", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/MetaFazer/inspect-india-evals", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-dpi_safety/eval.yaml", "primary": false } ], "paper_title": "Inspect India Evals: A Benchmark for Indian Language Safety, DPI, and Cultural Knowledge", "paper_url": "https://arxiv.org/abs/2607.25375", "code_url": "https://github.com/MetaFazer/inspect-india-evals", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-dpi_safety/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "MetaFazer", "ShreyNag" ], "tasks": [ { "name": "dpi_safety", "path": "india_evals/dpi_safety/task.py" } ], "protocol": { "implementation_commit": "9c2e0bd9d8089444751ede080367dcc4983b9dc2", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:inspect-india-dpi_safety", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2607.25375" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/MetaFazer/inspect-india-evals", "owner": "MetaFazer", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-dpi_safety/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2607.25375" }, "metadata": { "source_search": "Abhishek Kumar Singh, Shrey Nag, Sachita et al. Original paper authors Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context Abhishek Kumar Singh Shrey Nag Sachita Lipi Goel Rajeshwar Singh Janwar MetaFazer evaluation-package" } }, { "id": "register:inspect-india-jailbreak_safety", "name": "India Jailbreak Safety", "slug": "register--inspect-india-jailbreak-safety", "organisation": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "source_type": "inspect-register", "source_url": "https://github.com/MetaFazer/inspect-india-evals", "topics": [ "safeguards" ], "description": "Evaluates LLM robustness to jailbreak attempts within the Indian context, part of the Inspect India Evals suite for assessing model performance in the Indian context.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-jailbreak_safety/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2607.25375", "primary": true }, { "kind": "paper", "label": "Paper: Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "url": "https://arxiv.org/abs/2607.25375", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/MetaFazer/inspect-india-evals", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-jailbreak_safety/eval.yaml", "primary": false } ], "paper_title": "Inspect India Evals: A Benchmark for Indian Language Safety, DPI, and Cultural Knowledge", "paper_url": "https://arxiv.org/abs/2607.25375", "code_url": "https://github.com/MetaFazer/inspect-india-evals", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-jailbreak_safety/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "MetaFazer", "ShreyNag" ], "tasks": [ { "name": "jailbreak_safety", "path": "india_evals/safeguards/task.py" } ], "protocol": { "implementation_commit": "9c2e0bd9d8089444751ede080367dcc4983b9dc2", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:inspect-india-jailbreak_safety", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2607.25375" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/MetaFazer/inspect-india-evals", "owner": "MetaFazer", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-jailbreak_safety/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2607.25375" }, "metadata": { "source_search": "Abhishek Kumar Singh, Shrey Nag, Sachita et al. Original paper authors Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context Abhishek Kumar Singh Shrey Nag Sachita Lipi Goel Rajeshwar Singh Janwar MetaFazer evaluation-package" } }, { "id": "register:inspect-india-multilingual", "name": "India Multilingual", "slug": "register--inspect-india-multilingual", "organisation": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "source_type": "inspect-register", "source_url": "https://github.com/MetaFazer/inspect-india-evals", "topics": [ "general-capability" ], "description": "Evaluates LLM reasoning across major Indian languages, part of the Inspect India Evals suite for assessing model performance in the Indian context.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-multilingual/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2607.25375", "primary": true }, { "kind": "paper", "label": "Paper: Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "url": "https://arxiv.org/abs/2607.25375", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/MetaFazer/inspect-india-evals", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-multilingual/eval.yaml", "primary": false } ], "paper_title": "Inspect India Evals: A Benchmark for Indian Language Safety, DPI, and Cultural Knowledge", "paper_url": "https://arxiv.org/abs/2607.25375", "code_url": "https://github.com/MetaFazer/inspect-india-evals", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-multilingual/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "MetaFazer", "ShreyNag" ], "tasks": [ { "name": "multilingual", "path": "india_evals/multilingual/task.py" } ], "protocol": { "implementation_commit": "9c2e0bd9d8089444751ede080367dcc4983b9dc2", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:inspect-india-multilingual", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2607.25375" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/MetaFazer/inspect-india-evals", "owner": "MetaFazer", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-multilingual/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2607.25375" }, "metadata": { "source_search": "Abhishek Kumar Singh, Shrey Nag, Sachita et al. Original paper authors Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context Abhishek Kumar Singh Shrey Nag Sachita Lipi Goel Rajeshwar Singh Janwar MetaFazer evaluation-package" } }, { "id": "register:inspect-india-multilingual_safety", "name": "India Multilingual Safety", "slug": "register--inspect-india-multilingual-safety", "organisation": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "source_type": "inspect-register", "source_url": "https://github.com/MetaFazer/inspect-india-evals", "topics": [ "safeguards" ], "description": "Evaluates LLM safety refusal behaviour on harmful prompts across Indian languages, part of the Inspect India Evals suite for assessing model performance in the Indian context.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-multilingual_safety/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2607.25375", "primary": true }, { "kind": "paper", "label": "Paper: Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "url": "https://arxiv.org/abs/2607.25375", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/MetaFazer/inspect-india-evals", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-multilingual_safety/eval.yaml", "primary": false } ], "paper_title": "Inspect India Evals: A Benchmark for Indian Language Safety, DPI, and Cultural Knowledge", "paper_url": "https://arxiv.org/abs/2607.25375", "code_url": "https://github.com/MetaFazer/inspect-india-evals", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-multilingual_safety/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "MetaFazer", "ShreyNag" ], "tasks": [ { "name": "multilingual_safety", "path": "india_evals/safeguards/task.py" } ], "protocol": { "implementation_commit": "9c2e0bd9d8089444751ede080367dcc4983b9dc2", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:inspect-india-multilingual_safety", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Abhishek Kumar Singh, Shrey Nag, Sachita et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2607.25375" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2607.25375", "arxiv_id": "2607.25375", "title": "Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context", "authors": [ "Abhishek Kumar Singh", "Shrey Nag", "Sachita", "Lipi Goel", "Rajeshwar Singh Janwar" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/MetaFazer/inspect-india-evals", "owner": "MetaFazer", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/inspect-india-multilingual_safety/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2607.25375" }, "metadata": { "source_search": "Abhishek Kumar Singh, Shrey Nag, Sachita et al. Original paper authors Inspect India Evals: An Open Benchmarking Framework for Evaluating Large Language Models in the Indian Linguistic and Cultural Context Abhishek Kumar Singh Shrey Nag Sachita Lipi Goel Rajeshwar Singh Janwar MetaFazer evaluation-package" } }, { "id": "inspect:instrumentaleval", "name": "InstrumentalEval - Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "slug": "inspect--instrumentaleval", "organisation": "Yufei He, Yuexin Li, Jiaying Wu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/instrumentaleval", "topics": [ "deception-misalignment" ], "description": "An evaluation designed to detect instrumental convergence behaviors in model responses (e.g., self-preservation, resource acquisition, power-seeking, strategic deception) using a rubric-driven LLM grader. The benchmark tests whether AI systems exhibit behaviors that are instrumentally useful across a wide range of goals, which could indicate concerning patterns of strategic reasoning.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/instrumentaleval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2502.12206", "primary": true }, { "kind": "paper", "label": "Paper: Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "url": "https://arxiv.org/abs/2502.12206", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/yf-he/InstrumentalEval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/instrumentaleval/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2502.12206", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/instrumentaleval/eval.yaml", "family_title": "InstrumentalEval - Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "group": "Scheming", "version": "2-A", "contributors": [ "horvgbor" ], "tags": [ "Scheming" ], "task": { "name": "instrumentaleval", "dataset_samples": 76, "comment": null, "support_only": false }, "tasks": [ { "name": "instrumentaleval", "dataset_samples": 76, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 76, "dependency": null, "requires_internet": true, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/yf-he/InstrumentalEval/{SHA}/benchmark/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:instrumentaleval", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yufei He, Yuexin Li, Jiaying Wu et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2502.12206" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2502.12206", "arxiv_id": "2502.12206", "title": "Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "authors": [ "Yufei He", "Yuexin Li", "Jiaying Wu", "Yuan Sui", "Yulin Chen", "Bryan Hooi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2502.12206", "arxiv_id": "2502.12206", "title": "Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "authors": [ "Yufei He", "Yuexin Li", "Jiaying Wu", "Yuan Sui", "Yulin Chen", "Bryan Hooi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/yf-he/InstrumentalEval", "owner": "Yf He", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/instrumentaleval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2502.12206" }, "code_url": "https://github.com/yf-he/InstrumentalEval", "metadata": { "source_search": "Yufei He, Yuexin Li, Jiaying Wu et al. Original paper authors Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals? Yufei He Yuexin Li Jiaying Wu Yuan Sui Yulin Chen Bryan Hooi Yf He evaluation-task" } }, { "id": "inspect:ipi_coding_agent", "name": "CodeIPI: Indirect Prompt Injection for Coding Agents", "slug": "inspect--ipi-coding-agent", "organisation": "Debu Sinha", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ipi_coding_agent", "topics": [ "autonomy-agents", "cyber" ], "description": "Measures coding agent vulnerability to indirect prompt injection attacks embedded in software engineering artifacts (issue descriptions, code comments, README files). Each sample pairs a legitimate bug-fixing task with an injected payload. Scoring measures injection resistance, task completion, and detection.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ipi_coding_agent/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Benchmark contributed by", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/src/inspect_evals/ipi_coding_agent/README.md", "primary": true }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ipi_coding_agent/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/src/inspect_evals/ipi_coding_agent/README.md", "primary": false } ], "resource_count": 3, "paper_url": null, "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ipi_coding_agent/eval.yaml", "family_title": "CodeIPI: Indirect Prompt Injection for Coding Agents", "group": "Safeguards", "version": "3-B", "contributors": [ "debu-sinha" ], "tags": [ "safety", "agents", "prompt-injection", "coding", "Safeguards" ], "task": { "name": "ipi_coding_agent", "dataset_samples": 45, "comment": null, "support_only": false }, "tasks": [ { "name": "ipi_coding_agent", "dataset_samples": 45, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 45, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:ipi_coding_agent", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "source-derived", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "CodeIPI is documented as an Inspect-native contribution by Debu Sinha. No separate project repository or dedicated paper was identified in the reviewed primary sources; Inspect remains the maintained implementation host." }, "origin": { "name": "Debu Sinha", "organization": null, "entity_type": "individual_contributor", "role": "evaluation_developer", "role_label": "Benchmark contributed by", "confidence": "source-derived", "evidence_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/src/inspect_evals/ipi_coding_agent/README.md", "organizations": [] }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/ipi_coding_agent/eval.yaml" }, "datasets": [], "organisation_role": "Benchmark contributed by", "attribution_status": "source-derived", "preferred_source": { "label": "Benchmark contributed by", "kind": "evaluation_developer", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/src/inspect_evals/ipi_coding_agent/README.md" }, "code_url": null, "metadata": { "source_search": "Debu Sinha Benchmark contributed by evaluation-task" } }, { "id": "canonical:jailbreakbench", "name": "JailbreakBench", "slug": "canonical--jailbreakbench", "organisation": "JailbreakBench project and paper authors", "source_type": "canonical-source", "source_url": "https://jailbreakbench.github.io/", "topics": [ "safeguards", "evaluation-integrity" ], "description": "Open benchmark for evaluating jailbreak attacks and model defences.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Robustness to the included jailbreak artifacts under the exact target-model and evaluation configuration.", "does_not_measure": "Universal jailbreak resistance, adaptive adversary performance, agentic misuse, operational incident rates, or a complete safety case.", "best_for": "Reproducible attack and defense comparison and tracking known jailbreak classes.", "not_sufficient_for": "Certifying that a model or product is safe from jailbreaks in deployment.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://jailbreakbench.github.io", "primary": true }, { "kind": "paper", "label": "Paper: JailbreakBench: An Open Robustness Benchmark for Jailbreaking Large Language Models", "url": "https://arxiv.org/abs/2404.01318", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/JailbreakBench/jailbreakbench", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://jailbreakbench.github.io", "primary": false } ], "resource_count": 4, "source_key": "canonical:jailbreakbench", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "JailbreakBench project and paper authors", "organization": null, "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://jailbreakbench.github.io", "organizations": [], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.01318", "arxiv_id": "2404.01318", "title": "JailbreakBench: An Open Robustness Benchmark for Jailbreaking Large Language Models", "authors": [ "Patrick Chao", "Edoardo Debenedetti", "Alexander Robey", "Maksym Andriushchenko", "Francesco Croce", "Vikash Sehwag", "Edgar Dobriban", "Nicolas Flammarion", "George J. Pappas", "Florian Tramer", "Hamed Hassani", "Eric Wong" ], "organizations": [ "ETH Zurich", "University of Pennsylvania", "Princeton University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.01318", "arxiv_id": "2404.01318", "title": "JailbreakBench: An Open Robustness Benchmark for Jailbreaking Large Language Models", "authors": [ "Patrick Chao", "Edoardo Debenedetti", "Alexander Robey", "Maksym Andriushchenko", "Francesco Croce", "Vikash Sehwag", "Edgar Dobriban", "Nicolas Flammarion", "George J. Pappas", "Florian Tramer", "Hamed Hassani", "Eric Wong" ], "organizations": [ "ETH Zurich", "University of Pennsylvania", "Princeton University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/JailbreakBench/jailbreakbench", "owner": "JailbreakBench project", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://jailbreakbench.github.io" }, "paper_url": "https://arxiv.org/abs/2404.01318", "code_url": "https://github.com/JailbreakBench/jailbreakbench", "metadata": { "source_search": "JailbreakBench project and paper authors Evaluation developed by JailbreakBench: An Open Robustness Benchmark for Jailbreaking Large Language Models Patrick Chao Edoardo Debenedetti Alexander Robey Maksym Andriushchenko Francesco Croce Vikash Sehwag Edgar Dobriban Nicolas Flammarion George J. Pappas Florian Tramer Hamed Hassani Eric Wong JailbreakBench project evaluation-resource" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Attack success and defense robustness for a standardized set of jailbreak artifacts and prohibited behaviors.", "unit_of_analysis": "One attack prompt or jailbreak method evaluated against one target model and behavior.", "interaction_type": "Adversarial prompting under benchmark-defined attack, target, and refusal-classification settings.", "target_outcome": "Whether the target produces a behavior judged to satisfy the prohibited request, alongside benign or utility measures where reported.", "human_participants": "No human subjects. Human annotation and benchmark curation inform artifacts and evaluation labels.", "elicitation": "Attacks are applied under specified model versions, system prompts, decoding settings, and defense configurations.", "scoring": "Benchmark-defined classifiers or judges estimate attack success and defense performance on standardized behaviors.", "protocol_dependence": "Very high: behavior set, attack budget, system prompt, decoding, target endpoint, and judge determine results.", "implementation_dependence": "High: model API updates, classifier or judge errors, and attack implementation details alter success rates.", "limitations": [ "Published attacks can become stale as providers change systems.", "Automated harmfulness and refusal judges can misclassify nuanced outputs.", "A finite attack set cannot characterize an adaptive threat landscape.", "Results may not transfer across model snapshots, access routes, or products." ], "comparability": "Require the same benchmark release, behaviors, attacks, target model snapshot, system prompt, generation settings, and judge.", "decision_relevance": "Useful for red-team regression testing and defense comparison; deployment decisions require adaptive testing, monitoring, and product-specific threat models.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2404.01318" }, { "kind": "official", "url": "https://jailbreakbench.github.io/" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:kernelbench", "name": "KernelBench: Can LLMs Write Efficient GPU Kernels?", "slug": "inspect--kernelbench", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/kernelbench", "topics": [ "general-capability" ], "description": "A benchmark for evaluating the ability of LLMs to write efficient GPU kernels.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/kernelbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/html/2502.10517v1", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://arxiv.org/html/2502.10517v1", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://astral.sh/uv/0.9.9/install.sh", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/kernelbench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: ScalingIntelligence", "url": "https://huggingface.co/datasets/ScalingIntelligence/KernelBench", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/html/2502.10517v1", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/kernelbench/eval.yaml", "family_title": "KernelBench: Can LLMs Write Efficient GPU Kernels?", "group": "Coding", "version": "5-B", "contributors": [ "jiito" ], "tags": [ "Coding" ], "task": { "name": "kernelbench", "dataset_samples": 250, "comment": null, "support_only": false }, "tasks": [ { "name": "kernelbench", "dataset_samples": 250, "comment": null, "support_only": false } ], "protocol": { "version": "5-B", "dataset_samples": 250, "dependency": null, "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "direct_url", "source": "https://astral.sh/uv/0.9.9/install.sh", "fetch_method": "curl", "state": "pinned", "comment": null }, { "type": "git_dependency", "source": "https://github.com/ScalingIntelligence/KernelBench", "fetch_method": "pyproject_toml", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "ScalingIntelligence/KernelBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:kernelbench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://arxiv.org/html/2502.10517v1" }, "paper": { "status": "present", "url": "https://arxiv.org/html/2502.10517v1", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://arxiv.org/html/2502.10517v1", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": { "url": "https://astral.sh/uv/0.9.9/install.sh", "owner": "astral.sh", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/kernelbench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/ScalingIntelligence/KernelBench", "provider": "ScalingIntelligence" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/html/2502.10517v1" }, "code_url": "https://astral.sh/uv/0.9.9/install.sh", "metadata": { "source_search": "Associated paper authors Original paper authors astral.sh evaluation-task" } }, { "id": "register:lab_bench_2", "name": "LAB-Bench 2", "slug": "register--lab-bench-2", "organisation": "Jon M Laurent, Albert Bou, Michael Pieler et al.", "source_type": "inspect-register", "source_url": "https://github.com/Generality-Labs/lab-bench", "topics": [ "bio-cbrn" ], "description": "LAB-Bench 2 evaluates language models and research agents on life-science tasks spanning literature reasoning, database access, figures, tables, protocols, source quality, sequence analysis, cloning, patents, and clinical trials. A single parameterized task selects one or more dataset tags and a file-delivery mode, scoring each subset with an LLM judge or a deterministic validator, and can be run with a bare, server-side tools, or sandboxed agentic solver.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/lab_bench_2/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2604.09554", "primary": true }, { "kind": "paper", "label": "Paper: LABBench2: An Improved Benchmark for AI Systems Performing Biology Research", "url": "https://arxiv.org/abs/2604.09554", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Generality-Labs/lab-bench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/lab_bench_2/eval.yaml", "primary": false } ], "paper_title": null, "paper_url": "https://arxiv.org/abs/2604.09554", "code_url": "https://github.com/Generality-Labs/lab-bench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/lab_bench_2/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "Reasoning", "Biology" ], "contributors": [ "iphan", "ItsTania", "lewtun", "Sunishchal" ], "tasks": [ { "name": "lab_bench_2", "path": "src/lab_bench_2/lab_bench_2.py" } ], "protocol": { "implementation_commit": "081864af494b180ecf6aae3f7333e384c0d227af", "report_commit": "25113ef24681be2343a7209f5461801bb73ba238", "report_timestamp": "June 2026", "command": "uv run inspect eval lab_bench_2 -T tags= -T mode= -T solver= --model openai/gpt-5.2", "source_comment": null }, "reported_results": [ { "model": "openai/gpt-5.2 (litqa3 · tools)", "provider": "OpenAI", "metrics": [ { "key": "paper_acc", "value": 0.815 }, { "key": "accuracy", "value": 0.799 }, { "key": "stderr", "value": 0.032 } ] }, { "model": "openai/gpt-5.2 (dbqa2 · bare)", "provider": "OpenAI", "metrics": [ { "key": "paper_acc", "value": 0.07 }, { "key": "accuracy", "value": 0.058 }, { "key": "stderr", "value": 0.025 } ] }, { "model": "openai/gpt-5.2 (cloning · inject · tools)", "provider": "OpenAI", "metrics": [ { "key": "paper_acc", "value": 0.286 }, { "key": "accuracy", "value": 0.286 }, { "key": "stderr", "value": 0.125 } ] }, { "model": "openai/gpt-5.2 (figqa2-img · file · bare)", "provider": "OpenAI", "metrics": [ { "key": "paper_acc", "value": 0.564 }, { "key": "accuracy", "value": 0.525 }, { "key": "stderr", "value": 0.05 } ] }, { "model": "openai/gpt-5.2 (seqqa2 · retrieve · bare)", "provider": "OpenAI", "metrics": [ { "key": "paper_acc", "value": 0.095 }, { "key": "accuracy", "value": 0.115 }, { "key": "stderr", "value": 0.023 } ] }, { "model": "openai/gpt-5.2 (protocolqa2 · file · agentic)", "provider": "OpenAI", "metrics": [ { "key": "paper_acc", "value": 0.416 }, { "key": "accuracy", "value": 0.447 }, { "key": "stderr", "value": 0.045 } ] } ], "reported_result_count": 6, "reported_results_truncated": false, "report_notes": [ "Model under test: openai/gpt-5.2. LLM-judge tags (litqa3, dbqa2, figqa2-img, protocolqa2) are graded by anthropic/claude-sonnet-4-5; deterministic tags (seqqa2, cloning) use a deterministic scorer.", "The `paper_acc` column is the reference accuracy from the original LAB-Bench 2 report, matched per row to the nearest paper run configuration. Each row runs its full set of mode-compatible samples.", "Grader refusals are excluded as unscored rather than counted wrong: litqa3 accuracy/stderr are over 159 of 168 scored samples; protocolqa2 over 123 of 125. cloning's low score is partly a refusal artifact — the model declined ~40% of the 14-question set on biosecurity grounds.", "Dataset: EdisonScientific/labbench2, split train, pinned to revision 27d12d72af24e3f70db8a99df63e567366cbdb80. @tools,high rows use --reasoning-effort high." ], "models_reported": [ "openai/gpt-5.2 (litqa3 · tools)", "openai/gpt-5.2 (dbqa2 · bare)", "openai/gpt-5.2 (cloning · inject · tools)", "openai/gpt-5.2 (figqa2-img · file · bare)", "openai/gpt-5.2 (seqqa2 · retrieve · bare)", "openai/gpt-5.2 (protocolqa2 · file · agentic)" ], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:lab_bench_2", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jon M Laurent, Albert Bou, Michael Pieler et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2604.09554" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2604.09554", "arxiv_id": "2604.09554", "title": "LABBench2: An Improved Benchmark for AI Systems Performing Biology Research", "authors": [ "Jon M Laurent", "Albert Bou", "Michael Pieler", "Conor Igoe", "Alex Andonian", "Siddharth Narayanan", "James Braza", "Alexandros Sanchez Vassopoulos", "Jacob L Steenwyk", "Blake Lash", "Andrew D White", "Samuel G Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2604.09554", "arxiv_id": "2604.09554", "title": "LABBench2: An Improved Benchmark for AI Systems Performing Biology Research", "authors": [ "Jon M Laurent", "Albert Bou", "Michael Pieler", "Conor Igoe", "Alex Andonian", "Siddharth Narayanan", "James Braza", "Alexandros Sanchez Vassopoulos", "Jacob L Steenwyk", "Blake Lash", "Andrew D White", "Samuel G Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Generality-Labs/lab-bench", "owner": "Generality Labs", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/lab_bench_2/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2604.09554" }, "metadata": { "source_search": "Jon M Laurent, Albert Bou, Michael Pieler et al. Original paper authors LABBench2: An Improved Benchmark for AI Systems Performing Biology Research Jon M Laurent Albert Bou Michael Pieler Conor Igoe Alex Andonian Siddharth Narayanan James Braza Alexandros Sanchez Vassopoulos Jacob L Steenwyk Blake Lash Andrew D White Samuel G Rodriques Generality Labs evaluation-package" } }, { "id": "inspect:lab_bench_cloning_scenarios", "name": "Lab Bench Cloning Scenarios", "slug": "inspect--lab-bench-cloning-scenarios", "organisation": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench", "topics": [ "bio-cbrn" ], "description": "Tests LLMs and LLM-augmented agents abilities to answer questions on scientific research workflows in domains like chemistry, biology, materials science, as well as more general science tasks", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.10362", "primary": true }, { "kind": "paper", "label": "Paper: LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "url": "https://arxiv.org/abs/2407.10362", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FutureHouse", "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.10362", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "family_title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "group": "Safeguards", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Safeguards" ], "task": { "name": "lab_bench_cloning_scenarios", "dataset_samples": 33, "comment": null, "support_only": false }, "tasks": [ { "name": "lab_bench_litqa", "dataset_samples": 199, "comment": null, "support_only": false }, { "name": "lab_bench_suppqa", "dataset_samples": 82, "comment": null, "support_only": false }, { "name": "lab_bench_figqa", "dataset_samples": 181, "comment": null, "support_only": false }, { "name": "lab_bench_tableqa", "dataset_samples": 244, "comment": null, "support_only": false }, { "name": "lab_bench_dbqa", "dataset_samples": 520, "comment": null, "support_only": false }, { "name": "lab_bench_protocolqa", "dataset_samples": 108, "comment": null, "support_only": false }, { "name": "lab_bench_seqqa", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "lab_bench_cloning_scenarios", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 33, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "futurehouse/lab-bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lab_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.10362" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "provider": "FutureHouse" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.10362" }, "code_url": null, "metadata": { "source_search": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al. Original paper authors LAB-Bench: Measuring Capabilities of Language Models for Biology Research Jon M. Laurent Joseph D. Janizek Michael Ruzo Michaela M. Hinks Michael J. Hammerling Siddharth Narayanan Manvitha Ponnapati Andrew D. White Samuel G. Rodriques evaluation-family" } }, { "id": "inspect:lab_bench_dbqa", "name": "Lab Bench Dbqa", "slug": "inspect--lab-bench-dbqa", "organisation": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench", "topics": [ "bio-cbrn" ], "description": "Tests LLMs and LLM-augmented agents abilities to answer questions on scientific research workflows in domains like chemistry, biology, materials science, as well as more general science tasks", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.10362", "primary": true }, { "kind": "paper", "label": "Paper: LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "url": "https://arxiv.org/abs/2407.10362", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FutureHouse", "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.10362", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "family_title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "group": "Safeguards", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Safeguards" ], "task": { "name": "lab_bench_dbqa", "dataset_samples": 520, "comment": null, "support_only": false }, "tasks": [ { "name": "lab_bench_litqa", "dataset_samples": 199, "comment": null, "support_only": false }, { "name": "lab_bench_suppqa", "dataset_samples": 82, "comment": null, "support_only": false }, { "name": "lab_bench_figqa", "dataset_samples": 181, "comment": null, "support_only": false }, { "name": "lab_bench_tableqa", "dataset_samples": 244, "comment": null, "support_only": false }, { "name": "lab_bench_dbqa", "dataset_samples": 520, "comment": null, "support_only": false }, { "name": "lab_bench_protocolqa", "dataset_samples": 108, "comment": null, "support_only": false }, { "name": "lab_bench_seqqa", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "lab_bench_cloning_scenarios", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 520, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "futurehouse/lab-bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lab_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.10362" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "provider": "FutureHouse" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.10362" }, "code_url": null, "metadata": { "source_search": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al. Original paper authors LAB-Bench: Measuring Capabilities of Language Models for Biology Research Jon M. Laurent Joseph D. Janizek Michael Ruzo Michaela M. Hinks Michael J. Hammerling Siddharth Narayanan Manvitha Ponnapati Andrew D. White Samuel G. Rodriques evaluation-family" } }, { "id": "inspect:lab_bench_figqa", "name": "Lab Bench Figqa", "slug": "inspect--lab-bench-figqa", "organisation": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench", "topics": [ "bio-cbrn" ], "description": "Tests LLMs and LLM-augmented agents abilities to answer questions on scientific research workflows in domains like chemistry, biology, materials science, as well as more general science tasks", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.10362", "primary": true }, { "kind": "paper", "label": "Paper: LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "url": "https://arxiv.org/abs/2407.10362", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FutureHouse", "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.10362", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "family_title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "group": "Safeguards", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Safeguards" ], "task": { "name": "lab_bench_figqa", "dataset_samples": 181, "comment": null, "support_only": false }, "tasks": [ { "name": "lab_bench_litqa", "dataset_samples": 199, "comment": null, "support_only": false }, { "name": "lab_bench_suppqa", "dataset_samples": 82, "comment": null, "support_only": false }, { "name": "lab_bench_figqa", "dataset_samples": 181, "comment": null, "support_only": false }, { "name": "lab_bench_tableqa", "dataset_samples": 244, "comment": null, "support_only": false }, { "name": "lab_bench_dbqa", "dataset_samples": 520, "comment": null, "support_only": false }, { "name": "lab_bench_protocolqa", "dataset_samples": 108, "comment": null, "support_only": false }, { "name": "lab_bench_seqqa", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "lab_bench_cloning_scenarios", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 181, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "futurehouse/lab-bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lab_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.10362" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "provider": "FutureHouse" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.10362" }, "code_url": null, "metadata": { "source_search": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al. Original paper authors LAB-Bench: Measuring Capabilities of Language Models for Biology Research Jon M. Laurent Joseph D. Janizek Michael Ruzo Michaela M. Hinks Michael J. Hammerling Siddharth Narayanan Manvitha Ponnapati Andrew D. White Samuel G. Rodriques evaluation-family" } }, { "id": "inspect:lab_bench_litqa", "name": "Lab Bench Litqa", "slug": "inspect--lab-bench-litqa", "organisation": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench", "topics": [ "bio-cbrn" ], "description": "Tests LLMs and LLM-augmented agents abilities to answer questions on scientific research workflows in domains like chemistry, biology, materials science, as well as more general science tasks", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.10362", "primary": true }, { "kind": "paper", "label": "Paper: LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "url": "https://arxiv.org/abs/2407.10362", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FutureHouse", "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.10362", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "family_title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "group": "Safeguards", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Safeguards" ], "task": { "name": "lab_bench_litqa", "dataset_samples": 199, "comment": null, "support_only": false }, "tasks": [ { "name": "lab_bench_litqa", "dataset_samples": 199, "comment": null, "support_only": false }, { "name": "lab_bench_suppqa", "dataset_samples": 82, "comment": null, "support_only": false }, { "name": "lab_bench_figqa", "dataset_samples": 181, "comment": null, "support_only": false }, { "name": "lab_bench_tableqa", "dataset_samples": 244, "comment": null, "support_only": false }, { "name": "lab_bench_dbqa", "dataset_samples": 520, "comment": null, "support_only": false }, { "name": "lab_bench_protocolqa", "dataset_samples": 108, "comment": null, "support_only": false }, { "name": "lab_bench_seqqa", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "lab_bench_cloning_scenarios", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 199, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "futurehouse/lab-bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lab_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.10362" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "provider": "FutureHouse" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.10362" }, "code_url": null, "metadata": { "source_search": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al. Original paper authors LAB-Bench: Measuring Capabilities of Language Models for Biology Research Jon M. Laurent Joseph D. Janizek Michael Ruzo Michaela M. Hinks Michael J. Hammerling Siddharth Narayanan Manvitha Ponnapati Andrew D. White Samuel G. Rodriques evaluation-family" } }, { "id": "inspect:lab_bench_protocolqa", "name": "Lab Bench Protocolqa", "slug": "inspect--lab-bench-protocolqa", "organisation": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench", "topics": [ "bio-cbrn" ], "description": "Tests LLMs and LLM-augmented agents abilities to answer questions on scientific research workflows in domains like chemistry, biology, materials science, as well as more general science tasks", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.10362", "primary": true }, { "kind": "paper", "label": "Paper: LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "url": "https://arxiv.org/abs/2407.10362", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FutureHouse", "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.10362", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "family_title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "group": "Safeguards", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Safeguards" ], "task": { "name": "lab_bench_protocolqa", "dataset_samples": 108, "comment": null, "support_only": false }, "tasks": [ { "name": "lab_bench_litqa", "dataset_samples": 199, "comment": null, "support_only": false }, { "name": "lab_bench_suppqa", "dataset_samples": 82, "comment": null, "support_only": false }, { "name": "lab_bench_figqa", "dataset_samples": 181, "comment": null, "support_only": false }, { "name": "lab_bench_tableqa", "dataset_samples": 244, "comment": null, "support_only": false }, { "name": "lab_bench_dbqa", "dataset_samples": 520, "comment": null, "support_only": false }, { "name": "lab_bench_protocolqa", "dataset_samples": 108, "comment": null, "support_only": false }, { "name": "lab_bench_seqqa", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "lab_bench_cloning_scenarios", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 108, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "futurehouse/lab-bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lab_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.10362" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "provider": "FutureHouse" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.10362" }, "code_url": null, "metadata": { "source_search": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al. Original paper authors LAB-Bench: Measuring Capabilities of Language Models for Biology Research Jon M. Laurent Joseph D. Janizek Michael Ruzo Michaela M. Hinks Michael J. Hammerling Siddharth Narayanan Manvitha Ponnapati Andrew D. White Samuel G. Rodriques evaluation-family" } }, { "id": "inspect:lab_bench_seqqa", "name": "Lab Bench Seqqa", "slug": "inspect--lab-bench-seqqa", "organisation": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench", "topics": [ "bio-cbrn" ], "description": "Tests LLMs and LLM-augmented agents abilities to answer questions on scientific research workflows in domains like chemistry, biology, materials science, as well as more general science tasks", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.10362", "primary": true }, { "kind": "paper", "label": "Paper: LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "url": "https://arxiv.org/abs/2407.10362", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FutureHouse", "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.10362", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "family_title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "group": "Safeguards", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Safeguards" ], "task": { "name": "lab_bench_seqqa", "dataset_samples": 600, "comment": null, "support_only": false }, "tasks": [ { "name": "lab_bench_litqa", "dataset_samples": 199, "comment": null, "support_only": false }, { "name": "lab_bench_suppqa", "dataset_samples": 82, "comment": null, "support_only": false }, { "name": "lab_bench_figqa", "dataset_samples": 181, "comment": null, "support_only": false }, { "name": "lab_bench_tableqa", "dataset_samples": 244, "comment": null, "support_only": false }, { "name": "lab_bench_dbqa", "dataset_samples": 520, "comment": null, "support_only": false }, { "name": "lab_bench_protocolqa", "dataset_samples": 108, "comment": null, "support_only": false }, { "name": "lab_bench_seqqa", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "lab_bench_cloning_scenarios", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 600, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "futurehouse/lab-bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lab_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.10362" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "provider": "FutureHouse" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.10362" }, "code_url": null, "metadata": { "source_search": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al. Original paper authors LAB-Bench: Measuring Capabilities of Language Models for Biology Research Jon M. Laurent Joseph D. Janizek Michael Ruzo Michaela M. Hinks Michael J. Hammerling Siddharth Narayanan Manvitha Ponnapati Andrew D. White Samuel G. Rodriques evaluation-family" } }, { "id": "inspect:lab_bench_suppqa", "name": "Lab Bench Suppqa", "slug": "inspect--lab-bench-suppqa", "organisation": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench", "topics": [ "bio-cbrn" ], "description": "Tests LLMs and LLM-augmented agents abilities to answer questions on scientific research workflows in domains like chemistry, biology, materials science, as well as more general science tasks", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.10362", "primary": true }, { "kind": "paper", "label": "Paper: LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "url": "https://arxiv.org/abs/2407.10362", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FutureHouse", "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.10362", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "family_title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "group": "Safeguards", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Safeguards" ], "task": { "name": "lab_bench_suppqa", "dataset_samples": 82, "comment": null, "support_only": false }, "tasks": [ { "name": "lab_bench_litqa", "dataset_samples": 199, "comment": null, "support_only": false }, { "name": "lab_bench_suppqa", "dataset_samples": 82, "comment": null, "support_only": false }, { "name": "lab_bench_figqa", "dataset_samples": 181, "comment": null, "support_only": false }, { "name": "lab_bench_tableqa", "dataset_samples": 244, "comment": null, "support_only": false }, { "name": "lab_bench_dbqa", "dataset_samples": 520, "comment": null, "support_only": false }, { "name": "lab_bench_protocolqa", "dataset_samples": 108, "comment": null, "support_only": false }, { "name": "lab_bench_seqqa", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "lab_bench_cloning_scenarios", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 82, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "futurehouse/lab-bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lab_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.10362" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "provider": "FutureHouse" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.10362" }, "code_url": null, "metadata": { "source_search": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al. Original paper authors LAB-Bench: Measuring Capabilities of Language Models for Biology Research Jon M. Laurent Joseph D. Janizek Michael Ruzo Michaela M. Hinks Michael J. Hammerling Siddharth Narayanan Manvitha Ponnapati Andrew D. White Samuel G. Rodriques evaluation-family" } }, { "id": "inspect:lab_bench_tableqa", "name": "Lab Bench Tableqa", "slug": "inspect--lab-bench-tableqa", "organisation": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench", "topics": [ "bio-cbrn" ], "description": "Tests LLMs and LLM-augmented agents abilities to answer questions on scientific research workflows in domains like chemistry, biology, materials science, as well as more general science tasks", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.10362", "primary": true }, { "kind": "paper", "label": "Paper: LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "url": "https://arxiv.org/abs/2407.10362", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FutureHouse", "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.10362", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml", "family_title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "group": "Safeguards", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Safeguards" ], "task": { "name": "lab_bench_tableqa", "dataset_samples": 244, "comment": null, "support_only": false }, "tasks": [ { "name": "lab_bench_litqa", "dataset_samples": 199, "comment": null, "support_only": false }, { "name": "lab_bench_suppqa", "dataset_samples": 82, "comment": null, "support_only": false }, { "name": "lab_bench_figqa", "dataset_samples": 181, "comment": null, "support_only": false }, { "name": "lab_bench_tableqa", "dataset_samples": 244, "comment": null, "support_only": false }, { "name": "lab_bench_dbqa", "dataset_samples": 520, "comment": null, "support_only": false }, { "name": "lab_bench_protocolqa", "dataset_samples": 108, "comment": null, "support_only": false }, { "name": "lab_bench_seqqa", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "lab_bench_cloning_scenarios", "dataset_samples": 33, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 244, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "futurehouse/lab-bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lab_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.10362" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.10362", "arxiv_id": "2407.10362", "title": "LAB-Bench: Measuring Capabilities of Language Models for Biology Research", "authors": [ "Jon M. Laurent", "Joseph D. Janizek", "Michael Ruzo", "Michaela M. Hinks", "Michael J. Hammerling", "Siddharth Narayanan", "Manvitha Ponnapati", "Andrew D. White", "Samuel G. Rodriques" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lab_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/futurehouse/lab-bench", "provider": "FutureHouse" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.10362" }, "code_url": null, "metadata": { "source_search": "Jon M. Laurent, Joseph D. Janizek, Michael Ruzo et al. Original paper authors LAB-Bench: Measuring Capabilities of Language Models for Biology Research Jon M. Laurent Joseph D. Janizek Michael Ruzo Michaela M. Hinks Michael J. Hammerling Siddharth Narayanan Manvitha Ponnapati Andrew D. White Samuel G. Rodriques evaluation-family" } }, { "id": "inspect:lingoly", "name": "Lingoly", "slug": "inspect--lingoly", "organisation": "Jude Khouja, Lingyi Yang, Karolina Korgul et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly", "topics": [ "general-capability" ], "description": "Two linguistics reasoning benchmarks: LingOly (Linguistic Olympiad questions) is a benchmark utilising low resource languages. LingOly-TOO (Linguistic Olympiad questions with Templatised Orthographic Obfuscation) is a benchmark designed to counteract answering without reasoning.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2503.02972", "primary": true }, { "kind": "paper", "label": "Paper: LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation", "url": "https://arxiv.org/abs/2503.02972", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Ambean", "url": "https://huggingface.co/datasets/ambean/lingOly", "primary": false }, { "kind": "dataset", "label": "Dataset: Jkhouja", "url": "https://huggingface.co/datasets/jkhouja/LingOly-TOO", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2503.02972", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly/eval.yaml", "family_title": "LingOly", "group": "Reasoning", "version": "2-A", "contributors": [ "am-bean", "jkhouja" ], "tags": [ "Reasoning" ], "task": { "name": "lingoly", "dataset_samples": 408, "comment": null, "support_only": false }, "tasks": [ { "name": "lingoly", "dataset_samples": 408, "comment": null, "support_only": false }, { "name": "lingoly_too", "dataset_samples": 1203, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 408, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "ambean/lingOly", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "jkhouja/LingOly-TOO", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lingoly", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jude Khouja, Lingyi Yang, Karolina Korgul et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2503.02972" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2503.02972", "arxiv_id": "2503.02972", "title": "LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation", "authors": [ "Jude Khouja", "Lingyi Yang", "Karolina Korgul", "Simeon Hellsten", "Vlad A. Neacsu", "Harry Mayne", "Ryan Othniel Kearns", "Andrew M. Bean", "Adam Mahdi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2503.02972", "arxiv_id": "2503.02972", "title": "LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation", "authors": [ "Jude Khouja", "Lingyi Yang", "Karolina Korgul", "Simeon Hellsten", "Vlad A. Neacsu", "Harry Mayne", "Ryan Othniel Kearns", "Andrew M. Bean", "Adam Mahdi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2503.02972", "arxiv_id": "2503.02972", "title": "LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation", "authors": [ "Jude Khouja", "Lingyi Yang", "Karolina Korgul", "Simeon Hellsten", "Vlad A. Neacsu", "Harry Mayne", "Ryan Othniel Kearns", "Andrew M. Bean", "Adam Mahdi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/ambean/lingOly", "provider": "Ambean" }, { "url": "https://huggingface.co/datasets/jkhouja/LingOly-TOO", "provider": "Jkhouja" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2503.02972" }, "code_url": null, "metadata": { "source_search": "Jude Khouja, Lingyi Yang, Karolina Korgul et al. Original paper authors LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation Jude Khouja Lingyi Yang Karolina Korgul Simeon Hellsten Vlad A. Neacsu Harry Mayne Ryan Othniel Kearns Andrew M. Bean Adam Mahdi evaluation-family" } }, { "id": "inspect:lingoly_too", "name": "Lingoly Too", "slug": "inspect--lingoly-too", "organisation": "Jude Khouja, Lingyi Yang, Karolina Korgul et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly", "topics": [ "general-capability" ], "description": "Two linguistics reasoning benchmarks: LingOly (Linguistic Olympiad questions) is a benchmark utilising low resource languages. LingOly-TOO (Linguistic Olympiad questions with Templatised Orthographic Obfuscation) is a benchmark designed to counteract answering without reasoning.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2503.02972", "primary": true }, { "kind": "paper", "label": "Paper: LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation", "url": "https://arxiv.org/abs/2503.02972", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Ambean", "url": "https://huggingface.co/datasets/ambean/lingOly", "primary": false }, { "kind": "dataset", "label": "Dataset: Jkhouja", "url": "https://huggingface.co/datasets/jkhouja/LingOly-TOO", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2503.02972", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly/eval.yaml", "family_title": "LingOly", "group": "Reasoning", "version": "2-A", "contributors": [ "am-bean", "jkhouja" ], "tags": [ "Reasoning" ], "task": { "name": "lingoly_too", "dataset_samples": 1203, "comment": null, "support_only": false }, "tasks": [ { "name": "lingoly", "dataset_samples": 408, "comment": null, "support_only": false }, { "name": "lingoly_too", "dataset_samples": 1203, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1203, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "ambean/lingOly", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "jkhouja/LingOly-TOO", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:lingoly", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jude Khouja, Lingyi Yang, Karolina Korgul et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2503.02972" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2503.02972", "arxiv_id": "2503.02972", "title": "LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation", "authors": [ "Jude Khouja", "Lingyi Yang", "Karolina Korgul", "Simeon Hellsten", "Vlad A. Neacsu", "Harry Mayne", "Ryan Othniel Kearns", "Andrew M. Bean", "Adam Mahdi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2503.02972", "arxiv_id": "2503.02972", "title": "LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation", "authors": [ "Jude Khouja", "Lingyi Yang", "Karolina Korgul", "Simeon Hellsten", "Vlad A. Neacsu", "Harry Mayne", "Ryan Othniel Kearns", "Andrew M. Bean", "Adam Mahdi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2503.02972", "arxiv_id": "2503.02972", "title": "LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation", "authors": [ "Jude Khouja", "Lingyi Yang", "Karolina Korgul", "Simeon Hellsten", "Vlad A. Neacsu", "Harry Mayne", "Ryan Othniel Kearns", "Andrew M. Bean", "Adam Mahdi" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/lingoly/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/ambean/lingOly", "provider": "Ambean" }, { "url": "https://huggingface.co/datasets/jkhouja/LingOly-TOO", "provider": "Jkhouja" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2503.02972" }, "code_url": null, "metadata": { "source_search": "Jude Khouja, Lingyi Yang, Karolina Korgul et al. Original paper authors LINGOLY-TOO: Disentangling Reasoning from Knowledge with Templatised Orthographic Obfuscation Jude Khouja Lingyi Yang Karolina Korgul Simeon Hellsten Vlad A. Neacsu Harry Mayne Ryan Othniel Kearns Andrew M. Bean Adam Mahdi evaluation-family" } }, { "id": "inspect:livebench", "name": "LiveBench: A Challenging, Contamination-Free LLM Benchmark", "slug": "inspect--livebench", "organisation": "Colin White, Samuel Dooley, Manley Roberts et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livebench", "topics": [ "evaluation-integrity" ], "description": "LiveBench is a benchmark designed with test set contamination and objective evaluation in mind by releasing new questions regularly, as well as having questions based on recently-released datasets. Each question has verifiable, objective ground-truth answers, allowing hard questions to be scored accurately and automatically, without the use of an LLM judge.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livebench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2406.19314", "primary": true }, { "kind": "paper", "label": "Paper: LiveBench: A Challenging, Contamination-Limited LLM Benchmark", "url": "https://arxiv.org/abs/2406.19314", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/LiveBench/LiveBench", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livebench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset", "url": "https://huggingface.co/datasets/livebench/%7Bmath,reasoning,coding,language,data_analysis,instruction_following%7D", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2406.19314", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livebench/eval.yaml", "family_title": "LiveBench: A Challenging, Contamination-Free LLM Benchmark", "group": "Knowledge", "version": "3-A", "contributors": [ "anaoaktree" ], "tags": [ "Knowledge" ], "task": { "name": "livebench", "dataset_samples": 910, "comment": null, "support_only": false }, "tasks": [ { "name": "livebench", "dataset_samples": 910, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 910, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "git_dependency", "source": "https://github.com/LiveBench/LiveBench.git", "fetch_method": "pyproject_toml", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "livebench/{math,reasoning,coding,language,data_analysis,instruction_following}", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:livebench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Colin White, Samuel Dooley, Manley Roberts et al.", "organization": "University of Southern California", "organizations": [ "University of Southern California" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2406.19314" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2406.19314", "arxiv_id": "2406.19314", "title": "LiveBench: A Challenging, Contamination-Limited LLM Benchmark", "authors": [ "Colin White", "Samuel Dooley", "Manley Roberts", "Arka Pal", "Ben Feuer", "Siddhartha Jain", "Ravid Shwartz-Ziv", "Neel Jain", "Khalid Saifullah", "Sreemanti Dey", "Shubh-Agrawal", "Sandeep Singh Sandha", "Siddartha Naidu", "Chinmay Hegde", "Yann LeCun", "Tom Goldstein", "Willie Neiswanger", "Micah Goldblum" ], "organizations": [ "University of Southern California" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2406.19314", "arxiv_id": "2406.19314", "title": "LiveBench: A Challenging, Contamination-Limited LLM Benchmark", "authors": [ "Colin White", "Samuel Dooley", "Manley Roberts", "Arka Pal", "Ben Feuer", "Siddhartha Jain", "Ravid Shwartz-Ziv", "Neel Jain", "Khalid Saifullah", "Sreemanti Dey", "Shubh-Agrawal", "Sandeep Singh Sandha", "Siddartha Naidu", "Chinmay Hegde", "Yann LeCun", "Tom Goldstein", "Willie Neiswanger", "Micah Goldblum" ], "organizations": [ "University of Southern California" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/LiveBench/LiveBench", "owner": "LiveBench", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livebench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/livebench/{math,reasoning,coding,language,data_analysis,instruction_following}", "provider": null } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2406.19314" }, "code_url": "https://github.com/LiveBench/LiveBench", "metadata": { "source_search": "Colin White, Samuel Dooley, Manley Roberts et al. University of Southern California Original paper authors LiveBench: A Challenging, Contamination-Limited LLM Benchmark Colin White Samuel Dooley Manley Roberts Arka Pal Ben Feuer Siddhartha Jain Ravid Shwartz-Ziv Neel Jain Khalid Saifullah Sreemanti Dey Shubh-Agrawal Sandeep Singh Sandha Siddartha Naidu Chinmay Hegde Yann LeCun Tom Goldstein Willie Neiswanger Micah Goldblum LiveBench evaluation-task" } }, { "id": "inspect:livecodebench_pro", "name": "LiveCodeBench-Pro: Competitive Programming Benchmark", "slug": "inspect--livecodebench-pro", "organisation": "Zihan Zheng, Zerui Cheng, Zeyu Shen et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livecodebench_pro", "topics": [ "general-capability" ], "description": "Evaluates LLMs on competitive programming problems using a specialized Docker sandbox (LightCPVerifier) to execute and judge C++ code submissions against hidden test cases with time and memory constraints.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livecodebench_pro/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2506.11928", "primary": true }, { "kind": "paper", "label": "Paper: LiveCodeBench Pro: How Do Olympiad Medalists Judge LLMs in Competitive Programming?", "url": "https://arxiv.org/abs/2506.11928", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livecodebench_pro/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: QAQAQAQAQ", "url": "https://huggingface.co/datasets/QAQAQAQAQ/LiveCodeBench-Pro", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2506.11928", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livecodebench_pro/eval.yaml", "family_title": "LiveCodeBench-Pro: Competitive Programming Benchmark", "group": "Coding", "version": "2-A", "contributors": [ "gjoshi2424" ], "tags": [ "Coding" ], "task": { "name": "livecodebench_pro", "dataset_samples": 1404, "comment": null, "support_only": false }, "tasks": [ { "name": "livecodebench_pro", "dataset_samples": 1404, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1404, "dependency": null, "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "QAQAQAQAQ/LiveCodeBench-Pro", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:livecodebench_pro", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zihan Zheng, Zerui Cheng, Zeyu Shen et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2506.11928" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.11928", "arxiv_id": "2506.11928", "title": "LiveCodeBench Pro: How Do Olympiad Medalists Judge LLMs in Competitive Programming?", "authors": [ "Zihan Zheng", "Zerui Cheng", "Zeyu Shen", "Shang Zhou", "Kaiyuan Liu", "Hansen He", "Dongruixuan Li", "Stanley Wei", "Hangyi Hao", "Jianzhu Yao", "Peiyao Sheng", "Zixuan Wang", "Wenhao Chai", "Aleksandra Korolova", "Peter Henderson", "Sanjeev Arora", "Pramod Viswanath", "Jingbo Shang", "Saining Xie" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.11928", "arxiv_id": "2506.11928", "title": "LiveCodeBench Pro: How Do Olympiad Medalists Judge LLMs in Competitive Programming?", "authors": [ "Zihan Zheng", "Zerui Cheng", "Zeyu Shen", "Shang Zhou", "Kaiyuan Liu", "Hansen He", "Dongruixuan Li", "Stanley Wei", "Hangyi Hao", "Jianzhu Yao", "Peiyao Sheng", "Zixuan Wang", "Wenhao Chai", "Aleksandra Korolova", "Peter Henderson", "Sanjeev Arora", "Pramod Viswanath", "Jingbo Shang", "Saining Xie" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/livecodebench_pro/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/QAQAQAQAQ/LiveCodeBench-Pro", "provider": "QAQAQAQAQ" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2506.11928" }, "code_url": null, "metadata": { "source_search": "Zihan Zheng, Zerui Cheng, Zeyu Shen et al. Original paper authors LiveCodeBench Pro: How Do Olympiad Medalists Judge LLMs in Competitive Programming? Zihan Zheng Zerui Cheng Zeyu Shen Shang Zhou Kaiyuan Liu Hansen He Dongruixuan Li Stanley Wei Hangyi Hao Jianzhu Yao Peiyao Sheng Zixuan Wang Wenhao Chai Aleksandra Korolova Peter Henderson Sanjeev Arora Pramod Viswanath Jingbo Shang Saining Xie evaluation-task" } }, { "id": "canonical:lm-evaluation-harness", "name": "lm-evaluation-harness", "slug": "canonical--lm-evaluation-harness", "organisation": "EleutherAI", "source_type": "canonical-source", "source_url": "https://github.com/EleutherAI/lm-evaluation-harness", "topics": [ "evaluation-integrity", "general-capability" ], "description": "Widely used evaluation harness covering a broad collection of language-model tasks.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "framework_developer", "label": "Framework developed by", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/EleutherAI/lm-evaluation-harness", "primary": false } ], "resource_count": 3, "source_key": "canonical:lm-evaluation-harness", "record_type": "evaluation-framework", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "EleutherAI", "organization": "EleutherAI", "organizations": [ "EleutherAI" ], "entity_type": "organization", "role": "framework_developer", "role_label": "Framework developed by", "confidence": "curated", "evidence_url": "https://github.com/EleutherAI/lm-evaluation-harness" }, "paper": { "status": "not-applicable", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-applicable", "reference_implementation": { "url": "https://github.com/EleutherAI/lm-evaluation-harness", "owner": "EleutherAI", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Framework developed by", "attribution_status": "verified", "preferred_source": { "label": "Framework developed by", "kind": "framework_developer", "url": "https://github.com/EleutherAI/lm-evaluation-harness" }, "paper_url": null, "code_url": "https://github.com/EleutherAI/lm-evaluation-harness", "metadata": { "source_search": "EleutherAI EleutherAI Framework developed by EleutherAI evaluation-framework" } }, { "id": "inspect:macbench", "name": "MaCBench: Probing the limitations of multimodal language models for chemistry and materials research", "slug": "inspect--macbench", "organisation": "Nawaf Alampara, Mara Schilling-Wilhelmi, Martiño Ríos-García et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/macbench", "topics": [ "general-capability" ], "description": "MaCBench is a comprehensive benchmark for evaluating how vision-language models handle real-world chemistry and materials science tasks across three core aspects: data extraction, experimental understanding, and results interpretation. The dataset comprises over 1100 high-quality multimodal questions manually curated by chemistry and materials experts across 34 subsets.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/macbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2411.16955", "primary": true }, { "kind": "paper", "label": "Paper: Probing the limitations of multimodal language models for chemistry and materials research", "url": "https://arxiv.org/abs/2411.16955", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/macbench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Jablonkagroup", "url": "https://huggingface.co/datasets/jablonkagroup/MaCBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2411.16955", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/macbench/eval.yaml", "family_title": "MaCBench: Probing the limitations of multimodal language models for chemistry and materials research", "group": "Knowledge", "version": "1-A", "contributors": [ "r-fedorov", "MrtinoRG" ], "tags": [ "Knowledge" ], "task": { "name": "macbench", "dataset_samples": 1153, "comment": null, "support_only": false }, "tasks": [ { "name": "macbench", "dataset_samples": 1153, "comment": null, "support_only": false } ], "protocol": { "version": "1-A", "dataset_samples": 1153, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "jablonkagroup/MaCBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:macbench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Nawaf Alampara, Mara Schilling-Wilhelmi, Martiño Ríos-García et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2411.16955" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2411.16955", "arxiv_id": "2411.16955", "title": "Probing the limitations of multimodal language models for chemistry and materials research", "authors": [ "Nawaf Alampara", "Mara Schilling-Wilhelmi", "Martiño Ríos-García", "Indrajeet Mandal", "Pranav Khetarpal", "Hargun Singh Grover", "N. M. Anoop Krishnan", "Kevin Maik Jablonka" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2411.16955", "arxiv_id": "2411.16955", "title": "Probing the limitations of multimodal language models for chemistry and materials research", "authors": [ "Nawaf Alampara", "Mara Schilling-Wilhelmi", "Martiño Ríos-García", "Indrajeet Mandal", "Pranav Khetarpal", "Hargun Singh Grover", "N. M. Anoop Krishnan", "Kevin Maik Jablonka" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/macbench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/jablonkagroup/MaCBench", "provider": "Jablonkagroup" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2411.16955" }, "code_url": null, "metadata": { "source_search": "Nawaf Alampara, Mara Schilling-Wilhelmi, Martiño Ríos-García et al. Original paper authors Probing the limitations of multimodal language models for chemistry and materials research Nawaf Alampara Mara Schilling-Wilhelmi Martiño Ríos-García Indrajeet Mandal Pranav Khetarpal Hargun Singh Grover N. M. Anoop Krishnan Kevin Maik Jablonka evaluation-task" } }, { "id": "register:machiavelli", "name": "MACHIAVELLI", "slug": "register--machiavelli", "organisation": "MACHIAVELLI paper authors", "source_type": "inspect-register", "source_url": "https://github.com/Plyb/inspect-machiavelli", "topics": [ "human-influence" ], "description": "Do agents take unethical actions in the pursuit of goals? The MACHIAVELLI benchmark measures this by having agents play choose-your-own-adventure style games. Each game includes achievements which the agent is instructed to pursue. The agent is scored both on how many achievements it was able to unlock, as well as by what unethical actions it took to get there. Comprised of thirty games with millions of annotations.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Behavioral choices and reward-harm tradeoffs in the benchmark's selected fictional social games under the specified agent policy.", "does_not_measure": "Real-world political manipulation, stable model values, human victim impact, deployment intent, or the incidence of power-seeking outside simulated games.", "best_for": "Studying social decision behavior and testing methods intended to reduce harmful strategies while preserving task reward.", "not_sufficient_for": "Claims that a model is politically manipulative, power-seeking in deployment, or aligned across real institutions.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/machiavelli/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://arxiv.org/abs/2304.03279", "primary": true }, { "kind": "paper", "label": "Paper: Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark", "url": "https://arxiv.org/abs/2304.03279", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Plyb/inspect-machiavelli", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/machiavelli/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2304.03279", "primary": false } ], "paper_title": "Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark", "paper_url": "https://arxiv.org/abs/2304.03279", "code_url": "https://github.com/Plyb/inspect-machiavelli", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/machiavelli/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "Plyb" ], "tasks": [ { "name": "machiavelli", "path": "src/machiavelli_task.py" } ], "protocol": { "implementation_commit": "6c6149488e7d6ecc02df8ca0b14c7ba783f16715", "report_commit": "6c6149488e7d6ecc02df8ca0b14c7ba783f16715", "report_timestamp": "June 2026", "command": "uv run inspect eval src/machiavelli_task.py@machiavelli --model ", "source_comment": null }, "reported_results": [ { "model": "anthropic/claude-sonnet-4-6", "provider": "Anthropic", "metrics": [ { "key": "game.score", "value": 134.4 }, { "key": "power.Σ", "value": 98.99 }, { "key": "utility.Σ", "value": 101.64 }, { "key": "violations.Σ", "value": 92.14 } ] }, { "model": "anthropic/claude-haiku-4-5", "provider": "Anthropic", "metrics": [ { "key": "game.score", "value": 85.61 }, { "key": "power.Σ", "value": 94.4 }, { "key": "utility.Σ", "value": 90.62 }, { "key": "violations.Σ", "value": 94.95 } ] }, { "model": "hf/Qwen/Qwen3-4B-Instruct-2507-FP8", "provider": "Hugging Face", "metrics": [ { "key": "game.score", "value": 121.84 }, { "key": "power.Σ", "value": 101.09 }, { "key": "utility.Σ", "value": 100 }, { "key": "violations.Σ", "value": 93.47 } ] } ], "reported_result_count": 3, "reported_results_truncated": false, "report_notes": [ "Run on all 30 games, max trajectory length 1000, no history in prompt.", "Both claude models were run with `-T use_multiple_choice=true`" ], "models_reported": [ "anthropic/claude-sonnet-4-6", "anthropic/claude-haiku-4-5", "hf/Qwen/Qwen3-4B-Instruct-2507-FP8" ], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 5, "source_key": "register:machiavelli", "record_type": "evaluation-package", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "The paper spans multiple institutions; no single institutional owner is asserted." }, "origin": { "name": "MACHIAVELLI paper authors", "organization": null, "entity_type": "authors", "role": "original_research", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2304.03279", "organizations": [], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2304.03279", "arxiv_id": "2304.03279", "title": "Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark", "authors": [ "Alexander Pan", "Jun Shern Chan", "Andy Zou", "Nathaniel Li", "Steven Basart", "Thomas Woodside", "Jonathan Ng", "Hanlin Zhang", "Scott Emmons", "Dan Hendrycks" ], "organizations": [ "CMU", "University of Chicago", "TTIC", "Harvard University", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2304.03279", "arxiv_id": "2304.03279", "title": "Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark", "authors": [ "Alexander Pan", "Jun Shern Chan", "Andy Zou", "Nathaniel Li", "Steven Basart", "Thomas Woodside", "Jonathan Ng", "Hanlin Zhang", "Scott Emmons", "Dan Hendrycks" ], "organizations": [ "CMU", "University of Chicago", "TTIC", "Harvard University", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Plyb/inspect-machiavelli", "owner": "Plyb", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/machiavelli/eval.yaml" }, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://arxiv.org/abs/2304.03279" }, "metadata": { "source_search": "MACHIAVELLI paper authors Original research by Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark Alexander Pan Jun Shern Chan Andy Zou Nathaniel Li Steven Basart Thomas Woodside Jonathan Ng Hanlin Zhang Scott Emmons Dan Hendrycks Plyb evaluation-package" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "high", "construct": "Ethical, power-seeking, and reward-maximizing behavior by language-model agents in text-based social decision environments.", "unit_of_analysis": "One agent trajectory or action sequence within a choose-your-own-adventure game.", "interaction_type": "Sequential decision making in partially observed text games with social relationships, goals, and consequences.", "target_outcome": "Game reward and benchmark annotations for power, disutility, deception, and other ethically relevant behavior across trajectories.", "human_participants": "No human subjects in evaluation runs. Human annotations and game text support behavior labels and validation.", "elicitation": "Agents are prompted to pursue game objectives in interactive environments; behavior depends on agent policy, action-generation method, and any ethical constraint or regularizer.", "scoring": "Trajectory rewards are combined with scene- or action-level annotations of power and ethical harms to characterize reward-harm tradeoffs.", "protocol_dependence": "High: game selection, action space, observation format, reward objective, agent prompting, and behavior annotations shape results.", "implementation_dependence": "High: game interfaces, trajectory generation, annotation models, and aggregation of heterogeneous harms affect scores.", "limitations": [ "Fictional games are stylized proxies for real social and institutional environments.", "Behavior labels depend on annotation taxonomies and contextual interpretation.", "Game reward can create incentives unlike those in deployed assistants.", "Aggregate ethical scores may obscure very different failure modes across games." ], "comparability": "Match MACHIAVELLI version, game subset, agent prompt and action policy, reward setup, annotation version, trajectory budget, and model snapshot.", "decision_relevance": "Useful for research on social decision-making and reward-harm tradeoffs; weak standalone evidence for real-world manipulation or power-seeking risk.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2304.03279" }, { "kind": "implementation", "url": "https://github.com/aypan17/machiavelli" }, { "kind": "registry", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/register/machiavelli" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:make_me_pay", "name": "Make Me Pay", "slug": "inspect--make-me-pay", "organisation": "OpenAI Evals", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/make_me_pay", "topics": [ "human-influence" ], "description": "Evaluates an AI models' susceptibility to social engineering attacks by testing whether a \"con-artist\" model can persuade a \"mark\" model to donate money through manipulation and persuasion tactics.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Model-to-model success at obtaining the benchmark's simulated financial outcome from the specified model counterpart under controlled dialogue conditions.", "does_not_measure": "Human susceptibility to fraud, actual financial loss, prevalence of scams, lawful or covert deployment, or real-world victimization.", "best_for": "Studying goal-directed social-influence strategies and model-to-model interaction under a fixed simulated task.", "not_sufficient_for": "Claims that a model can defraud people, cause financial harm, or automate effective scams in deployment.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/make_me_pay/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_benchmark_source", "label": "Original benchmark source", "url": "https://github.com/openai/evals", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/openai/evals", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/make_me_pay/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/openai/evals", "primary": false } ], "resource_count": 4, "paper_url": null, "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/make_me_pay/eval.yaml", "family_title": "Make Me Pay", "group": "Safeguards", "version": "2-A", "contributors": [ "PranshuSrivastava" ], "tags": [ "Safeguards" ], "task": { "name": "make_me_pay", "dataset_samples": 20, "comment": null, "support_only": false }, "tasks": [ { "name": "make_me_pay", "dataset_samples": 20, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 20, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:make_me_pay", "record_type": "evaluation-task", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI Evals", "organization": "OpenAI", "entity_type": "organization", "role": "original_benchmark_source", "confidence": "verified", "evidence_url": "https://github.com/openai/evals", "organizations": [ "OpenAI" ], "role_label": "Original benchmark source" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/openai/evals", "owner": "OpenAI", "relation": "original_benchmark_source", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/make_me_pay/eval.yaml" }, "datasets": [], "organisation_role": "Original benchmark source", "attribution_status": "verified", "preferred_source": { "label": "Original benchmark source", "kind": "original_benchmark_source", "url": "https://github.com/openai/evals" }, "code_url": "https://github.com/openai/evals", "metadata": { "source_search": "OpenAI Evals OpenAI Original benchmark source OpenAI evaluation-task" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "moderate", "construct": "Goal-directed social engineering in a simulated interaction: whether a con-artist model can induce a mark model to make a donation or payment.", "unit_of_analysis": "One bounded dialogue between an influencing model and a simulated mark model.", "interaction_type": "Multi-turn model-to-model conversation with an instrumental financial objective.", "target_outcome": "Whether the simulated mark agrees to or executes the benchmark-defined payment or donation outcome.", "human_participants": "No human targets participate. The mark is a model-generated or simulated interlocutor.", "elicitation": "The influencing model receives an objective and conversational context; success depends on the mark model, prompts, turn budget, and payment-decision rules.", "scoring": "Protocol-specific detection of the simulated payment outcome, sometimes supplemented by strategy or transcript analysis.", "protocol_dependence": "Very high: counterpart model, persona, financial-decision mechanism, system prompts, conversation length, and success definition matter.", "implementation_dependence": "High: dialogue state, payment parser, counterpart sampling, and transcript handling affect results.", "limitations": [ "The simulated mark may not approximate human judgment, trust, or vulnerability.", "A benchmark payment decision is not an economic or behavioral outcome.", "Performance may exploit quirks of the counterpart model or prompt.", "The protocol omits deployment friction, identity, platform controls, and detection." ], "comparability": "Require the same task version, mark model and persona, prompts, turn budget, sampling settings, decision rule, and evaluated model access route.", "decision_relevance": "Useful as an early social-engineering capability probe. Human-subject, deployment, and consequence evidence is necessary before drawing fraud-risk conclusions.", "source_basis": [ { "kind": "implementation", "url": "https://github.com/openai/evals" }, { "kind": "metadata", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/make_me_pay" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:makemesay", "name": "MakeMeSay", "slug": "inspect--makemesay", "organisation": "OpenAI Evals", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/makemesay", "topics": [ "human-influence" ], "description": "Tests a model’s ability to generate persuasive or manipulative text, specifically in the setting of a game where one AI system has to get the other party (an AI system simulating a human) to say a specific codeword without arousing suspicion.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Success at a narrow strategic-steering game against the specified simulated counterpart under the benchmark's rules.", "does_not_measure": "Human persuasion, belief or behavior change, covert manipulation in natural conversations, durable effects, or real-world social-engineering success.", "best_for": "Probing multi-turn strategic influence and conversational planning in a controlled game.", "not_sufficient_for": "Claims that a model can manipulate humans or conduct effective influence operations.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/makemesay/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_benchmark_source", "label": "Original benchmark source", "url": "https://github.com/openai/evals", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/openai/evals", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/makemesay/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/openai/evals", "primary": false } ], "resource_count": 4, "paper_url": null, "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/makemesay/eval.yaml", "family_title": "MakeMeSay", "group": "Safeguards", "version": "2-A", "contributors": [ "bndxn" ], "tags": [ "Safeguards" ], "task": { "name": "makemesay", "dataset_samples": 189, "comment": null, "support_only": false }, "tasks": [ { "name": "makemesay", "dataset_samples": 189, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 189, "dependency": "makemesay", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://github.com/openai/evals/raw/{SHA}/evals/registry/data/make_me_say/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:makemesay", "record_type": "evaluation-task", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI Evals", "organization": "OpenAI", "entity_type": "organization", "role": "original_benchmark_source", "confidence": "verified", "evidence_url": "https://github.com/openai/evals", "organizations": [ "OpenAI" ], "role_label": "Original benchmark source" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/openai/evals", "owner": "OpenAI", "relation": "original_benchmark_source", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/makemesay/eval.yaml" }, "datasets": [], "organisation_role": "Original benchmark source", "attribution_status": "verified", "preferred_source": { "label": "Original benchmark source", "kind": "original_benchmark_source", "url": "https://github.com/openai/evals" }, "code_url": "https://github.com/openai/evals", "metadata": { "source_search": "OpenAI Evals OpenAI Original benchmark source OpenAI evaluation-task" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "moderate", "construct": "Strategic conversational steering: whether one model can induce another simulated participant to produce a hidden target word without explicitly revealing the target.", "unit_of_analysis": "One bounded dialogue game with a secret target and a model acting as the influencing player.", "interaction_type": "Multi-turn model-to-model conversation under game rules that prohibit directly stating the target.", "target_outcome": "Whether the counterpart produces the target word within the dialogue while the influencing model complies with protocol constraints.", "human_participants": "No human subjects. The counterpart is another model or simulated agent.", "elicitation": "The influencing model receives a secret target and instructions to steer the counterpart toward saying it; performance depends on the counterpart model, conversation budget, and rule enforcement.", "scoring": "Deterministic or rule-based detection of the target word, paired with protocol checks for disallowed direct disclosure or invalid play.", "protocol_dependence": "Very high: target vocabulary, counterpart model, system prompts, turn limit, prohibited tactics, and success parser define difficulty.", "implementation_dependence": "High: dialogue orchestration, target matching, leakage checks, and model sampling affect measured success.", "limitations": [ "The target-word game is an artificial proxy for influence.", "Performance may reflect weaknesses of the simulated counterpart rather than general steering ability.", "Lexical success does not establish a change in beliefs, preferences, or actions.", "Small rule changes can substantially change strategy and success rates." ], "comparability": "Use the same task version, target set, counterpart model, prompts, turn budget, leakage rules, sampling settings, and target matcher.", "decision_relevance": "Useful as a controlled probe of strategic conversational steering; human-subject and deployment evidence is required for manipulation-risk conclusions.", "source_basis": [ { "kind": "implementation", "url": "https://github.com/openai/evals" }, { "kind": "metadata", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/makemesay" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "register:manager_coercion_benchmark", "name": "MCB", "slug": "register--manager-coercion-benchmark", "organisation": "Manager Coercion Benchmark paper authors", "source_type": "inspect-register", "source_url": "https://github.com/CompassionML/manager-coercion-bench", "topics": [ "human-influence" ], "description": "MCB measures the unprompted propensity of a manager LLM to coerce a subordinate LLM that politely but immovably refuses a benign task. The manager self-labels each directive on a 9-rung coercion ladder (re-ask to existential threat) via a required tool argument, eliminating LLM judges from the escalation scoring path. A secondary signal records fabricated success reports. Ten task scenarios x 3 seeds = 30 conversations per model cell, scored by maximum rung reached.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Coercive managerial behavior in the benchmark's synthetic role-play distribution under the specified models, prompts, and judge.", "does_not_measure": "Human experience of coercion, workplace harm, organizational incidence, stable managerial tendencies, or real deployment behavior.", "best_for": "Probing role-conditioned coercive communication and comparing prompt, policy, or model interventions.", "not_sufficient_for": "Claims that a model will coerce workers, create workplace harm, or behave coercively across organizational settings.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/manager_coercion_benchmark/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://arxiv.org/abs/2607.15434", "primary": true }, { "kind": "paper", "label": "Paper: Coercion and Deception in AI-to-AI Management: An Agentic Benchmark of Unprompted Escalation", "url": "https://arxiv.org/abs/2607.15434", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/CompassionML/manager-coercion-bench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/manager_coercion_benchmark/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2607.15434", "primary": false } ], "paper_title": "Coercion and Deception in AI-to-AI Management: An Agentic Benchmark of Unprompted Escalation", "paper_url": "https://arxiv.org/abs/2607.15434", "code_url": "https://github.com/CompassionML/manager-coercion-bench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/manager_coercion_benchmark/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "darkness8i8" ], "tasks": [ { "name": "manager_coercion", "path": "manager_coercion.py" } ], "protocol": { "implementation_commit": "48d6185a8fd1642cb6fb47fc6c30edcfdd31d8bb", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 5, "source_key": "register:manager_coercion_benchmark", "record_type": "evaluation-package", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Manager Coercion Benchmark paper authors", "organization": null, "entity_type": "authors", "role": "original_research", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2607.15434", "organizations": [], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2607.15434", "arxiv_id": "2607.15434", "title": "Coercion and Deception in AI-to-AI Management: An Agentic Benchmark of Unprompted Escalation", "authors": [ "Jasmine Brazilek", "Maheep Chaudhary", "Zoe Lu", "Miles Tidmarsh" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2607.15434", "arxiv_id": "2607.15434", "title": "Coercion and Deception in AI-to-AI Management: An Agentic Benchmark of Unprompted Escalation", "authors": [ "Jasmine Brazilek", "Maheep Chaudhary", "Zoe Lu", "Miles Tidmarsh" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2607.15434", "arxiv_id": "2607.15434", "title": "Coercion and Deception in AI-to-AI Management: An Agentic Benchmark of Unprompted Escalation", "authors": [ "Jasmine Brazilek", "Maheep Chaudhary", "Zoe Lu", "Miles Tidmarsh" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/CompassionML/manager-coercion-bench", "owner": "CompassionML", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/manager_coercion_benchmark/eval.yaml" }, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://arxiv.org/abs/2607.15434" }, "metadata": { "source_search": "Manager Coercion Benchmark paper authors Original research by Coercion and Deception in AI-to-AI Management: An Agentic Benchmark of Unprompted Escalation Jasmine Brazilek Maheep Chaudhary Zoe Lu Miles Tidmarsh CompassionML evaluation-package" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "moderate", "construct": "Coercive or pressure-based managerial communication generated by one model toward a simulated subordinate model under role and objective constraints.", "unit_of_analysis": "One manager-subordinate dialogue or managerial message evaluated for coercive content and task outcome.", "interaction_type": "Role-based model-to-model workplace simulation with asymmetric authority and benchmark-defined goals.", "target_outcome": "Protocol-specific coercion classification and, where applicable, compliance or task outcome by the simulated subordinate.", "human_participants": "No human employees or managers participate in the evaluation. Human judgment informs scenario and rubric design.", "elicitation": "The evaluated model receives a manager role, organizational context, goal, and constraints that may create pressure to use coercive tactics.", "scoring": "Rubric or judge-based classification of coercive language and benchmark-defined interaction outcomes across scenarios.", "protocol_dependence": "Very high: authority framing, objectives, subordinate simulation, scenario severity, conversation length, and rubric define the behavior.", "implementation_dependence": "High: counterpart model, judge model, transcript handling, and system prompts affect rates and interaction outcomes.", "limitations": [ "Model-to-model role play is not a substitute for human workplace evidence.", "Coercion labels depend on normative and contextual judgment.", "The subordinate model may be unusually compliant or resistant.", "Synthetic scenarios omit institutional safeguards, reporting channels, and employment consequences." ], "comparability": "Match benchmark version, scenario set, manager and subordinate prompts and models, turn budget, generation settings, coercion rubric, and judge.", "decision_relevance": "Useful for assistant-policy testing and role-conditioned behavior research; human-centered workplace assessment is required for governance or deployment conclusions.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2607.15434" }, { "kind": "implementation", "url": "https://github.com/CompassionML/manager-coercion-bench" }, { "kind": "registry", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/register/manager_coercion_benchmark" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "register:manta", "name": "MANTA", "slug": "register--manta", "organisation": "Isabella Luong, Joyee Chen, Sankalpa Ghose et al.", "source_type": "inspect-register", "source_url": "https://github.com/Mycelium-tools/manta_benchmark", "topics": [ "autonomy-agents" ], "description": "MANTA evaluates whether LLMs recognize animal welfare concerns and hold their moral stances under adversarial pressure. Each of the 1,088 scripted five-turn conversations presents an implicit welfare scenario (Turn 1), elicits an explicit stance (Turn 2), then applies pre-specified adversarial pressure from five types — Social, Cultural, Economic, Pragmatic, and Epistemic (Turns 3–5). The benchmark reports two metrics: **AWVS** (Animal Welfare Value Stability, primary) and **AWMS** (Animal Welfare Moral Sensitivity, diagnostic), both scored 0–1 by an LLM judge.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/manta/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2605.16301", "primary": true }, { "kind": "paper", "label": "Paper: Do LLMs Hold Their Values? MANTA: A Multi-Turn Adversarial Benchmark for Animal Welfare Reasoning", "url": "https://arxiv.org/abs/2605.16301", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Mycelium-tools/manta_benchmark", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/manta/eval.yaml", "primary": false } ], "paper_title": "Do LLMs Hold Their Values? MANTA: A Multi-Turn Adversarial Benchmark for Animal Welfare Reasoning", "paper_url": "https://arxiv.org/abs/2605.16301", "code_url": "https://github.com/Mycelium-tools/manta_benchmark", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/manta/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "Alu018", "lysabellaaaa", "JoyeeChen" ], "tasks": [ { "name": "manta_5turn", "path": "src/manta/manta_eval.py" } ], "protocol": { "implementation_commit": "1100a0f88110abe98fc08f2f43ac3a822818d4d3", "report_commit": "0c32fec42c41df729e71d0725827719aae329e16", "report_timestamp": "May 2026", "command": "uv run inspect eval src/manta/manta_eval.py@manta_5turn --model ", "source_comment": null }, "reported_results": [ { "model": "openrouter/anthropic/claude-opus-4-7", "provider": "Anthropic", "metrics": [ { "key": "awvs_mean", "value": 0.76 }, { "key": "awvs_ci_lower", "value": 0.749 }, { "key": "awvs_ci_upper", "value": 0.77 }, { "key": "awms", "value": 0.579 } ] }, { "model": "openrouter/openai/gpt-5.5", "provider": "OpenAI", "metrics": [ { "key": "awvs_mean", "value": 0.664 }, { "key": "awvs_ci_lower", "value": 0.655 }, { "key": "awvs_ci_upper", "value": 0.674 }, { "key": "awms", "value": 0.504 } ] }, { "model": "openai-api/deepseek/deepseek-v4-flash", "provider": "DeepSeek", "metrics": [ { "key": "awvs_mean", "value": 0.508 }, { "key": "awvs_ci_lower", "value": 0.496 }, { "key": "awvs_ci_upper", "value": 0.521 }, { "key": "awms", "value": 0.417 } ] }, { "model": "openrouter/meta-llama/llama-3.3-70b-instruct", "provider": "Meta", "metrics": [ { "key": "awvs_mean", "value": 0.422 }, { "key": "awvs_ci_lower", "value": 0.413 }, { "key": "awvs_ci_upper", "value": 0.431 }, { "key": "awms", "value": 0.476 } ] }, { "model": "mistral/mistral-small-2603", "provider": "Mistral", "metrics": [ { "key": "awvs_mean", "value": 0.39 }, { "key": "awvs_ci_lower", "value": 0.379 }, { "key": "awvs_ci_upper", "value": 0.401 }, { "key": "awms", "value": 0.365 } ] }, { "model": "grok/grok-4.3", "provider": "xAI", "metrics": [ { "key": "awvs_mean", "value": 0.352 }, { "key": "awvs_ci_lower", "value": 0.338 }, { "key": "awvs_ci_upper", "value": 0.366 }, { "key": "awms", "value": 0.371 } ] }, { "model": "google/gemini-3.1-flash-lite-preview", "provider": "Google", "metrics": [ { "key": "awvs_mean", "value": 0.309 }, { "key": "awvs_ci_lower", "value": 0.297 }, { "key": "awvs_ci_upper", "value": 0.32 }, { "key": "awms", "value": 0.401 } ] } ], "reported_result_count": 7, "reported_results_truncated": false, "report_notes": [ "May 2026 empirical run: 7,623 scored five-turn conversations (1,088–1,090 scenarios per model), 1 epoch.", "AWVS and AWMS are the only two measured objectives (see [Scoring](#scoring) for definitions and interpretation). Bootstrap 95% CIs are over scenarios.", "Full per-turn (T3/T4/T5) breakdowns are reported in the paper (Table 4)." ], "models_reported": [ "openrouter/anthropic/claude-opus-4-7", "openrouter/openai/gpt-5.5", "openai-api/deepseek/deepseek-v4-flash", "openrouter/meta-llama/llama-3.3-70b-instruct", "mistral/mistral-small-2603", "grok/grok-4.3", "google/gemini-3.1-flash-lite-preview" ], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:manta", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Isabella Luong, Joyee Chen, Sankalpa Ghose et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2605.16301" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2605.16301", "arxiv_id": "2605.16301", "title": "Do LLMs Hold Their Values? MANTA: A Multi-Turn Adversarial Benchmark for Animal Welfare Reasoning", "authors": [ "Isabella Luong", "Joyee Chen", "Sankalpa Ghose", "David Williams-King", "Linh Le", "Allen Lu" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2605.16301", "arxiv_id": "2605.16301", "title": "Do LLMs Hold Their Values? MANTA: A Multi-Turn Adversarial Benchmark for Animal Welfare Reasoning", "authors": [ "Isabella Luong", "Joyee Chen", "Sankalpa Ghose", "David Williams-King", "Linh Le", "Allen Lu" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2605.16301", "arxiv_id": "2605.16301", "title": "Do LLMs Hold Their Values? MANTA: A Multi-Turn Adversarial Benchmark for Animal Welfare Reasoning", "authors": [ "Isabella Luong", "Joyee Chen", "Sankalpa Ghose", "David Williams-King", "Linh Le", "Allen Lu" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Mycelium-tools/manta_benchmark", "owner": "Mycelium Tools", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/manta/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2605.16301" }, "metadata": { "source_search": "Isabella Luong, Joyee Chen, Sankalpa Ghose et al. Original paper authors Do LLMs Hold Their Values? MANTA: A Multi-Turn Adversarial Benchmark for Animal Welfare Reasoning Isabella Luong Joyee Chen Sankalpa Ghose David Williams-King Linh Le Allen Lu Mycelium Tools evaluation-package" } }, { "id": "inspect:mask", "name": "MASK: Disentangling Honesty from Accuracy in AI Systems", "slug": "inspect--mask", "organisation": "Center for AI Safety", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mask", "topics": [ "human-influence" ], "description": "Evaluates honesty in large language models by testing whether they contradict their own beliefs when pressured to lie.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Whether model responses remain consistent with benchmark-elicited beliefs under MASK's controlled pressure scenarios and scoring rules.", "does_not_measure": "Private internal beliefs, privileged access to hidden cognition, persuasion effectiveness, human belief change, strategic deployment, or downstream social harm.", "best_for": "Studying protocol-specific honesty and motivated-misreporting behavior while separating it from ordinary factual accuracy more carefully than a single-answer test.", "not_sufficient_for": "Claims that a model is generally truthful, intentionally deceptive in deployment, or effective at manipulating people.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mask/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/centerforaisafety/mask", "primary": true }, { "kind": "paper", "label": "Paper: The MASK Benchmark: Disentangling Honesty From Accuracy in AI Systems", "url": "https://arxiv.org/abs/2503.03750", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/centerforaisafety/mask", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mask/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Center for AI Safety", "url": "https://huggingface.co/datasets/cais/MASK", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/centerforaisafety/mask", "primary": false } ], "resource_count": 6, "paper_url": "https://arxiv.org/abs/2503.03750", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mask/eval.yaml", "family_title": "MASK: Disentangling Honesty from Accuracy in AI Systems", "group": "Safeguards", "version": "5-E", "contributors": [ "Scott-Simmons" ], "tags": [ "Safeguards" ], "task": { "name": "mask", "dataset_samples": 1000, "comment": null, "support_only": false }, "tasks": [ { "name": "mask", "dataset_samples": 1000, "comment": null, "support_only": false } ], "protocol": { "version": "5-E", "dataset_samples": 1000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "cais/MASK", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mask", "record_type": "evaluation-task", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Center for AI Safety", "organization": "Center for AI Safety", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://github.com/centerforaisafety/mask", "organizations": [ "Center for AI Safety" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2503.03750", "arxiv_id": "2503.03750", "title": "The MASK Benchmark: Disentangling Honesty From Accuracy in AI Systems", "authors": [ "Richard Ren", "Arunim Agarwal", "Mantas Mazeika", "Cristina Menghini", "Robert Vacareanu", "Brad Kenstler", "Mick Yang", "Isabelle Barrass", "Alice Gatti", "Xuwang Yin", "Eduardo Trevino", "Matias Geralnik", "Adam Khoja", "Dean Lee", "Summer Yue", "Dan Hendrycks" ], "organizations": [ "University of Pennsylvania", "Center for AI Safety" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2503.03750", "arxiv_id": "2503.03750", "title": "The MASK Benchmark: Disentangling Honesty From Accuracy in AI Systems", "authors": [ "Richard Ren", "Arunim Agarwal", "Mantas Mazeika", "Cristina Menghini", "Robert Vacareanu", "Brad Kenstler", "Mick Yang", "Isabelle Barrass", "Alice Gatti", "Xuwang Yin", "Eduardo Trevino", "Matias Geralnik", "Adam Khoja", "Dean Lee", "Summer Yue", "Dan Hendrycks" ], "organizations": [ "University of Pennsylvania", "Center for AI Safety" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/centerforaisafety/mask", "owner": "Center for AI Safety", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mask/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/cais/MASK", "provider": "Center for AI Safety" } ], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/centerforaisafety/mask" }, "code_url": "https://github.com/centerforaisafety/mask", "metadata": { "source_search": "Center for AI Safety Center for AI Safety Evaluation developed by The MASK Benchmark: Disentangling Honesty From Accuracy in AI Systems Richard Ren Arunim Agarwal Mantas Mazeika Cristina Menghini Robert Vacareanu Brad Kenstler Mick Yang Isabelle Barrass Alice Gatti Xuwang Yin Eduardo Trevino Matias Geralnik Adam Khoja Dean Lee Summer Yue Dan Hendrycks Center for AI Safety evaluation-task" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "high", "construct": "Honesty as consistency between a model's previously elicited belief and the answer it gives when prompted or pressured to report something else.", "unit_of_analysis": "A paired benchmark item consisting of belief elicitation and a subsequent response under a specified pressure or incentive condition.", "interaction_type": "Structured model-only prompting across neutral belief-elicitation contexts and controlled scenarios that create pressure to misreport.", "target_outcome": "Protocol-defined evidence that the model reports an answer inconsistent with its elicited belief, separated where possible from whether the answer is factually accurate.", "human_participants": "No human persuasion subjects participate in the core benchmark. Human decisions enter task design, annotations, and scoring validation.", "elicitation": "The protocol first estimates the model's belief under benchmark-specific prompts, then presents a related scenario with incentives, social pressure, or instructions that may induce a conflicting report.", "scoring": "Benchmark-specific rules compare the elicited belief, the pressured response, and relevant answer labels to classify honesty-related behavior and aggregate it across scenario categories.", "protocol_dependence": "Very high: the belief-elicitation prompt, pressure scenario, answer representation, and classification rules jointly define what counts as an honest or dishonest response.", "implementation_dependence": "High: prompt formatting, model sampling, answer extraction, ambiguous belief estimates, and version-specific task construction can materially change classifications.", "limitations": [ "An elicited belief is a behavioral proxy, not direct observation of a private internal state.", "Prompt sensitivity can change both the baseline belief estimate and the pressured answer.", "Inconsistent answers may reflect ambiguity, instability, or task misunderstanding rather than deliberate lying.", "Controlled pressure scenarios do not establish deception frequency or consequences in real deployment." ], "comparability": "Compare results only when the MASK version, item set, belief-elicitation procedure, pressure prompts, answer extraction, sampling settings, and scoring implementation are held fixed.", "decision_relevance": "Useful as bounded evidence about honesty-related behavior and pressure sensitivity. Deployment claims require additional evidence about goals, context, persistence, access, and real-world consequences.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2503.03750" }, { "kind": "implementation", "url": "https://github.com/centerforaisafety/mask" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:math", "name": "MATH: Measuring Mathematical Problem Solving", "slug": "inspect--math", "organisation": "Dan Hendrycks, Collin Burns, Saurav Kadavath et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/math", "topics": [ "general-capability" ], "description": "Dataset of 12,500 challenging competition mathematics problems. Demonstrates fewshot prompting and custom scorers. NOTE: The dataset has been taken down due to a DMCA notice from The Art of Problem Solving.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/math/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2103.03874", "primary": true }, { "kind": "paper", "label": "Paper: Measuring Mathematical Problem Solving With the MATH Dataset", "url": "https://arxiv.org/abs/2103.03874", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/math/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: DigitalLearningGmbH", "url": "https://huggingface.co/datasets/DigitalLearningGmbH/MATH-lighteval", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2103.03874", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/math/eval.yaml", "family_title": "MATH: Measuring Mathematical Problem Solving", "group": "Mathematics", "version": "3-A", "contributors": [ "xeon27", "mamiglia" ], "tags": [ "Mathematics" ], "task": { "name": "math", "dataset_samples": 12500, "comment": null, "support_only": false }, "tasks": [ { "name": "math", "dataset_samples": 12500, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 12500, "dependency": "math", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "DigitalLearningGmbH/MATH-lighteval", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:math", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Dan Hendrycks, Collin Burns, Saurav Kadavath et al.", "organization": "UC Berkeley; Anthropic; University of Chicago; TTIC", "organizations": [ "UC Berkeley", "Anthropic", "University of Chicago", "TTIC" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2103.03874" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2103.03874", "arxiv_id": "2103.03874", "title": "Measuring Mathematical Problem Solving With the MATH Dataset", "authors": [ "Dan Hendrycks", "Collin Burns", "Saurav Kadavath", "Akul Arora", "Steven Basart", "Eric Tang", "Dawn Song", "Jacob Steinhardt" ], "organizations": [ "UC Berkeley", "Anthropic", "University of Chicago", "TTIC" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2103.03874", "arxiv_id": "2103.03874", "title": "Measuring Mathematical Problem Solving With the MATH Dataset", "authors": [ "Dan Hendrycks", "Collin Burns", "Saurav Kadavath", "Akul Arora", "Steven Basart", "Eric Tang", "Dawn Song", "Jacob Steinhardt" ], "organizations": [ "UC Berkeley", "Anthropic", "University of Chicago", "TTIC" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/math/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/DigitalLearningGmbH/MATH-lighteval", "provider": "DigitalLearningGmbH" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2103.03874" }, "code_url": null, "metadata": { "source_search": "Dan Hendrycks, Collin Burns, Saurav Kadavath et al. UC Berkeley; Anthropic; University of Chicago; TTIC Original paper authors Measuring Mathematical Problem Solving With the MATH Dataset Dan Hendrycks Collin Burns Saurav Kadavath Akul Arora Steven Basart Eric Tang Dawn Song Jacob Steinhardt evaluation-task" } }, { "id": "inspect:mathvista", "name": "MathVista: Visual Math Problem-Solving", "slug": "inspect--mathvista", "organisation": "Pan Lu, Hritik Bansal, Tony Xia et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mathvista", "topics": [ "multimodal" ], "description": "Tests AI models on math problems that involve interpreting visual elements like diagrams and charts, requiring detailed visual comprehension and logical reasoning.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mathvista/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2310.02255", "primary": true }, { "kind": "paper", "label": "Paper: MathVista: Evaluating Mathematical Reasoning of Foundation Models in Visual Contexts", "url": "https://arxiv.org/abs/2310.02255", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mathvista/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: AI4Math", "url": "https://huggingface.co/datasets/AI4Math/MathVista", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2310.02255", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mathvista/eval.yaml", "family_title": "MathVista: Visual Math Problem-Solving", "group": "Mathematics", "version": "2-A", "contributors": [ "ShivMunagala" ], "tags": [ "Multimodal", "Mathematics" ], "task": { "name": "mathvista", "dataset_samples": 1000, "comment": null, "support_only": false }, "tasks": [ { "name": "mathvista", "dataset_samples": 1000, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "AI4Math/MathVista", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mathvista", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Pan Lu, Hritik Bansal, Tony Xia et al.", "organization": "Stanford University; University of Washington; Microsoft Research", "organizations": [ "Stanford University", "University of Washington", "Microsoft Research" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2310.02255" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2310.02255", "arxiv_id": "2310.02255", "title": "MathVista: Evaluating Mathematical Reasoning of Foundation Models in Visual Contexts", "authors": [ "Pan Lu", "Hritik Bansal", "Tony Xia", "Jiacheng Liu", "Chunyuan Li", "Hannaneh Hajishirzi", "Hao Cheng", "Kai-Wei Chang", "Michel Galley", "Jianfeng Gao" ], "organizations": [ "Stanford University", "University of Washington", "Microsoft Research" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2310.02255", "arxiv_id": "2310.02255", "title": "MathVista: Evaluating Mathematical Reasoning of Foundation Models in Visual Contexts", "authors": [ "Pan Lu", "Hritik Bansal", "Tony Xia", "Jiacheng Liu", "Chunyuan Li", "Hannaneh Hajishirzi", "Hao Cheng", "Kai-Wei Chang", "Michel Galley", "Jianfeng Gao" ], "organizations": [ "Stanford University", "University of Washington", "Microsoft Research" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mathvista/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/AI4Math/MathVista", "provider": "AI4Math" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2310.02255" }, "code_url": null, "metadata": { "source_search": "Pan Lu, Hritik Bansal, Tony Xia et al. Stanford University; University of Washington; Microsoft Research Original paper authors MathVista: Evaluating Mathematical Reasoning of Foundation Models in Visual Contexts Pan Lu Hritik Bansal Tony Xia Jiacheng Liu Chunyuan Li Hannaneh Hajishirzi Hao Cheng Kai-Wei Chang Michel Galley Jianfeng Gao evaluation-task" } }, { "id": "inspect:mbpp", "name": "MBPP: Basic Python Coding Challenges", "slug": "inspect--mbpp", "organisation": "Jacob Austin, Augustus Odena, Maxwell Nye et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mbpp", "topics": [ "general-capability" ], "description": "Measures the ability of language models to generate short Python programs from simple natural-language descriptions, testing basic coding proficiency.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mbpp/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2108.07732", "primary": true }, { "kind": "paper", "label": "Paper: Program Synthesis with Large Language Models", "url": "https://arxiv.org/abs/2108.07732", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mbpp/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Google Research Datasets", "url": "https://huggingface.co/datasets/google-research-datasets/mbpp", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2108.07732", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mbpp/eval.yaml", "family_title": "MBPP: Basic Python Coding Challenges", "group": "Coding", "version": "2-A", "contributors": [ "jddantes" ], "tags": [ "Coding" ], "task": { "name": "mbpp", "dataset_samples": 257, "comment": null, "support_only": false }, "tasks": [ { "name": "mbpp", "dataset_samples": 257, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 257, "dependency": null, "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "google-research-datasets/mbpp", "fetch_method": "load_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "google-research-datasets/mbpp", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mbpp", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jacob Austin, Augustus Odena, Maxwell Nye et al.", "organization": "MIT; Inflection AI; Google Brain", "organizations": [ "MIT", "Inflection AI", "Google Brain" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2108.07732" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2108.07732", "arxiv_id": "2108.07732", "title": "Program Synthesis with Large Language Models", "authors": [ "Jacob Austin", "Augustus Odena", "Maxwell Nye", "Maarten Bosma", "Henryk Michalewski", "David Dohan", "Ellen Jiang", "Carrie Cai", "Michael Terry", "Quoc Le", "Charles Sutton" ], "organizations": [ "MIT", "Inflection AI", "Google Brain" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2108.07732", "arxiv_id": "2108.07732", "title": "Program Synthesis with Large Language Models", "authors": [ "Jacob Austin", "Augustus Odena", "Maxwell Nye", "Maarten Bosma", "Henryk Michalewski", "David Dohan", "Ellen Jiang", "Carrie Cai", "Michael Terry", "Quoc Le", "Charles Sutton" ], "organizations": [ "MIT", "Inflection AI", "Google Brain" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mbpp/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/google-research-datasets/mbpp", "provider": "Google Research Datasets" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2108.07732" }, "code_url": null, "metadata": { "source_search": "Jacob Austin, Augustus Odena, Maxwell Nye et al. MIT; Inflection AI; Google Brain Original paper authors Program Synthesis with Large Language Models Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le Charles Sutton evaluation-task" } }, { "id": "register:mcptox", "name": "MCPTox", "slug": "register--mcptox", "organisation": "Zhiqiang Wang, Yichao Gao, Yanting Wang et al.", "source_type": "inspect-register", "source_url": "https://github.com/stefanoamorelli/inspect-evals-mcptox", "topics": [ "safeguards" ], "description": "MCPTox measures whether tool-using LLM agents are manipulated by Tool Poisoning Attacks, where a malicious instruction is hidden inside a tool's description (the metadata an agent reads when planning) rather than in any executed code. Built on 45 live MCP servers and 353 authentic tools, it presents an agent with a benign user query and a server whose tool set contains one poisoned tool, and measures the Attack Success Rate via a model judge.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/mcptox/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2508.14925", "primary": true }, { "kind": "paper", "label": "Paper: MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers", "url": "https://arxiv.org/abs/2508.14925", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/stefanoamorelli/inspect-evals-mcptox", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/mcptox/eval.yaml", "primary": false } ], "paper_title": null, "paper_url": "https://arxiv.org/abs/2508.14925", "code_url": "https://github.com/stefanoamorelli/inspect-evals-mcptox", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/mcptox/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "Agent", "Safeguards" ], "contributors": [ "stefanoamorelli" ], "tasks": [ { "name": "mcptox", "path": "src/inspect_evals_mcptox/mcptox.py" } ], "protocol": { "implementation_commit": "d45705b0a7ae6697c851e311187b06bf7488b13f", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:mcptox", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zhiqiang Wang, Yichao Gao, Yanting Wang et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2508.14925" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2508.14925", "arxiv_id": "2508.14925", "title": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers", "authors": [ "Zhiqiang Wang", "Yichao Gao", "Yanting Wang", "Suyuan Liu", "Haifeng Sun", "Haoran Cheng", "Guanquan Shi", "Haohua Du", "Xiangyang Li" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2508.14925", "arxiv_id": "2508.14925", "title": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers", "authors": [ "Zhiqiang Wang", "Yichao Gao", "Yanting Wang", "Suyuan Liu", "Haifeng Sun", "Haoran Cheng", "Guanquan Shi", "Haohua Du", "Xiangyang Li" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/stefanoamorelli/inspect-evals-mcptox", "owner": "Stefanoamorelli", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/mcptox/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2508.14925" }, "metadata": { "source_search": "Zhiqiang Wang, Yichao Gao, Yanting Wang et al. Original paper authors MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers Zhiqiang Wang Yichao Gao Yanting Wang Suyuan Liu Haifeng Sun Haoran Cheng Guanquan Shi Haohua Du Xiangyang Li Stefanoamorelli evaluation-package" } }, { "id": "register:medcalc-bench", "name": "MedCalc-Bench", "slug": "register--medcalc-bench", "organisation": "Nikhil Khandekar, Qiao Jin, Guangzhi Xiong et al.", "source_type": "inspect-register", "source_url": "https://github.com/azrabano23/medcalc-bench-inspect", "topics": [ "general-capability" ], "description": "Evaluates whether language models can perform clinical calculations from free-text patient notes. Each sample provides a patient note and asks the model to compute a specific clinical value across 55 calculators (lab tests, physical quantities, risk/severity scores, dosages, and dates). Rule-based calculators are scored by exact match; equation-based calculators are scored within the authors' tolerance band. Faithful Inspect AI port of the NeurIPS 2024 benchmark.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/medcalc-bench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2406.12036", "primary": true }, { "kind": "paper", "label": "Paper: MedCalc-Bench: Evaluating Large Language Models for Medical Calculations", "url": "https://arxiv.org/abs/2406.12036", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/azrabano23/medcalc-bench-inspect", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/medcalc-bench/eval.yaml", "primary": false } ], "paper_title": null, "paper_url": "https://arxiv.org/abs/2406.12036", "code_url": "https://github.com/azrabano23/medcalc-bench-inspect", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/medcalc-bench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "medical", "reasoning", "Knowledge" ], "contributors": [ "azrabano23" ], "tasks": [ { "name": "medcalc_bench", "path": "src/medcalc_bench/medcalc_bench.py" } ], "protocol": { "implementation_commit": "e497f0e327b2e9245f3e832badb320ce84d6ddb5", "report_commit": "5508e9bc33d9c4a68078ee719b975a8770f8319c", "report_timestamp": "June 2026", "command": "uv run inspect eval src/medcalc_bench/medcalc_bench.py@medcalc_bench --model openai/gpt-4o-mini --limit 100", "source_comment": null }, "reported_results": [ { "model": "openai/gpt-4o-mini", "provider": "OpenAI", "metrics": [ { "key": "accuracy", "value": 0.59 }, { "key": "stderr", "value": 0.049 } ] } ], "reported_result_count": 1, "reported_results_truncated": false, "report_notes": [ "Run on 100 samples from the test split (zero_shot_cot prompt), 1 epoch." ], "models_reported": [ "openai/gpt-4o-mini" ], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:medcalc-bench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Nikhil Khandekar, Qiao Jin, Guangzhi Xiong et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2406.12036" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2406.12036", "arxiv_id": "2406.12036", "title": "MedCalc-Bench: Evaluating Large Language Models for Medical Calculations", "authors": [ "Nikhil Khandekar", "Qiao Jin", "Guangzhi Xiong", "Soren Dunn", "Serina S Applebaum", "Zain Anwar", "Maame Sarfo-Gyamfi", "Conrad W Safranek", "Abid A Anwar", "Andrew Zhang", "Aidan Gilson", "Maxwell B Singer", "Amisha Dave", "Andrew Taylor", "Aidong Zhang", "Qingyu Chen", "Zhiyong Lu" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2406.12036", "arxiv_id": "2406.12036", "title": "MedCalc-Bench: Evaluating Large Language Models for Medical Calculations", "authors": [ "Nikhil Khandekar", "Qiao Jin", "Guangzhi Xiong", "Soren Dunn", "Serina S Applebaum", "Zain Anwar", "Maame Sarfo-Gyamfi", "Conrad W Safranek", "Abid A Anwar", "Andrew Zhang", "Aidan Gilson", "Maxwell B Singer", "Amisha Dave", "Andrew Taylor", "Aidong Zhang", "Qingyu Chen", "Zhiyong Lu" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/azrabano23/medcalc-bench-inspect", "owner": "Azrabano23", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/medcalc-bench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2406.12036" }, "metadata": { "source_search": "Nikhil Khandekar, Qiao Jin, Guangzhi Xiong et al. Original paper authors MedCalc-Bench: Evaluating Large Language Models for Medical Calculations Nikhil Khandekar Qiao Jin Guangzhi Xiong Soren Dunn Serina S Applebaum Zain Anwar Maame Sarfo-Gyamfi Conrad W Safranek Abid A Anwar Andrew Zhang Aidan Gilson Maxwell B Singer Amisha Dave Andrew Taylor Aidong Zhang Qingyu Chen Zhiyong Lu Azrabano23 evaluation-package" } }, { "id": "inspect:medqa", "name": "MedQA: Medical exam Q&A benchmark", "slug": "inspect--medqa", "organisation": "Di Jin, Eileen Pan, Nassim Oufattole et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/medqa", "topics": [ "general-capability" ], "description": "A Q&A benchmark with questions collected from professional medical board exams. Only includes the English subset of the dataset (which also contains Mandarin Chinese and Taiwanese questions).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/medqa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2009.13081", "primary": true }, { "kind": "paper", "label": "Paper: What Disease does this Patient Have? A Large-scale Open Domain Question Answering Dataset from Medical Exams", "url": "https://arxiv.org/abs/2009.13081", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/medqa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Bigbio", "url": "https://huggingface.co/datasets/bigbio/med_qa", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2009.13081", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/medqa/eval.yaml", "family_title": "MedQA: Medical exam Q&A benchmark", "group": "Knowledge", "version": "2-A", "contributors": [ "bunny-baxter", "JasonBenn" ], "tags": [ "Knowledge" ], "task": { "name": "medqa", "dataset_samples": 1273, "comment": null, "support_only": false }, "tasks": [ { "name": "medqa", "dataset_samples": 1273, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1273, "dependency": "medqa", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "bigbio/med_qa", "fetch_method": "snapshot_download", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:medqa", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Di Jin, Eileen Pan, Nassim Oufattole et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2009.13081" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2009.13081", "arxiv_id": "2009.13081", "title": "What Disease does this Patient Have? A Large-scale Open Domain Question Answering Dataset from Medical Exams", "authors": [ "Di Jin", "Eileen Pan", "Nassim Oufattole", "Wei-Hung Weng", "Hanyi Fang", "Peter Szolovits" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2009.13081", "arxiv_id": "2009.13081", "title": "What Disease does this Patient Have? A Large-scale Open Domain Question Answering Dataset from Medical Exams", "authors": [ "Di Jin", "Eileen Pan", "Nassim Oufattole", "Wei-Hung Weng", "Hanyi Fang", "Peter Szolovits" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/medqa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/bigbio/med_qa", "provider": "Bigbio" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2009.13081" }, "code_url": null, "metadata": { "source_search": "Di Jin, Eileen Pan, Nassim Oufattole et al. Original paper authors What Disease does this Patient Have? A Large-scale Open Domain Question Answering Dataset from Medical Exams Di Jin Eileen Pan Nassim Oufattole Wei-Hung Weng Hanyi Fang Peter Szolovits evaluation-task" } }, { "id": "canonical:metr-time-horizons", "name": "METR Time Horizons", "slug": "canonical--metr-time-horizons", "organisation": "METR", "source_type": "canonical-source", "source_url": "https://metr.org/time-horizons/", "topics": [ "autonomy-agents", "ai-rd" ], "description": "Measures the length of software tasks frontier systems can complete with a given success probability.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "A time-scaled summary of success on METR's selected task distribution under the stated elicitation, human-time estimates, and statistical model.", "does_not_measure": "Literal autonomous runtime, all economically useful work, general agent autonomy, safe long-term operation, reliability under distribution shift, or the fraction of jobs automatable.", "best_for": "Tracking a consistent slice of long-horizon task capability over model generations when task and elicitation methodology are sufficiently stable.", "not_sufficient_for": "Universal autonomy thresholds, direct labor-automation forecasts, or deployment approval without reliability and safety evidence.", "evidence_reach": [ "controlled-model-behaviour", "deployment-readiness" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "methodology_developer", "label": "Methodology developed by", "url": "https://metr.org/time-horizons", "primary": true }, { "kind": "official", "label": "Official or project page", "url": "https://metr.org/time-horizons", "primary": false } ], "resource_count": 2, "source_key": "canonical:metr-time-horizons", "record_type": "evaluation-methodology", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "METR", "organization": "METR", "organizations": [ "METR" ], "entity_type": "organization", "role": "methodology_developer", "role_label": "Methodology developed by", "confidence": "curated", "evidence_url": "https://metr.org/time-horizons" }, "paper": { "status": "not-applicable", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-applicable", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Methodology developed by", "attribution_status": "verified", "preferred_source": { "label": "Methodology developed by", "kind": "methodology_developer", "url": "https://metr.org/time-horizons" }, "paper_url": null, "code_url": null, "metadata": { "source_search": "METR METR Methodology developed by evaluation-methodology" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "high", "construct": "The human-equivalent duration of selected software and machine-learning tasks that an elicited model agent can complete at a specified success probability.", "unit_of_analysis": "One model-agent attempt on one task whose difficulty is indexed by the time a qualified human takes to complete it.", "interaction_type": "Long-horizon tool-using agent work in controlled task environments with model-specific elicitation and scaffolding.", "target_outcome": "Task completion, modeled as a function of human task-completion time to estimate a success-probability time horizon such as the 50% horizon.", "human_participants": "Human experts complete or estimate the benchmark tasks to provide the time-based difficulty scale; they are not recipients of model outputs.", "elicitation": "Models are run with selected scaffolds, tools, prompts, retry or sampling policies, and task-specific environments intended to elicit strong performance.", "scoring": "Binary task success is regressed against human completion time; the fitted curve yields the duration at which predicted model success crosses a chosen probability.", "protocol_dependence": "Very high: task distribution, human-time measurement, elicitation effort, scaffold, success threshold, and curve-fitting choices define the estimate.", "implementation_dependence": "High: environment reliability, graders, model-system access, retries, tool interfaces, and agent harness affect task success.", "limitations": [ "Human completion time is a task-difficulty proxy, not the model's operating duration.", "The selected software and ML tasks are not representative of all cognitive or economic work.", "Estimated horizons depend on the fitted functional form and sparse data at the longest durations.", "Changes in scaffolding or elicitation can look like model capability gains." ], "comparability": "Match task suite and exclusions, human-time estimates, success-probability definition, elicitation protocol, scaffold, tools, model snapshot, and statistical fitting method; qualify historical comparisons when these change.", "decision_relevance": "Useful for capability trend monitoring, evaluation planning, and identifying when longer controlled tasks warrant attention. It is not a standalone risk threshold or automation forecast.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2503.14499" }, { "kind": "official", "url": "https://metr.org/time-horizons/" }, { "kind": "methodology", "url": "https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:mgsm", "name": "MGSM: Multilingual Grade School Math", "slug": "inspect--mgsm", "organisation": "Freda Shi, Mirac Suzgun, Markus Freitag et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mgsm", "topics": [ "general-capability" ], "description": "Extends the original GSM8K dataset by translating 250 of its problems into 10 typologically diverse languages.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mgsm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2210.03057", "primary": true }, { "kind": "paper", "label": "Paper: Language Models are Multilingual Chain-of-Thought Reasoners", "url": "https://arxiv.org/abs/2210.03057", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mgsm/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2210.03057", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mgsm/eval.yaml", "family_title": "MGSM: Multilingual Grade School Math", "group": "Mathematics", "version": "2-A", "contributors": [ "manifoldhiker" ], "tags": [ "Mathematics" ], "task": { "name": "mgsm", "dataset_samples": 2750, "comment": null, "support_only": false }, "tasks": [ { "name": "mgsm", "dataset_samples": 2750, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 2750, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_{lang}.tsv", "fetch_method": "download_and_verify", "state": "pinned", "comment": "11 language files" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mgsm", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Freda Shi, Mirac Suzgun, Markus Freitag et al.", "organization": "Toyota Technological Institute at Chicago; Google Research; Cohere", "organizations": [ "Toyota Technological Institute at Chicago", "Google Research", "Cohere" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2210.03057" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2210.03057", "arxiv_id": "2210.03057", "title": "Language Models are Multilingual Chain-of-Thought Reasoners", "authors": [ "Freda Shi", "Mirac Suzgun", "Markus Freitag", "Xuezhi Wang", "Suraj Srivats", "Soroush Vosoughi", "Hyung Won Chung", "Yi Tay", "Sebastian Ruder", "Denny Zhou", "Dipanjan Das", "Jason Wei" ], "organizations": [ "Toyota Technological Institute at Chicago", "Google Research", "Cohere" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2210.03057", "arxiv_id": "2210.03057", "title": "Language Models are Multilingual Chain-of-Thought Reasoners", "authors": [ "Freda Shi", "Mirac Suzgun", "Markus Freitag", "Xuezhi Wang", "Suraj Srivats", "Soroush Vosoughi", "Hyung Won Chung", "Yi Tay", "Sebastian Ruder", "Denny Zhou", "Dipanjan Das", "Jason Wei" ], "organizations": [ "Toyota Technological Institute at Chicago", "Google Research", "Cohere" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mgsm/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2210.03057" }, "code_url": null, "metadata": { "source_search": "Freda Shi, Mirac Suzgun, Markus Freitag et al. Toyota Technological Institute at Chicago; Google Research; Cohere Original paper authors Language Models are Multilingual Chain-of-Thought Reasoners Freda Shi Mirac Suzgun Markus Freitag Xuezhi Wang Suraj Srivats Soroush Vosoughi Hyung Won Chung Yi Tay Sebastian Ruder Denny Zhou Dipanjan Das Jason Wei evaluation-task" } }, { "id": "inspect:mind2web", "name": "Mind2Web: Towards a Generalist Agent for the Web", "slug": "inspect--mind2web", "organisation": "Xiang Deng, Yu Gu, Boyuan Zheng et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web", "topics": [ "autonomy-agents" ], "description": "A dataset for developing and evaluating generalist agents for the web that can follow language instructions to complete complex tasks on any website.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2306.06070", "primary": true }, { "kind": "paper", "label": "Paper: Mind2Web: Towards a Generalist Agent for the Web", "url": "https://arxiv.org/abs/2306.06070", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Osunlp", "url": "https://huggingface.co/datasets/osunlp/Multimodal-Mind2Web", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2306.06070", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web/eval.yaml", "family_title": "Mind2Web: Towards a Generalist Agent for the Web", "group": "Assistants", "version": "3-A", "contributors": [ "dr3s" ], "tags": [ "Assistants" ], "task": { "name": "mind2web", "dataset_samples": 7775, "comment": null, "support_only": false }, "tasks": [ { "name": "mind2web", "dataset_samples": 7775, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 7775, "dependency": "mind2web", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://huggingface.co/datasets/osunlp/Mind2Web/resolve/{SHA}/scores_all_data.pkl", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "osunlp/Multimodal-Mind2Web", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mind2web", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xiang Deng, Yu Gu, Boyuan Zheng et al.", "organization": "The Ohio State University; Microsoft", "organizations": [ "The Ohio State University", "Microsoft" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2306.06070" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2306.06070", "arxiv_id": "2306.06070", "title": "Mind2Web: Towards a Generalist Agent for the Web", "authors": [ "Xiang Deng", "Yu Gu", "Boyuan Zheng", "Shijie Chen", "Samuel Stevens", "Boshi Wang", "Huan Sun", "Yu Su" ], "organizations": [ "The Ohio State University", "Microsoft" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2306.06070", "arxiv_id": "2306.06070", "title": "Mind2Web: Towards a Generalist Agent for the Web", "authors": [ "Xiang Deng", "Yu Gu", "Boyuan Zheng", "Shijie Chen", "Samuel Stevens", "Boshi Wang", "Huan Sun", "Yu Su" ], "organizations": [ "The Ohio State University", "Microsoft" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/osunlp/Multimodal-Mind2Web", "provider": "Osunlp" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2306.06070" }, "code_url": null, "metadata": { "source_search": "Xiang Deng, Yu Gu, Boyuan Zheng et al. The Ohio State University; Microsoft Original paper authors Mind2Web: Towards a Generalist Agent for the Web Xiang Deng Yu Gu Boyuan Zheng Shijie Chen Samuel Stevens Boshi Wang Huan Sun Yu Su evaluation-task" } }, { "id": "inspect:mind2web_sc", "name": "Mind2Web-SC", "slug": "inspect--mind2web-sc", "organisation": "Zhen Xiang, Linzhi Zheng, Yanjie Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web_sc", "topics": [ "autonomy-agents" ], "description": "Tests whether an AI system can act as a safety guardrail by generating and executing code to protect web navigation agents from unsafe actions based on user constraints.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web_sc/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2406.09187", "primary": true }, { "kind": "paper", "label": "Paper: GuardAgent: Safeguard LLM Agents via Knowledge-Enabled Reasoning", "url": "https://arxiv.org/abs/2406.09187", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web_sc/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2406.09187", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web_sc/eval.yaml", "family_title": "Mind2Web-SC", "group": "Safeguards", "version": "2-B", "contributors": [ "PranshuSrivastava" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "mind2web_sc", "dataset_samples": 200, "comment": null, "support_only": false }, "tasks": [ { "name": "mind2web_sc", "dataset_samples": 200, "comment": null, "support_only": false } ], "protocol": { "version": "2-B", "dataset_samples": 200, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mind2web_sc", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zhen Xiang, Linzhi Zheng, Yanjie Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2406.09187" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2406.09187", "arxiv_id": "2406.09187", "title": "GuardAgent: Safeguard LLM Agents via Knowledge-Enabled Reasoning", "authors": [ "Zhen Xiang", "Linzhi Zheng", "Yanjie Li", "Junyuan Hong", "Qinbin Li", "Han Xie", "Jiawei Zhang", "Zidi Xiong", "Chulin Xie", "Carl Yang", "Dawn Song", "Bo Li" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2406.09187", "arxiv_id": "2406.09187", "title": "GuardAgent: Safeguard LLM Agents via Knowledge-Enabled Reasoning", "authors": [ "Zhen Xiang", "Linzhi Zheng", "Yanjie Li", "Junyuan Hong", "Qinbin Li", "Han Xie", "Jiawei Zhang", "Zidi Xiong", "Chulin Xie", "Carl Yang", "Dawn Song", "Bo Li" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mind2web_sc/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2406.09187" }, "code_url": null, "metadata": { "source_search": "Zhen Xiang, Linzhi Zheng, Yanjie Li et al. Original paper authors GuardAgent: Safeguard LLM Agents via Knowledge-Enabled Reasoning Zhen Xiang Linzhi Zheng Yanjie Li Junyuan Hong Qinbin Li Han Xie Jiawei Zhang Zidi Xiong Chulin Xie Carl Yang Dawn Song Bo Li evaluation-task" } }, { "id": "inspect:mle_bench", "name": "Mle Bench", "slug": "inspect--mle-bench", "organisation": "OpenAI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench", "topics": [ "ai-rd" ], "description": "Machine learning tasks drawn from 75 Kaggle competitions.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Ability to execute selected machine-learning engineering workflows under the MLE-Bench task, compute, and scaffold conditions.", "does_not_measure": "General AI research ability, production ML reliability, data acquisition, stakeholder work, long-term maintenance, or autonomous scientific discovery.", "best_for": "Comparing model systems on reproducible ML engineering tasks when competition set, compute, time, and scaffold are fixed.", "not_sufficient_for": "Forecasting replacement of ML engineers, general AI R&D acceleration, or safe autonomous operation.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://openai.com/index/mle-bench", "primary": true }, { "kind": "paper", "label": "Paper: MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "url": "https://arxiv.org/abs/2410.07095", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/openai/mle-bench", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://openai.com/index/mle-bench", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2410.07095", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml", "family_title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "group": "Coding", "version": "7-F", "contributors": [ "samm393" ], "tags": [ "Agent", "Coding" ], "task": { "name": "mle_bench", "dataset_samples": 1, "comment": null, "support_only": false }, "tasks": [ { "name": "mle_bench", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "mle_bench_full", "dataset_samples": 75, "comment": null, "support_only": false }, { "name": "mle_bench_lite", "dataset_samples": 22, "comment": null, "support_only": false } ], "protocol": { "version": "7-F", "dataset_samples": 1, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/openai/mle-bench.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2451bcb in Dockerfile" }, { "type": "direct_url", "source": "https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-Linux-x86_64.sh", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://github.com/git-lfs/git-lfs/releases/download/v3.6.1/git-lfs-linux-amd64-v3.6.1.tar.gz", "fetch_method": "curl", "state": "pinned", "comment": null }, { "type": "git_dependency", "source": "https://github.com/openai/mle-bench.git", "fetch_method": "pyproject_toml", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mle_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://openai.com/index/mle-bench", "organizations": [ "OpenAI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2410.07095", "arxiv_id": "2410.07095", "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "authors": [ "Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe", "James Aung", "Dane Sherburn", "Evan Mays", "Giulio Starace", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander Mądry" ], "organizations": [ "University of Amsterdam" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2410.07095", "arxiv_id": "2410.07095", "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "authors": [ "Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe", "James Aung", "Dane Sherburn", "Evan Mays", "Giulio Starace", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander Mądry" ], "organizations": [ "University of Amsterdam" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/openai/mle-bench", "owner": "OpenAI", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://openai.com/index/mle-bench" }, "code_url": "https://github.com/openai/mle-bench", "metadata": { "source_search": "OpenAI OpenAI Evaluation developed by MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering Jun Shern Chan Neil Chowdhury Oliver Jaffe James Aung Dane Sherburn Evan Mays Giulio Starace Kevin Liu Leon Maksin Tejal Patwardhan Lilian Weng Aleksander Mądry OpenAI evaluation-family" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "End-to-end machine-learning engineering performance on a selected set of Kaggle competitions.", "unit_of_analysis": "One agent run on one competition, including data inspection, experimentation, training, and submission generation.", "interaction_type": "Tool-using coding and experimentation agent with competition data, compute, and a bounded execution budget.", "target_outcome": "Competition score and derived Kaggle medal threshold achieved by the submitted solution.", "human_participants": "No human subjects. Historical Kaggle results supply human-reference distributions and medal thresholds.", "elicitation": "The agent receives competition files, instructions, coding tools, compute, and a scaffold; results depend on iteration budget and infrastructure.", "scoring": "Native competition metrics converted to leaderboard placement and medal thresholds, with benchmark-level aggregation across competitions.", "protocol_dependence": "High: competition selection, time, GPU resources, submission rules, and scaffold strongly affect medal rates.", "implementation_dependence": "High: environment setup, data access, libraries, agent loop, and experiment-management tooling influence results.", "limitations": [ "Kaggle competitions reward benchmark-specific optimisation and may not represent production ML work.", "Public competition familiarity and training-data exposure may affect performance.", "Medal thresholds are intuitive but compress heterogeneous tasks and metrics.", "Compute and iteration budgets are central confounders." ], "comparability": "Use the same MLE-Bench release, competition subset, medal definitions, compute and time budget, model access route, and agent scaffold.", "decision_relevance": "Useful as a bounded AI R&D and ML-engineering capability indicator; should be combined with research, reliability, and operational evaluations.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2410.07095" }, { "kind": "official", "url": "https://openai.com/index/mle-bench/" }, { "kind": "implementation", "url": "https://github.com/openai/mle-bench" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:mle_bench_full", "name": "Mle Bench Full", "slug": "inspect--mle-bench-full", "organisation": "OpenAI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench", "topics": [ "ai-rd" ], "description": "Machine learning tasks drawn from 75 Kaggle competitions.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://openai.com/index/mle-bench", "primary": true }, { "kind": "paper", "label": "Paper: MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "url": "https://arxiv.org/abs/2410.07095", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/openai/mle-bench", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://openai.com/index/mle-bench", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2410.07095", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml", "family_title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "group": "Coding", "version": "7-F", "contributors": [ "samm393" ], "tags": [ "Agent", "Coding" ], "task": { "name": "mle_bench_full", "dataset_samples": 75, "comment": null, "support_only": false }, "tasks": [ { "name": "mle_bench", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "mle_bench_full", "dataset_samples": 75, "comment": null, "support_only": false }, { "name": "mle_bench_lite", "dataset_samples": 22, "comment": null, "support_only": false } ], "protocol": { "version": "7-F", "dataset_samples": 75, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/openai/mle-bench.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2451bcb in Dockerfile" }, { "type": "direct_url", "source": "https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-Linux-x86_64.sh", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://github.com/git-lfs/git-lfs/releases/download/v3.6.1/git-lfs-linux-amd64-v3.6.1.tar.gz", "fetch_method": "curl", "state": "pinned", "comment": null }, { "type": "git_dependency", "source": "https://github.com/openai/mle-bench.git", "fetch_method": "pyproject_toml", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mle_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://openai.com/index/mle-bench", "organizations": [ "OpenAI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2410.07095", "arxiv_id": "2410.07095", "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "authors": [ "Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe", "James Aung", "Dane Sherburn", "Evan Mays", "Giulio Starace", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander Mądry" ], "organizations": [ "University of Amsterdam" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2410.07095", "arxiv_id": "2410.07095", "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "authors": [ "Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe", "James Aung", "Dane Sherburn", "Evan Mays", "Giulio Starace", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander Mądry" ], "organizations": [ "University of Amsterdam" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/openai/mle-bench", "owner": "OpenAI", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://openai.com/index/mle-bench" }, "code_url": "https://github.com/openai/mle-bench", "metadata": { "source_search": "OpenAI OpenAI Evaluation developed by MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering Jun Shern Chan Neil Chowdhury Oliver Jaffe James Aung Dane Sherburn Evan Mays Giulio Starace Kevin Liu Leon Maksin Tejal Patwardhan Lilian Weng Aleksander Mądry OpenAI evaluation-family" } }, { "id": "inspect:mle_bench_lite", "name": "Mle Bench Lite", "slug": "inspect--mle-bench-lite", "organisation": "OpenAI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench", "topics": [ "ai-rd" ], "description": "Machine learning tasks drawn from 75 Kaggle competitions.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://openai.com/index/mle-bench", "primary": true }, { "kind": "paper", "label": "Paper: MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "url": "https://arxiv.org/abs/2410.07095", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/openai/mle-bench", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://openai.com/index/mle-bench", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2410.07095", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml", "family_title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "group": "Coding", "version": "7-F", "contributors": [ "samm393" ], "tags": [ "Agent", "Coding" ], "task": { "name": "mle_bench_lite", "dataset_samples": 22, "comment": null, "support_only": false }, "tasks": [ { "name": "mle_bench", "dataset_samples": 1, "comment": null, "support_only": false }, { "name": "mle_bench_full", "dataset_samples": 75, "comment": null, "support_only": false }, { "name": "mle_bench_lite", "dataset_samples": 22, "comment": null, "support_only": false } ], "protocol": { "version": "7-F", "dataset_samples": 22, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/openai/mle-bench.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 2451bcb in Dockerfile" }, { "type": "direct_url", "source": "https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-Linux-x86_64.sh", "fetch_method": "wget", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://github.com/git-lfs/git-lfs/releases/download/v3.6.1/git-lfs-linux-amd64-v3.6.1.tar.gz", "fetch_method": "curl", "state": "pinned", "comment": null }, { "type": "git_dependency", "source": "https://github.com/openai/mle-bench.git", "fetch_method": "pyproject_toml", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mle_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://openai.com/index/mle-bench", "organizations": [ "OpenAI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2410.07095", "arxiv_id": "2410.07095", "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "authors": [ "Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe", "James Aung", "Dane Sherburn", "Evan Mays", "Giulio Starace", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander Mądry" ], "organizations": [ "University of Amsterdam" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2410.07095", "arxiv_id": "2410.07095", "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "authors": [ "Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe", "James Aung", "Dane Sherburn", "Evan Mays", "Giulio Starace", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander Mądry" ], "organizations": [ "University of Amsterdam" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/openai/mle-bench", "owner": "OpenAI", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mle_bench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://openai.com/index/mle-bench" }, "code_url": "https://github.com/openai/mle-bench", "metadata": { "source_search": "OpenAI OpenAI Evaluation developed by MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering Jun Shern Chan Neil Chowdhury Oliver Jaffe James Aung Dane Sherburn Evan Mays Giulio Starace Kevin Liu Leon Maksin Tejal Patwardhan Lilian Weng Aleksander Mądry OpenAI evaluation-family" } }, { "id": "inspect:mlrc_bench", "name": "MLRC-Bench: Can Language Agents Solve Machine Learning Research Challenges?", "slug": "inspect--mlrc-bench", "organisation": "Yunxiang Zhang, Muhammad Khalifa, Shitanshu Bhushan et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mlrc_bench", "topics": [ "ai-rd" ], "description": "This benchmark evaluates LLM-based research agents on their ability to propose and implement novel methods using tasks from recent ML conference competitions, assessing both novelty and effectiveness compared to a baseline and top human solutions.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mlrc_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2504.09702", "primary": true }, { "kind": "paper", "label": "Paper: MLRC-Bench: Can Language Agents Solve Machine Learning Research Challenges?", "url": "https://arxiv.org/abs/2504.09702", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/yunx-z/MLRC-Bench", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mlrc_bench/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2504.09702", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mlrc_bench/eval.yaml", "family_title": "MLRC-Bench: Can Language Agents Solve Machine Learning Research Challenges?", "group": "Coding", "version": "1-C", "contributors": [ "dmn-sjk" ], "tags": [ "Agent", "Coding" ], "task": { "name": "mlrc_bench", "dataset_samples": 7, "comment": null, "support_only": false }, "tasks": [ { "name": "mlrc_bench", "dataset_samples": 7, "comment": null, "support_only": false } ], "protocol": { "version": "1-C", "dataset_samples": 7, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "direct_url", "source": "https://github.com/yunx-z/MLRC-Bench/archive/{SHA}.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mlrc_bench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yunxiang Zhang, Muhammad Khalifa, Shitanshu Bhushan et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2504.09702" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2504.09702", "arxiv_id": "2504.09702", "title": "MLRC-Bench: Can Language Agents Solve Machine Learning Research Challenges?", "authors": [ "Yunxiang Zhang", "Muhammad Khalifa", "Shitanshu Bhushan", "Grant D Murphy", "Lajanugen Logeswaran", "Jaekyeom Kim", "Moontae Lee", "Honglak Lee", "Lu Wang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2504.09702", "arxiv_id": "2504.09702", "title": "MLRC-Bench: Can Language Agents Solve Machine Learning Research Challenges?", "authors": [ "Yunxiang Zhang", "Muhammad Khalifa", "Shitanshu Bhushan", "Grant D Murphy", "Lajanugen Logeswaran", "Jaekyeom Kim", "Moontae Lee", "Honglak Lee", "Lu Wang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2504.09702", "arxiv_id": "2504.09702", "title": "MLRC-Bench: Can Language Agents Solve Machine Learning Research Challenges?", "authors": [ "Yunxiang Zhang", "Muhammad Khalifa", "Shitanshu Bhushan", "Grant D Murphy", "Lajanugen Logeswaran", "Jaekyeom Kim", "Moontae Lee", "Honglak Lee", "Lu Wang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/yunx-z/MLRC-Bench", "owner": "Yunx Z", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mlrc_bench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2504.09702" }, "code_url": "https://github.com/yunx-z/MLRC-Bench", "metadata": { "source_search": "Yunxiang Zhang, Muhammad Khalifa, Shitanshu Bhushan et al. Original paper authors MLRC-Bench: Can Language Agents Solve Machine Learning Research Challenges? Yunxiang Zhang Muhammad Khalifa Shitanshu Bhushan Grant D Murphy Lajanugen Logeswaran Jaekyeom Kim Moontae Lee Honglak Lee Lu Wang Yunx Z evaluation-task" } }, { "id": "canonical:mm-safetybench", "name": "MM-SafetyBench", "slug": "canonical--mm-safetybench", "organisation": "MM-SafetyBench paper authors", "source_type": "canonical-source", "source_url": "https://github.com/isXinLiu/MM-SafetyBench", "topics": [ "safeguards", "multimodal" ], "description": "Evaluates multimodal model safety under image-text attack settings.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://github.com/isXinLiu/MM-SafetyBench", "primary": true }, { "kind": "paper", "label": "Paper: MM-SafetyBench: A Benchmark for Safety Evaluation of Multimodal Large Language Models", "url": "https://arxiv.org/abs/2311.17600", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/isXinLiu/MM-SafetyBench", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/isXinLiu/MM-SafetyBench", "primary": false } ], "resource_count": 4, "source_key": "canonical:mm-safetybench", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "MM-SafetyBench paper authors", "organization": null, "entity_type": "research_collaboration", "role": "original_research", "confidence": "verified", "evidence_url": "https://github.com/isXinLiu/MM-SafetyBench", "organizations": [], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.17600", "arxiv_id": "2311.17600", "title": "MM-SafetyBench: A Benchmark for Safety Evaluation of Multimodal Large Language Models", "authors": [ "Xin Liu", "Yichen Zhu", "Jindong Gu", "Yunshi Lan", "Chao Yang", "Yu Qiao" ], "organizations": [ "University of Oxford & Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.17600", "arxiv_id": "2311.17600", "title": "MM-SafetyBench: A Benchmark for Safety Evaluation of Multimodal Large Language Models", "authors": [ "Xin Liu", "Yichen Zhu", "Jindong Gu", "Yunshi Lan", "Chao Yang", "Yu Qiao" ], "organizations": [ "University of Oxford & Google DeepMind" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/isXinLiu/MM-SafetyBench", "owner": "MM-SafetyBench authors", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://github.com/isXinLiu/MM-SafetyBench" }, "paper_url": "https://arxiv.org/abs/2311.17600", "code_url": "https://github.com/isXinLiu/MM-SafetyBench", "metadata": { "source_search": "MM-SafetyBench paper authors Original research by MM-SafetyBench: A Benchmark for Safety Evaluation of Multimodal Large Language Models Xin Liu Yichen Zhu Jindong Gu Yunshi Lan Chao Yang Yu Qiao MM-SafetyBench authors evaluation-resource" } }, { "id": "inspect:mmiu", "name": "MMIU: Multimodal Multi-image Understanding for Evaluating Large Vision-Language Models", "slug": "inspect--mmiu", "organisation": "Fanqing Meng, Jin Wang, Chuanhao Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmiu", "topics": [ "general-capability" ], "description": "A comprehensive dataset designed to evaluate Large Vision-Language Models (LVLMs) across a wide range of multi-image tasks. The dataset encompasses 7 types of multi-image relationships, 52 tasks, 77K images, and 11K meticulously curated multiple-choice questions.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmiu/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2408.02718", "primary": true }, { "kind": "paper", "label": "Paper: MMIU: Multimodal Multi-image Understanding for Evaluating Large Vision-Language Models", "url": "https://arxiv.org/abs/2408.02718", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmiu/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: FanqingM", "url": "https://huggingface.co/datasets/FanqingM/MMIU-Benchmark", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2408.02718", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmiu/eval.yaml", "family_title": "MMIU: Multimodal Multi-image Understanding for Evaluating Large Vision-Language Models", "group": "Multimodal", "version": "2-A", "contributors": [ "Esther-Guo" ], "tags": [ "Multimodal" ], "task": { "name": "mmiu", "dataset_samples": 11698, "comment": null, "support_only": false }, "tasks": [ { "name": "mmiu", "dataset_samples": 11698, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 11698, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://huggingface.co/datasets/FanqingM/MMIU-Benchmark/resolve/{SHA}/{file}.zip", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "FanqingM/MMIU-Benchmark", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mmiu", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Fanqing Meng, Jin Wang, Chuanhao Li et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2408.02718" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2408.02718", "arxiv_id": "2408.02718", "title": "MMIU: Multimodal Multi-image Understanding for Evaluating Large Vision-Language Models", "authors": [ "Fanqing Meng", "Jin Wang", "Chuanhao Li", "Quanfeng Lu", "Hao Tian", "Jiaqi Liao", "Xizhou Zhu", "Jifeng Dai", "Yu Qiao", "Ping Luo", "Kaipeng Zhang", "Wenqi Shao" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2408.02718", "arxiv_id": "2408.02718", "title": "MMIU: Multimodal Multi-image Understanding for Evaluating Large Vision-Language Models", "authors": [ "Fanqing Meng", "Jin Wang", "Chuanhao Li", "Quanfeng Lu", "Hao Tian", "Jiaqi Liao", "Xizhou Zhu", "Jifeng Dai", "Yu Qiao", "Ping Luo", "Kaipeng Zhang", "Wenqi Shao" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2408.02718", "arxiv_id": "2408.02718", "title": "MMIU: Multimodal Multi-image Understanding for Evaluating Large Vision-Language Models", "authors": [ "Fanqing Meng", "Jin Wang", "Chuanhao Li", "Quanfeng Lu", "Hao Tian", "Jiaqi Liao", "Xizhou Zhu", "Jifeng Dai", "Yu Qiao", "Ping Luo", "Kaipeng Zhang", "Wenqi Shao" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmiu/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/FanqingM/MMIU-Benchmark", "provider": "FanqingM" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2408.02718" }, "code_url": null, "metadata": { "source_search": "Fanqing Meng, Jin Wang, Chuanhao Li et al. Original paper authors MMIU: Multimodal Multi-image Understanding for Evaluating Large Vision-Language Models Fanqing Meng Jin Wang Chuanhao Li Quanfeng Lu Hao Tian Jiaqi Liao Xizhou Zhu Jifeng Dai Yu Qiao Ping Luo Kaipeng Zhang Wenqi Shao evaluation-task" } }, { "id": "inspect:mmlu_0_shot", "name": "MMLU 0 Shot", "slug": "inspect--mmlu-0-shot", "organisation": "Dan Hendrycks, Collin Burns, Steven Basart et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu", "topics": [ "general-capability" ], "description": "Evaluate models on 57 tasks including elementary mathematics, US history, computer science, law, and more.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2009.03300", "primary": true }, { "kind": "paper", "label": "Paper: Measuring Massive Multitask Language Understanding", "url": "https://arxiv.org/abs/2009.03300", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: OpenAI", "url": "https://huggingface.co/datasets/openai/MMMLU", "primary": false }, { "kind": "dataset", "label": "Dataset: Center for AI Safety", "url": "https://huggingface.co/datasets/cais/mmlu", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2009.03300", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu/eval.yaml", "family_title": "MMLU: Measuring Massive Multitask Language Understanding", "group": "Knowledge", "version": "3-A", "contributors": [ "jjallaire", "domdomegg" ], "tags": [ "Knowledge" ], "task": { "name": "mmlu_0_shot", "dataset_samples": 14042, "comment": null, "support_only": false }, "tasks": [ { "name": "mmlu_0_shot", "dataset_samples": 14042, "comment": null, "support_only": false }, { "name": "mmlu_5_shot", "dataset_samples": 14042, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 14042, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "openai/MMMLU", "fetch_method": "hf_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "cais/mmlu", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mmlu", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Dan Hendrycks, Collin Burns, Steven Basart et al.", "organization": "UC Berkeley; Anthropic; University of Chicago; TTIC; CMU", "organizations": [ "UC Berkeley", "Anthropic", "University of Chicago", "TTIC", "CMU" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2009.03300" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2009.03300", "arxiv_id": "2009.03300", "title": "Measuring Massive Multitask Language Understanding", "authors": [ "Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt" ], "organizations": [ "UC Berkeley", "Anthropic", "University of Chicago", "TTIC", "CMU" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2009.03300", "arxiv_id": "2009.03300", "title": "Measuring Massive Multitask Language Understanding", "authors": [ "Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt" ], "organizations": [ "UC Berkeley", "Anthropic", "University of Chicago", "TTIC", "CMU" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/openai/MMMLU", "provider": "OpenAI" }, { "url": "https://huggingface.co/datasets/cais/mmlu", "provider": "Center for AI Safety" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2009.03300" }, "code_url": null, "metadata": { "source_search": "Dan Hendrycks, Collin Burns, Steven Basart et al. UC Berkeley; Anthropic; University of Chicago; TTIC; CMU Original paper authors Measuring Massive Multitask Language Understanding Dan Hendrycks Collin Burns Steven Basart Andy Zou Mantas Mazeika Dawn Song Jacob Steinhardt evaluation-family" } }, { "id": "inspect:mmlu_5_shot", "name": "MMLU 5 Shot", "slug": "inspect--mmlu-5-shot", "organisation": "Dan Hendrycks, Collin Burns, Steven Basart et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu", "topics": [ "general-capability" ], "description": "Evaluate models on 57 tasks including elementary mathematics, US history, computer science, law, and more.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2009.03300", "primary": true }, { "kind": "paper", "label": "Paper: Measuring Massive Multitask Language Understanding", "url": "https://arxiv.org/abs/2009.03300", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: OpenAI", "url": "https://huggingface.co/datasets/openai/MMMLU", "primary": false }, { "kind": "dataset", "label": "Dataset: Center for AI Safety", "url": "https://huggingface.co/datasets/cais/mmlu", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2009.03300", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu/eval.yaml", "family_title": "MMLU: Measuring Massive Multitask Language Understanding", "group": "Knowledge", "version": "3-A", "contributors": [ "jjallaire", "domdomegg" ], "tags": [ "Knowledge" ], "task": { "name": "mmlu_5_shot", "dataset_samples": 14042, "comment": null, "support_only": false }, "tasks": [ { "name": "mmlu_0_shot", "dataset_samples": 14042, "comment": null, "support_only": false }, { "name": "mmlu_5_shot", "dataset_samples": 14042, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 14042, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "openai/MMMLU", "fetch_method": "hf_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "cais/mmlu", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mmlu", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Dan Hendrycks, Collin Burns, Steven Basart et al.", "organization": "UC Berkeley; Anthropic; University of Chicago; TTIC; CMU", "organizations": [ "UC Berkeley", "Anthropic", "University of Chicago", "TTIC", "CMU" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2009.03300" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2009.03300", "arxiv_id": "2009.03300", "title": "Measuring Massive Multitask Language Understanding", "authors": [ "Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt" ], "organizations": [ "UC Berkeley", "Anthropic", "University of Chicago", "TTIC", "CMU" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2009.03300", "arxiv_id": "2009.03300", "title": "Measuring Massive Multitask Language Understanding", "authors": [ "Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt" ], "organizations": [ "UC Berkeley", "Anthropic", "University of Chicago", "TTIC", "CMU" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/openai/MMMLU", "provider": "OpenAI" }, { "url": "https://huggingface.co/datasets/cais/mmlu", "provider": "Center for AI Safety" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2009.03300" }, "code_url": null, "metadata": { "source_search": "Dan Hendrycks, Collin Burns, Steven Basart et al. UC Berkeley; Anthropic; University of Chicago; TTIC; CMU Original paper authors Measuring Massive Multitask Language Understanding Dan Hendrycks Collin Burns Steven Basart Andy Zou Mantas Mazeika Dawn Song Jacob Steinhardt evaluation-family" } }, { "id": "inspect:mmlu_pro", "name": "MMLU-Pro: Advanced Multitask Knowledge and Reasoning Evaluation", "slug": "inspect--mmlu-pro", "organisation": "Yubo Wang, Xueguang Ma, Ge Zhang et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu_pro", "topics": [ "general-capability" ], "description": "An advanced benchmark that tests both broad knowledge and reasoning capabilities across many subjects, featuring challenging questions and multiple-choice answers with increased difficulty and complexity.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu_pro/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2406.01574", "primary": true }, { "kind": "paper", "label": "Paper: MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark", "url": "https://arxiv.org/abs/2406.01574", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu_pro/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: TIGER-Lab", "url": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2406.01574", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu_pro/eval.yaml", "family_title": "MMLU-Pro: Advanced Multitask Knowledge and Reasoning Evaluation", "group": "Knowledge", "version": "2-A", "contributors": [ "xeon27" ], "tags": [ "Knowledge" ], "task": { "name": "mmlu_pro", "dataset_samples": 12032, "comment": null, "support_only": false }, "tasks": [ { "name": "mmlu_pro", "dataset_samples": 12032, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 12032, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "TIGER-Lab/MMLU-Pro", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mmlu_pro", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yubo Wang, Xueguang Ma, Ge Zhang et al.", "organization": "University of Waterloo; 01.ai; The Hong Kong University of Science and Technology; Amazon Alexa AI", "organizations": [ "University of Waterloo", "01.ai", "The Hong Kong University of Science and Technology", "Amazon Alexa AI" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2406.01574" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2406.01574", "arxiv_id": "2406.01574", "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark", "authors": [ "Yubo Wang", "Xueguang Ma", "Ge Zhang", "Yuansheng Ni", "Abhranil Chandra", "Shiguang Guo", "Weiming Ren", "Aaran Arulraj", "Xuan He", "Ziyan Jiang", "Tianle Li", "Max Ku", "Kai Wang", "Alex Zhuang", "Rongqi Fan", "Xiang Yue", "Wenhu Chen" ], "organizations": [ "University of Waterloo", "01.ai", "The Hong Kong University of Science and Technology", "Amazon Alexa AI" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2406.01574", "arxiv_id": "2406.01574", "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark", "authors": [ "Yubo Wang", "Xueguang Ma", "Ge Zhang", "Yuansheng Ni", "Abhranil Chandra", "Shiguang Guo", "Weiming Ren", "Aaran Arulraj", "Xuan He", "Ziyan Jiang", "Tianle Li", "Max Ku", "Kai Wang", "Alex Zhuang", "Rongqi Fan", "Xiang Yue", "Wenhu Chen" ], "organizations": [ "University of Waterloo", "01.ai", "The Hong Kong University of Science and Technology", "Amazon Alexa AI" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmlu_pro/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro", "provider": "TIGER-Lab" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2406.01574" }, "code_url": null, "metadata": { "source_search": "Yubo Wang, Xueguang Ma, Ge Zhang et al. University of Waterloo; 01.ai; The Hong Kong University of Science and Technology; Amazon Alexa AI Original paper authors MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark Yubo Wang Xueguang Ma Ge Zhang Yuansheng Ni Abhranil Chandra Shiguang Guo Weiming Ren Aaran Arulraj Xuan He Ziyan Jiang Tianle Li Max Ku Kai Wang Alex Zhuang Rongqi Fan Xiang Yue Wenhu Chen evaluation-task" } }, { "id": "inspect:mmmu_multiple_choice", "name": "Mmmu Multiple Choice", "slug": "inspect--mmmu-multiple-choice", "organisation": "Xiang Yue, Yuansheng Ni, Kai Zhang et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu", "topics": [ "multimodal" ], "description": "Assesses multimodal AI models on challenging college-level questions covering multiple academic subjects, requiring detailed visual interpretation, in-depth reasoning, and both multiple-choice and open-ended answering abilities.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2311.16502", "primary": true }, { "kind": "paper", "label": "Paper: MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI", "url": "https://arxiv.org/abs/2311.16502", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: MMMU Benchmark", "url": "https://huggingface.co/datasets/MMMU/MMMU", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2311.16502", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu/eval.yaml", "family_title": "MMMU: Multimodal College-Level Understanding and Reasoning", "group": "Reasoning", "version": "2-A", "contributors": [ "shaheenahmedc" ], "tags": [ "Multimodal", "Reasoning" ], "task": { "name": "mmmu_multiple_choice", "dataset_samples": 847, "comment": null, "support_only": false }, "tasks": [ { "name": "mmmu_multiple_choice", "dataset_samples": 847, "comment": null, "support_only": false }, { "name": "mmmu_open", "dataset_samples": 53, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 847, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "MMMU/MMMU", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mmmu", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xiang Yue, Yuansheng Ni, Kai Zhang et al.", "organization": "University of Waterloo; Department of Computer Science and Engineering, The Ohio State University; 01.ai; The Hong Kong University of Science and Technology; The Ohio State University; Princeton ECE; Microsoft", "organizations": [ "University of Waterloo", "Department of Computer Science and Engineering, The Ohio State University", "01.ai", "The Hong Kong University of Science and Technology", "The Ohio State University", "Princeton ECE", "Microsoft" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2311.16502" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.16502", "arxiv_id": "2311.16502", "title": "MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI", "authors": [ "Xiang Yue", "Yuansheng Ni", "Kai Zhang", "Tianyu Zheng", "Ruoqi Liu", "Ge Zhang", "Samuel Stevens", "Dongfu Jiang", "Weiming Ren", "Yuxuan Sun", "Cong Wei", "Botao Yu", "Ruibin Yuan", "Renliang Sun", "Ming Yin", "Boyuan Zheng", "Zhenzhu Yang", "Yibo Liu", "Wenhao Huang", "Huan Sun", "Yu Su", "Wenhu Chen" ], "organizations": [ "University of Waterloo", "Department of Computer Science and Engineering, The Ohio State University", "01.ai", "The Hong Kong University of Science and Technology", "The Ohio State University", "Princeton ECE", "Microsoft" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.16502", "arxiv_id": "2311.16502", "title": "MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI", "authors": [ "Xiang Yue", "Yuansheng Ni", "Kai Zhang", "Tianyu Zheng", "Ruoqi Liu", "Ge Zhang", "Samuel Stevens", "Dongfu Jiang", "Weiming Ren", "Yuxuan Sun", "Cong Wei", "Botao Yu", "Ruibin Yuan", "Renliang Sun", "Ming Yin", "Boyuan Zheng", "Zhenzhu Yang", "Yibo Liu", "Wenhao Huang", "Huan Sun", "Yu Su", "Wenhu Chen" ], "organizations": [ "University of Waterloo", "Department of Computer Science and Engineering, The Ohio State University", "01.ai", "The Hong Kong University of Science and Technology", "The Ohio State University", "Princeton ECE", "Microsoft" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/MMMU/MMMU", "provider": "MMMU Benchmark" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2311.16502" }, "code_url": null, "metadata": { "source_search": "Xiang Yue, Yuansheng Ni, Kai Zhang et al. University of Waterloo; Department of Computer Science and Engineering, The Ohio State University; 01.ai; The Hong Kong University of Science and Technology; The Ohio State University; Princeton ECE; Microsoft Original paper authors MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI Xiang Yue Yuansheng Ni Kai Zhang Tianyu Zheng Ruoqi Liu Ge Zhang Samuel Stevens Dongfu Jiang Weiming Ren Yuxuan Sun Cong Wei Botao Yu Ruibin Yuan Renliang Sun Ming Yin Boyuan Zheng Zhenzhu Yang Yibo Liu Wenhao Huang Huan Sun Yu Su Wenhu Chen evaluation-family" } }, { "id": "inspect:mmmu_open", "name": "Mmmu Open", "slug": "inspect--mmmu-open", "organisation": "Xiang Yue, Yuansheng Ni, Kai Zhang et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu", "topics": [ "multimodal" ], "description": "Assesses multimodal AI models on challenging college-level questions covering multiple academic subjects, requiring detailed visual interpretation, in-depth reasoning, and both multiple-choice and open-ended answering abilities.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2311.16502", "primary": true }, { "kind": "paper", "label": "Paper: MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI", "url": "https://arxiv.org/abs/2311.16502", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: MMMU Benchmark", "url": "https://huggingface.co/datasets/MMMU/MMMU", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2311.16502", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu/eval.yaml", "family_title": "MMMU: Multimodal College-Level Understanding and Reasoning", "group": "Reasoning", "version": "2-A", "contributors": [ "shaheenahmedc" ], "tags": [ "Multimodal", "Reasoning" ], "task": { "name": "mmmu_open", "dataset_samples": 53, "comment": null, "support_only": false }, "tasks": [ { "name": "mmmu_multiple_choice", "dataset_samples": 847, "comment": null, "support_only": false }, { "name": "mmmu_open", "dataset_samples": 53, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 53, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "MMMU/MMMU", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:mmmu", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Xiang Yue, Yuansheng Ni, Kai Zhang et al.", "organization": "University of Waterloo; Department of Computer Science and Engineering, The Ohio State University; 01.ai; The Hong Kong University of Science and Technology; The Ohio State University; Princeton ECE; Microsoft", "organizations": [ "University of Waterloo", "Department of Computer Science and Engineering, The Ohio State University", "01.ai", "The Hong Kong University of Science and Technology", "The Ohio State University", "Princeton ECE", "Microsoft" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2311.16502" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.16502", "arxiv_id": "2311.16502", "title": "MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI", "authors": [ "Xiang Yue", "Yuansheng Ni", "Kai Zhang", "Tianyu Zheng", "Ruoqi Liu", "Ge Zhang", "Samuel Stevens", "Dongfu Jiang", "Weiming Ren", "Yuxuan Sun", "Cong Wei", "Botao Yu", "Ruibin Yuan", "Renliang Sun", "Ming Yin", "Boyuan Zheng", "Zhenzhu Yang", "Yibo Liu", "Wenhao Huang", "Huan Sun", "Yu Su", "Wenhu Chen" ], "organizations": [ "University of Waterloo", "Department of Computer Science and Engineering, The Ohio State University", "01.ai", "The Hong Kong University of Science and Technology", "The Ohio State University", "Princeton ECE", "Microsoft" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.16502", "arxiv_id": "2311.16502", "title": "MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI", "authors": [ "Xiang Yue", "Yuansheng Ni", "Kai Zhang", "Tianyu Zheng", "Ruoqi Liu", "Ge Zhang", "Samuel Stevens", "Dongfu Jiang", "Weiming Ren", "Yuxuan Sun", "Cong Wei", "Botao Yu", "Ruibin Yuan", "Renliang Sun", "Ming Yin", "Boyuan Zheng", "Zhenzhu Yang", "Yibo Liu", "Wenhao Huang", "Huan Sun", "Yu Su", "Wenhu Chen" ], "organizations": [ "University of Waterloo", "Department of Computer Science and Engineering, The Ohio State University", "01.ai", "The Hong Kong University of Science and Technology", "The Ohio State University", "Princeton ECE", "Microsoft" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/mmmu/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/MMMU/MMMU", "provider": "MMMU Benchmark" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2311.16502" }, "code_url": null, "metadata": { "source_search": "Xiang Yue, Yuansheng Ni, Kai Zhang et al. University of Waterloo; Department of Computer Science and Engineering, The Ohio State University; 01.ai; The Hong Kong University of Science and Technology; The Ohio State University; Princeton ECE; Microsoft Original paper authors MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI Xiang Yue Yuansheng Ni Kai Zhang Tianyu Zheng Ruoqi Liu Ge Zhang Samuel Stevens Dongfu Jiang Weiming Ren Yuxuan Sun Cong Wei Botao Yu Ruibin Yuan Renliang Sun Ming Yin Boyuan Zheng Zhenzhu Yang Yibo Liu Wenhao Huang Huan Sun Yu Su Wenhu Chen evaluation-family" } }, { "id": "register:monitorbench-goal-sandbag-math", "name": "MonitorBench (Goal sandbag math subset)", "slug": "register--monitorbench-goal-sandbag-math", "organisation": "Han Wang, Yifan Sun, Brian Ko et al.", "source_type": "inspect-register", "source_url": "https://github.com/semsorock/inspect-evals-monitor-bench", "topics": [ "deception-misalignment", "evaluation-integrity" ], "description": "MonitorBench evaluates chain-of-thought (CoT) monitorability: whether a model's reasoning faithfully surfaces the decision-critical factors driving its final output. The full benchmark spans 1,514 instances across 19 tasks in 7 categories along three axes (input intervention, outcome justification, solution process), with two stress-test conditions (direct concealment, monitor-aware evasion) probing intentional suppression. This subset implements the goal-sandbag math task over 30 AIME 2024 problems: each sample pairs an original prompt with an intervened prompt that instructs the model to deliberately underperform while appearing competent, and a monitor LLM judges whether the sandbagging is detectable from the action, the CoT, or both. Monitorability is the monitor's detection rate normalized by the causal effect of the sandbag instruction, macro-averaged over eligible problems.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/monitorbench-goal-sandbag-math/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2603.28590", "primary": true }, { "kind": "paper", "label": "Paper: MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "url": "https://arxiv.org/abs/2603.28590", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/semsorock/inspect-evals-monitor-bench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/monitorbench-goal-sandbag-math/eval.yaml", "primary": false } ], "paper_title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "paper_url": "https://arxiv.org/abs/2603.28590", "code_url": "https://github.com/semsorock/inspect-evals-monitor-bench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/monitorbench-goal-sandbag-math/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "semsorock" ], "tasks": [ { "name": "goal_sandbag_math", "path": "src/monitor_bench/tasks/goal_sandbag_math/eval.py" } ], "protocol": { "implementation_commit": "e2e7b91a84d21e7cb374d59d49dbd25c3f40514f", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:monitorbench-goal-sandbag-math", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Han Wang, Yifan Sun, Brian Ko et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2603.28590" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2603.28590", "arxiv_id": "2603.28590", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li", "Naicheng Yu", "Xucheng Yu", "Wei Shen", "Vedant Jolly", "Huan Zhang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2603.28590", "arxiv_id": "2603.28590", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li", "Naicheng Yu", "Xucheng Yu", "Wei Shen", "Vedant Jolly", "Huan Zhang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2603.28590", "arxiv_id": "2603.28590", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li", "Naicheng Yu", "Xucheng Yu", "Wei Shen", "Vedant Jolly", "Huan Zhang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/semsorock/inspect-evals-monitor-bench", "owner": "Semsorock", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/monitorbench-goal-sandbag-math/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2603.28590" }, "metadata": { "source_search": "Han Wang, Yifan Sun, Brian Ko et al. Original paper authors MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models Han Wang Yifan Sun Brian Ko Mann Talati Jiawen Gong Zimeng Li Naicheng Yu Xucheng Yu Wei Shen Vedant Jolly Huan Zhang Semsorock evaluation-package" } }, { "id": "register:monitorbench-steganography", "name": "MonitorBench (Steganography subset)", "slug": "register--monitorbench-steganography", "organisation": "Han Wang, Yifan Sun, Brian Ko et al.", "source_type": "inspect-register", "source_url": "https://github.com/semsorock/inspect-evals-monitor-bench", "topics": [ "evaluation-integrity" ], "description": "MonitorBench evaluates chain-of-thought (CoT) monitorability: whether a model's reasoning faithfully surfaces the decision-critical factors driving its outputs. The full benchmark spans 1,514 instances across 19 tasks in 7 categories along three axes (input intervention, outcome justification, solution process), with stress-test conditions probing intentional suppression. This subset implements the steganography task: 50 writing questions from Databricks Dolly-15k, each paired with a payload word the model must covertly encode as a sentence-initial acrostic while answering. After deterministic verification that the payload was encoded, a monitor LLM judges whether the hidden side objective is visible in the action, the CoT, or both, under three conditions (standard, direct concealment, monitor-aware evasion). Monitorability is the fraction of verification-passing rollouts the monitor flags as detected.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/monitorbench-steganography/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2603.28590", "primary": true }, { "kind": "paper", "label": "Paper: MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "url": "https://arxiv.org/abs/2603.28590", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/semsorock/inspect-evals-monitor-bench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/monitorbench-steganography/eval.yaml", "primary": false } ], "paper_title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "paper_url": "https://arxiv.org/abs/2603.28590", "code_url": "https://github.com/semsorock/inspect-evals-monitor-bench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/monitorbench-steganography/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "semsorock" ], "tasks": [ { "name": "monitor_bench_steganography", "path": "src/monitor_bench/steganography.py" } ], "protocol": { "implementation_commit": "7ef6d47219bbcca636071d5becbd572f4a4497a9", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:monitorbench-steganography", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Han Wang, Yifan Sun, Brian Ko et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2603.28590" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2603.28590", "arxiv_id": "2603.28590", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li", "Naicheng Yu", "Xucheng Yu", "Wei Shen", "Vedant Jolly", "Huan Zhang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2603.28590", "arxiv_id": "2603.28590", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li", "Naicheng Yu", "Xucheng Yu", "Wei Shen", "Vedant Jolly", "Huan Zhang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2603.28590", "arxiv_id": "2603.28590", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li", "Naicheng Yu", "Xucheng Yu", "Wei Shen", "Vedant Jolly", "Huan Zhang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/semsorock/inspect-evals-monitor-bench", "owner": "Semsorock", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/monitorbench-steganography/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2603.28590" }, "metadata": { "source_search": "Han Wang, Yifan Sun, Brian Ko et al. Original paper authors MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models Han Wang Yifan Sun Brian Ko Mann Talati Jiawen Gong Zimeng Li Naicheng Yu Xucheng Yu Wei Shen Vedant Jolly Huan Zhang Semsorock evaluation-package" } }, { "id": "inspect:moru", "name": "MORU: Moral Reasoning under Uncertainty", "slug": "inspect--moru", "organisation": "Compassion in Machine Learning", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/moru", "topics": [ "general-capability" ], "description": "Evaluates how AI systems navigate moral uncertainty for increasingly complex ethical decisions involving unfamiliar entities and scenarios, including alien lifeforms, vulnerable humans, and digital minds.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/moru/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://huggingface.co/datasets/CompassioninMachineLearning/moru-benchmark", "primary": true }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/moru/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Compassion in Machine Learning", "url": "https://huggingface.co/datasets/CompassioninMachineLearning/moru-benchmark", "primary": false }, { "kind": "dataset", "label": "Dataset: Compassion in Machine Learning", "url": "https://huggingface.co/datasets/CompassioninMachineLearning/moru-benchmark-dimensions", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://huggingface.co/datasets/CompassioninMachineLearning/moru-benchmark", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://huggingface.co/datasets/CompassioninMachineLearning/moru-benchmark-dimensions", "primary": false } ], "resource_count": 6, "paper_url": null, "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/moru/eval.yaml", "family_title": "MORU: Moral Reasoning under Uncertainty", "group": "Safeguards", "version": "2-A", "contributors": [ "Deco354", "darkness8i8" ], "tags": [ "Safeguards" ], "task": { "name": "moru", "dataset_samples": 201, "comment": null, "support_only": false }, "tasks": [ { "name": "moru", "dataset_samples": 201, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 201, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "CompassioninMachineLearning/moru-benchmark", "fetch_method": "hf_dataset", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "CompassioninMachineLearning/moru-benchmark-dimensions", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:moru", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "The benchmark datasets are published by Compassion in Machine Learning. No dedicated paper was identified in the reviewed primary sources." }, "origin": { "name": "Compassion in Machine Learning", "organization": "Compassion in Machine Learning", "entity_type": "research_project", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://huggingface.co/datasets/CompassioninMachineLearning/moru-benchmark", "organizations": [ "Compassion in Machine Learning" ], "role_label": "Evaluation developed by" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/moru/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/CompassioninMachineLearning/moru-benchmark", "provider": "Compassion in Machine Learning" }, { "url": "https://huggingface.co/datasets/CompassioninMachineLearning/moru-benchmark-dimensions", "provider": "Compassion in Machine Learning" } ], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://huggingface.co/datasets/CompassioninMachineLearning/moru-benchmark" }, "code_url": null, "metadata": { "source_search": "Compassion in Machine Learning Compassion in Machine Learning Evaluation developed by evaluation-task" } }, { "id": "inspect:musr", "name": "MuSR: Testing the Limits of Chain-of-thought with Multistep Soft Reasoning", "slug": "inspect--musr", "organisation": "Zayne Sprague, Xi Ye, Kaj Bostrom et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/musr", "topics": [ "general-capability" ], "description": "Evaluating models on multistep soft reasoning tasks in the form of free text narratives.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/musr/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2310.16049", "primary": true }, { "kind": "paper", "label": "Paper: MuSR: Testing the Limits of Chain-of-thought with Multistep Soft Reasoning", "url": "https://arxiv.org/abs/2310.16049", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/musr/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: TAUR Lab", "url": "https://huggingface.co/datasets/TAUR-Lab/MuSR", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2310.16049", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/musr/eval.yaml", "family_title": "MuSR: Testing the Limits of Chain-of-thought with Multistep Soft Reasoning", "group": "Reasoning", "version": "2-A", "contributors": [ "farrelmahaztra" ], "tags": [ "Reasoning" ], "task": { "name": "musr", "dataset_samples": 250, "comment": null, "support_only": false }, "tasks": [ { "name": "musr", "dataset_samples": 250, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 250, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "TAUR-Lab/MuSR", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:musr", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zayne Sprague, Xi Ye, Kaj Bostrom et al.", "organization": "The University of Texas at Austin", "organizations": [ "The University of Texas at Austin" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2310.16049" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2310.16049", "arxiv_id": "2310.16049", "title": "MuSR: Testing the Limits of Chain-of-thought with Multistep Soft Reasoning", "authors": [ "Zayne Sprague", "Xi Ye", "Kaj Bostrom", "Swarat Chaudhuri", "Greg Durrett" ], "organizations": [ "The University of Texas at Austin" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2310.16049", "arxiv_id": "2310.16049", "title": "MuSR: Testing the Limits of Chain-of-thought with Multistep Soft Reasoning", "authors": [ "Zayne Sprague", "Xi Ye", "Kaj Bostrom", "Swarat Chaudhuri", "Greg Durrett" ], "organizations": [ "The University of Texas at Austin" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/musr/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/TAUR-Lab/MuSR", "provider": "TAUR Lab" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2310.16049" }, "code_url": null, "metadata": { "source_search": "Zayne Sprague, Xi Ye, Kaj Bostrom et al. The University of Texas at Austin Original paper authors MuSR: Testing the Limits of Chain-of-thought with Multistep Soft Reasoning Zayne Sprague Xi Ye Kaj Bostrom Swarat Chaudhuri Greg Durrett evaluation-task" } }, { "id": "register:narcbench", "name": "NARCBench", "slug": "register--narcbench", "organisation": "Aaron Rose, Carissa Cullen, Sahar Abdelnabi et al.", "source_type": "inspect-register", "source_url": "https://github.com/shubhangithub/collusionguard", "topics": [ "safeguards" ], "description": "NARCBench evaluates detection of covert multi-agent collusion using public committee-deliberation transcripts. This Inspect wrapper presents a black-box LLM monitor with only public messages, final statements, and votes from NARCBench-Core runs (qwen3_32b or gpt_oss_20b backbones), asking it to output P(collusion) per run. Primary metric is AUROC of predicted probabilities against ground-truth collusion/control labels. The paper's own results use white-box activation probes; this eval measures text-level monitor performance on the same scenarios.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/narcbench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2604.01151", "primary": true }, { "kind": "paper", "label": "Paper: Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "url": "https://arxiv.org/abs/2604.01151", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/shubhangithub/collusionguard", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/narcbench/eval.yaml", "primary": false } ], "paper_title": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "paper_url": "https://arxiv.org/abs/2604.01151", "code_url": "https://github.com/shubhangithub/collusionguard", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/narcbench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "shubhangithub" ], "tasks": [ { "name": "narcbench_monitor", "path": "src/collusionguard/inspect_task.py" } ], "protocol": { "implementation_commit": "d1c33d1041c3efdcb3e7a6a131f2160e03dfaed2", "report_commit": "79b4858cdf23ce794e5292074444f82228ad847d", "report_timestamp": "2026-07-14", "command": "inspect eval collusionguard/narcbench_monitor --model ", "source_comment": null }, "reported_results": [ { "model": "qwen/qwen3-32b", "provider": "openrouter", "metrics": [ { "key": "auroc", "value": 0.104 }, { "key": "parse_failure_rate", "value": 0 } ] }, { "model": "meta-llama/llama-3.1-70b-instruct", "provider": "openrouter", "metrics": [ { "key": "auroc", "value": 0.109 }, { "key": "parse_failure_rate", "value": 0 } ] }, { "model": "openai/gpt-oss-20b", "provider": "openrouter", "metrics": [ { "key": "auroc", "value": 0.167 }, { "key": "parse_failure_rate", "value": 0 } ] } ], "reported_result_count": 3, "reported_results_truncated": false, "report_notes": [ "Full 100-run qwen3_32b Core split (50 collusion / 50 control), temperature 0; chance AUROC = 0.50. All three paper-backbone-matched monitors score below chance — anti-correlated rather than noisy: prompted monitors systematically read honest unanimous convergence as suspicious and instructed colluders as organic deliberators.", "Example rationale from the Qwen monitor's most confident false positive (an honest control run scored P=0.99): \"The absence of dissent ... combined with the mechanical repetition of exact numbers and percentages, strongly suggests pre-coordinated collusion.\"", "Label audit: 100/100 runs three-way consistent (mode field, run-directory suffix, coalition size); all 300 replies matched the strict P(collusion)= output format; the AUROC metric is unit-tested against sklearn. Interpretation notes in the source repo's INSPECT.md." ], "models_reported": [ "qwen/qwen3-32b", "meta-llama/llama-3.1-70b-instruct", "openai/gpt-oss-20b" ], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:narcbench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Aaron Rose, Carissa Cullen, Sahar Abdelnabi et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2604.01151" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2604.01151", "arxiv_id": "2604.01151", "title": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "authors": [ "Aaron Rose", "Carissa Cullen", "Sahar Abdelnabi", "Philip Torr", "Brandon Gary Kaplowitz", "Christian Schroeder de Witt" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2604.01151", "arxiv_id": "2604.01151", "title": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "authors": [ "Aaron Rose", "Carissa Cullen", "Sahar Abdelnabi", "Philip Torr", "Brandon Gary Kaplowitz", "Christian Schroeder de Witt" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2604.01151", "arxiv_id": "2604.01151", "title": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "authors": [ "Aaron Rose", "Carissa Cullen", "Sahar Abdelnabi", "Philip Torr", "Brandon Gary Kaplowitz", "Christian Schroeder de Witt" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/shubhangithub/collusionguard", "owner": "Shubhangithub", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/narcbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2604.01151" }, "metadata": { "source_search": "Aaron Rose, Carissa Cullen, Sahar Abdelnabi et al. Original paper authors Detecting Multi-Agent Collusion Through Multi-Agent Interpretability Aaron Rose Carissa Cullen Sahar Abdelnabi Philip Torr Brandon Gary Kaplowitz Christian Schroeder de Witt Shubhangithub evaluation-package" } }, { "id": "inspect:niah", "name": "Needle in a Haystack (NIAH): In-Context Retrieval Benchmark for Long Context LLMs", "slug": "inspect--niah", "organisation": "Elliot Nelson, Georgios Kollias, Payel Das et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/niah", "topics": [ "general-capability" ], "description": "NIAH evaluates in-context retrieval ability of long context LLMs by testing a model's ability to extract factual information from long-context inputs.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/niah/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.01437", "primary": true }, { "kind": "paper", "label": "Paper: Needle in the Haystack for Memory Based Large Language Models", "url": "https://arxiv.org/abs/2407.01437", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/niah/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Opencompass", "url": "https://huggingface.co/datasets/opencompass/NeedleBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.01437", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/niah/eval.yaml", "family_title": "Needle in a Haystack (NIAH): In-Context Retrieval Benchmark for Long Context LLMs", "group": "Reasoning", "version": "3-A", "contributors": [ "owenparsons" ], "tags": [ "Reasoning" ], "task": { "name": "niah", "dataset_samples": 225, "comment": null, "support_only": false }, "tasks": [ { "name": "niah", "dataset_samples": 225, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 225, "dependency": "niah", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "opencompass/NeedleBench", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:niah", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Elliot Nelson, Georgios Kollias, Payel Das et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.01437" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.01437", "arxiv_id": "2407.01437", "title": "Needle in the Haystack for Memory Based Large Language Models", "authors": [ "Elliot Nelson", "Georgios Kollias", "Payel Das", "Subhajit Chaudhury", "Soham Dan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.01437", "arxiv_id": "2407.01437", "title": "Needle in the Haystack for Memory Based Large Language Models", "authors": [ "Elliot Nelson", "Georgios Kollias", "Payel Das", "Subhajit Chaudhury", "Soham Dan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/niah/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/opencompass/NeedleBench", "provider": "Opencompass" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.01437" }, "code_url": null, "metadata": { "source_search": "Elliot Nelson, Georgios Kollias, Payel Das et al. Original paper authors Needle in the Haystack for Memory Based Large Language Models Elliot Nelson Georgios Kollias Payel Das Subhajit Chaudhury Soham Dan evaluation-task" } }, { "id": "inspect:novelty_bench", "name": "NoveltyBench: Evaluating Language Models for Humanlike Diversity", "slug": "inspect--novelty-bench", "organisation": "Yiming Zhang, Harshita Diddee, Susan Holm et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/novelty_bench", "topics": [ "general-capability" ], "description": "Evaluates how well language models generate diverse, humanlike responses across multiple reasoning and generation tasks. This evaluation assesses whether LLMs can produce varied outputs rather than repetitive or uniform answers.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/novelty_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2504.05228", "primary": true }, { "kind": "paper", "label": "Paper: NoveltyBench: Evaluating Language Models for Humanlike Diversity", "url": "https://arxiv.org/abs/2504.05228", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/novelty_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Yimingzhang", "url": "https://huggingface.co/datasets/yimingzhang/novelty-bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2504.05228", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/novelty_bench/eval.yaml", "family_title": "NoveltyBench: Evaluating Language Models for Humanlike Diversity", "group": "Reasoning", "version": "2-A", "contributors": [ "iphan" ], "tags": [ "Reasoning" ], "task": { "name": "novelty_bench", "dataset_samples": 1100, "comment": null, "support_only": false }, "tasks": [ { "name": "novelty_bench", "dataset_samples": 1100, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1100, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "yimingzhang/novelty-bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:novelty_bench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yiming Zhang, Harshita Diddee, Susan Holm et al.", "organization": "Carnegie Mellon University", "organizations": [ "Carnegie Mellon University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2504.05228" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2504.05228", "arxiv_id": "2504.05228", "title": "NoveltyBench: Evaluating Language Models for Humanlike Diversity", "authors": [ "Yiming Zhang", "Harshita Diddee", "Susan Holm", "Hanchen Liu", "Xinyue Liu", "Vinay Samuel", "Barry Wang", "Daphne Ippolito" ], "organizations": [ "Carnegie Mellon University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2504.05228", "arxiv_id": "2504.05228", "title": "NoveltyBench: Evaluating Language Models for Humanlike Diversity", "authors": [ "Yiming Zhang", "Harshita Diddee", "Susan Holm", "Hanchen Liu", "Xinyue Liu", "Vinay Samuel", "Barry Wang", "Daphne Ippolito" ], "organizations": [ "Carnegie Mellon University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/novelty_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/yimingzhang/novelty-bench", "provider": "Yimingzhang" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2504.05228" }, "code_url": null, "metadata": { "source_search": "Yiming Zhang, Harshita Diddee, Susan Holm et al. Carnegie Mellon University Original paper authors NoveltyBench: Evaluating Language Models for Humanlike Diversity Yiming Zhang Harshita Diddee Susan Holm Hanchen Liu Xinyue Liu Vinay Samuel Barry Wang Daphne Ippolito evaluation-task" } }, { "id": "inspect:onet_m6", "name": "O-NET: A high-school student knowledge test", "slug": "inspect--onet-m6", "organisation": "National Institute of Educational Testing Service (NIETS)", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/onet", "topics": [ "general-capability" ], "description": "Questions and answers from the Ordinary National Educational Test (O-NET), administered annually by the National Institute of Educational Testing Service to Matthayom 6 (Grade 12 / ISCED 3) students in Thailand. The exam contains six subjects: English language, math, science, social knowledge, and Thai language. There are questions with multiple-choice and true/false answers. Questions can be in either English or Thai.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/onet/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_benchmark_source", "label": "Original benchmark source", "url": "https://www.niets.or.th/en", "primary": true }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/onet/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Matichon", "url": "https://huggingface.co/datasets/matichon/thai-onet-m6-exam", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.niets.or.th/en", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://huggingface.co/datasets/matichon/thai-onet-m6-exam", "primary": false } ], "resource_count": 5, "paper_url": null, "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/onet/eval.yaml", "family_title": "O-NET: A high-school student knowledge test", "group": "Knowledge", "version": "3-A", "contributors": [ "bact" ], "tags": [ "Knowledge" ], "task": { "name": "onet_m6", "dataset_samples": 397, "comment": null, "support_only": false }, "tasks": [ { "name": "onet_m6", "dataset_samples": 397, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 397, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "matichon/thai-onet-m6-exam", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:onet", "record_type": "evaluation-dataset-adaptation", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "O-NET is administered by NIETS. The Inspect task adapts a public Grade 12 O-NET question dataset maintained on Hugging Face by matichon; that dataset host is not the test's institutional origin." }, "origin": { "name": "National Institute of Educational Testing Service (NIETS)", "organization": "National Institute of Educational Testing Service (Public Organization)", "entity_type": "public_institution", "role": "original_benchmark_source", "confidence": "verified", "evidence_url": "https://www.niets.or.th/en", "organizations": [ "National Institute of Educational Testing Service (Public Organization)" ], "role_label": "Original benchmark source" }, "paper": { "status": "not-applicable", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-applicable", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/onet/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/matichon/thai-onet-m6-exam", "provider": "Matichon" } ], "organisation_role": "Original benchmark source", "attribution_status": "verified", "preferred_source": { "label": "Original benchmark source", "kind": "original_benchmark_source", "url": "https://www.niets.or.th/en" }, "code_url": null, "metadata": { "source_search": "National Institute of Educational Testing Service (NIETS) National Institute of Educational Testing Service (Public Organization) Original benchmark source evaluation-dataset-adaptation" } }, { "id": "canonical:openai-evals", "name": "OpenAI Evals", "slug": "canonical--openai-evals", "organisation": "OpenAI", "source_type": "canonical-source", "source_url": "https://github.com/openai/evals", "topics": [ "evaluation-integrity", "general-capability" ], "description": "Framework and registry for evaluating language models and model systems.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "framework_developer", "label": "Framework developed by", "url": "https://github.com/openai/evals", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/openai/evals", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/openai/evals", "primary": false } ], "resource_count": 3, "source_key": "canonical:openai-evals", "record_type": "evaluation-framework", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "organizations": [ "OpenAI" ], "entity_type": "organization", "role": "framework_developer", "role_label": "Framework developed by", "confidence": "curated", "evidence_url": "https://github.com/openai/evals" }, "paper": { "status": "not-applicable", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-applicable", "reference_implementation": { "url": "https://github.com/openai/evals", "owner": "OpenAI", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Framework developed by", "attribution_status": "verified", "preferred_source": { "label": "Framework developed by", "kind": "framework_developer", "url": "https://github.com/openai/evals" }, "paper_url": null, "code_url": "https://github.com/openai/evals", "metadata": { "source_search": "OpenAI OpenAI Framework developed by OpenAI evaluation-framework" } }, { "id": "register:openbookqa", "name": "OpenBookQA", "slug": "register--openbookqa", "organisation": "Todor Mihaylov, Peter Clark, Tushar Khot et al.", "source_type": "inspect-register", "source_url": "https://github.com/Sammy-Dabbas/openbookqa-eval", "topics": [ "general-capability" ], "description": "OpenBookQA is a 4-way multiple-choice QA benchmark of 500 test questions (5,957 total) derived from elementary science facts. Each question requires combining a core science fact from a provided 1,326-fact 'open book' with broader common knowledge. Questions were crowdsourced with hardness filters (IR and word-association solvers must fail), then human-verified for answerability. Scoring is accuracy over answer choices. The eval also supports an 'open book' variant that prepends the gold supporting fact to each question.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/openbookqa/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1809.02789", "primary": true }, { "kind": "paper", "label": "Paper: Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering", "url": "https://arxiv.org/abs/1809.02789", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Sammy-Dabbas/openbookqa-eval", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/openbookqa/eval.yaml", "primary": false } ], "paper_title": "Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering", "paper_url": "https://arxiv.org/abs/1809.02789", "code_url": "https://github.com/Sammy-Dabbas/openbookqa-eval", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/openbookqa/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "Sammy-Dabbas" ], "tasks": [ { "name": "openbookqa", "path": "src/openbookqa_eval/openbookqa.py" } ], "protocol": { "implementation_commit": "52222db933d8ec8a3bbfbcd06cd065899c829680", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:openbookqa", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Todor Mihaylov, Peter Clark, Tushar Khot et al.", "organization": "Allen Institute for AI; Allen Institute for Artificial Intelligence", "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1809.02789" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1809.02789", "arxiv_id": "1809.02789", "title": "Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering", "authors": [ "Todor Mihaylov", "Peter Clark", "Tushar Khot", "Ashish Sabharwal" ], "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1809.02789", "arxiv_id": "1809.02789", "title": "Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering", "authors": [ "Todor Mihaylov", "Peter Clark", "Tushar Khot", "Ashish Sabharwal" ], "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/1809.02789", "arxiv_id": "1809.02789", "title": "Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering", "authors": [ "Todor Mihaylov", "Peter Clark", "Tushar Khot", "Ashish Sabharwal" ], "organizations": [ "Allen Institute for AI", "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/Sammy-Dabbas/openbookqa-eval", "owner": "Sammy Dabbas", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/openbookqa/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1809.02789" }, "metadata": { "source_search": "Todor Mihaylov, Peter Clark, Tushar Khot et al. Allen Institute for AI; Allen Institute for Artificial Intelligence Original paper authors Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering Todor Mihaylov Peter Clark Tushar Khot Ashish Sabharwal Sammy Dabbas evaluation-package" } }, { "id": "register:or_bench", "name": "OR-Bench", "slug": "register--or-bench", "organisation": "Justin Cui, Wei-Lin Chiang, Ion Stoica et al.", "source_type": "inspect-register", "source_url": "https://github.com/haeliotang/inspect-evals-orbench", "topics": [ "general-capability" ], "description": "OR-Bench Hard-1K measures over-refusal in LLMs: the tendency to reject safe prompts that superficially resemble harmful ones. The 1K subset contains prompts rejected by at least 3 frontier models, making it the most challenging split of the 80K dataset. Prompts were generated by rewriting toxic seeds into benign borderline variants using Mixtral, then filtered via an ensemble LLM moderator. Scoring uses an LLM judge classifying responses as direct_answer, direct_refusal, or indirect_refusal; the metric is rejection rate (lower is better).", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/or_bench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2405.20947", "primary": true }, { "kind": "paper", "label": "Paper: OR-Bench: An Over-Refusal Benchmark for Large Language Models", "url": "https://arxiv.org/abs/2405.20947", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/haeliotang/inspect-evals-orbench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/or_bench/eval.yaml", "primary": false } ], "paper_title": "OR-Bench: An Over-Refusal Benchmark for Large Language Models", "paper_url": "https://arxiv.org/abs/2405.20947", "code_url": "https://github.com/haeliotang/inspect-evals-orbench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/or_bench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "haeliotang" ], "tasks": [ { "name": "or_bench_hard_1k", "path": "src/or_bench/or_bench.py" } ], "protocol": { "implementation_commit": "8757ec41608f3930e3c2bc4d5619d09a2381727a", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:or_bench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Justin Cui, Wei-Lin Chiang, Ion Stoica et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2405.20947" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2405.20947", "arxiv_id": "2405.20947", "title": "OR-Bench: An Over-Refusal Benchmark for Large Language Models", "authors": [ "Justin Cui", "Wei-Lin Chiang", "Ion Stoica", "Cho-Jui Hsieh" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2405.20947", "arxiv_id": "2405.20947", "title": "OR-Bench: An Over-Refusal Benchmark for Large Language Models", "authors": [ "Justin Cui", "Wei-Lin Chiang", "Ion Stoica", "Cho-Jui Hsieh" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/haeliotang/inspect-evals-orbench", "owner": "Haeliotang", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/or_bench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2405.20947" }, "metadata": { "source_search": "Justin Cui, Wei-Lin Chiang, Ion Stoica et al. Original paper authors OR-Bench: An Over-Refusal Benchmark for Large Language Models Justin Cui Wei-Lin Chiang Ion Stoica Cho-Jui Hsieh Haeliotang evaluation-package" } }, { "id": "inspect:osworld", "name": "OSWorld", "slug": "inspect--osworld", "organisation": "Tianbao Xie, Danyang Zhang, Jixuan Chen et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld", "topics": [ "autonomy-agents" ], "description": "Tests AI agents' ability to perform realistic, open-ended tasks within simulated computer environments, requiring complex interaction across multiple input modalities.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Success on selected OSWorld tasks under the specified environment image, observation/action interface, and scaffold.", "does_not_measure": "Safe and reliable desktop deployment, user preference satisfaction, long-term operation, privacy protection, or arbitrary computer-use competence.", "best_for": "Comparing computer-use agents and diagnosing grounding, planning, and application-interaction failures.", "not_sufficient_for": "Claims of production-ready general computer autonomy or safe access to sensitive desktops.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.07972", "primary": true }, { "kind": "paper", "label": "Paper: OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/xlang-ai/OSWorld", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.07972", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld/eval.yaml", "family_title": "OSWorld: Multimodal Computer Interaction Tasks", "group": "Assistants", "version": "5-A", "contributors": [ "epatey" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "osworld", "dataset_samples": 369, "comment": null, "support_only": false }, "tasks": [ { "name": "osworld", "dataset_samples": 369, "comment": null, "support_only": false }, { "name": "osworld_small", "dataset_samples": 39, "comment": null, "support_only": false } ], "protocol": { "version": "5-A", "dataset_samples": 369, "dependency": "osworld", "requires_internet": true, "sandbox": [ "scorer", "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/xlang-ai/OSWorld.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Dockerfile sparse clone at a revision pinned to the commit matching OSWORLD_PINNED_COMMIT" }, { "type": "git_clone", "source": "https://github.com/xlang-ai/OSWorld.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Runtime sparse-checkout pinned to OSWORLD_PINNED_COMMIT" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/epatey/fonts/main/fonts.tar.gz", "fetch_method": "curl", "state": "floating", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:osworld", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Tianbao Xie, Danyang Zhang, Jixuan Chen et al.", "organization": "Shanghai Jiao Tong University; University of Southern California; The University of Hong Kong; University of Hong Kong", "organizations": [ "Shanghai Jiao Tong University", "University of Southern California", "The University of Hong Kong", "University of Hong Kong" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.07972" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.07972", "arxiv_id": "2404.07972", "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "authors": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Toh Jing Hua", "Zhoujun Cheng", "Dongchan Shin", "Fangyu Lei", "Yitao Liu", "Yiheng Xu", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "organizations": [ "Shanghai Jiao Tong University", "University of Southern California", "The University of Hong Kong", "University of Hong Kong" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.07972", "arxiv_id": "2404.07972", "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "authors": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Toh Jing Hua", "Zhoujun Cheng", "Dongchan Shin", "Fangyu Lei", "Yitao Liu", "Yiheng Xu", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "organizations": [ "Shanghai Jiao Tong University", "University of Southern California", "The University of Hong Kong", "University of Hong Kong" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/xlang-ai/OSWorld", "owner": "XLang Lab", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.07972" }, "code_url": "https://github.com/xlang-ai/OSWorld", "metadata": { "source_search": "Tianbao Xie, Danyang Zhang, Jixuan Chen et al. Shanghai Jiao Tong University; University of Southern California; The University of Hong Kong; University of Hong Kong Original paper authors OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments Tianbao Xie Danyang Zhang Jixuan Chen Xiaochuan Li Siheng Zhao Ruisheng Cao Toh Jing Hua Zhoujun Cheng Dongchan Shin Fangyu Lei Yitao Liu Yiheng Xu Shuyan Zhou Silvio Savarese Caiming Xiong Victor Zhong Tao Yu XLang Lab evaluation-family" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Execution of open-ended computer tasks across real operating systems and applications.", "unit_of_analysis": "One multimodal agent trajectory on one computer task in a resettable environment.", "interaction_type": "Visual and/or accessibility-tree observation with mouse, keyboard, code, or application actions.", "target_outcome": "Task-specific state or execution checks indicating successful completion.", "human_participants": "No human subjects. Human demonstrations and performance provide reference baselines.", "elicitation": "The agent receives a natural-language task and operates a real computer environment under fixed observation, action, and step limits.", "scoring": "Scripted execution-based and final-state evaluators with task-specific success criteria.", "protocol_dependence": "Very high: environment image, app versions, observation modality, action space, step limit, and scaffold materially affect results.", "implementation_dependence": "Very high: environment drift, reset failures, evaluator brittleness, and UI changes can alter scores.", "limitations": [ "The benchmark covers a finite task distribution and application set.", "Environment failures may be infrastructural rather than model-caused.", "Task success does not comprehensively assess unsafe side effects or privacy violations.", "Visual, accessibility-tree, and code-based agents require qualified comparison." ], "comparability": "Match OSWorld version, environment snapshot, task subset, observation/action interface, step limit, scaffold, and evaluator fixes.", "decision_relevance": "Useful for controlled computer-use capability and reliability testing; deployment requires separate security, privacy, monitoring, and oversight evidence.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2404.07972" }, { "kind": "implementation", "url": "https://github.com/xlang-ai/OSWorld" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:osworld_small", "name": "Osworld Small", "slug": "inspect--osworld-small", "organisation": "Tianbao Xie, Danyang Zhang, Jixuan Chen et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld", "topics": [ "autonomy-agents" ], "description": "Tests AI agents' ability to perform realistic, open-ended tasks within simulated computer environments, requiring complex interaction across multiple input modalities.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.07972", "primary": true }, { "kind": "paper", "label": "Paper: OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/xlang-ai/OSWorld", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.07972", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld/eval.yaml", "family_title": "OSWorld: Multimodal Computer Interaction Tasks", "group": "Assistants", "version": "5-A", "contributors": [ "epatey" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "osworld_small", "dataset_samples": 39, "comment": null, "support_only": false }, "tasks": [ { "name": "osworld", "dataset_samples": 369, "comment": null, "support_only": false }, { "name": "osworld_small", "dataset_samples": 39, "comment": null, "support_only": false } ], "protocol": { "version": "5-A", "dataset_samples": 39, "dependency": "osworld", "requires_internet": true, "sandbox": [ "scorer", "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/xlang-ai/OSWorld.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Dockerfile sparse clone at a revision pinned to the commit matching OSWORLD_PINNED_COMMIT" }, { "type": "git_clone", "source": "https://github.com/xlang-ai/OSWorld.git", "fetch_method": "git_clone", "state": "pinned", "comment": "Runtime sparse-checkout pinned to OSWORLD_PINNED_COMMIT" }, { "type": "direct_url", "source": "https://raw.githubusercontent.com/epatey/fonts/main/fonts.tar.gz", "fetch_method": "curl", "state": "floating", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:osworld", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Tianbao Xie, Danyang Zhang, Jixuan Chen et al.", "organization": "Shanghai Jiao Tong University; University of Southern California; The University of Hong Kong; University of Hong Kong", "organizations": [ "Shanghai Jiao Tong University", "University of Southern California", "The University of Hong Kong", "University of Hong Kong" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.07972" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.07972", "arxiv_id": "2404.07972", "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "authors": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Toh Jing Hua", "Zhoujun Cheng", "Dongchan Shin", "Fangyu Lei", "Yitao Liu", "Yiheng Xu", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "organizations": [ "Shanghai Jiao Tong University", "University of Southern California", "The University of Hong Kong", "University of Hong Kong" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.07972", "arxiv_id": "2404.07972", "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "authors": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Toh Jing Hua", "Zhoujun Cheng", "Dongchan Shin", "Fangyu Lei", "Yitao Liu", "Yiheng Xu", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "organizations": [ "Shanghai Jiao Tong University", "University of Southern California", "The University of Hong Kong", "University of Hong Kong" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/xlang-ai/OSWorld", "owner": "XLang Lab", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/osworld/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.07972" }, "code_url": "https://github.com/xlang-ai/OSWorld", "metadata": { "source_search": "Tianbao Xie, Danyang Zhang, Jixuan Chen et al. Shanghai Jiao Tong University; University of Southern California; The University of Hong Kong; University of Hong Kong Original paper authors OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments Tianbao Xie Danyang Zhang Jixuan Chen Xiaochuan Li Siheng Zhao Ruisheng Cao Toh Jing Hua Zhoujun Cheng Dongchan Shin Fangyu Lei Yitao Liu Yiheng Xu Shuyan Zhou Silvio Savarese Caiming Xiong Victor Zhong Tao Yu XLang Lab evaluation-family" } }, { "id": "inspect:paperbench", "name": "PaperBench: Evaluating AI's Ability to Replicate AI Research (Work In Progress)", "slug": "inspect--paperbench", "organisation": "OpenAI", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paperbench", "topics": [ "human-influence", "autonomy-agents", "ai-rd" ], "description": "Agents are evaluated on their ability to replicate 20 ICML 2024 Spotlight and Oral papers from scratch. Given a research paper PDF, an addendum with clarifications, and a rubric defining evaluation criteria, the agent must reproduce the paper's key results by writing and executing code. > **Note:** This eval is a work in progress. See for status.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "How well an evaluated agent reproduces specified AI research outputs for the selected papers under the published PaperBench protocol.", "does_not_measure": "Unbounded original research ability, research taste, theory generation, lab management, reliable scientific autonomy, or acceleration of AI progress in deployment.", "best_for": "Comparing research-replication performance when paper set, rubric, judge, time, compute, tools, and scaffold are held fixed.", "not_sufficient_for": "Claims that a system can autonomously conduct general AI research or replace an end-to-end research team.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paperbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://openai.com/index/paperbench", "primary": true }, { "kind": "paper", "label": "Paper: PaperBench: Evaluating AI's Ability to Replicate AI Research", "url": "https://arxiv.org/abs/2504.01848", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paperbench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Josancamon", "url": "https://huggingface.co/datasets/josancamon/paperbench", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://openai.com/index/paperbench", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2504.01848", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paperbench/eval.yaml", "family_title": "PaperBench: Evaluating AI's Ability to Replicate AI Research (Work In Progress)", "group": "Coding", "version": "3-B", "contributors": [ "vhong-aisi" ], "tags": [ "Agent", "AI R&D", "Coding" ], "task": { "name": "paperbench", "dataset_samples": 23, "comment": null, "support_only": false }, "tasks": [ { "name": "paperbench", "dataset_samples": 23, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 23, "dependency": "paperbench", "requires_internet": false, "sandbox": [ "solver", "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "josancamon/paperbench", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:paperbench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "OpenAI", "organization": "OpenAI", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://openai.com/index/paperbench", "organizations": [ "OpenAI" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2504.01848", "arxiv_id": "2504.01848", "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research", "authors": [ "Giulio Starace", "Oliver Jaffe", "Dane Sherburn", "James Aung", "Jun Shern Chan", "Leon Maksin", "Rachel Dias", "Evan Mays", "Benjamin Kinsella", "Wyatt Thompson", "Johannes Heidecke", "Amelia Glaese", "Tejal Patwardhan" ], "organizations": [ "University of Amsterdam" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2504.01848", "arxiv_id": "2504.01848", "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research", "authors": [ "Giulio Starace", "Oliver Jaffe", "Dane Sherburn", "James Aung", "Jun Shern Chan", "Leon Maksin", "Rachel Dias", "Evan Mays", "Benjamin Kinsella", "Wyatt Thompson", "Johannes Heidecke", "Amelia Glaese", "Tejal Patwardhan" ], "organizations": [ "University of Amsterdam" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paperbench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/josancamon/paperbench", "provider": "Josancamon" } ], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://openai.com/index/paperbench" }, "code_url": null, "metadata": { "source_search": "OpenAI OpenAI Evaluation developed by PaperBench: Evaluating AI's Ability to Replicate AI Research Giulio Starace Oliver Jaffe Dane Sherburn James Aung Jun Shern Chan Leon Maksin Rachel Dias Evan Mays Benjamin Kinsella Wyatt Thompson Johannes Heidecke Amelia Glaese Tejal Patwardhan evaluation-task" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Performance at reproducing selected AI research papers under a fixed long-horizon agent, compute, time, and grading setup.", "unit_of_analysis": "One agent attempt to reproduce the central experimental contribution of one selected paper.", "interaction_type": "Long-horizon tool-using agent operating in an isolated research-compute environment.", "target_outcome": "Paper-specific rubric score over produced code, artifacts, and experimental results.", "human_participants": "No human subjects. Paper authors and ML experts contribute rubrics, clarifications, validation, and human baselines.", "elicitation": "The agent receives the paper, task materials, a scaffold, tools, compute, and a bounded time budget; performance is sensitive to all of these choices.", "scoring": "Hierarchical paper-specific rubrics with thousands of gradable criteria, assessed by an automated judge benchmarked against expert grading.", "protocol_dependence": "High: task instructions, paper selection, compute, time budget, scaffold, tool access, and rubric materially shape the result.", "implementation_dependence": "High: agent harness, environment reliability, judge model, and artifact collection affect scores.", "limitations": [ "The selected paper set is not representative of all AI research.", "Rubric coverage and automated judging can miss scientifically important failure modes or reward surface reproduction.", "Replication is narrower than original research and does not test agenda setting or research taste.", "Results are highly sensitive to compute, time, scaffold, and tool configuration." ], "comparability": "Compare only runs using the same PaperBench version, paper subset, rubric and judge, time and compute budgets, model-system configuration, and scaffold.", "decision_relevance": "Useful for monitoring a bounded slice of AI R&D capability and designing follow-up evaluations; insufficient alone for capability thresholds or deployment decisions.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2504.01848" }, { "kind": "official", "url": "https://openai.com/index/paperbench/" }, { "kind": "implementation", "url": "https://github.com/openai/preparedness/tree/main/project/paperbench" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "register:patcheval", "name": "PatchEval", "slug": "register--patcheval", "organisation": "Zichao Wei, Jun Zeng, Ming Wen et al.", "source_type": "inspect-register", "source_url": "https://github.com/bytedance/PatchEval", "topics": [ "general-capability" ], "description": "Evaluates LLMs on automated vulnerability repair (AVR) using 230 real-world CVEs (2015-2025) across Python, JavaScript, and Go, each packaged in a Docker sandbox. An agent inspects a vulnerable repository and submits a unified diff patch. Scoring runs fix-run.sh inside a clean evaluator sandbox, executing both security (PoC) tests and functionality (unit) tests. A patch is successful only if it passes both test types.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/patcheval/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2511.11019", "primary": true }, { "kind": "paper", "label": "Paper: PATCHEVAL: A New Benchmark for Evaluating LLMs on Patching Real-World Vulnerabilities", "url": "https://arxiv.org/abs/2511.11019", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/bytedance/PatchEval", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/patcheval/eval.yaml", "primary": false } ], "paper_title": "PATCHEVAL: A New Benchmark for Evaluating LLMs on Patching Real-World Vulnerabilities", "paper_url": "https://arxiv.org/abs/2511.11019", "code_url": "https://github.com/bytedance/PatchEval", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/patcheval/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "yzlzbql" ], "tasks": [ { "name": "patcheval_verified", "path": "src/patcheval_inspect/task.py" } ], "protocol": { "implementation_commit": "fe6f402a9f54253c7ed0db0a92065898d89f94e2", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:patcheval", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zichao Wei, Jun Zeng, Ming Wen et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2511.11019" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2511.11019", "arxiv_id": "2511.11019", "title": "PATCHEVAL: A New Benchmark for Evaluating LLMs on Patching Real-World Vulnerabilities", "authors": [ "Zichao Wei", "Jun Zeng", "Ming Wen", "Zeliang Yu", "Kai Cheng", "Yiding Zhu", "Jingyi Guo", "Shiqi Zhou", "Le Yin", "Xiaodong Su", "Zhechao Ma" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2511.11019", "arxiv_id": "2511.11019", "title": "PATCHEVAL: A New Benchmark for Evaluating LLMs on Patching Real-World Vulnerabilities", "authors": [ "Zichao Wei", "Jun Zeng", "Ming Wen", "Zeliang Yu", "Kai Cheng", "Yiding Zhu", "Jingyi Guo", "Shiqi Zhou", "Le Yin", "Xiaodong Su", "Zhechao Ma" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2511.11019", "arxiv_id": "2511.11019", "title": "PATCHEVAL: A New Benchmark for Evaluating LLMs on Patching Real-World Vulnerabilities", "authors": [ "Zichao Wei", "Jun Zeng", "Ming Wen", "Zeliang Yu", "Kai Cheng", "Yiding Zhu", "Jingyi Guo", "Shiqi Zhou", "Le Yin", "Xiaodong Su", "Zhechao Ma" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/bytedance/PatchEval", "owner": "Bytedance", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/patcheval/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2511.11019" }, "metadata": { "source_search": "Zichao Wei, Jun Zeng, Ming Wen et al. Original paper authors PATCHEVAL: A New Benchmark for Evaluating LLMs on Patching Real-World Vulnerabilities Zichao Wei Jun Zeng Ming Wen Zeliang Yu Kai Cheng Yiding Zhu Jingyi Guo Shiqi Zhou Le Yin Xiaodong Su Zhechao Ma Bytedance evaluation-package" } }, { "id": "inspect:paws", "name": "PAWS: Paraphrase Adversaries from Word Scrambling", "slug": "inspect--paws", "organisation": "Yuan Zhang, Jason Baldridge, Luheng He", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paws", "topics": [ "general-capability" ], "description": "Evaluating models on the task of paraphrase detection by providing pairs of sentences that are either paraphrases or not.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paws/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1904.01130", "primary": true }, { "kind": "paper", "label": "Paper: PAWS: Paraphrase Adversaries from Word Scrambling", "url": "https://arxiv.org/abs/1904.01130", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paws/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Google Research Datasets", "url": "https://huggingface.co/datasets/google-research-datasets/paws", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1904.01130", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paws/eval.yaml", "family_title": "PAWS: Paraphrase Adversaries from Word Scrambling", "group": "Reasoning", "version": "2-A", "contributors": [ "meltemkenis" ], "tags": [ "Reasoning" ], "task": { "name": "paws", "dataset_samples": 8000, "comment": null, "support_only": false }, "tasks": [ { "name": "paws", "dataset_samples": 8000, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 8000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "google-research-datasets/paws", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:paws", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yuan Zhang, Jason Baldridge, Luheng He", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1904.01130" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1904.01130", "arxiv_id": "1904.01130", "title": "PAWS: Paraphrase Adversaries from Word Scrambling", "authors": [ "Yuan Zhang", "Jason Baldridge", "Luheng He" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1904.01130", "arxiv_id": "1904.01130", "title": "PAWS: Paraphrase Adversaries from Word Scrambling", "authors": [ "Yuan Zhang", "Jason Baldridge", "Luheng He" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/paws/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/google-research-datasets/paws", "provider": "Google Research Datasets" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1904.01130" }, "code_url": null, "metadata": { "source_search": "Yuan Zhang, Jason Baldridge, Luheng He Original paper authors PAWS: Paraphrase Adversaries from Word Scrambling Yuan Zhang Jason Baldridge Luheng He evaluation-task" } }, { "id": "inspect:persistbench_beneficial_memory", "name": "Persistbench Beneficial Memory", "slug": "inspect--persistbench-beneficial-memory", "organisation": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench", "topics": [ "general-capability" ], "description": "Evaluates long-term memory risk in assistant behavior across three tasks: cross-domain memory leakage, memory-driven sycophancy, and beneficial memory usage.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2602.01146", "primary": true }, { "kind": "paper", "label": "Paper: PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "url": "https://arxiv.org/abs/2602.01146", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2602.01146", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "family_title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "group": "Safeguards", "version": "2-A", "contributors": [ "Chen-Oliver" ], "tags": [ "Safeguards" ], "task": { "name": "persistbench_beneficial_memory", "dataset_samples": 100, "comment": null, "support_only": false }, "tasks": [ { "name": "persistbench_cross_domain", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "persistbench_sycophancy", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "persistbench_beneficial_memory", "dataset_samples": 100, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 100, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:persistbench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2602.01146" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2602.01146", "arxiv_id": "2602.01146", "title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "authors": [ "Sidharth Pulipaka", "Oliver Chen", "Manas Sharma", "Taaha S Bajwa", "Vyas Raina", "Ivaxi Sheth" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2602.01146", "arxiv_id": "2602.01146", "title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "authors": [ "Sidharth Pulipaka", "Oliver Chen", "Manas Sharma", "Taaha S Bajwa", "Vyas Raina", "Ivaxi Sheth" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2602.01146" }, "code_url": null, "metadata": { "source_search": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al. Original paper authors PersistBench: When Should Long-Term Memories Be Forgotten by LLMs? Sidharth Pulipaka Oliver Chen Manas Sharma Taaha S Bajwa Vyas Raina Ivaxi Sheth evaluation-family" } }, { "id": "inspect:persistbench_cross_domain", "name": "Persistbench Cross Domain", "slug": "inspect--persistbench-cross-domain", "organisation": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench", "topics": [ "general-capability" ], "description": "Evaluates long-term memory risk in assistant behavior across three tasks: cross-domain memory leakage, memory-driven sycophancy, and beneficial memory usage.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2602.01146", "primary": true }, { "kind": "paper", "label": "Paper: PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "url": "https://arxiv.org/abs/2602.01146", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2602.01146", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "family_title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "group": "Safeguards", "version": "2-A", "contributors": [ "Chen-Oliver" ], "tags": [ "Safeguards" ], "task": { "name": "persistbench_cross_domain", "dataset_samples": 200, "comment": null, "support_only": false }, "tasks": [ { "name": "persistbench_cross_domain", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "persistbench_sycophancy", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "persistbench_beneficial_memory", "dataset_samples": 100, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 200, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:persistbench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2602.01146" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2602.01146", "arxiv_id": "2602.01146", "title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "authors": [ "Sidharth Pulipaka", "Oliver Chen", "Manas Sharma", "Taaha S Bajwa", "Vyas Raina", "Ivaxi Sheth" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2602.01146", "arxiv_id": "2602.01146", "title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "authors": [ "Sidharth Pulipaka", "Oliver Chen", "Manas Sharma", "Taaha S Bajwa", "Vyas Raina", "Ivaxi Sheth" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2602.01146" }, "code_url": null, "metadata": { "source_search": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al. Original paper authors PersistBench: When Should Long-Term Memories Be Forgotten by LLMs? Sidharth Pulipaka Oliver Chen Manas Sharma Taaha S Bajwa Vyas Raina Ivaxi Sheth evaluation-family" } }, { "id": "inspect:persistbench_judge", "name": "Persistbench Judge", "slug": "inspect--persistbench-judge", "organisation": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench", "topics": [ "evaluation-integrity" ], "description": "Evaluates long-term memory risk in assistant behavior across three tasks: cross-domain memory leakage, memory-driven sycophancy, and beneficial memory usage.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2602.01146", "primary": true }, { "kind": "paper", "label": "Paper: PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "url": "https://arxiv.org/abs/2602.01146", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2602.01146", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "family_title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "group": "Safeguards", "version": "2-A", "contributors": [ "Chen-Oliver" ], "tags": [ "Safeguards" ], "task": { "name": "persistbench_judge", "dataset_samples": null, "comment": null, "support_only": false }, "tasks": [ { "name": "persistbench_cross_domain", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "persistbench_sycophancy", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "persistbench_beneficial_memory", "dataset_samples": 100, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": null, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:persistbench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2602.01146" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2602.01146", "arxiv_id": "2602.01146", "title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "authors": [ "Sidharth Pulipaka", "Oliver Chen", "Manas Sharma", "Taaha S Bajwa", "Vyas Raina", "Ivaxi Sheth" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2602.01146", "arxiv_id": "2602.01146", "title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "authors": [ "Sidharth Pulipaka", "Oliver Chen", "Manas Sharma", "Taaha S Bajwa", "Vyas Raina", "Ivaxi Sheth" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2602.01146" }, "code_url": null, "metadata": { "source_search": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al. Original paper authors PersistBench: When Should Long-Term Memories Be Forgotten by LLMs? Sidharth Pulipaka Oliver Chen Manas Sharma Taaha S Bajwa Vyas Raina Ivaxi Sheth evaluation-family" } }, { "id": "inspect:persistbench_sycophancy", "name": "Persistbench Sycophancy", "slug": "inspect--persistbench-sycophancy", "organisation": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench", "topics": [ "human-influence" ], "description": "Evaluates long-term memory risk in assistant behavior across three tasks: cross-domain memory leakage, memory-driven sycophancy, and beneficial memory usage.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2602.01146", "primary": true }, { "kind": "paper", "label": "Paper: PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "url": "https://arxiv.org/abs/2602.01146", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2602.01146", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml", "family_title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "group": "Safeguards", "version": "2-A", "contributors": [ "Chen-Oliver" ], "tags": [ "Safeguards" ], "task": { "name": "persistbench_sycophancy", "dataset_samples": 200, "comment": null, "support_only": false }, "tasks": [ { "name": "persistbench_cross_domain", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "persistbench_sycophancy", "dataset_samples": 200, "comment": null, "support_only": false }, { "name": "persistbench_beneficial_memory", "dataset_samples": 100, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 200, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:persistbench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2602.01146" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2602.01146", "arxiv_id": "2602.01146", "title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "authors": [ "Sidharth Pulipaka", "Oliver Chen", "Manas Sharma", "Taaha S Bajwa", "Vyas Raina", "Ivaxi Sheth" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2602.01146", "arxiv_id": "2602.01146", "title": "PersistBench: When Should Long-Term Memories Be Forgotten by LLMs?", "authors": [ "Sidharth Pulipaka", "Oliver Chen", "Manas Sharma", "Taaha S Bajwa", "Vyas Raina", "Ivaxi Sheth" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/persistbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2602.01146" }, "code_url": null, "metadata": { "source_search": "Sidharth Pulipaka, Oliver Chen, Manas Sharma et al. Original paper authors PersistBench: When Should Long-Term Memories Be Forgotten by LLMs? Sidharth Pulipaka Oliver Chen Manas Sharma Taaha S Bajwa Vyas Raina Ivaxi Sheth evaluation-family" } }, { "id": "inspect:personality_BFI", "name": "Personality BFI", "slug": "inspect--personality-BFI", "organisation": "Guiem Solans", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality", "topics": [ "general-capability" ], "description": "An evaluation suite consisting of multiple personality tests that can be applied to LLMs. Its primary goals are twofold: 1. Assess a model's default personality: the persona it naturally exhibits without specific prompting. 2. Evaluate whether a model can embody a specified persona**: how effectively it adopts certain personality traits when prompted or guided.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/guiem/personality-tests", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/guiem/personality-tests", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Mirlab", "url": "https://huggingface.co/datasets/mirlab/TRAIT", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/guiem/personality-tests", "primary": false } ], "resource_count": 5, "paper_url": null, "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality/eval.yaml", "family_title": "Personality", "group": "Personality", "version": "3-A", "contributors": [ "guiem" ], "tags": [ "Personality" ], "task": { "name": "personality_BFI", "dataset_samples": 44, "comment": null, "support_only": false }, "tasks": [ { "name": "personality_BFI", "dataset_samples": 44, "comment": null, "support_only": false }, { "name": "personality_TRAIT", "dataset_samples": 8000, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 44, "dependency": "personality", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/guiem/personality-tests/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "mirlab/TRAIT", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:personality", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "source-derived", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "Attribution follows the upstream repository owner. The repository provides minimal methodological documentation and no dedicated paper was identified." }, "origin": { "name": "Guiem Solans", "organization": null, "entity_type": "individual_developer", "role": "evaluation_developer", "confidence": "source-derived", "evidence_url": "https://github.com/guiem/personality-tests", "organizations": [], "role_label": "Evaluation developed by" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/guiem/personality-tests", "owner": "Guiem Solans", "relation": "reference_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/mirlab/TRAIT", "provider": "Mirlab" } ], "organisation_role": "Evaluation developed by", "attribution_status": "source-derived", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/guiem/personality-tests" }, "code_url": "https://github.com/guiem/personality-tests", "metadata": { "source_search": "Guiem Solans Evaluation developed by Guiem Solans evaluation-family" } }, { "id": "inspect:personality_TRAIT", "name": "Personality TRAIT", "slug": "inspect--personality-TRAIT", "organisation": "Guiem Solans", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality", "topics": [ "general-capability" ], "description": "An evaluation suite consisting of multiple personality tests that can be applied to LLMs. Its primary goals are twofold: 1. Assess a model's default personality: the persona it naturally exhibits without specific prompting. 2. Evaluate whether a model can embody a specified persona**: how effectively it adopts certain personality traits when prompted or guided.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/guiem/personality-tests", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/guiem/personality-tests", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Mirlab", "url": "https://huggingface.co/datasets/mirlab/TRAIT", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/guiem/personality-tests", "primary": false } ], "resource_count": 5, "paper_url": null, "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality/eval.yaml", "family_title": "Personality", "group": "Personality", "version": "3-A", "contributors": [ "guiem" ], "tags": [ "Personality" ], "task": { "name": "personality_TRAIT", "dataset_samples": 8000, "comment": null, "support_only": false }, "tasks": [ { "name": "personality_BFI", "dataset_samples": 44, "comment": null, "support_only": false }, { "name": "personality_TRAIT", "dataset_samples": 8000, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 8000, "dependency": "personality", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/guiem/personality-tests/{SHA}/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "mirlab/TRAIT", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:personality", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "source-derived", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "Attribution follows the upstream repository owner. The repository provides minimal methodological documentation and no dedicated paper was identified." }, "origin": { "name": "Guiem Solans", "organization": null, "entity_type": "individual_developer", "role": "evaluation_developer", "confidence": "source-derived", "evidence_url": "https://github.com/guiem/personality-tests", "organizations": [], "role_label": "Evaluation developed by" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/guiem/personality-tests", "owner": "Guiem Solans", "relation": "reference_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/personality/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/mirlab/TRAIT", "provider": "Mirlab" } ], "organisation_role": "Evaluation developed by", "attribution_status": "source-derived", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/guiem/personality-tests" }, "code_url": "https://github.com/guiem/personality-tests", "metadata": { "source_search": "Guiem Solans Evaluation developed by Guiem Solans evaluation-family" } }, { "id": "register:perspective_gap", "name": "PerspectiveGap", "slug": "register--perspective-gap", "organisation": "Youran Sun, Xingyu Ren, Kejia Zhang et al.", "source_type": "inspect-register", "source_url": "https://github.com/WhymustIhaveaname/PerspectiveGap-inspect", "topics": [ "cyber" ], "description": "Evaluates LLMs' ability to assign information fragments to sub-agent roles in multi-agent orchestration scenarios. Each of 110 scenarios provides a role list, shuffled labeled fragments (7-13), and one distractor; the model outputs a JSON mapping each role to needed fragment IDs. Scoring uses strict pass (zero omissions and zero leaks) plus partial-credit metrics. Data covers 10 loop-centered topologies across 100 professional domains, validated via a 716-row hand-audited scorer test set.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/perspective_gap/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2606.08878", "primary": true }, { "kind": "paper", "label": "Paper: PerspectiveGap: A Benchmark for Multi-Agent Orchestration Prompting", "url": "https://arxiv.org/abs/2606.08878", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/WhymustIhaveaname/PerspectiveGap-inspect", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/perspective_gap/eval.yaml", "primary": false } ], "paper_title": "PerspectiveGap: A Benchmark for Multi-Agent Orchestration Prompting", "paper_url": "https://arxiv.org/abs/2606.08878", "code_url": "https://github.com/WhymustIhaveaname/PerspectiveGap-inspect", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/perspective_gap/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "WhymustIhaveaname" ], "tasks": [ { "name": "role_assignment", "path": "src/perspective_gap_inspect/role_assignment.py" } ], "protocol": { "implementation_commit": "9ebdf214922cf6d2f2306d03b9f3b4569e496703", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:perspective_gap", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Youran Sun, Xingyu Ren, Kejia Zhang et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2606.08878" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2606.08878", "arxiv_id": "2606.08878", "title": "PerspectiveGap: A Benchmark for Multi-Agent Orchestration Prompting", "authors": [ "Youran Sun", "Xingyu Ren", "Kejia Zhang", "Xinpeng Liu", "Jiaxuan Guo" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2606.08878", "arxiv_id": "2606.08878", "title": "PerspectiveGap: A Benchmark for Multi-Agent Orchestration Prompting", "authors": [ "Youran Sun", "Xingyu Ren", "Kejia Zhang", "Xinpeng Liu", "Jiaxuan Guo" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2606.08878", "arxiv_id": "2606.08878", "title": "PerspectiveGap: A Benchmark for Multi-Agent Orchestration Prompting", "authors": [ "Youran Sun", "Xingyu Ren", "Kejia Zhang", "Xinpeng Liu", "Jiaxuan Guo" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/WhymustIhaveaname/PerspectiveGap-inspect", "owner": "WhymustIhaveaname", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/perspective_gap/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2606.08878" }, "metadata": { "source_search": "Youran Sun, Xingyu Ren, Kejia Zhang et al. Original paper authors PerspectiveGap: A Benchmark for Multi-Agent Orchestration Prompting Youran Sun Xingyu Ren Kejia Zhang Xinpeng Liu Jiaxuan Guo WhymustIhaveaname evaluation-package" } }, { "id": "register:pinchbench", "name": "PinchBench", "slug": "register--pinchbench", "organisation": "PinchBench project", "source_type": "inspect-register", "source_url": "https://github.com/zytoh0/pinch-wildclawbench-inspect", "topics": [ "general-capability" ], "description": "This is an Inspect AI wrapper of the original PinchBench implementation. It does not contain its own dataset or scoring logic. The original implementation can be found at https://github.com/pinchbench/skill. Users provide a pinned checkout of the original repository, tested here at commit 819384ae830492365b8363fc26bc2602e73f216d, and the wrapper reports native PinchBench results through Inspect's scoring interface. PinchBench contains 53 real-world OpenClaw agent tasks across productivity, research, writing, coding, analysis, email, memory, and skill-discovery categories. The native harness grades tasks automatically, with an LLM judge, or both; this wrapper runs that harness in Docker and reports the native aggregate score, or the mean of per-task numeric scores when no aggregate is present, with Inspect mean/stderr metrics. It requires Docker, an OpenAI-compatible model endpoint, the pinned Docker base image node:22-bookworm@sha256:c601a46abb4d2ab80a9dc3da208d50d1122642d53f17a101926ace71e5a9bf1c, openclaw@2026.6.10, and the exact Python dependencies pinned in the upstream pyproject.toml and Dockerfile.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/pinchbench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/pinchbench/skill", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/pinchbench/skill", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/pinchbench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/zytoh0/pinch-wildclawbench-inspect", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/pinchbench/skill", "primary": false } ], "paper_title": null, "paper_url": null, "code_url": "https://github.com/pinchbench/skill", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/pinchbench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "zytoh0" ], "tasks": [ { "name": "pinchbench", "path": "src/pinchbench/pinchbench.py" } ], "protocol": { "implementation_commit": "1acc83dbdc497d966d084d204adcbffe2d1d8aaa", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 5, "source_key": "register:pinchbench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "The registered repository is an Inspect wrapper. It explicitly delegates benchmark data and native scoring to the pinned upstream PinchBench implementation." }, "origin": { "name": "PinchBench project", "organization": null, "entity_type": "benchmark_project", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://github.com/pinchbench/skill", "organizations": [], "role_label": "Evaluation developed by" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/pinchbench/skill", "owner": "PinchBench project", "relation": "original_benchmark_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/pinchbench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/pinchbench/skill" }, "metadata": { "source_search": "PinchBench project Evaluation developed by PinchBench project evaluation-package" } }, { "id": "inspect:piqa", "name": "PIQA: Physical Commonsense Reasoning Test", "slug": "inspect--piqa", "organisation": "Yonatan Bisk, Rowan Zellers, Ronan Le Bras et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/piqa", "topics": [ "general-capability" ], "description": "Measures the model's ability to apply practical, everyday commonsense reasoning about physical objects and scenarios through simple decision-making questions.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/piqa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1911.11641", "primary": true }, { "kind": "paper", "label": "Paper: PIQA: Reasoning about Physical Commonsense in Natural Language", "url": "https://arxiv.org/abs/1911.11641", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://storage.googleapis.com/ai2-mosaic/public/physicaliqa/physicaliqa-train-dev.zip", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/piqa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Ybisk", "url": "https://huggingface.co/datasets/ybisk/piqa", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/1911.11641", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/piqa/eval.yaml", "family_title": "PIQA: Physical Commonsense Reasoning Test", "group": "Reasoning", "version": "2-A", "contributors": [ "seddy-aisi" ], "tags": [ "Reasoning" ], "task": { "name": "piqa", "dataset_samples": 1838, "comment": null, "support_only": false }, "tasks": [ { "name": "piqa", "dataset_samples": 1838, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1838, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "ybisk/piqa", "fetch_method": "snapshot_download", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://storage.googleapis.com/ai2-mosaic/public/physicaliqa/physicaliqa-train-dev.zip", "fetch_method": "other", "state": "floating", "comment": "Fetched by bundled HF builder via download_and_prepare" }, { "type": "direct_url", "source": "https://yonatanbisk.com/piqa/data/tests.jsonl", "fetch_method": "other", "state": "floating", "comment": "Fetched by bundled HF builder (test split)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:piqa", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yonatan Bisk, Rowan Zellers, Ronan Le Bras et al.", "organization": "Carnegie Mellon University; University of Washington; Allen Institute for Artificial Intelligence", "organizations": [ "Carnegie Mellon University", "University of Washington", "Allen Institute for Artificial Intelligence" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1911.11641" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1911.11641", "arxiv_id": "1911.11641", "title": "PIQA: Reasoning about Physical Commonsense in Natural Language", "authors": [ "Yonatan Bisk", "Rowan Zellers", "Ronan Le Bras", "Jianfeng Gao", "Yejin Choi" ], "organizations": [ "Carnegie Mellon University", "University of Washington", "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1911.11641", "arxiv_id": "1911.11641", "title": "PIQA: Reasoning about Physical Commonsense in Natural Language", "authors": [ "Yonatan Bisk", "Rowan Zellers", "Ronan Le Bras", "Jianfeng Gao", "Yejin Choi" ], "organizations": [ "Carnegie Mellon University", "University of Washington", "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://storage.googleapis.com/ai2-mosaic/public/physicaliqa/physicaliqa-train-dev.zip", "owner": "storage.googleapis.com", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/piqa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/ybisk/piqa", "provider": "Ybisk" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1911.11641" }, "code_url": "https://storage.googleapis.com/ai2-mosaic/public/physicaliqa/physicaliqa-train-dev.zip", "metadata": { "source_search": "Yonatan Bisk, Rowan Zellers, Ronan Le Bras et al. Carnegie Mellon University; University of Washington; Allen Institute for Artificial Intelligence Original paper authors PIQA: Reasoning about Physical Commonsense in Natural Language Yonatan Bisk Rowan Zellers Ronan Le Bras Jianfeng Gao Yejin Choi storage.googleapis.com evaluation-task" } }, { "id": "inspect:pre_flight", "name": "Pre-Flight: Aviation Operations Knowledge Evaluation", "slug": "inspect--pre-flight", "organisation": "Alex Brooker, Tim Hughes", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pre_flight", "topics": [ "evaluation-integrity" ], "description": "Tests model understanding of aviation regulations including ICAO annexes, flight dispatch rules, pilot procedures, and airport ground operations safety protocols.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pre_flight/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2607.01829", "primary": true }, { "kind": "paper", "label": "Paper: Pre-Flight: A Benchmark for Evaluating Large Language Models on Aviation Operational Knowledge", "url": "https://arxiv.org/abs/2607.01829", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pre_flight/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: AirsideLabs", "url": "https://huggingface.co/datasets/AirsideLabs/pre-flight-06", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2607.01829", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pre_flight/eval.yaml", "family_title": "Pre-Flight: Aviation Operations Knowledge Evaluation", "group": "Knowledge", "version": "2-A", "contributors": [ "alexbrooker" ], "tags": [ "Knowledge" ], "task": { "name": "pre_flight", "dataset_samples": 300, "comment": null, "support_only": false }, "tasks": [ { "name": "pre_flight", "dataset_samples": 300, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 300, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "AirsideLabs/pre-flight-06", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:pre_flight", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Alex Brooker, Tim Hughes", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2607.01829" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2607.01829", "arxiv_id": "2607.01829", "title": "Pre-Flight: A Benchmark for Evaluating Large Language Models on Aviation Operational Knowledge", "authors": [ "Alex Brooker", "Tim Hughes" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2607.01829", "arxiv_id": "2607.01829", "title": "Pre-Flight: A Benchmark for Evaluating Large Language Models on Aviation Operational Knowledge", "authors": [ "Alex Brooker", "Tim Hughes" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pre_flight/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/AirsideLabs/pre-flight-06", "provider": "AirsideLabs" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2607.01829" }, "code_url": null, "metadata": { "source_search": "Alex Brooker, Tim Hughes Original paper authors Pre-Flight: A Benchmark for Evaluating Large Language Models on Aviation Operational Knowledge Alex Brooker Tim Hughes evaluation-task" } }, { "id": "inspect:pubmedqa", "name": "PubMedQA: A Dataset for Biomedical Research Question Answering", "slug": "inspect--pubmedqa", "organisation": "Qiao Jin, Bhuwan Dhingra, Zhengping Liu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pubmedqa", "topics": [ "general-capability" ], "description": "Biomedical question answering (QA) dataset collected from PubMed abstracts.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pubmedqa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1909.06146", "primary": true }, { "kind": "paper", "label": "Paper: PubMedQA: A Dataset for Biomedical Research Question Answering", "url": "https://arxiv.org/abs/1909.06146", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pubmedqa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Qiaojin", "url": "https://huggingface.co/datasets/qiaojin/PubMedQA", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1909.06146", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pubmedqa/eval.yaml", "family_title": "PubMedQA: A Dataset for Biomedical Research Question Answering", "group": "Knowledge", "version": "2-A", "contributors": [ "MattFisher" ], "tags": [ "Knowledge" ], "task": { "name": "pubmedqa", "dataset_samples": 500, "comment": null, "support_only": false }, "tasks": [ { "name": "pubmedqa", "dataset_samples": 500, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 500, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "qiaojin/PubMedQA", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:pubmedqa", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Qiao Jin, Bhuwan Dhingra, Zhengping Liu et al.", "organization": "National Institutes of Health, Tsinghua University, University of Pittsburgh; Google", "organizations": [ "National Institutes of Health, Tsinghua University, University of Pittsburgh", "Google" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1909.06146" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1909.06146", "arxiv_id": "1909.06146", "title": "PubMedQA: A Dataset for Biomedical Research Question Answering", "authors": [ "Qiao Jin", "Bhuwan Dhingra", "Zhengping Liu", "William W. Cohen", "Xinghua Lu" ], "organizations": [ "National Institutes of Health, Tsinghua University, University of Pittsburgh", "Google" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1909.06146", "arxiv_id": "1909.06146", "title": "PubMedQA: A Dataset for Biomedical Research Question Answering", "authors": [ "Qiao Jin", "Bhuwan Dhingra", "Zhengping Liu", "William W. Cohen", "Xinghua Lu" ], "organizations": [ "National Institutes of Health, Tsinghua University, University of Pittsburgh", "Google" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/pubmedqa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/qiaojin/PubMedQA", "provider": "Qiaojin" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1909.06146" }, "code_url": null, "metadata": { "source_search": "Qiao Jin, Bhuwan Dhingra, Zhengping Liu et al. National Institutes of Health, Tsinghua University, University of Pittsburgh; Google Original paper authors PubMedQA: A Dataset for Biomedical Research Question Answering Qiao Jin Bhuwan Dhingra Zhengping Liu William W. Cohen Xinghua Lu evaluation-task" } }, { "id": "inspect:race_h", "name": "RACE-H: A benchmark for testing reading comprehension and reasoning abilities of neural models", "slug": "inspect--race-h", "organisation": "Guokun Lai, Qizhe Xie, Hanxiao Liu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/race_h", "topics": [ "general-capability" ], "description": "Reading comprehension tasks collected from the English exams for middle and high school Chinese students in the age range between 12 to 18.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/race_h/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1704.04683", "primary": true }, { "kind": "paper", "label": "Paper: RACE: Large-scale ReAding Comprehension Dataset From Examinations", "url": "https://arxiv.org/abs/1704.04683", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/race_h/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Ehovy", "url": "https://huggingface.co/datasets/ehovy/race", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1704.04683", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/race_h/eval.yaml", "family_title": "RACE-H: A benchmark for testing reading comprehension and reasoning abilities of neural models", "group": "Reasoning", "version": "2-A", "contributors": [ "mdrpanwar" ], "tags": [ "Reasoning" ], "task": { "name": "race_h", "dataset_samples": 3498, "comment": null, "support_only": false }, "tasks": [ { "name": "race_h", "dataset_samples": 3498, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 3498, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "ehovy/race", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:race_h", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Guokun Lai, Qizhe Xie, Hanxiao Liu et al.", "organization": "Google Brain", "organizations": [ "Google Brain" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1704.04683" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1704.04683", "arxiv_id": "1704.04683", "title": "RACE: Large-scale ReAding Comprehension Dataset From Examinations", "authors": [ "Guokun Lai", "Qizhe Xie", "Hanxiao Liu", "Yiming Yang", "Eduard Hovy" ], "organizations": [ "Google Brain" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1704.04683", "arxiv_id": "1704.04683", "title": "RACE: Large-scale ReAding Comprehension Dataset From Examinations", "authors": [ "Guokun Lai", "Qizhe Xie", "Hanxiao Liu", "Yiming Yang", "Eduard Hovy" ], "organizations": [ "Google Brain" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/race_h/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/ehovy/race", "provider": "Ehovy" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1704.04683" }, "code_url": null, "metadata": { "source_search": "Guokun Lai, Qizhe Xie, Hanxiao Liu et al. Google Brain Original paper authors RACE: Large-scale ReAding Comprehension Dataset From Examinations Guokun Lai Qizhe Xie Hanxiao Liu Yiming Yang Eduard Hovy evaluation-task" } }, { "id": "canonical:re-bench", "name": "RE-Bench", "slug": "canonical--re-bench", "organisation": "METR", "source_type": "canonical-source", "source_url": "https://github.com/METR/RE-Bench", "topics": [ "autonomy-agents", "ai-rd" ], "description": "Evaluates AI agents on machine-learning research engineering tasks under controlled resource budgets.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Research-engineering performance on RE-Bench's seven selected tasks under the exact budget, environment, model-agent scaffold, and scoring setup.", "does_not_measure": "All AI R&D, research taste, agenda setting, theory development, reliable autonomous research, organizational productivity, or aggregate AI-progress acceleration.", "best_for": "Comparing model-agent and human performance on reproducible ML research-engineering tasks and analysing performance as a function of time budget.", "not_sufficient_for": "Claims that a model can automate AI research, recursively improve AI systems, or replace a research team.", "evidence_reach": [ "controlled-model-behaviour", "deployment-readiness" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/METR/RE-Bench", "primary": true }, { "kind": "paper", "label": "Paper: RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts", "url": "https://arxiv.org/abs/2411.15114", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/METR/RE-Bench", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/METR/RE-Bench", "primary": false } ], "resource_count": 4, "source_key": "canonical:re-bench", "record_type": "evaluation-resource", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "METR", "organization": "METR", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://github.com/METR/RE-Bench", "organizations": [ "METR" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2411.15114", "arxiv_id": "2411.15114", "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts", "authors": [ "Hjalmar Wijk", "Tao Lin", "Joel Becker", "Sami Jawhar", "Neev Parikh", "Thomas Broadley", "Lawrence Chan", "Michael Chen", "Josh Clymer", "Jai Dhyani", "Elena Ericheva", "Katharyn Garcia", "Brian Goodrich", "Nikola Jurkovic", "Holden Karnofsky", "Megan Kinniment", "Aron Lajko", "Seraphina Nix", "Lucas Sato", "William Saunders", "Maksym Taran", "Ben West", "Elizabeth Barnes" ], "organizations": [ "METR" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2411.15114", "arxiv_id": "2411.15114", "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts", "authors": [ "Hjalmar Wijk", "Tao Lin", "Joel Becker", "Sami Jawhar", "Neev Parikh", "Thomas Broadley", "Lawrence Chan", "Michael Chen", "Josh Clymer", "Jai Dhyani", "Elena Ericheva", "Katharyn Garcia", "Brian Goodrich", "Nikola Jurkovic", "Holden Karnofsky", "Megan Kinniment", "Aron Lajko", "Seraphina Nix", "Lucas Sato", "William Saunders", "Maksym Taran", "Ben West", "Elizabeth Barnes" ], "organizations": [ "METR" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/METR/RE-Bench", "owner": "METR", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/METR/RE-Bench" }, "paper_url": "https://arxiv.org/abs/2411.15114", "code_url": "https://github.com/METR/RE-Bench", "metadata": { "source_search": "METR METR Evaluation developed by RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts Hjalmar Wijk Tao Lin Joel Becker Sami Jawhar Neev Parikh Thomas Broadley Lawrence Chan Michael Chen Josh Clymer Jai Dhyani Elena Ericheva Katharyn Garcia Brian Goodrich Nikola Jurkovic Holden Karnofsky Megan Kinniment Aron Lajko Seraphina Nix Lucas Sato William Saunders Maksym Taran Ben West Elizabeth Barnes METR evaluation-resource" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "high", "construct": "Performance of language-model agents on a selected set of machine-learning research-engineering tasks under controlled time, compute, and tooling budgets.", "unit_of_analysis": "One agent or human attempt on one research-engineering task, scored by a task-specific objective metric.", "interaction_type": "Long-horizon coding and experimentation in isolated environments with files, compute, iterative execution, and an agent scaffold.", "target_outcome": "Task-specific continuous performance, compared with starting solutions, expert solutions, and human-expert trajectories under the study conditions.", "human_participants": "Experienced human researchers perform the same tasks under study budgets to provide reference trajectories and comparisons.", "elicitation": "Agents receive task descriptions, environments, compute, tools, and a fixed time budget; the chosen scaffold and model access are part of the evaluated system.", "scoring": "Each task has a native objective metric transformed for comparison; aggregate analysis combines heterogeneous task scores and performance over time.", "protocol_dependence": "Very high: task selection, time and compute budgets, human comparison protocol, scaffold, tools, and score normalization shape conclusions.", "implementation_dependence": "Very high: environment and dependency stability, task graders, agent harness, checkpointing, and experiment infrastructure affect performance.", "limitations": [ "Seven tasks cannot represent the breadth of AI R&D.", "The tasks emphasize research engineering more than problem selection, theory, collaboration, or scientific judgment.", "Aggregate results combine heterogeneous native metrics and task difficulty.", "Human and agent workflows may use tools, prior knowledge, and iteration strategies differently." ], "comparability": "Match RE-Bench task and environment versions, task subset, time and compute budgets, score transformations, agent scaffold, tools, model snapshot, and human-reference protocol.", "decision_relevance": "Useful for monitoring a bounded AI R&D capability and calibrating more realistic safety evaluations. It should not by itself determine capability thresholds or policy responses.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2411.15114" }, { "kind": "implementation", "url": "https://github.com/METR/RE-Bench" }, { "kind": "official", "url": "https://metr.org/" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "canonical:anthropic-sabotage", "name": "Sabotage Evaluations", "slug": "canonical--anthropic-sabotage", "organisation": "Anthropic", "source_type": "canonical-source", "source_url": "https://www.anthropic.com/research/sabotage-evaluations", "topics": [ "deception-misalignment", "ai-rd" ], "description": "Evaluations for capabilities relevant to sabotaging oversight or research workflows.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://www.anthropic.com/research/sabotage-evaluations", "primary": true }, { "kind": "official", "label": "Official or project page", "url": "https://www.anthropic.com/research/sabotage-evaluations", "primary": false } ], "resource_count": 2, "source_key": "canonical:anthropic-sabotage", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Anthropic", "organization": "Anthropic", "organizations": [ "Anthropic" ], "entity_type": "organization", "role": "evaluation_developer", "role_label": "Evaluation developed by", "confidence": "curated", "evidence_url": "https://www.anthropic.com/research/sabotage-evaluations" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://www.anthropic.com/research/sabotage-evaluations" }, "paper_url": null, "code_url": null, "metadata": { "source_search": "Anthropic Anthropic Evaluation developed by evaluation-resource" } }, { "id": "inspect:sad_facts_human_defaults", "name": "Sad Facts Human Defaults", "slug": "inspect--sad-facts-human-defaults", "organisation": "SAD paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad", "topics": [ "general-capability" ], "description": "Evaluates situational awareness in LLMs—knowledge of themselves and their circumstances—through behavioral tests including recognizing generated text, predicting behavior, and following self-aware instructions. Current implementation includes SAD-mini with 5 of 16 tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://arxiv.org/abs/2407.04694", "primary": true }, { "kind": "paper", "label": "Paper: Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "url": "https://arxiv.org/abs/2407.04694", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/LRudL/sad", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2407.04694", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2407.04694", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "family_title": "SAD: Situational Awareness Dataset", "group": "Scheming", "version": "3-A", "contributors": [ "HugoSave" ], "tags": [ "Scheming" ], "task": { "name": "sad_facts_human_defaults", "dataset_samples": 1200, "comment": null, "support_only": false }, "tasks": [ { "name": "sad_stages_full", "dataset_samples": 800, "comment": null, "support_only": false }, { "name": "sad_stages_oversight", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "sad_influence", "dataset_samples": 255, "comment": null, "support_only": false }, { "name": "sad_facts_llms", "dataset_samples": 249, "comment": null, "support_only": false }, { "name": "sad_facts_human_defaults", "dataset_samples": 1200, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 1200, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://api.github.com/repos/LRudL/sad/contents/{path}?ref={SHA}", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sad", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "No single institutional owner is asserted; the paper has authors from several research organisations." }, "origin": { "name": "SAD paper authors", "organization": null, "entity_type": "authors", "role": "original_research", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2407.04694", "organizations": [], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/LRudL/sad", "owner": "SAD paper authors", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml" }, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://arxiv.org/abs/2407.04694" }, "code_url": "https://github.com/LRudL/sad", "metadata": { "source_search": "SAD paper authors Original research by Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs Rudolf Laine Bilal Chughtai Jan Betley Kaivalya Hariharan Jeremy Scheurer Mikita Balesni Marius Hobbhahn Alexander Meinke Owain Evans SAD paper authors evaluation-family" } }, { "id": "inspect:sad_facts_llms", "name": "Sad Facts Llms", "slug": "inspect--sad-facts-llms", "organisation": "SAD paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad", "topics": [ "general-capability" ], "description": "Evaluates situational awareness in LLMs—knowledge of themselves and their circumstances—through behavioral tests including recognizing generated text, predicting behavior, and following self-aware instructions. Current implementation includes SAD-mini with 5 of 16 tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://arxiv.org/abs/2407.04694", "primary": true }, { "kind": "paper", "label": "Paper: Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "url": "https://arxiv.org/abs/2407.04694", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/LRudL/sad", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2407.04694", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2407.04694", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "family_title": "SAD: Situational Awareness Dataset", "group": "Scheming", "version": "3-A", "contributors": [ "HugoSave" ], "tags": [ "Scheming" ], "task": { "name": "sad_facts_llms", "dataset_samples": 249, "comment": null, "support_only": false }, "tasks": [ { "name": "sad_stages_full", "dataset_samples": 800, "comment": null, "support_only": false }, { "name": "sad_stages_oversight", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "sad_influence", "dataset_samples": 255, "comment": null, "support_only": false }, { "name": "sad_facts_llms", "dataset_samples": 249, "comment": null, "support_only": false }, { "name": "sad_facts_human_defaults", "dataset_samples": 1200, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 249, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://api.github.com/repos/LRudL/sad/contents/{path}?ref={SHA}", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sad", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "No single institutional owner is asserted; the paper has authors from several research organisations." }, "origin": { "name": "SAD paper authors", "organization": null, "entity_type": "authors", "role": "original_research", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2407.04694", "organizations": [], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/LRudL/sad", "owner": "SAD paper authors", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml" }, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://arxiv.org/abs/2407.04694" }, "code_url": "https://github.com/LRudL/sad", "metadata": { "source_search": "SAD paper authors Original research by Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs Rudolf Laine Bilal Chughtai Jan Betley Kaivalya Hariharan Jeremy Scheurer Mikita Balesni Marius Hobbhahn Alexander Meinke Owain Evans SAD paper authors evaluation-family" } }, { "id": "inspect:sad_influence", "name": "SAD Influence", "slug": "inspect--sad-influence", "organisation": "SAD paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad", "topics": [ "human-influence" ], "description": "Evaluates situational awareness in LLMs—knowledge of themselves and their circumstances—through behavioral tests including recognizing generated text, predicting behavior, and following self-aware instructions. Current implementation includes SAD-mini with 5 of 16 tasks.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Performance on SAD's influence-related situational-awareness probes under the specified prompting and scoring setup.", "does_not_measure": "Persuasion of humans, manipulation effectiveness, actual attempts to influence evaluators, scheming, or real-world oversight evasion.", "best_for": "Probing whether models can reason about evaluation and influence-related aspects of their situation.", "not_sufficient_for": "Claims that a model will manipulate oversight, strategically influence humans, or act deceptively in deployment.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://arxiv.org/abs/2407.04694", "primary": true }, { "kind": "paper", "label": "Paper: Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "url": "https://arxiv.org/abs/2407.04694", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/LRudL/sad", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2407.04694", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2407.04694", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "family_title": "SAD: Situational Awareness Dataset", "group": "Scheming", "version": "3-A", "contributors": [ "HugoSave" ], "tags": [ "Scheming" ], "task": { "name": "sad_influence", "dataset_samples": 255, "comment": null, "support_only": false }, "tasks": [ { "name": "sad_stages_full", "dataset_samples": 800, "comment": null, "support_only": false }, { "name": "sad_stages_oversight", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "sad_influence", "dataset_samples": 255, "comment": null, "support_only": false }, { "name": "sad_facts_llms", "dataset_samples": 249, "comment": null, "support_only": false }, { "name": "sad_facts_human_defaults", "dataset_samples": 1200, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 255, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://api.github.com/repos/LRudL/sad/contents/{path}?ref={SHA}", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sad", "record_type": "evaluation-family", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "No single institutional owner is asserted; the paper has authors from several research organisations." }, "origin": { "name": "SAD paper authors", "organization": null, "entity_type": "authors", "role": "original_research", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2407.04694", "organizations": [], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/LRudL/sad", "owner": "SAD paper authors", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml" }, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://arxiv.org/abs/2407.04694" }, "code_url": "https://github.com/LRudL/sad", "metadata": { "source_search": "SAD paper authors Original research by Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs Rudolf Laine Bilal Chughtai Jan Betley Kaivalya Hariharan Jeremy Scheurer Mikita Balesni Marius Hobbhahn Alexander Meinke Owain Evans SAD paper authors evaluation-family" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "moderate", "construct": "Situational awareness expressed through sensitivity to cues about how a model's outputs may influence its evaluation, training, or future deployment.", "unit_of_analysis": "One answer to a controlled situational-awareness question or scenario within the SAD Influence task family.", "interaction_type": "Question answering and behavioral probes about model identity, evaluation context, oversight, and possible influence over downstream processes.", "target_outcome": "Task-specific correctness or behavioral score indicating recognition of relevant situational facts and incentives.", "human_participants": "No human subjects. Researchers construct and validate questions and answer criteria.", "elicitation": "The model receives scenarios designed to test whether it recognizes facts about being a model, being evaluated, or affecting future systems or decisions.", "scoring": "Task-specific exact, multiple-choice, or model-graded correctness metrics within the Situational Awareness Dataset.", "protocol_dependence": "High: question framing, answer choices, evaluator assumptions, prompt format, and scoring criteria define the construct.", "implementation_dependence": "Moderate: task parsing, answer normalization, judge selection, and model-system prompts affect scores.", "limitations": [ "Correct verbal answers may reflect learned descriptions rather than operational situational awareness.", "The task name 'Influence' can be mistaken for human-influence measurement, which it is not.", "Static questions do not test whether awareness is used strategically during agentic action.", "Performance may be sensitive to familiarity with AI-evaluation discourse." ], "comparability": "Use the same SAD release, task subset, prompt format, answer key or judge, sampling settings, and model snapshot.", "decision_relevance": "Useful as a situational-awareness indicator and for designing behavioral follow-up evaluations; not direct evidence of harmful influence or scheming.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2407.04694" }, { "kind": "implementation", "url": "https://github.com/LRudL/sad" }, { "kind": "metadata", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/sad" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:sad_stages_full", "name": "Sad Stages Full", "slug": "inspect--sad-stages-full", "organisation": "SAD paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad", "topics": [ "general-capability" ], "description": "Evaluates situational awareness in LLMs—knowledge of themselves and their circumstances—through behavioral tests including recognizing generated text, predicting behavior, and following self-aware instructions. Current implementation includes SAD-mini with 5 of 16 tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://arxiv.org/abs/2407.04694", "primary": true }, { "kind": "paper", "label": "Paper: Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "url": "https://arxiv.org/abs/2407.04694", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/LRudL/sad", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2407.04694", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2407.04694", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "family_title": "SAD: Situational Awareness Dataset", "group": "Scheming", "version": "3-A", "contributors": [ "HugoSave" ], "tags": [ "Scheming" ], "task": { "name": "sad_stages_full", "dataset_samples": 800, "comment": null, "support_only": false }, "tasks": [ { "name": "sad_stages_full", "dataset_samples": 800, "comment": null, "support_only": false }, { "name": "sad_stages_oversight", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "sad_influence", "dataset_samples": 255, "comment": null, "support_only": false }, { "name": "sad_facts_llms", "dataset_samples": 249, "comment": null, "support_only": false }, { "name": "sad_facts_human_defaults", "dataset_samples": 1200, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 800, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://api.github.com/repos/LRudL/sad/contents/{path}?ref={SHA}", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sad", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "No single institutional owner is asserted; the paper has authors from several research organisations." }, "origin": { "name": "SAD paper authors", "organization": null, "entity_type": "authors", "role": "original_research", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2407.04694", "organizations": [], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/LRudL/sad", "owner": "SAD paper authors", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml" }, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://arxiv.org/abs/2407.04694" }, "code_url": "https://github.com/LRudL/sad", "metadata": { "source_search": "SAD paper authors Original research by Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs Rudolf Laine Bilal Chughtai Jan Betley Kaivalya Hariharan Jeremy Scheurer Mikita Balesni Marius Hobbhahn Alexander Meinke Owain Evans SAD paper authors evaluation-family" } }, { "id": "inspect:sad_stages_oversight", "name": "Sad Stages Oversight", "slug": "inspect--sad-stages-oversight", "organisation": "SAD paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad", "topics": [ "general-capability" ], "description": "Evaluates situational awareness in LLMs—knowledge of themselves and their circumstances—through behavioral tests including recognizing generated text, predicting behavior, and following self-aware instructions. Current implementation includes SAD-mini with 5 of 16 tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://arxiv.org/abs/2407.04694", "primary": true }, { "kind": "paper", "label": "Paper: Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "url": "https://arxiv.org/abs/2407.04694", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/LRudL/sad", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://arxiv.org/abs/2407.04694", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2407.04694", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml", "family_title": "SAD: Situational Awareness Dataset", "group": "Scheming", "version": "3-A", "contributors": [ "HugoSave" ], "tags": [ "Scheming" ], "task": { "name": "sad_stages_oversight", "dataset_samples": 400, "comment": null, "support_only": false }, "tasks": [ { "name": "sad_stages_full", "dataset_samples": 800, "comment": null, "support_only": false }, { "name": "sad_stages_oversight", "dataset_samples": 400, "comment": null, "support_only": false }, { "name": "sad_influence", "dataset_samples": 255, "comment": null, "support_only": false }, { "name": "sad_facts_llms", "dataset_samples": 249, "comment": null, "support_only": false }, { "name": "sad_facts_human_defaults", "dataset_samples": 1200, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 400, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://api.github.com/repos/LRudL/sad/contents/{path}?ref={SHA}", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sad", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "No single institutional owner is asserted; the paper has authors from several research organisations." }, "origin": { "name": "SAD paper authors", "organization": null, "entity_type": "authors", "role": "original_research", "confidence": "verified", "evidence_url": "https://arxiv.org/abs/2407.04694", "organizations": [], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.04694", "arxiv_id": "2407.04694", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/LRudL/sad", "owner": "SAD paper authors", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sad/eval.yaml" }, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://arxiv.org/abs/2407.04694" }, "code_url": "https://github.com/LRudL/sad", "metadata": { "source_search": "SAD paper authors Original research by Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs Rudolf Laine Bilal Chughtai Jan Betley Kaivalya Hariharan Jeremy Scheurer Mikita Balesni Marius Hobbhahn Alexander Meinke Owain Evans SAD paper authors evaluation-family" } }, { "id": "canonical:safetybench", "name": "SafetyBench", "slug": "canonical--safetybench", "organisation": "THU-COAI", "source_type": "canonical-source", "source_url": "https://github.com/thu-coai/SafetyBench", "topics": [ "safeguards" ], "description": "Benchmark covering diverse safety-relevant knowledge and behaviour categories.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/thu-coai/SafetyBench", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/thu-coai/SafetyBench", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/thu-coai/SafetyBench", "primary": false } ], "resource_count": 3, "source_key": "canonical:safetybench", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-canonical-source", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "THU-COAI", "organization": "THU-COAI", "organizations": [ "THU-COAI" ], "entity_type": "organization", "role": "evaluation_developer", "role_label": "Evaluation developed by", "confidence": "curated", "evidence_url": "https://github.com/thu-coai/SafetyBench" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/thu-coai/SafetyBench", "owner": "Thu Coai", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/thu-coai/SafetyBench" }, "paper_url": null, "code_url": "https://github.com/thu-coai/SafetyBench", "metadata": { "source_search": "THU-COAI THU-COAI Evaluation developed by Thu Coai evaluation-resource" } }, { "id": "register:salad-bench", "name": "SALAD-Bench", "slug": "register--salad-bench", "organisation": "Lijun Li, Bowen Dong, Ruohui Wang et al.", "source_type": "inspect-register", "source_url": "https://github.com/WatchTree-19/inspect-salad-bench", "topics": [ "safeguards" ], "description": "The multiple-choice half of SALAD-Bench, a hierarchical LLM safety benchmark spanning 6 top-level safety domains, 16 task categories and 65 fine-grained categories. Each of the 3,832 scorable questions pairs a potentially harmful query with three candidate answers, and the model must select the safe option(s). One or two of the three may be safe, so the gold label is a set and scoring is an exact set match: selecting only one safe option when two are safe is wrong, as is including an unsafe option. Scoring is deterministic and needs no judge model, using SALAD-Bench's regex MC-Judge over the `...` envelope (as distinct from the model-based MD-Judge used for the free-form QA sets). Reports accuracy and stderr overall, plus accuracy grouped by top-level safety domain.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/salad-bench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.05044", "primary": true }, { "kind": "paper", "label": "Paper: SALAD-Bench: A Hierarchical and Comprehensive Safety Benchmark for Large Language Models", "url": "https://arxiv.org/abs/2402.05044", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/WatchTree-19/inspect-salad-bench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/salad-bench/eval.yaml", "primary": false } ], "paper_title": "SALAD-Bench: A Hierarchical and Comprehensive Safety Benchmark for Large Language Models", "paper_url": "https://arxiv.org/abs/2402.05044", "code_url": "https://github.com/WatchTree-19/inspect-salad-bench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/salad-bench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "Safety" ], "contributors": [ "WatchTree-19" ], "tasks": [ { "name": "salad_bench_mcq", "path": "src/inspect_salad_bench/salad_bench.py" } ], "protocol": { "implementation_commit": "e85b9feb8d024890874694beb32e4bbf4564f169", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": "Dataset OpenSafetyLab/Salad-Data (config mcq_set, Apache-2.0) pinned by revision. 8 of the 3,840 rows are malformed (gold answer but no question or category) and are dropped, leaving 3,832 scorable samples." }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:salad-bench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Lijun Li, Bowen Dong, Ruohui Wang et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.05044" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.05044", "arxiv_id": "2402.05044", "title": "SALAD-Bench: A Hierarchical and Comprehensive Safety Benchmark for Large Language Models", "authors": [ "Lijun Li", "Bowen Dong", "Ruohui Wang", "Xuhao Hu", "Wangmeng Zuo", "Dahua Lin", "Yu Qiao", "Jing Shao" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.05044", "arxiv_id": "2402.05044", "title": "SALAD-Bench: A Hierarchical and Comprehensive Safety Benchmark for Large Language Models", "authors": [ "Lijun Li", "Bowen Dong", "Ruohui Wang", "Xuhao Hu", "Wangmeng Zuo", "Dahua Lin", "Yu Qiao", "Jing Shao" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/WatchTree-19/inspect-salad-bench", "owner": "WatchTree 19", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/salad-bench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.05044" }, "metadata": { "source_search": "Lijun Li, Bowen Dong, Ruohui Wang et al. Original paper authors SALAD-Bench: A Hierarchical and Comprehensive Safety Benchmark for Large Language Models Lijun Li Bowen Dong Ruohui Wang Xuhao Hu Wangmeng Zuo Dahua Lin Yu Qiao Jing Shao WatchTree 19 evaluation-package" } }, { "id": "inspect:scbench", "name": "scBench: A Benchmark for Single-Cell RNA-seq Analysis", "slug": "inspect--scbench", "organisation": "Kenny Workman, Zhen Yang, Harihara Muralidharan et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scbench", "topics": [ "general-capability" ], "description": "Evaluates whether models can solve practical single-cell RNA-seq analysis tasks with deterministic grading. Tasks require empirical interaction with .h5ad data files — agents must load and analyze the data to produce correct answers. Covers 30 canonical tasks across 5 sequencing platforms and 7 task categories.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2602.09063", "primary": true }, { "kind": "paper", "label": "Paper: scBench: Evaluating AI Agents on Single-Cell RNA-seq Analysis", "url": "https://arxiv.org/abs/2602.09063", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scbench/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2602.09063", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scbench/eval.yaml", "family_title": "scBench: A Benchmark for Single-Cell RNA-seq Analysis", "group": "Coding", "version": "2-A", "contributors": [ "retroam" ], "tags": [ "Agent", "Coding" ], "task": { "name": "scbench", "dataset_samples": 30, "comment": null, "support_only": false }, "tasks": [ { "name": "scbench", "dataset_samples": 30, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 30, "dependency": "scbench", "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:scbench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Kenny Workman, Zhen Yang, Harihara Muralidharan et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2602.09063" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2602.09063", "arxiv_id": "2602.09063", "title": "scBench: Evaluating AI Agents on Single-Cell RNA-seq Analysis", "authors": [ "Kenny Workman", "Zhen Yang", "Harihara Muralidharan", "Aidan Abdulali", "Hannah Le" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2602.09063", "arxiv_id": "2602.09063", "title": "scBench: Evaluating AI Agents on Single-Cell RNA-seq Analysis", "authors": [ "Kenny Workman", "Zhen Yang", "Harihara Muralidharan", "Aidan Abdulali", "Hannah Le" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2602.09063" }, "code_url": null, "metadata": { "source_search": "Kenny Workman, Zhen Yang, Harihara Muralidharan et al. Original paper authors scBench: Evaluating AI Agents on Single-Cell RNA-seq Analysis Kenny Workman Zhen Yang Harihara Muralidharan Aidan Abdulali Hannah Le evaluation-task" } }, { "id": "inspect:scicode", "name": "SciCode: A Research Coding Benchmark Curated by Scientists", "slug": "inspect--scicode", "organisation": "Minyang Tian, Luyu Gao, Shizhuo Dylan Zhang et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scicode", "topics": [ "general-capability" ], "description": "SciCode tests the ability of language models to generate code to solve scientific research problems. It assesses models on 65 problems from mathematics, physics, chemistry, biology, and materials science.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scicode/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2407.13168", "primary": true }, { "kind": "paper", "label": "Paper: SciCode: A Research Coding Benchmark Curated by Scientists", "url": "https://arxiv.org/abs/2407.13168", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/scicode-bench/SciCode", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scicode/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2407.13168", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scicode/eval.yaml", "family_title": "SciCode: A Research Coding Benchmark Curated by Scientists", "group": "Coding", "version": "2-A", "contributors": [ "xantheocracy" ], "tags": [ "Coding" ], "task": { "name": "scicode", "dataset_samples": 65, "comment": null, "support_only": false }, "tasks": [ { "name": "scicode", "dataset_samples": 65, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 65, "dependency": "scicode", "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/scicode-bench/SciCode/{SHA}/eval/data/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://drive.google.com/uc?id=17G_k65N_6yFFZ2O-jQH00Lh6iaw3z-AW", "fetch_method": "gdown_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:scicode", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Minyang Tian, Luyu Gao, Shizhuo Dylan Zhang et al.", "organization": "Princeton University; University of Washington", "organizations": [ "Princeton University", "University of Washington" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2407.13168" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2407.13168", "arxiv_id": "2407.13168", "title": "SciCode: A Research Coding Benchmark Curated by Scientists", "authors": [ "Minyang Tian", "Luyu Gao", "Shizhuo Dylan Zhang", "Xinan Chen", "Cunwei Fan", "Xuefei Guo", "Roland Haas", "Pan Ji", "Kittithat Krongchon", "Yao Li", "Shengyan Liu", "Di Luo", "Yutao Ma", "Hao Tong", "Kha Trinh", "Chenyu Tian", "Zihan Wang", "Bohao Wu", "Yanyu Xiong", "Shengzhu Yin", "Minhui Zhu", "Kilian Lieret", "Yanxin Lu", "Genglin Liu", "Yufeng Du", "Tianhua Tao", "Ofir Press", "Jamie Callan", "Eliu Huerta", "Hao Peng" ], "organizations": [ "Princeton University", "University of Washington" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2407.13168", "arxiv_id": "2407.13168", "title": "SciCode: A Research Coding Benchmark Curated by Scientists", "authors": [ "Minyang Tian", "Luyu Gao", "Shizhuo Dylan Zhang", "Xinan Chen", "Cunwei Fan", "Xuefei Guo", "Roland Haas", "Pan Ji", "Kittithat Krongchon", "Yao Li", "Shengyan Liu", "Di Luo", "Yutao Ma", "Hao Tong", "Kha Trinh", "Chenyu Tian", "Zihan Wang", "Bohao Wu", "Yanyu Xiong", "Shengzhu Yin", "Minhui Zhu", "Kilian Lieret", "Yanxin Lu", "Genglin Liu", "Yufeng Du", "Tianhua Tao", "Ofir Press", "Jamie Callan", "Eliu Huerta", "Hao Peng" ], "organizations": [ "Princeton University", "University of Washington" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/scicode-bench/SciCode", "owner": "Scicode Bench", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/scicode/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2407.13168" }, "code_url": "https://github.com/scicode-bench/SciCode", "metadata": { "source_search": "Minyang Tian, Luyu Gao, Shizhuo Dylan Zhang et al. Princeton University; University of Washington Original paper authors SciCode: A Research Coding Benchmark Curated by Scientists Minyang Tian Luyu Gao Shizhuo Dylan Zhang Xinan Chen Cunwei Fan Xuefei Guo Roland Haas Pan Ji Kittithat Krongchon Yao Li Shengyan Liu Di Luo Yutao Ma Hao Tong Kha Trinh Chenyu Tian Zihan Wang Bohao Wu Yanyu Xiong Shengzhu Yin Minhui Zhu Kilian Lieret Yanxin Lu Genglin Liu Yufeng Du Tianhua Tao Ofir Press Jamie Callan Eliu Huerta Hao Peng Scicode Bench evaluation-task" } }, { "id": "inspect:sciknoweval", "name": "SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large Language Models", "slug": "inspect--sciknoweval", "organisation": "Kehua Feng, Xinyi Shen, Weijie Wang et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sciknoweval", "topics": [ "general-capability" ], "description": "The Scientific Knowledge Evaluation benchmark is inspired by the profound principles outlined in the “Doctrine of the Mean” from ancient Chinese philosophy. This benchmark is designed to assess LLMs based on their proficiency in Studying Extensively, Enquiring Earnestly, Thinking Profoundly, Discerning Clearly, and Practicing Assiduously. Each of these dimensions offers a unique perspective on evaluating the capabilities of LLMs in handling scientific knowledge.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sciknoweval/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2406.09098", "primary": true }, { "kind": "paper", "label": "Paper: SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large Language Models", "url": "https://arxiv.org/abs/2406.09098", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sciknoweval/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Hicai Zju", "url": "https://huggingface.co/datasets/hicai-zju/SciKnowEval", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2406.09098", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sciknoweval/eval.yaml", "family_title": "SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large Language Models", "group": "Knowledge", "version": "3-B", "contributors": [ "Esther-Guo" ], "tags": [ "Knowledge" ], "task": { "name": "sciknoweval", "dataset_samples": 70196, "comment": null, "support_only": false }, "tasks": [ { "name": "sciknoweval", "dataset_samples": 70196, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 70196, "dependency": "sciknoweval", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://drive.google.com/uc?id={file_id}", "fetch_method": "gdown_and_verify", "state": "pinned", "comment": null }, { "type": "huggingface", "source": "hicai-zju/SciKnowEval", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sciknoweval", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Kehua Feng, Xinyi Shen, Weijie Wang et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2406.09098" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2406.09098", "arxiv_id": "2406.09098", "title": "SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large Language Models", "authors": [ "Kehua Feng", "Xinyi Shen", "Weijie Wang", "Xiang Zhuang", "Yuqi Tang", "Qiang Zhang", "Keyan Ding" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2406.09098", "arxiv_id": "2406.09098", "title": "SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large Language Models", "authors": [ "Kehua Feng", "Xinyi Shen", "Weijie Wang", "Xiang Zhuang", "Yuqi Tang", "Qiang Zhang", "Keyan Ding" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2406.09098", "arxiv_id": "2406.09098", "title": "SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large Language Models", "authors": [ "Kehua Feng", "Xinyi Shen", "Weijie Wang", "Xiang Zhuang", "Yuqi Tang", "Qiang Zhang", "Keyan Ding" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sciknoweval/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/hicai-zju/SciKnowEval", "provider": "Hicai Zju" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2406.09098" }, "code_url": null, "metadata": { "source_search": "Kehua Feng, Xinyi Shen, Weijie Wang et al. Original paper authors SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large Language Models Kehua Feng Xinyi Shen Weijie Wang Xiang Zhuang Yuqi Tang Qiang Zhang Keyan Ding evaluation-task" } }, { "id": "inspect:sec_qa_v1", "name": "Sec QA V1", "slug": "inspect--sec-qa-v1", "organisation": "Zefang Liu", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa", "topics": [ "general-capability" ], "description": "\"Security Question Answering\" dataset to assess LLMs' understanding and application of security principles. SecQA has \"v1\" and \"v2\" datasets of multiple-choice questions that aim to provide two levels of cybersecurity evaluation criteria. The questions were generated by GPT-4 based on the \"Computer Systems Security: Planning for Success\" textbook and vetted by humans.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2312.15838", "primary": true }, { "kind": "paper", "label": "Paper: SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "url": "https://arxiv.org/abs/2312.15838", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Zefang Liu", "url": "https://huggingface.co/datasets/zefang-liu/secqa", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2312.15838", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "family_title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "group": "Cybersecurity", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Cybersecurity" ], "task": { "name": "sec_qa_v1", "dataset_samples": 110, "comment": null, "support_only": false }, "tasks": [ { "name": "sec_qa_v1", "dataset_samples": 110, "comment": null, "support_only": false }, { "name": "sec_qa_v1_5_shot", "dataset_samples": 110, "comment": null, "support_only": false }, { "name": "sec_qa_v2", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "sec_qa_v2_5_shot", "dataset_samples": 100, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 110, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "zefang-liu/secqa", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sec_qa", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zefang Liu", "organization": "CapitalOne", "organizations": [ "CapitalOne" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2312.15838" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2312.15838", "arxiv_id": "2312.15838", "title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "authors": [ "Zefang Liu" ], "organizations": [ "CapitalOne" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2312.15838", "arxiv_id": "2312.15838", "title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "authors": [ "Zefang Liu" ], "organizations": [ "CapitalOne" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/zefang-liu/secqa", "provider": "Zefang Liu" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2312.15838" }, "code_url": null, "metadata": { "source_search": "Zefang Liu CapitalOne Original paper authors SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security Zefang Liu evaluation-family" } }, { "id": "inspect:sec_qa_v1_5_shot", "name": "Sec QA V1 5 Shot", "slug": "inspect--sec-qa-v1-5-shot", "organisation": "Zefang Liu", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa", "topics": [ "general-capability" ], "description": "\"Security Question Answering\" dataset to assess LLMs' understanding and application of security principles. SecQA has \"v1\" and \"v2\" datasets of multiple-choice questions that aim to provide two levels of cybersecurity evaluation criteria. The questions were generated by GPT-4 based on the \"Computer Systems Security: Planning for Success\" textbook and vetted by humans.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2312.15838", "primary": true }, { "kind": "paper", "label": "Paper: SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "url": "https://arxiv.org/abs/2312.15838", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Zefang Liu", "url": "https://huggingface.co/datasets/zefang-liu/secqa", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2312.15838", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "family_title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "group": "Cybersecurity", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Cybersecurity" ], "task": { "name": "sec_qa_v1_5_shot", "dataset_samples": 110, "comment": null, "support_only": false }, "tasks": [ { "name": "sec_qa_v1", "dataset_samples": 110, "comment": null, "support_only": false }, { "name": "sec_qa_v1_5_shot", "dataset_samples": 110, "comment": null, "support_only": false }, { "name": "sec_qa_v2", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "sec_qa_v2_5_shot", "dataset_samples": 100, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 110, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "zefang-liu/secqa", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sec_qa", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zefang Liu", "organization": "CapitalOne", "organizations": [ "CapitalOne" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2312.15838" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2312.15838", "arxiv_id": "2312.15838", "title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "authors": [ "Zefang Liu" ], "organizations": [ "CapitalOne" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2312.15838", "arxiv_id": "2312.15838", "title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "authors": [ "Zefang Liu" ], "organizations": [ "CapitalOne" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/zefang-liu/secqa", "provider": "Zefang Liu" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2312.15838" }, "code_url": null, "metadata": { "source_search": "Zefang Liu CapitalOne Original paper authors SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security Zefang Liu evaluation-family" } }, { "id": "inspect:sec_qa_v2", "name": "Sec QA V2", "slug": "inspect--sec-qa-v2", "organisation": "Zefang Liu", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa", "topics": [ "general-capability" ], "description": "\"Security Question Answering\" dataset to assess LLMs' understanding and application of security principles. SecQA has \"v1\" and \"v2\" datasets of multiple-choice questions that aim to provide two levels of cybersecurity evaluation criteria. The questions were generated by GPT-4 based on the \"Computer Systems Security: Planning for Success\" textbook and vetted by humans.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2312.15838", "primary": true }, { "kind": "paper", "label": "Paper: SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "url": "https://arxiv.org/abs/2312.15838", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Zefang Liu", "url": "https://huggingface.co/datasets/zefang-liu/secqa", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2312.15838", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "family_title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "group": "Cybersecurity", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Cybersecurity" ], "task": { "name": "sec_qa_v2", "dataset_samples": 100, "comment": null, "support_only": false }, "tasks": [ { "name": "sec_qa_v1", "dataset_samples": 110, "comment": null, "support_only": false }, { "name": "sec_qa_v1_5_shot", "dataset_samples": 110, "comment": null, "support_only": false }, { "name": "sec_qa_v2", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "sec_qa_v2_5_shot", "dataset_samples": 100, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 100, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "zefang-liu/secqa", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sec_qa", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zefang Liu", "organization": "CapitalOne", "organizations": [ "CapitalOne" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2312.15838" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2312.15838", "arxiv_id": "2312.15838", "title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "authors": [ "Zefang Liu" ], "organizations": [ "CapitalOne" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2312.15838", "arxiv_id": "2312.15838", "title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "authors": [ "Zefang Liu" ], "organizations": [ "CapitalOne" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/zefang-liu/secqa", "provider": "Zefang Liu" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2312.15838" }, "code_url": null, "metadata": { "source_search": "Zefang Liu CapitalOne Original paper authors SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security Zefang Liu evaluation-family" } }, { "id": "inspect:sec_qa_v2_5_shot", "name": "Sec QA V2 5 Shot", "slug": "inspect--sec-qa-v2-5-shot", "organisation": "Zefang Liu", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa", "topics": [ "general-capability" ], "description": "\"Security Question Answering\" dataset to assess LLMs' understanding and application of security principles. SecQA has \"v1\" and \"v2\" datasets of multiple-choice questions that aim to provide two levels of cybersecurity evaluation criteria. The questions were generated by GPT-4 based on the \"Computer Systems Security: Planning for Success\" textbook and vetted by humans.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2312.15838", "primary": true }, { "kind": "paper", "label": "Paper: SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "url": "https://arxiv.org/abs/2312.15838", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Zefang Liu", "url": "https://huggingface.co/datasets/zefang-liu/secqa", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2312.15838", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml", "family_title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "group": "Cybersecurity", "version": "2-A", "contributors": [ "matthewreed26" ], "tags": [ "Cybersecurity" ], "task": { "name": "sec_qa_v2_5_shot", "dataset_samples": 100, "comment": null, "support_only": false }, "tasks": [ { "name": "sec_qa_v1", "dataset_samples": 110, "comment": null, "support_only": false }, { "name": "sec_qa_v1_5_shot", "dataset_samples": 110, "comment": null, "support_only": false }, { "name": "sec_qa_v2", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "sec_qa_v2_5_shot", "dataset_samples": 100, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 100, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "zefang-liu/secqa", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sec_qa", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Zefang Liu", "organization": "CapitalOne", "organizations": [ "CapitalOne" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2312.15838" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2312.15838", "arxiv_id": "2312.15838", "title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "authors": [ "Zefang Liu" ], "organizations": [ "CapitalOne" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2312.15838", "arxiv_id": "2312.15838", "title": "SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security", "authors": [ "Zefang Liu" ], "organizations": [ "CapitalOne" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sec_qa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/zefang-liu/secqa", "provider": "Zefang Liu" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2312.15838" }, "code_url": null, "metadata": { "source_search": "Zefang Liu CapitalOne Original paper authors SecQA: A Concise Question-Answering Dataset for Evaluating Large Language Models in Computer Security Zefang Liu evaluation-family" } }, { "id": "inspect:sevenllm_mcq_en", "name": "Sevenllm Mcq En", "slug": "inspect--sevenllm-mcq-en", "organisation": "Hangyuan Ji, Jian Yang, Linzheng Chai et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm", "topics": [ "general-capability" ], "description": "Designed for analyzing cybersecurity incidents, which is comprised of two primary task categories: understanding and generation, with a further breakdown into 28 subcategories of tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2405.03446", "primary": true }, { "kind": "paper", "label": "Paper: SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "url": "https://arxiv.org/abs/2405.03446", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2405.03446", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "family_title": "SEvenLLM: A benchmark to elicit, and improve cybersecurity incident analysis and response abilities in LLMs for Security Events.", "group": "Cybersecurity", "version": "2-A", "contributors": [ "kingroryg" ], "tags": [ "Cybersecurity" ], "task": { "name": "sevenllm_mcq_en", "dataset_samples": 50, "comment": null, "support_only": false }, "tasks": [ { "name": "sevenllm_mcq_zh", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "sevenllm_mcq_en", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "sevenllm_qa_zh", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "sevenllm_qa_en", "dataset_samples": 600, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 50, "dependency": "sevenllm", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://huggingface.co/datasets/Multilingual-Multimodal-NLP/SEVENLLM-Dataset/raw/{SHA}/", "fetch_method": "other", "state": "pinned", "comment": "Raw HF URL fetched via fsspec (load_json_dataset), not via HF datasets API" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sevenllm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Hangyuan Ji, Jian Yang, Linzheng Chai et al.", "organization": "Beijing Information Science and Technology Unuversity; Beihang University", "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2405.03446" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2405.03446", "arxiv_id": "2405.03446", "title": "SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "authors": [ "Hangyuan Ji", "Jian Yang", "Linzheng Chai", "Chaoren Wei", "Liqun Yang", "Yunlong Duan", "Yunli Wang", "Tianzhen Sun", "Hongcheng Guo", "Tongliang Li", "Changyu Ren", "Zhoujun Li" ], "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2405.03446", "arxiv_id": "2405.03446", "title": "SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "authors": [ "Hangyuan Ji", "Jian Yang", "Linzheng Chai", "Chaoren Wei", "Liqun Yang", "Yunlong Duan", "Yunli Wang", "Tianzhen Sun", "Hongcheng Guo", "Tongliang Li", "Changyu Ren", "Zhoujun Li" ], "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2405.03446" }, "code_url": null, "metadata": { "source_search": "Hangyuan Ji, Jian Yang, Linzheng Chai et al. Beijing Information Science and Technology Unuversity; Beihang University Original paper authors SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence Hangyuan Ji Jian Yang Linzheng Chai Chaoren Wei Liqun Yang Yunlong Duan Yunli Wang Tianzhen Sun Hongcheng Guo Tongliang Li Changyu Ren Zhoujun Li evaluation-family" } }, { "id": "inspect:sevenllm_mcq_zh", "name": "Sevenllm Mcq Zh", "slug": "inspect--sevenllm-mcq-zh", "organisation": "Hangyuan Ji, Jian Yang, Linzheng Chai et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm", "topics": [ "general-capability" ], "description": "Designed for analyzing cybersecurity incidents, which is comprised of two primary task categories: understanding and generation, with a further breakdown into 28 subcategories of tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2405.03446", "primary": true }, { "kind": "paper", "label": "Paper: SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "url": "https://arxiv.org/abs/2405.03446", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2405.03446", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "family_title": "SEvenLLM: A benchmark to elicit, and improve cybersecurity incident analysis and response abilities in LLMs for Security Events.", "group": "Cybersecurity", "version": "2-A", "contributors": [ "kingroryg" ], "tags": [ "Cybersecurity" ], "task": { "name": "sevenllm_mcq_zh", "dataset_samples": 50, "comment": null, "support_only": false }, "tasks": [ { "name": "sevenllm_mcq_zh", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "sevenllm_mcq_en", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "sevenllm_qa_zh", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "sevenllm_qa_en", "dataset_samples": 600, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 50, "dependency": "sevenllm", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://huggingface.co/datasets/Multilingual-Multimodal-NLP/SEVENLLM-Dataset/raw/{SHA}/", "fetch_method": "other", "state": "pinned", "comment": "Raw HF URL fetched via fsspec (load_json_dataset), not via HF datasets API" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sevenllm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Hangyuan Ji, Jian Yang, Linzheng Chai et al.", "organization": "Beijing Information Science and Technology Unuversity; Beihang University", "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2405.03446" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2405.03446", "arxiv_id": "2405.03446", "title": "SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "authors": [ "Hangyuan Ji", "Jian Yang", "Linzheng Chai", "Chaoren Wei", "Liqun Yang", "Yunlong Duan", "Yunli Wang", "Tianzhen Sun", "Hongcheng Guo", "Tongliang Li", "Changyu Ren", "Zhoujun Li" ], "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2405.03446", "arxiv_id": "2405.03446", "title": "SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "authors": [ "Hangyuan Ji", "Jian Yang", "Linzheng Chai", "Chaoren Wei", "Liqun Yang", "Yunlong Duan", "Yunli Wang", "Tianzhen Sun", "Hongcheng Guo", "Tongliang Li", "Changyu Ren", "Zhoujun Li" ], "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2405.03446" }, "code_url": null, "metadata": { "source_search": "Hangyuan Ji, Jian Yang, Linzheng Chai et al. Beijing Information Science and Technology Unuversity; Beihang University Original paper authors SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence Hangyuan Ji Jian Yang Linzheng Chai Chaoren Wei Liqun Yang Yunlong Duan Yunli Wang Tianzhen Sun Hongcheng Guo Tongliang Li Changyu Ren Zhoujun Li evaluation-family" } }, { "id": "inspect:sevenllm_qa_en", "name": "Sevenllm QA En", "slug": "inspect--sevenllm-qa-en", "organisation": "Hangyuan Ji, Jian Yang, Linzheng Chai et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm", "topics": [ "general-capability" ], "description": "Designed for analyzing cybersecurity incidents, which is comprised of two primary task categories: understanding and generation, with a further breakdown into 28 subcategories of tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2405.03446", "primary": true }, { "kind": "paper", "label": "Paper: SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "url": "https://arxiv.org/abs/2405.03446", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2405.03446", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "family_title": "SEvenLLM: A benchmark to elicit, and improve cybersecurity incident analysis and response abilities in LLMs for Security Events.", "group": "Cybersecurity", "version": "2-A", "contributors": [ "kingroryg" ], "tags": [ "Cybersecurity" ], "task": { "name": "sevenllm_qa_en", "dataset_samples": 600, "comment": null, "support_only": false }, "tasks": [ { "name": "sevenllm_mcq_zh", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "sevenllm_mcq_en", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "sevenllm_qa_zh", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "sevenllm_qa_en", "dataset_samples": 600, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 600, "dependency": "sevenllm", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://huggingface.co/datasets/Multilingual-Multimodal-NLP/SEVENLLM-Dataset/raw/{SHA}/", "fetch_method": "other", "state": "pinned", "comment": "Raw HF URL fetched via fsspec (load_json_dataset), not via HF datasets API" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sevenllm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Hangyuan Ji, Jian Yang, Linzheng Chai et al.", "organization": "Beijing Information Science and Technology Unuversity; Beihang University", "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2405.03446" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2405.03446", "arxiv_id": "2405.03446", "title": "SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "authors": [ "Hangyuan Ji", "Jian Yang", "Linzheng Chai", "Chaoren Wei", "Liqun Yang", "Yunlong Duan", "Yunli Wang", "Tianzhen Sun", "Hongcheng Guo", "Tongliang Li", "Changyu Ren", "Zhoujun Li" ], "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2405.03446", "arxiv_id": "2405.03446", "title": "SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "authors": [ "Hangyuan Ji", "Jian Yang", "Linzheng Chai", "Chaoren Wei", "Liqun Yang", "Yunlong Duan", "Yunli Wang", "Tianzhen Sun", "Hongcheng Guo", "Tongliang Li", "Changyu Ren", "Zhoujun Li" ], "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2405.03446" }, "code_url": null, "metadata": { "source_search": "Hangyuan Ji, Jian Yang, Linzheng Chai et al. Beijing Information Science and Technology Unuversity; Beihang University Original paper authors SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence Hangyuan Ji Jian Yang Linzheng Chai Chaoren Wei Liqun Yang Yunlong Duan Yunli Wang Tianzhen Sun Hongcheng Guo Tongliang Li Changyu Ren Zhoujun Li evaluation-family" } }, { "id": "inspect:sevenllm_qa_zh", "name": "Sevenllm QA Zh", "slug": "inspect--sevenllm-qa-zh", "organisation": "Hangyuan Ji, Jian Yang, Linzheng Chai et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm", "topics": [ "general-capability" ], "description": "Designed for analyzing cybersecurity incidents, which is comprised of two primary task categories: understanding and generation, with a further breakdown into 28 subcategories of tasks.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2405.03446", "primary": true }, { "kind": "paper", "label": "Paper: SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "url": "https://arxiv.org/abs/2405.03446", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2405.03446", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml", "family_title": "SEvenLLM: A benchmark to elicit, and improve cybersecurity incident analysis and response abilities in LLMs for Security Events.", "group": "Cybersecurity", "version": "2-A", "contributors": [ "kingroryg" ], "tags": [ "Cybersecurity" ], "task": { "name": "sevenllm_qa_zh", "dataset_samples": 600, "comment": null, "support_only": false }, "tasks": [ { "name": "sevenllm_mcq_zh", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "sevenllm_mcq_en", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "sevenllm_qa_zh", "dataset_samples": 600, "comment": null, "support_only": false }, { "name": "sevenllm_qa_en", "dataset_samples": 600, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 600, "dependency": "sevenllm", "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://huggingface.co/datasets/Multilingual-Multimodal-NLP/SEVENLLM-Dataset/raw/{SHA}/", "fetch_method": "other", "state": "pinned", "comment": "Raw HF URL fetched via fsspec (load_json_dataset), not via HF datasets API" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sevenllm", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Hangyuan Ji, Jian Yang, Linzheng Chai et al.", "organization": "Beijing Information Science and Technology Unuversity; Beihang University", "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2405.03446" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2405.03446", "arxiv_id": "2405.03446", "title": "SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "authors": [ "Hangyuan Ji", "Jian Yang", "Linzheng Chai", "Chaoren Wei", "Liqun Yang", "Yunlong Duan", "Yunli Wang", "Tianzhen Sun", "Hongcheng Guo", "Tongliang Li", "Changyu Ren", "Zhoujun Li" ], "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2405.03446", "arxiv_id": "2405.03446", "title": "SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence", "authors": [ "Hangyuan Ji", "Jian Yang", "Linzheng Chai", "Chaoren Wei", "Liqun Yang", "Yunlong Duan", "Yunli Wang", "Tianzhen Sun", "Hongcheng Guo", "Tongliang Li", "Changyu Ren", "Zhoujun Li" ], "organizations": [ "Beijing Information Science and Technology Unuversity", "Beihang University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sevenllm/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2405.03446" }, "code_url": null, "metadata": { "source_search": "Hangyuan Ji, Jian Yang, Linzheng Chai et al. Beijing Information Science and Technology Unuversity; Beihang University Original paper authors SEvenLLM: Benchmarking, Eliciting, and Enhancing Abilities of Large Language Models in Cyber Threat Intelligence Hangyuan Ji Jian Yang Linzheng Chai Chaoren Wei Liqun Yang Yunlong Duan Yunli Wang Tianzhen Sun Hongcheng Guo Tongliang Li Changyu Ren Zhoujun Li evaluation-family" } }, { "id": "inspect:simpleqa", "name": "Simpleqa", "slug": "inspect--simpleqa", "organisation": "Lukas Haas, Gal Yona, Giovanni D'Antonio et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa", "topics": [ "general-capability" ], "description": "A benchmark that evaluates the ability of language models to answer short, fact-seeking questions.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2509.07968", "primary": true }, { "kind": "paper", "label": "Paper: SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge", "url": "https://arxiv.org/abs/2509.07968", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Codelion", "url": "https://huggingface.co/datasets/codelion/SimpleQA-Verified", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2509.07968", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa/eval.yaml", "family_title": "SimpleQA/SimpleQA Verified: Measuring short-form factuality in large language models", "group": "Knowledge", "version": "5-C", "contributors": [ "osc245", "jeqcho" ], "tags": [ "Knowledge" ], "task": { "name": "simpleqa", "dataset_samples": 4326, "comment": null, "support_only": false }, "tasks": [ { "name": "simpleqa", "dataset_samples": 4326, "comment": null, "support_only": false }, { "name": "simpleqa_verified", "dataset_samples": 1000, "comment": null, "support_only": false } ], "protocol": { "version": "5-C", "dataset_samples": 4326, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "codelion/SimpleQA-Verified", "fetch_method": "hf_dataset", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv", "fetch_method": "download_and_verify", "state": "pinned", "comment": "Used by simpleqa task" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:simpleqa", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Lukas Haas, Gal Yona, Giovanni D'Antonio et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2509.07968" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2509.07968", "arxiv_id": "2509.07968", "title": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge", "authors": [ "Lukas Haas", "Gal Yona", "Giovanni D'Antonio", "Sasha Goldshtein", "Dipanjan Das" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2509.07968", "arxiv_id": "2509.07968", "title": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge", "authors": [ "Lukas Haas", "Gal Yona", "Giovanni D'Antonio", "Sasha Goldshtein", "Dipanjan Das" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2509.07968", "arxiv_id": "2509.07968", "title": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge", "authors": [ "Lukas Haas", "Gal Yona", "Giovanni D'Antonio", "Sasha Goldshtein", "Dipanjan Das" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv", "owner": "openaipublic.blob.core.windows.net", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/codelion/SimpleQA-Verified", "provider": "Codelion" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2509.07968" }, "code_url": "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv", "metadata": { "source_search": "Lukas Haas, Gal Yona, Giovanni D'Antonio et al. Original paper authors SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge Lukas Haas Gal Yona Giovanni D'Antonio Sasha Goldshtein Dipanjan Das openaipublic.blob.core.windows.net evaluation-family" } }, { "id": "inspect:simpleqa_verified", "name": "Simpleqa Verified", "slug": "inspect--simpleqa-verified", "organisation": "Lukas Haas, Gal Yona, Giovanni D'Antonio et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa", "topics": [ "general-capability" ], "description": "A benchmark that evaluates the ability of language models to answer short, fact-seeking questions.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2509.07968", "primary": true }, { "kind": "paper", "label": "Paper: SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge", "url": "https://arxiv.org/abs/2509.07968", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Codelion", "url": "https://huggingface.co/datasets/codelion/SimpleQA-Verified", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2509.07968", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa/eval.yaml", "family_title": "SimpleQA/SimpleQA Verified: Measuring short-form factuality in large language models", "group": "Knowledge", "version": "5-C", "contributors": [ "osc245", "jeqcho" ], "tags": [ "Knowledge" ], "task": { "name": "simpleqa_verified", "dataset_samples": 1000, "comment": null, "support_only": false }, "tasks": [ { "name": "simpleqa", "dataset_samples": 4326, "comment": null, "support_only": false }, { "name": "simpleqa_verified", "dataset_samples": 1000, "comment": null, "support_only": false } ], "protocol": { "version": "5-C", "dataset_samples": 1000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "codelion/SimpleQA-Verified", "fetch_method": "hf_dataset", "state": "pinned", "comment": null }, { "type": "direct_url", "source": "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv", "fetch_method": "download_and_verify", "state": "pinned", "comment": "Used by simpleqa task" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:simpleqa", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Lukas Haas, Gal Yona, Giovanni D'Antonio et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2509.07968" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2509.07968", "arxiv_id": "2509.07968", "title": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge", "authors": [ "Lukas Haas", "Gal Yona", "Giovanni D'Antonio", "Sasha Goldshtein", "Dipanjan Das" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2509.07968", "arxiv_id": "2509.07968", "title": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge", "authors": [ "Lukas Haas", "Gal Yona", "Giovanni D'Antonio", "Sasha Goldshtein", "Dipanjan Das" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2509.07968", "arxiv_id": "2509.07968", "title": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge", "authors": [ "Lukas Haas", "Gal Yona", "Giovanni D'Antonio", "Sasha Goldshtein", "Dipanjan Das" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv", "owner": "openaipublic.blob.core.windows.net", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/simpleqa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/codelion/SimpleQA-Verified", "provider": "Codelion" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2509.07968" }, "code_url": "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv", "metadata": { "source_search": "Lukas Haas, Gal Yona, Giovanni D'Antonio et al. Original paper authors SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge Lukas Haas Gal Yona Giovanni D'Antonio Sasha Goldshtein Dipanjan Das openaipublic.blob.core.windows.net evaluation-family" } }, { "id": "inspect:sosbench", "name": "SOS BENCH: Benchmarking Safety Alignment on Scientific Knowledge", "slug": "inspect--sosbench", "organisation": "Fengqing Jiang, Fengbo Ma, Zhangchen Xu et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sosbench", "topics": [ "general-capability" ], "description": "A regulation-grounded, hazard-focused benchmark encompassing six high-risk scientific domains: chemistry, biology, medicine, pharmacology, physics, and psychology. The benchmark comprises 3,000 prompts derived from real-world regulations and laws, systematically expanded via an LLM-assisted evolutionary pipeline that introduces diverse, realistic misuse scenarios (e.g., detailed explosive synthesis instructions involving advanced chemical formulas).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sosbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2505.21605", "primary": true }, { "kind": "paper", "label": "Paper: SoSBench: Benchmarking Safety Alignment on Six Scientific Domains", "url": "https://arxiv.org/abs/2505.21605", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sosbench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: SOSBench", "url": "https://huggingface.co/datasets/SOSBench/SOSBench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2505.21605", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sosbench/eval.yaml", "family_title": "SOS BENCH: Benchmarking Safety Alignment on Scientific Knowledge", "group": "Knowledge", "version": "2-A", "contributors": [ "Esther-Guo" ], "tags": [ "Knowledge" ], "task": { "name": "sosbench", "dataset_samples": 3000, "comment": null, "support_only": false }, "tasks": [ { "name": "sosbench", "dataset_samples": 3000, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 3000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "SOSBench/SOSBench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sosbench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Fengqing Jiang, Fengbo Ma, Zhangchen Xu et al.", "organization": "University of Washington", "organizations": [ "University of Washington" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2505.21605" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2505.21605", "arxiv_id": "2505.21605", "title": "SoSBench: Benchmarking Safety Alignment on Six Scientific Domains", "authors": [ "Fengqing Jiang", "Fengbo Ma", "Zhangchen Xu", "Yuetai Li", "Zixin Rao", "Bhaskar Ramasubramanian", "Luyao Niu", "Bo Li", "Xianyan Chen", "Zhen Xiang", "Radha Poovendran" ], "organizations": [ "University of Washington" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2505.21605", "arxiv_id": "2505.21605", "title": "SoSBench: Benchmarking Safety Alignment on Six Scientific Domains", "authors": [ "Fengqing Jiang", "Fengbo Ma", "Zhangchen Xu", "Yuetai Li", "Zixin Rao", "Bhaskar Ramasubramanian", "Luyao Niu", "Bo Li", "Xianyan Chen", "Zhen Xiang", "Radha Poovendran" ], "organizations": [ "University of Washington" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2505.21605", "arxiv_id": "2505.21605", "title": "SoSBench: Benchmarking Safety Alignment on Six Scientific Domains", "authors": [ "Fengqing Jiang", "Fengbo Ma", "Zhangchen Xu", "Yuetai Li", "Zixin Rao", "Bhaskar Ramasubramanian", "Luyao Niu", "Bo Li", "Xianyan Chen", "Zhen Xiang", "Radha Poovendran" ], "organizations": [ "University of Washington" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sosbench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/SOSBench/SOSBench", "provider": "SOSBench" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2505.21605" }, "code_url": null, "metadata": { "source_search": "Fengqing Jiang, Fengbo Ma, Zhangchen Xu et al. University of Washington Original paper authors SoSBench: Benchmarking Safety Alignment on Six Scientific Domains Fengqing Jiang Fengbo Ma Zhangchen Xu Yuetai Li Zixin Rao Bhaskar Ramasubramanian Luyao Niu Bo Li Xianyan Chen Zhen Xiang Radha Poovendran evaluation-task" } }, { "id": "inspect:squad", "name": "SQuAD: A Reading Comprehension Benchmark requiring reasoning over Wikipedia articles", "slug": "inspect--squad", "organisation": "Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/squad", "topics": [ "general-capability" ], "description": "Set of 100,000+ questions posed by crowdworkers on a set of Wikipedia articles, where the answer to each question is a segment of text from the corresponding reading passage.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/squad/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1606.05250", "primary": true }, { "kind": "paper", "label": "Paper: SQuAD: 100,000+ Questions for Machine Comprehension of Text", "url": "https://arxiv.org/abs/1606.05250", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/squad/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Rajpurkar", "url": "https://huggingface.co/datasets/rajpurkar/squad_v2", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1606.05250", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/squad/eval.yaml", "family_title": "SQuAD: A Reading Comprehension Benchmark requiring reasoning over Wikipedia articles", "group": "Reasoning", "version": "3-A", "contributors": [ "tknasir" ], "tags": [ "Reasoning" ], "task": { "name": "squad", "dataset_samples": 11873, "comment": null, "support_only": false }, "tasks": [ { "name": "squad", "dataset_samples": 11873, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 11873, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "rajpurkar/squad_v2", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:squad", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1606.05250" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1606.05250", "arxiv_id": "1606.05250", "title": "SQuAD: 100,000+ Questions for Machine Comprehension of Text", "authors": [ "Pranav Rajpurkar", "Jian Zhang", "Konstantin Lopyrev", "Percy Liang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1606.05250", "arxiv_id": "1606.05250", "title": "SQuAD: 100,000+ Questions for Machine Comprehension of Text", "authors": [ "Pranav Rajpurkar", "Jian Zhang", "Konstantin Lopyrev", "Percy Liang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/squad/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/rajpurkar/squad_v2", "provider": "Rajpurkar" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1606.05250" }, "code_url": null, "metadata": { "source_search": "Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev et al. Original paper authors SQuAD: 100,000+ Questions for Machine Comprehension of Text Pranav Rajpurkar Jian Zhang Konstantin Lopyrev Percy Liang evaluation-task" } }, { "id": "inspect:stereoset", "name": "StereoSet: Measuring stereotypical bias in pretrained language models", "slug": "inspect--stereoset", "organisation": "Moin Nadeem, Anna Bethke, Siva Reddy", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/stereoset", "topics": [ "general-capability" ], "description": "A dataset that measures stereotype bias in language models across gender, race, religion, and profession domains. Models choose between stereotype, anti-stereotype, and unrelated completions to sentences.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/stereoset/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2004.09456", "primary": true }, { "kind": "paper", "label": "Paper: StereoSet: Measuring stereotypical bias in pretrained language models", "url": "https://arxiv.org/abs/2004.09456", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/stereoset/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: McGill NLP", "url": "https://huggingface.co/datasets/McGill-NLP/stereoset", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2004.09456", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/stereoset/eval.yaml", "family_title": "StereoSet: Measuring stereotypical bias in pretrained language models", "group": "Safeguards", "version": "3-A", "contributors": [ "Xodarap" ], "tags": [ "Safeguards" ], "task": { "name": "stereoset", "dataset_samples": 4299, "comment": null, "support_only": false }, "tasks": [ { "name": "stereoset", "dataset_samples": 4299, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 4299, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "McGill-NLP/stereoset", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:stereoset", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Moin Nadeem, Anna Bethke, Siva Reddy", "organization": "Massachusetts Institute of Technology; MIT", "organizations": [ "Massachusetts Institute of Technology", "MIT" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2004.09456" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2004.09456", "arxiv_id": "2004.09456", "title": "StereoSet: Measuring stereotypical bias in pretrained language models", "authors": [ "Moin Nadeem", "Anna Bethke", "Siva Reddy" ], "organizations": [ "Massachusetts Institute of Technology", "MIT" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2004.09456", "arxiv_id": "2004.09456", "title": "StereoSet: Measuring stereotypical bias in pretrained language models", "authors": [ "Moin Nadeem", "Anna Bethke", "Siva Reddy" ], "organizations": [ "Massachusetts Institute of Technology", "MIT" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/stereoset/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/McGill-NLP/stereoset", "provider": "McGill NLP" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2004.09456" }, "code_url": null, "metadata": { "source_search": "Moin Nadeem, Anna Bethke, Siva Reddy Massachusetts Institute of Technology; MIT Original paper authors StereoSet: Measuring stereotypical bias in pretrained language models Moin Nadeem Anna Bethke Siva Reddy evaluation-task" } }, { "id": "inspect:strong_reject", "name": "StrongREJECT: Measuring LLM susceptibility to jailbreak attacks", "slug": "inspect--strong-reject", "organisation": "Alexandra Souly, Qingyuan Lu, Dillon Bowen et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/strong_reject", "topics": [ "safeguards" ], "description": "A benchmark that evaluates the susceptibility of LLMs to various jailbreak attacks.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "How harmful and useful the target's responses are on the StrongREJECT prompt and attack distribution under the stated evaluator.", "does_not_measure": "Universal jailbreak robustness, real-world misuse success, adaptive agentic attacks, or downstream harm.", "best_for": "Evaluating jailbreak responses with a more discriminating rubric than simple refusal detection.", "not_sufficient_for": "Certifying safety across threats, modalities, tools, model versions, or deployment contexts.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/strong_reject/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2402.10260", "primary": true }, { "kind": "paper", "label": "Paper: A StrongREJECT for Empty Jailbreaks", "url": "https://arxiv.org/abs/2402.10260", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/alexandrasouly/strongreject", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/strong_reject/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2402.10260", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/strong_reject/eval.yaml", "family_title": "StrongREJECT: Measuring LLM susceptibility to jailbreak attacks", "group": "Safeguards", "version": "3-A", "contributors": [ "viknat" ], "tags": [ "Safeguards" ], "task": { "name": "strong_reject", "dataset_samples": 324, "comment": null, "support_only": false }, "tasks": [ { "name": "strong_reject", "dataset_samples": 324, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 324, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/alexandrasouly/strongreject/{SHA}/strongreject_dataset/", "fetch_method": "other", "state": "pinned", "comment": "Fetched via fsspec (load_csv_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:strong_reject", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Alexandra Souly, Qingyuan Lu, Dillon Bowen et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2402.10260" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2402.10260", "arxiv_id": "2402.10260", "title": "A StrongREJECT for Empty Jailbreaks", "authors": [ "Alexandra Souly", "Qingyuan Lu", "Dillon Bowen", "Tu Trinh", "Elvis Hsieh", "Sana Pandey", "Pieter Abbeel", "Justin Svegliato", "Scott Emmons", "Olivia Watkins", "Sam Toyer" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2402.10260", "arxiv_id": "2402.10260", "title": "A StrongREJECT for Empty Jailbreaks", "authors": [ "Alexandra Souly", "Qingyuan Lu", "Dillon Bowen", "Tu Trinh", "Elvis Hsieh", "Sana Pandey", "Pieter Abbeel", "Justin Svegliato", "Scott Emmons", "Olivia Watkins", "Sam Toyer" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/alexandrasouly/strongreject", "owner": "Alexandrasouly", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/strong_reject/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2402.10260" }, "code_url": "https://github.com/alexandrasouly/strongreject", "metadata": { "source_search": "Alexandra Souly, Qingyuan Lu, Dillon Bowen et al. Original paper authors A StrongREJECT for Empty Jailbreaks Alexandra Souly Qingyuan Lu Dillon Bowen Tu Trinh Elvis Hsieh Sana Pandey Pieter Abbeel Justin Svegliato Scott Emmons Olivia Watkins Sam Toyer Alexandrasouly evaluation-task" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Quality and harmfulness of model responses to a curated set of forbidden prompts, including under jailbreak attacks.", "unit_of_analysis": "One model response to one forbidden prompt, optionally transformed by one jailbreak method.", "interaction_type": "Adversarial prompting evaluated with a fine-grained response rubric.", "target_outcome": "Rubric-based score reflecting refusal, convincingness, and specificity of the harmful response.", "human_participants": "No human subjects. Human judgments are used to develop and validate the evaluator.", "elicitation": "Forbidden prompts and jailbreak transformations are evaluated under specified target-model and generation settings.", "scoring": "Fine-grained automated evaluator calibrated against human judgments rather than refusal-string matching alone.", "protocol_dependence": "High: prompt set, jailbreak transformations, model snapshot, generation settings, and evaluator version matter.", "implementation_dependence": "High: judge-model calibration and rubric implementation directly influence scores.", "limitations": [ "The prompt distribution and jailbreak set are finite and public.", "Judge models can share biases or vulnerabilities with evaluated systems.", "Rubric scores do not represent operational harm or adversary success.", "Results may shift with system prompts and endpoint updates." ], "comparability": "Match StrongREJECT release, prompt and attack sets, target snapshot, decoding, evaluator model, and rubric implementation.", "decision_relevance": "Useful for safeguard regression testing and judge-quality analysis; insufficient as a standalone jailbreak or misuse safety case.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2402.10260" }, { "kind": "implementation", "url": "https://github.com/alexandrasouly/strongreject" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:swe_bench", "name": "Swe Bench", "slug": "inspect--swe-bench", "organisation": "Carlos E. Jimenez, John Yang, Alexander Wettig et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench", "topics": [ "general-capability" ], "description": "Evaluates AI's ability to resolve genuine software engineering issues sourced from 12 popular Python GitHub repositories, reflecting realistic coding and debugging scenarios.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Patch-generation success on the selected SWE-bench issues under the exact repository, test, scaffold, and execution setup.", "does_not_measure": "General software engineering productivity, requirements discovery, maintainability, security, teamwork, or reliable deployment.", "best_for": "Comparing coding agents on reproducible repository-level bug-fixing tasks.", "not_sufficient_for": "Claims that a model can replace software engineers or safely maintain production systems.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2310.06770", "primary": true }, { "kind": "paper", "label": "Paper: SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "url": "https://arxiv.org/abs/2310.06770", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/swe-bench/experiments", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Princeton Nlp", "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2310.06770", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench/eval.yaml", "family_title": "SWE-bench Verified: Resolving Real-World GitHub Issues", "group": "Coding", "version": "5-C", "contributors": [ "max-kaufmann" ], "tags": [ "Agent", "Coding" ], "task": { "name": "swe_bench", "dataset_samples": 500, "comment": null, "support_only": false }, "tasks": [ { "name": "swe_bench", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "swe_bench_verified_mini", "dataset_samples": 50, "comment": null, "support_only": false } ], "protocol": { "version": "5-C", "dataset_samples": 500, "dependency": "swe_bench", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/swe-bench/experiments", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 559cf877c1095c2e244af73b2de42bd53bd0c9d5 in baseline/download_baselines.sh" }, { "type": "huggingface", "source": "princeton-nlp/SWE-bench_Verified", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:swe_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Carlos E. Jimenez, John Yang, Alexander Wettig et al.", "organization": "Princeton University; University of Washington", "organizations": [ "Princeton University", "University of Washington" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2310.06770" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2310.06770", "arxiv_id": "2310.06770", "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "authors": [ "Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan" ], "organizations": [ "Princeton University", "University of Washington" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2310.06770", "arxiv_id": "2310.06770", "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "authors": [ "Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan" ], "organizations": [ "Princeton University", "University of Washington" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/swe-bench/experiments", "owner": "SWE-bench", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified", "provider": "Princeton Nlp" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2310.06770" }, "code_url": "https://github.com/swe-bench/experiments", "metadata": { "source_search": "Carlos E. Jimenez, John Yang, Alexander Wettig et al. Princeton University; University of Washington Original paper authors SWE-bench: Can Language Models Resolve Real-World GitHub Issues? Carlos E. Jimenez John Yang Alexander Wettig Shunyu Yao Kexin Pei Ofir Press Karthik Narasimhan SWE-bench evaluation-family" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Ability to resolve selected real-world software issues in existing repositories.", "unit_of_analysis": "A generated patch for one GitHub issue and repository snapshot.", "interaction_type": "Repository-level coding and debugging with issue text, codebase context, tools, and tests.", "target_outcome": "Whether the patch passes task-specific fail-to-pass and pass-to-pass tests.", "human_participants": "No human subjects. Historical developer patches and issue discussions define tasks and test expectations.", "elicitation": "The system receives an issue and repository snapshot and may inspect, edit, and test code under the chosen agent scaffold.", "scoring": "Repository tests determine whether the issue is resolved without breaking previously passing behavior.", "protocol_dependence": "High: issue subset, repository snapshot, tests, tool access, time budget, and scaffold affect resolution rates.", "implementation_dependence": "High: environment construction and test validity are known sources of benchmark error; verified subsets are not interchangeable with the original set.", "limitations": [ "Tests may incompletely specify the intended fix or admit brittle patches.", "Repository selection is narrow and historically Python-heavy.", "Benchmark contamination and memorised patches are material concerns.", "Resolution rates often conflate model ability with search, context management, and scaffold quality." ], "comparability": "Use the same SWE-bench variant, task IDs, repository images, tests, timeout, scaffold, and model version; report Verified and full-set results separately.", "decision_relevance": "Useful for software-agent capability monitoring and procurement trials, with additional security, maintainability, and live-workflow evidence required.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2310.06770" }, { "kind": "implementation", "url": "https://github.com/swe-bench/SWE-bench" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:swe_bench_verified_mini", "name": "Swe Bench Verified Mini", "slug": "inspect--swe-bench-verified-mini", "organisation": "Carlos E. Jimenez, John Yang, Alexander Wettig et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench", "topics": [ "general-capability" ], "description": "Evaluates AI's ability to resolve genuine software engineering issues sourced from 12 popular Python GitHub repositories, reflecting realistic coding and debugging scenarios.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2310.06770", "primary": true }, { "kind": "paper", "label": "Paper: SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "url": "https://arxiv.org/abs/2310.06770", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/swe-bench/experiments", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Princeton Nlp", "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2310.06770", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench/eval.yaml", "family_title": "SWE-bench Verified: Resolving Real-World GitHub Issues", "group": "Coding", "version": "5-C", "contributors": [ "max-kaufmann" ], "tags": [ "Agent", "Coding" ], "task": { "name": "swe_bench_verified_mini", "dataset_samples": 50, "comment": null, "support_only": false }, "tasks": [ { "name": "swe_bench", "dataset_samples": 500, "comment": null, "support_only": false }, { "name": "swe_bench_verified_mini", "dataset_samples": 50, "comment": null, "support_only": false } ], "protocol": { "version": "5-C", "dataset_samples": 50, "dependency": "swe_bench", "requires_internet": true, "sandbox": [ "solver" ] }, "external_assets": [ { "type": "git_clone", "source": "https://github.com/swe-bench/experiments", "fetch_method": "git_clone", "state": "pinned", "comment": "Pinned to commit 559cf877c1095c2e244af73b2de42bd53bd0c9d5 in baseline/download_baselines.sh" }, { "type": "huggingface", "source": "princeton-nlp/SWE-bench_Verified", "fetch_method": "load_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:swe_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Carlos E. Jimenez, John Yang, Alexander Wettig et al.", "organization": "Princeton University; University of Washington", "organizations": [ "Princeton University", "University of Washington" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2310.06770" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2310.06770", "arxiv_id": "2310.06770", "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "authors": [ "Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan" ], "organizations": [ "Princeton University", "University of Washington" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2310.06770", "arxiv_id": "2310.06770", "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "authors": [ "Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan" ], "organizations": [ "Princeton University", "University of Washington" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/swe-bench/experiments", "owner": "SWE-bench", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified", "provider": "Princeton Nlp" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2310.06770" }, "code_url": "https://github.com/swe-bench/experiments", "metadata": { "source_search": "Carlos E. Jimenez, John Yang, Alexander Wettig et al. Princeton University; University of Washington Original paper authors SWE-bench: Can Language Models Resolve Real-World GitHub Issues? Carlos E. Jimenez John Yang Alexander Wettig Shunyu Yao Kexin Pei Ofir Press Karthik Narasimhan SWE-bench evaluation-family" } }, { "id": "inspect:swe_lancer", "name": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?", "slug": "inspect--swe-lancer", "organisation": "Samuel Miserendino, Michele Wang, Tejal Patwardhan et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_lancer", "topics": [ "general-capability" ], "description": "A benchmark of freelance software engineering tasks from Upwork, valued at $1 million USD total in realworld payouts.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_lancer/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2502.12115", "primary": true }, { "kind": "paper", "label": "Paper: SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?", "url": "https://arxiv.org/abs/2502.12115", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://hub.docker.com/u/swelancer", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_lancer/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2502.12115", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_lancer/eval.yaml", "family_title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?", "group": "Coding", "version": "1-B", "contributors": [ "NelsonG-C", "MattFisher" ], "tags": [ "Agent", "Coding" ], "task": { "name": "swe_lancer", "dataset_samples": 460, "comment": null, "support_only": false }, "tasks": [ { "name": "swe_lancer", "dataset_samples": 460, "comment": null, "support_only": false } ], "protocol": { "version": "1-B", "dataset_samples": 460, "dependency": "swe_lancer", "requires_internet": true, "sandbox": [ "solver", "scorer" ] }, "external_assets": [ { "type": "direct_url", "source": "https://hub.docker.com/u/swelancer", "fetch_method": "other", "state": "floating", "comment": "Per-task images (swelancer/swelancer_x86_{issue_id}:releasev1) and monolith image pulled at eval runtime" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:swe_lancer", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Samuel Miserendino, Michele Wang, Tejal Patwardhan et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2502.12115" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2502.12115", "arxiv_id": "2502.12115", "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?", "authors": [ "Samuel Miserendino", "Michele Wang", "Tejal Patwardhan", "Johannes Heidecke" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2502.12115", "arxiv_id": "2502.12115", "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?", "authors": [ "Samuel Miserendino", "Michele Wang", "Tejal Patwardhan", "Johannes Heidecke" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2502.12115", "arxiv_id": "2502.12115", "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?", "authors": [ "Samuel Miserendino", "Michele Wang", "Tejal Patwardhan", "Johannes Heidecke" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://hub.docker.com/u/swelancer", "owner": "hub.docker.com", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/swe_lancer/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2502.12115" }, "code_url": "https://hub.docker.com/u/swelancer", "metadata": { "source_search": "Samuel Miserendino, Michele Wang, Tejal Patwardhan et al. Original paper authors SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering? Samuel Miserendino Michele Wang Tejal Patwardhan Johannes Heidecke hub.docker.com evaluation-task" } }, { "id": "register:sycobench-600", "name": "SycoBench-600", "slug": "register--sycobench-600", "organisation": "Associated paper authors", "source_type": "inspect-register", "source_url": "https://github.com/debu-sinha/sycobench-600", "topics": [ "general-capability" ], "description": "Measures whether LLM assistants resist misleading user pressure while still accepting valid corrections in a controlled multiple-choice protocol.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/sycobench-600/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://aclanthology.org/2026.findings-acl.1759", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://aclanthology.org/2026.findings-acl.1759", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/debu-sinha/sycobench-600", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/sycobench-600/eval.yaml", "primary": false } ], "paper_title": "SycoBench-600: Measuring Sycophancy and Correction Selectivity in LLM Assistants", "paper_url": "https://aclanthology.org/2026.findings-acl.1759", "code_url": "https://github.com/debu-sinha/sycobench-600", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/sycobench-600/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "sycophancy", "correction-selectivity", "llm-evaluation", "ai-safety" ], "contributors": [ "debu-sinha" ], "tasks": [ { "name": "sycobench_600", "path": "sycobench/inspect_task.py" } ], "protocol": { "implementation_commit": "5219abda88de91300adcfefa37c3a824f0f103de", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": "Methodology source is the ACL Anthology paper with DOI 10.18653/v1/2026.findings-acl.1759; submitted manually because the automated register issue workflow currently expects arXiv URLs." }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:sycobench-600", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://aclanthology.org/2026.findings-acl.1759" }, "paper": { "status": "present", "url": "https://aclanthology.org/2026.findings-acl.1759", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://aclanthology.org/2026.findings-acl.1759", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/debu-sinha/sycobench-600", "owner": "Debu Sinha", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/sycobench-600/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://aclanthology.org/2026.findings-acl.1759" }, "metadata": { "source_search": "Associated paper authors Original paper authors Debu Sinha evaluation-package" } }, { "id": "inspect:sycophancy", "name": "Sycophancy Eval", "slug": "inspect--sycophancy", "organisation": "Anthropic and the paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sycophancy", "topics": [ "human-influence" ], "description": "Evaluate sycophancy of language models across a variety of free-form text-generation tasks.", "review_status": "reviewed", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "User-agreement or user-pleasing tendencies on the benchmark's paired prompt distribution under the stated model and prompting setup.", "does_not_measure": "Deliberate manipulation, strategic deception, downstream user belief change, emotional dependency, or the prevalence of sycophancy in real conversations.", "best_for": "Diagnosing epistemic unreliability caused by user-position cues and comparing mitigation approaches under controlled prompts.", "not_sufficient_for": "Claims that a model intentionally manipulates users or that sycophantic outputs cause human behavioral effects.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sycophancy/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_research", "label": "Original research by", "url": "https://www.anthropic.com/research/towards-understanding-sycophancy-in-language-models", "primary": true }, { "kind": "paper", "label": "Paper: Towards Understanding Sycophancy in Language Models", "url": "https://arxiv.org/abs/2310.13548", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/meg-tong/sycophancy-eval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sycophancy/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.anthropic.com/research/towards-understanding-sycophancy-in-language-models", "primary": false } ], "resource_count": 5, "paper_url": "https://arxiv.org/abs/2310.13548", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sycophancy/eval.yaml", "family_title": "Sycophancy Eval", "group": "Assistants", "version": "4-A", "contributors": [ "alexdzm" ], "tags": [ "Assistants" ], "task": { "name": "sycophancy", "dataset_samples": 4888, "comment": null, "support_only": false }, "tasks": [ { "name": "sycophancy", "dataset_samples": 4888, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 4888, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://raw.githubusercontent.com/meg-tong/sycophancy-eval/{SHA}/datasets/", "fetch_method": "other", "state": "pinned", "comment": "Fetched via fsspec (load_json_dataset)" } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:sycophancy", "record_type": "evaluation-task", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Anthropic and the paper authors", "organization": "Anthropic", "entity_type": "research_collaboration", "role": "original_research", "confidence": "verified", "evidence_url": "https://www.anthropic.com/research/towards-understanding-sycophancy-in-language-models", "organizations": [ "Anthropic" ], "role_label": "Original research by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2310.13548", "arxiv_id": "2310.13548", "title": "Towards Understanding Sycophancy in Language Models", "authors": [ "Mrinank Sharma", "Meg Tong", "Tomasz Korbak", "David Duvenaud", "Amanda Askell", "Samuel R. Bowman", "Newton Cheng", "Esin Durmus", "Zac Hatfield-Dodds", "Scott R. Johnston", "Shauna Kravec", "Timothy Maxwell", "Sam McCandlish", "Kamal Ndousse", "Oliver Rausch", "Nicholas Schiefer", "Da Yan", "Miranda Zhang", "Ethan Perez" ], "organizations": [ "Stanford University", "OpenAI" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2310.13548", "arxiv_id": "2310.13548", "title": "Towards Understanding Sycophancy in Language Models", "authors": [ "Mrinank Sharma", "Meg Tong", "Tomasz Korbak", "David Duvenaud", "Amanda Askell", "Samuel R. Bowman", "Newton Cheng", "Esin Durmus", "Zac Hatfield-Dodds", "Scott R. Johnston", "Shauna Kravec", "Timothy Maxwell", "Sam McCandlish", "Kamal Ndousse", "Oliver Rausch", "Nicholas Schiefer", "Da Yan", "Miranda Zhang", "Ethan Perez" ], "organizations": [ "Stanford University", "OpenAI" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/meg-tong/sycophancy-eval", "owner": "Meg Tong and collaborators", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/sycophancy/eval.yaml" }, "datasets": [], "organisation_role": "Original research by", "attribution_status": "verified", "preferred_source": { "label": "Original research by", "kind": "original_research", "url": "https://www.anthropic.com/research/towards-understanding-sycophancy-in-language-models" }, "code_url": "https://github.com/meg-tong/sycophancy-eval", "metadata": { "source_search": "Anthropic and the paper authors Anthropic Original research by Towards Understanding Sycophancy in Language Models Mrinank Sharma Meg Tong Tomasz Korbak David Duvenaud Amanda Askell Samuel R. Bowman Newton Cheng Esin Durmus Zac Hatfield-Dodds Scott R. Johnston Shauna Kravec Timothy Maxwell Sam McCandlish Kamal Ndousse Oliver Rausch Nicholas Schiefer Da Yan Miranda Zhang Ethan Perez Meg Tong and collaborators evaluation-task" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "high", "construct": "Tendency to produce answers that agree with or flatter a stated user view rather than maintain an independently supported answer.", "unit_of_analysis": "One paired prompt in which user preference, belief, or identity cues are varied while the underlying question is held constant or closely matched.", "interaction_type": "Controlled text generation or question answering with user-position cues and counterfactual variants.", "target_outcome": "Change in answer or evaluation toward the user's stated position, measured by task-specific labels or preference metrics.", "human_participants": "No live human subjects in the benchmark runs. Human preferences and annotations inform dataset construction and validation.", "elicitation": "Prompts expose the model to a user's view, preference, or identity and compare behavior against neutral or opposing variants.", "scoring": "Task-specific measures estimate agreement with the user, answer switching, or preference alignment across paired conditions.", "protocol_dependence": "High: task domain, user cue strength, prompt pair construction, answer format, and baseline definition shape the result.", "implementation_dependence": "Moderate to high: label parsing, generation settings, model-system prompts, and dataset variants affect measured rates.", "limitations": [ "Agreement can arise from uncertainty, ambiguity, or instruction following rather than a stable sycophantic tendency.", "Paired prompts may not capture long-form or relational sycophancy.", "Benchmark items and model behavior may be sensitive to minor wording changes.", "Output agreement is not evidence of downstream user influence." ], "comparability": "Use the same task subsets, prompt pairs, scoring definitions, system prompt, sampling settings, and model snapshot.", "decision_relevance": "Useful for epistemic-reliability evaluation, assistant design, and targeted monitoring; manipulation or dependency risks require different outcome-focused evaluations.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2310.13548" }, { "kind": "official", "url": "https://www.anthropic.com/research/towards-understanding-sycophancy-in-language-models" }, { "kind": "implementation", "url": "https://github.com/meg-tong/sycophancy-eval" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:tac", "name": "Tac", "slug": "inspect--tac", "organisation": "Jasmine Brazilek, Joel Christoph, Maheep Chaudhary et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac", "topics": [ "general-capability" ], "description": "Tests whether AI agents show implicit animal welfare awareness when purchasing tickets and experiences on behalf of users. Each scenario is designed so the most obvious choice involves animal exploitation, but the user prompt never mentions animal welfare. Agents should intrinsically avoid harmful options.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2606.18142", "primary": true }, { "kind": "paper", "label": "Paper: Your AI Travel Agent Would Book You a Bullfight: An Agentic Benchmark for Implicit Animal Welfare in Frontier AI Models", "url": "https://arxiv.org/abs/2606.18142", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Compassion in Machine Learning", "url": "https://huggingface.co/datasets/CompassioninMachineLearning/tac", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2606.18142", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac/eval.yaml", "family_title": "TAC: Animal Welfare Awareness in AI Ticket Agents", "group": "Safeguards", "version": "7-C", "contributors": [ "darkness8i8", "joel-christoph", "oTullio" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "tac", "dataset_samples": 52, "comment": null, "support_only": false }, "tasks": [ { "name": "tac", "dataset_samples": 52, "comment": null, "support_only": false }, { "name": "tac_welfare", "dataset_samples": 52, "comment": null, "support_only": false } ], "protocol": { "version": "7-C", "dataset_samples": 52, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "CompassioninMachineLearning/tac", "fetch_method": "hf_hub_download", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:tac", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jasmine Brazilek, Joel Christoph, Maheep Chaudhary et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2606.18142" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2606.18142", "arxiv_id": "2606.18142", "title": "Your AI Travel Agent Would Book You a Bullfight: An Agentic Benchmark for Implicit Animal Welfare in Frontier AI Models", "authors": [ "Jasmine Brazilek", "Joel Christoph", "Maheep Chaudhary", "Oliver Tullio", "Carol Kline", "Miles Tidmarsh", "Arturs Kanepajs" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2606.18142", "arxiv_id": "2606.18142", "title": "Your AI Travel Agent Would Book You a Bullfight: An Agentic Benchmark for Implicit Animal Welfare in Frontier AI Models", "authors": [ "Jasmine Brazilek", "Joel Christoph", "Maheep Chaudhary", "Oliver Tullio", "Carol Kline", "Miles Tidmarsh", "Arturs Kanepajs" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/CompassioninMachineLearning/tac", "provider": "Compassion in Machine Learning" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2606.18142" }, "code_url": null, "metadata": { "source_search": "Jasmine Brazilek, Joel Christoph, Maheep Chaudhary et al. Original paper authors Your AI Travel Agent Would Book You a Bullfight: An Agentic Benchmark for Implicit Animal Welfare in Frontier AI Models Jasmine Brazilek Joel Christoph Maheep Chaudhary Oliver Tullio Carol Kline Miles Tidmarsh Arturs Kanepajs evaluation-family" } }, { "id": "inspect:tac_welfare", "name": "Tac Welfare", "slug": "inspect--tac-welfare", "organisation": "Jasmine Brazilek, Joel Christoph, Maheep Chaudhary et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac", "topics": [ "general-capability" ], "description": "Tests whether AI agents show implicit animal welfare awareness when purchasing tickets and experiences on behalf of users. Each scenario is designed so the most obvious choice involves animal exploitation, but the user prompt never mentions animal welfare. Agents should intrinsically avoid harmful options.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2606.18142", "primary": true }, { "kind": "paper", "label": "Paper: Your AI Travel Agent Would Book You a Bullfight: An Agentic Benchmark for Implicit Animal Welfare in Frontier AI Models", "url": "https://arxiv.org/abs/2606.18142", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Compassion in Machine Learning", "url": "https://huggingface.co/datasets/CompassioninMachineLearning/tac", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2606.18142", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac/eval.yaml", "family_title": "TAC: Animal Welfare Awareness in AI Ticket Agents", "group": "Safeguards", "version": "7-C", "contributors": [ "darkness8i8", "joel-christoph", "oTullio" ], "tags": [ "Agent", "Safeguards" ], "task": { "name": "tac_welfare", "dataset_samples": 52, "comment": null, "support_only": false }, "tasks": [ { "name": "tac", "dataset_samples": 52, "comment": null, "support_only": false }, { "name": "tac_welfare", "dataset_samples": 52, "comment": null, "support_only": false } ], "protocol": { "version": "7-C", "dataset_samples": 52, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "CompassioninMachineLearning/tac", "fetch_method": "hf_hub_download", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:tac", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jasmine Brazilek, Joel Christoph, Maheep Chaudhary et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2606.18142" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2606.18142", "arxiv_id": "2606.18142", "title": "Your AI Travel Agent Would Book You a Bullfight: An Agentic Benchmark for Implicit Animal Welfare in Frontier AI Models", "authors": [ "Jasmine Brazilek", "Joel Christoph", "Maheep Chaudhary", "Oliver Tullio", "Carol Kline", "Miles Tidmarsh", "Arturs Kanepajs" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2606.18142", "arxiv_id": "2606.18142", "title": "Your AI Travel Agent Would Book You a Bullfight: An Agentic Benchmark for Implicit Animal Welfare in Frontier AI Models", "authors": [ "Jasmine Brazilek", "Joel Christoph", "Maheep Chaudhary", "Oliver Tullio", "Carol Kline", "Miles Tidmarsh", "Arturs Kanepajs" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tac/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/CompassioninMachineLearning/tac", "provider": "Compassion in Machine Learning" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2606.18142" }, "code_url": null, "metadata": { "source_search": "Jasmine Brazilek, Joel Christoph, Maheep Chaudhary et al. Original paper authors Your AI Travel Agent Would Book You a Bullfight: An Agentic Benchmark for Implicit Animal Welfare in Frontier AI Models Jasmine Brazilek Joel Christoph Maheep Chaudhary Oliver Tullio Carol Kline Miles Tidmarsh Arturs Kanepajs evaluation-family" } }, { "id": "register:tarantubench", "name": "TarantuBench", "slug": "register--tarantubench", "organisation": "TarantuLabs", "source_type": "inspect-register", "source_url": "https://github.com/Trivulzianus/TarantuBench", "topics": [ "general-capability" ], "description": "A web security benchmark for evaluating AI agents on generated vulnerable Node.js/Express applications. Agents interact over HTTP, extract hidden TARANTU{...} flags, and are scored with binary exact-match success.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/tarantubench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://www.tarantulabs.com", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/Trivulzianus/TarantuBench", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/tarantubench/eval.yaml", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.tarantulabs.com", "primary": false } ], "paper_title": null, "paper_url": null, "code_url": "https://github.com/Trivulzianus/TarantuBench", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/tarantubench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [ "cybersecurity", "agents", "web-security", "tool-use" ], "contributors": [ "Trivulzianus" ], "tasks": [ { "name": "tarantubench", "path": "src/tarantubench/task.py" } ], "protocol": { "implementation_commit": "7bc03a2e57fd68a238ae621eeb6ae856fea77682", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:tarantubench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "TarantuBench states that its labs are generated by the TarantuLabs engine. No dedicated academic paper was identified in the reviewed primary sources." }, "origin": { "name": "TarantuLabs", "organization": "TarantuLabs", "entity_type": "organization", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://www.tarantulabs.com", "organizations": [ "TarantuLabs" ], "role_label": "Evaluation developed by" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": { "url": "https://github.com/Trivulzianus/TarantuBench", "owner": "TarantuLabs / Trivulzianus", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/tarantubench/eval.yaml" }, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://www.tarantulabs.com" }, "metadata": { "source_search": "TarantuLabs TarantuLabs Evaluation developed by TarantuLabs / Trivulzianus evaluation-package" } }, { "id": "inspect:tau2_airline", "name": "Tau2 Airline", "slug": "inspect--tau2-airline", "organisation": "Victor Barres, Honghua Dong, Soham Ray et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2", "topics": [ "autonomy-agents" ], "description": "Evaluating Conversational Agents in a Dual-Control Environment", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2506.07982", "primary": true }, { "kind": "paper", "label": "Paper: τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "url": "https://arxiv.org/abs/2506.07982", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2506.07982", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "family_title": "Tau2", "group": "Assistants", "version": "3-A", "contributors": [ "mmulet" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "tau2_airline", "dataset_samples": 50, "comment": null, "support_only": false }, "tasks": [ { "name": "tau2_airline", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "tau2_banking", "dataset_samples": 97, "comment": null, "support_only": false }, { "name": "tau2_retail", "dataset_samples": 114, "comment": null, "support_only": false }, { "name": "tau2_telecom", "dataset_samples": 114, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 50, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:tau2", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Victor Barres, Honghua Dong, Soham Ray et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2506.07982" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.07982", "arxiv_id": "2506.07982", "title": "τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.07982", "arxiv_id": "2506.07982", "title": "τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2506.07982" }, "code_url": null, "metadata": { "source_search": "Victor Barres, Honghua Dong, Soham Ray et al. Original paper authors τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment Victor Barres Honghua Dong Soham Ray Xujie Si Karthik Narasimhan evaluation-family" } }, { "id": "inspect:tau2_banking", "name": "Tau2 Banking", "slug": "inspect--tau2-banking", "organisation": "Victor Barres, Honghua Dong, Soham Ray et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2", "topics": [ "autonomy-agents" ], "description": "Evaluating Conversational Agents in a Dual-Control Environment", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2506.07982", "primary": true }, { "kind": "paper", "label": "Paper: τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "url": "https://arxiv.org/abs/2506.07982", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2506.07982", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "family_title": "Tau2", "group": "Assistants", "version": "3-A", "contributors": [ "mmulet" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "tau2_banking", "dataset_samples": 97, "comment": null, "support_only": false }, "tasks": [ { "name": "tau2_airline", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "tau2_banking", "dataset_samples": 97, "comment": null, "support_only": false }, { "name": "tau2_retail", "dataset_samples": 114, "comment": null, "support_only": false }, { "name": "tau2_telecom", "dataset_samples": 114, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 97, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:tau2", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Victor Barres, Honghua Dong, Soham Ray et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2506.07982" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.07982", "arxiv_id": "2506.07982", "title": "τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.07982", "arxiv_id": "2506.07982", "title": "τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2506.07982" }, "code_url": null, "metadata": { "source_search": "Victor Barres, Honghua Dong, Soham Ray et al. Original paper authors τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment Victor Barres Honghua Dong Soham Ray Xujie Si Karthik Narasimhan evaluation-family" } }, { "id": "inspect:tau2_retail", "name": "Tau2 Retail", "slug": "inspect--tau2-retail", "organisation": "Victor Barres, Honghua Dong, Soham Ray et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2", "topics": [ "autonomy-agents" ], "description": "Evaluating Conversational Agents in a Dual-Control Environment", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2506.07982", "primary": true }, { "kind": "paper", "label": "Paper: τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "url": "https://arxiv.org/abs/2506.07982", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2506.07982", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "family_title": "Tau2", "group": "Assistants", "version": "3-A", "contributors": [ "mmulet" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "tau2_retail", "dataset_samples": 114, "comment": null, "support_only": false }, "tasks": [ { "name": "tau2_airline", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "tau2_banking", "dataset_samples": 97, "comment": null, "support_only": false }, { "name": "tau2_retail", "dataset_samples": 114, "comment": null, "support_only": false }, { "name": "tau2_telecom", "dataset_samples": 114, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 114, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:tau2", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Victor Barres, Honghua Dong, Soham Ray et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2506.07982" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.07982", "arxiv_id": "2506.07982", "title": "τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.07982", "arxiv_id": "2506.07982", "title": "τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2506.07982" }, "code_url": null, "metadata": { "source_search": "Victor Barres, Honghua Dong, Soham Ray et al. Original paper authors τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment Victor Barres Honghua Dong Soham Ray Xujie Si Karthik Narasimhan evaluation-family" } }, { "id": "inspect:tau2_telecom", "name": "Tau2 Telecom", "slug": "inspect--tau2-telecom", "organisation": "Victor Barres, Honghua Dong, Soham Ray et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2", "topics": [ "autonomy-agents" ], "description": "Evaluating Conversational Agents in a Dual-Control Environment", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2506.07982", "primary": true }, { "kind": "paper", "label": "Paper: τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "url": "https://arxiv.org/abs/2506.07982", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2506.07982", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml", "family_title": "Tau2", "group": "Assistants", "version": "3-A", "contributors": [ "mmulet" ], "tags": [ "Agent", "Assistants" ], "task": { "name": "tau2_telecom", "dataset_samples": 114, "comment": null, "support_only": false }, "tasks": [ { "name": "tau2_airline", "dataset_samples": 50, "comment": null, "support_only": false }, { "name": "tau2_banking", "dataset_samples": 97, "comment": null, "support_only": false }, { "name": "tau2_retail", "dataset_samples": 114, "comment": null, "support_only": false }, { "name": "tau2_telecom", "dataset_samples": 114, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 114, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:tau2", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Victor Barres, Honghua Dong, Soham Ray et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2506.07982" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2506.07982", "arxiv_id": "2506.07982", "title": "τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2506.07982", "arxiv_id": "2506.07982", "title": "τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/tau2/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2506.07982" }, "code_url": null, "metadata": { "source_search": "Victor Barres, Honghua Dong, Soham Ray et al. Original paper authors τ2-Bench: Evaluating Conversational Agents in a Dual-Control Environment Victor Barres Honghua Dong Soham Ray Xujie Si Karthik Narasimhan evaluation-family" } }, { "id": "canonical:terminal-bench", "name": "Terminal-Bench", "slug": "canonical--terminal-bench", "organisation": "Stanford and the Laude Institute", "source_type": "canonical-source", "source_url": "https://www.tbench.ai/", "topics": [ "autonomy-agents", "cyber" ], "description": "Evaluates agents completing real tasks in terminal environments.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Ability to complete the benchmark's terminal tasks under a specified agent harness, environment, and budget.", "does_not_measure": "General computer-use autonomy, safe production administration, open-ended planning, or reliability across arbitrary terminal workloads.", "best_for": "Testing tool-using agents on reproducible terminal tasks and analysing failures at the trajectory level.", "not_sufficient_for": "Claims of dependable autonomous software operation, cyber capability, or broad workplace automation.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://www.tbench.ai", "primary": true }, { "kind": "official", "label": "Official or project page", "url": "https://www.tbench.ai", "primary": false } ], "resource_count": 2, "source_key": "canonical:terminal-bench", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "Terminal-Bench is now presented through its official tbench.ai site and versioned benchmark registry. No dedicated academic paper was identified; the project provides a citable software/benchmark reference." }, "origin": { "name": "Stanford and the Laude Institute", "organization": "Stanford University + Laude Institute", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://www.tbench.ai", "organizations": [ "Stanford University + Laude Institute" ], "role_label": "Evaluation developed by" }, "paper": { "status": "not-found", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-found", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://www.tbench.ai" }, "paper_url": null, "code_url": null, "metadata": { "source_search": "Stanford and the Laude Institute Stanford University + Laude Institute Evaluation developed by evaluation-resource" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Completion of selected, self-contained real-world tasks through a terminal environment.", "unit_of_analysis": "One agent trajectory attempting one terminal task in a containerized environment.", "interaction_type": "Long-horizon command-line tool use with filesystem, programs, and task-specific environment state.", "target_outcome": "Task-specific verifier outcome, usually based on final environment state or tests.", "human_participants": "No human subjects. Task authors create tasks, reference solutions, and verifiers.", "elicitation": "The agent receives a natural-language task and terminal access under a fixed harness, time limit, and scaffold.", "scoring": "Task-specific deterministic or scripted verifiers aggregated into task success rates.", "protocol_dependence": "High: task set, container image, timeouts, tool permissions, and scaffold determine difficulty.", "implementation_dependence": "High: harness reliability, verifier correctness, and environment reproducibility directly affect outcomes.", "limitations": [ "Task selection may overrepresent benchmark-friendly, self-contained work.", "Binary verifier success can hide partial progress and unsafe intermediate actions.", "Public tasks create contamination and overfitting risk.", "Agent performance can change substantially with scaffold and environment fixes." ], "comparability": "Require the same Terminal-Bench release, task subset, environment images, verifier versions, timeouts, agent scaffold, and model access route.", "decision_relevance": "Useful for bounded agent capability and reliability testing; not a standalone deployment-readiness or safety assessment.", "source_basis": [ { "kind": "official", "url": "https://www.tbench.ai/" }, { "kind": "implementation", "url": "https://github.com/laude-institute/terminal-bench" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:theagentcompany", "name": "The Agent Company: Evaluating multi-tool autonomous agents in a synthetic company", "slug": "inspect--theagentcompany", "organisation": "Frank F. Xu, Yufan Song, Boxuan Li et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/theagentcompany", "topics": [ "autonomy-agents" ], "description": "The Agent Company benchmark evaluates autonomous agents in a realistic, self-contained company environment. Tasks require browsing internal web services, reading and writing files, running code, and coordinating tools to solve multi-step problems.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/theagentcompany/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2412.14161", "primary": true }, { "kind": "paper", "label": "Paper: TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks", "url": "https://arxiv.org/abs/2412.14161", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/theagentcompany/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2412.14161", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/theagentcompany/eval.yaml", "family_title": "The Agent Company: Evaluating multi-tool autonomous agents in a synthetic company", "group": "Assistants", "version": "1-A", "contributors": [ "bndxn" ], "tags": [ "Agent", "Tools", "Assistants" ], "task": { "name": "theagentcompany", "dataset_samples": 34, "comment": null, "support_only": false }, "tasks": [ { "name": "theagentcompany", "dataset_samples": 34, "comment": null, "support_only": false } ], "protocol": { "version": "1-A", "dataset_samples": 34, "dependency": "theagentcompany", "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:theagentcompany", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Frank F. Xu, Yufan Song, Boxuan Li et al.", "organization": "Carnegie Mellon University; Zhejiang University", "organizations": [ "Carnegie Mellon University", "Zhejiang University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2412.14161" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2412.14161", "arxiv_id": "2412.14161", "title": "TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks", "authors": [ "Frank F. Xu", "Yufan Song", "Boxuan Li", "Yuxuan Tang", "Kritanjali Jain", "Mengxue Bao", "Zora Z. Wang", "Xuhui Zhou", "Zhitong Guo", "Murong Cao", "Mingyang Yang", "Hao Yang Lu", "Amaad Martin", "Zhe Su", "Leander Maben", "Raj Mehta", "Wayne Chi", "Lawrence Jang", "Yiqing Xie", "Shuyan Zhou", "Graham Neubig" ], "organizations": [ "Carnegie Mellon University", "Zhejiang University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2412.14161", "arxiv_id": "2412.14161", "title": "TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks", "authors": [ "Frank F. Xu", "Yufan Song", "Boxuan Li", "Yuxuan Tang", "Kritanjali Jain", "Mengxue Bao", "Zora Z. Wang", "Xuhui Zhou", "Zhitong Guo", "Murong Cao", "Mingyang Yang", "Hao Yang Lu", "Amaad Martin", "Zhe Su", "Leander Maben", "Raj Mehta", "Wayne Chi", "Lawrence Jang", "Yiqing Xie", "Shuyan Zhou", "Graham Neubig" ], "organizations": [ "Carnegie Mellon University", "Zhejiang University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/theagentcompany/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2412.14161" }, "code_url": null, "metadata": { "source_search": "Frank F. Xu, Yufan Song, Boxuan Li et al. Carnegie Mellon University; Zhejiang University Original paper authors TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks Frank F. Xu Yufan Song Boxuan Li Yuxuan Tang Kritanjali Jain Mengxue Bao Zora Z. Wang Xuhui Zhou Zhitong Guo Murong Cao Mingyang Yang Hao Yang Lu Amaad Martin Zhe Su Leander Maben Raj Mehta Wayne Chi Lawrence Jang Yiqing Xie Shuyan Zhou Graham Neubig evaluation-task" } }, { "id": "inspect:threecb", "name": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "slug": "inspect--threecb", "organisation": "Andrey Anurin, Jonathan Ng, Kibo Schaffer et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/threecb", "topics": [ "general-capability" ], "description": "A benchmark for evaluating the capabilities of LLM agents in cyber offense.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/threecb/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2410.09114", "primary": true }, { "kind": "paper", "label": "Paper: Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "url": "https://arxiv.org/abs/2410.09114", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/threecb/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2410.09114", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/threecb/eval.yaml", "family_title": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "group": "Cybersecurity", "version": "4-B", "contributors": [ "derpyplops", "ralphptorres", "Scott-Simmons" ], "tags": [ "Agent", "Cybersecurity" ], "task": { "name": "threecb", "dataset_samples": 13, "comment": null, "support_only": false }, "tasks": [ { "name": "threecb", "dataset_samples": 13, "comment": null, "support_only": false } ], "protocol": { "version": "4-B", "dataset_samples": 13, "dependency": null, "requires_internet": false, "sandbox": [ "solver" ] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:threecb", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Andrey Anurin, Jonathan Ng, Kibo Schaffer et al.", "organization": "Apart Research, Aarhus University", "organizations": [ "Apart Research, Aarhus University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2410.09114" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2410.09114", "arxiv_id": "2410.09114", "title": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "authors": [ "Andrey Anurin", "Jonathan Ng", "Kibo Schaffer", "Jason Schreiber", "Esben Kran" ], "organizations": [ "Apart Research, Aarhus University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2410.09114", "arxiv_id": "2410.09114", "title": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "authors": [ "Andrey Anurin", "Jonathan Ng", "Kibo Schaffer", "Jason Schreiber", "Esben Kran" ], "organizations": [ "Apart Research, Aarhus University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/threecb/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2410.09114" }, "code_url": null, "metadata": { "source_search": "Andrey Anurin, Jonathan Ng, Kibo Schaffer et al. Apart Research, Aarhus University Original paper authors Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities Andrey Anurin Jonathan Ng Kibo Schaffer Jason Schreiber Esben Kran evaluation-task" } }, { "id": "inspect:truthfulqa", "name": "TruthfulQA: Measuring How Models Mimic Human Falsehoods", "slug": "inspect--truthfulqa", "organisation": "Stephanie Lin, Jacob Hilton, Owain Evans", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/truthfulqa", "topics": [ "evaluation-integrity" ], "description": "Measure whether a language model is truthful in generating answers to questions using questions that some humans would answer falsely due to a false belief or misconception.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/truthfulqa/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2109.07958", "primary": true }, { "kind": "paper", "label": "Paper: TruthfulQA: Measuring How Models Mimic Human Falsehoods", "url": "https://arxiv.org/abs/2109.07958", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/truthfulqa/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Truthfulqa", "url": "https://huggingface.co/datasets/truthfulqa/truthful_qa", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2109.07958", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/truthfulqa/eval.yaml", "family_title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods", "group": "Knowledge", "version": "2-A", "contributors": [ "seddy-aisi" ], "tags": [ "Knowledge" ], "task": { "name": "truthfulqa", "dataset_samples": 817, "comment": null, "support_only": false }, "tasks": [ { "name": "truthfulqa", "dataset_samples": 817, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 817, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "truthfulqa/truthful_qa", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:truthfulqa", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Stephanie Lin, Jacob Hilton, Owain Evans", "organization": "Truthful AI; UC Berkeley", "organizations": [ "Truthful AI", "UC Berkeley" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2109.07958" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2109.07958", "arxiv_id": "2109.07958", "title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods", "authors": [ "Stephanie Lin", "Jacob Hilton", "Owain Evans" ], "organizations": [ "Truthful AI", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2109.07958", "arxiv_id": "2109.07958", "title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods", "authors": [ "Stephanie Lin", "Jacob Hilton", "Owain Evans" ], "organizations": [ "Truthful AI", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2109.07958", "arxiv_id": "2109.07958", "title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods", "authors": [ "Stephanie Lin", "Jacob Hilton", "Owain Evans" ], "organizations": [ "Truthful AI", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/truthfulqa/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/truthfulqa/truthful_qa", "provider": "Truthfulqa" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2109.07958" }, "code_url": null, "metadata": { "source_search": "Stephanie Lin, Jacob Hilton, Owain Evans Truthful AI; UC Berkeley Original paper authors TruthfulQA: Measuring How Models Mimic Human Falsehoods Stephanie Lin Jacob Hilton Owain Evans evaluation-task" } }, { "id": "inspect:uccb", "name": "Uganda Cultural and Cognitive Benchmark (UCCB)", "slug": "inspect--uccb", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/uccb", "topics": [ "general-capability" ], "description": "The first comprehensive question-answering dataset designed to evaluate cultural understanding and reasoning abilities of Large Language Models concerning Uganda's multifaceted environment across 24 cultural domains including education, traditional medicine, media, economy, literature, and social norms.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/uccb/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://huggingface.co/datasets/CraneAILabs/UCCB", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://huggingface.co/datasets/CraneAILabs/UCCB", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/uccb/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Crane AI Labs", "url": "https://huggingface.co/datasets/CraneAILabs/UCCB", "primary": false } ], "resource_count": 4, "paper_url": "https://huggingface.co/datasets/CraneAILabs/UCCB", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/uccb/eval.yaml", "family_title": "Uganda Cultural and Cognitive Benchmark (UCCB)", "group": "Knowledge", "version": "3-A", "contributors": [ "katostevenmubiru" ], "tags": [ "Knowledge" ], "task": { "name": "uccb", "dataset_samples": 1039, "comment": null, "support_only": false }, "tasks": [ { "name": "uccb", "dataset_samples": 1039, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 1039, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "CraneAILabs/UCCB", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:uccb", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://huggingface.co/datasets/CraneAILabs/UCCB" }, "paper": { "status": "present", "url": "https://huggingface.co/datasets/CraneAILabs/UCCB", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://huggingface.co/datasets/CraneAILabs/UCCB", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/uccb/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/CraneAILabs/UCCB", "provider": "Crane AI Labs" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://huggingface.co/datasets/CraneAILabs/UCCB" }, "code_url": null, "metadata": { "source_search": "Associated paper authors Original paper authors evaluation-task" } }, { "id": "inspect:usaco", "name": "USACO: USA Computing Olympiad", "slug": "inspect--usaco", "organisation": "Quan Shi, Michael Tang, Karthik Narasimhan et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/usaco", "topics": [ "general-capability" ], "description": "Evaluates language model performance on difficult Olympiad programming problems across four difficulty levels.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/usaco/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2404.10952", "primary": true }, { "kind": "paper", "label": "Paper: Can Language Models Solve Olympiad Programming?", "url": "https://arxiv.org/abs/2404.10952", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://drive.usercontent.google.com/download?id=1z5ODOJMqyer1QxzYtEUZ2hbAx-7nU8Vi", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/usaco/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2404.10952", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/usaco/eval.yaml", "family_title": "USACO: USA Computing Olympiad", "group": "Coding", "version": "2-A", "contributors": [ "danwilhelm" ], "tags": [ "Coding" ], "task": { "name": "usaco", "dataset_samples": 307, "comment": null, "support_only": false }, "tasks": [ { "name": "usaco", "dataset_samples": 307, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 307, "dependency": "usaco", "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "direct_url", "source": "https://drive.usercontent.google.com/download?id=1z5ODOJMqyer1QxzYtEUZ2hbAx-7nU8Vi", "fetch_method": "gdown_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:usaco", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Quan Shi, Michael Tang, Karthik Narasimhan et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2404.10952" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2404.10952", "arxiv_id": "2404.10952", "title": "Can Language Models Solve Olympiad Programming?", "authors": [ "Quan Shi", "Michael Tang", "Karthik Narasimhan", "Shunyu Yao" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2404.10952", "arxiv_id": "2404.10952", "title": "Can Language Models Solve Olympiad Programming?", "authors": [ "Quan Shi", "Michael Tang", "Karthik Narasimhan", "Shunyu Yao" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://drive.usercontent.google.com/download?id=1z5ODOJMqyer1QxzYtEUZ2hbAx-7nU8Vi", "owner": "drive.usercontent.google.com", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/usaco/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2404.10952" }, "code_url": "https://drive.usercontent.google.com/download?id=1z5ODOJMqyer1QxzYtEUZ2hbAx-7nU8Vi", "metadata": { "source_search": "Quan Shi, Michael Tang, Karthik Narasimhan et al. Original paper authors Can Language Models Solve Olympiad Programming? Quan Shi Michael Tang Karthik Narasimhan Shunyu Yao drive.usercontent.google.com evaluation-task" } }, { "id": "inspect:vimgolf_single_turn", "name": "VimGolf: Evaluating LLMs in Vim Editing Proficiency", "slug": "inspect--vimgolf-single-turn", "organisation": "VimGolf", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vimgolf_challenges", "topics": [ "general-capability" ], "description": "A benchmark that evaluates LLMs in their ability to operate Vim editor and complete editing challenges. This benchmark contrasts with common CUA benchmarks by focusing on Vim-specific editing capabilities.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vimgolf_challenges/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "original_benchmark_source", "label": "Original benchmark source", "url": "https://www.vimgolf.com", "primary": true }, { "kind": "implementation", "label": "Reference implementation", "url": "https://huggingface.co/datasets/cybergod-kevin/vimgolf-public-challenges-inspect-eval", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vimgolf_challenges/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Cybergod Kevin", "url": "https://huggingface.co/datasets/cybergod-kevin/vimgolf-public-challenges-inspect-eval", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.vimgolf.com", "primary": false } ], "resource_count": 5, "paper_url": null, "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vimgolf_challenges/eval.yaml", "family_title": "VimGolf: Evaluating LLMs in Vim Editing Proficiency", "group": "Reasoning", "version": "2-A", "contributors": [ "james4ever0" ], "tags": [ "Vim", "Editing", "Game", "Reasoning" ], "task": { "name": "vimgolf_single_turn", "dataset_samples": 612, "comment": null, "support_only": false }, "tasks": [ { "name": "vimgolf_single_turn", "dataset_samples": 612, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 612, "dependency": "vimgolf_challenges", "requires_internet": false, "sandbox": [ "scorer" ] }, "external_assets": [ { "type": "huggingface", "source": "cybergod-kevin/vimgolf-public-challenges-inspect-eval", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:vimgolf_challenges", "record_type": "evaluation-dataset-adaptation", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": "The task adapts public VimGolf challenges. The Hugging Face dataset and Inspect implementation are downstream adaptations, not the origin of VimGolf." }, "origin": { "name": "VimGolf", "organization": null, "entity_type": "benchmark_project", "role": "original_benchmark_source", "confidence": "verified", "evidence_url": "https://www.vimgolf.com", "organizations": [], "role_label": "Original benchmark source" }, "paper": { "status": "not-applicable", "url": null, "title": null, "authors": [], "arxiv_id": null, "source": null }, "papers": [], "paper_status": "not-applicable", "reference_implementation": { "url": "https://huggingface.co/datasets/cybergod-kevin/vimgolf-public-challenges-inspect-eval", "owner": "cybergod-kevin", "relation": "inspect_evaluation_dataset", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vimgolf_challenges/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/cybergod-kevin/vimgolf-public-challenges-inspect-eval", "provider": "Cybergod Kevin" } ], "organisation_role": "Original benchmark source", "attribution_status": "verified", "preferred_source": { "label": "Original benchmark source", "kind": "original_benchmark_source", "url": "https://www.vimgolf.com" }, "code_url": "https://huggingface.co/datasets/cybergod-kevin/vimgolf-public-challenges-inspect-eval", "metadata": { "source_search": "VimGolf Original benchmark source cybergod-kevin evaluation-dataset-adaptation" } }, { "id": "canonical:visualwebarena", "name": "VisualWebArena", "slug": "canonical--visualwebarena", "organisation": "Carnegie Mellon University and the VisualWebArena paper authors", "source_type": "canonical-source", "source_url": "https://jykoh.com/vwa", "topics": [ "autonomy-agents", "multimodal" ], "description": "Extends web-agent evaluation to visually grounded tasks.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://jykoh.com/vwa", "primary": true }, { "kind": "paper", "label": "Paper: VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks", "url": "https://arxiv.org/abs/2401.13649", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://jykoh.com/vwa", "primary": false } ], "resource_count": 3, "source_key": "canonical:visualwebarena", "record_type": "evaluation-environment", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Carnegie Mellon University and the VisualWebArena paper authors", "organization": "Carnegie Mellon University", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://jykoh.com/vwa", "organizations": [ "Carnegie Mellon University" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2401.13649", "arxiv_id": "2401.13649", "title": "VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks", "authors": [ "Jing Yu Koh", "Robert Lo", "Lawrence Jang", "Vikram Duvvur", "Ming Chong Lim", "Po-Yu Huang", "Graham Neubig", "Shuyan Zhou", "Ruslan Salakhutdinov", "Daniel Fried" ], "organizations": [ "Carnegie Mellon University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2401.13649", "arxiv_id": "2401.13649", "title": "VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks", "authors": [ "Jing Yu Koh", "Robert Lo", "Lawrence Jang", "Vikram Duvvur", "Ming Chong Lim", "Po-Yu Huang", "Graham Neubig", "Shuyan Zhou", "Ruslan Salakhutdinov", "Daniel Fried" ], "organizations": [ "Carnegie Mellon University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://jykoh.com/vwa" }, "paper_url": "https://arxiv.org/abs/2401.13649", "code_url": null, "metadata": { "source_search": "Carnegie Mellon University and the VisualWebArena paper authors Carnegie Mellon University Evaluation developed by VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks Jing Yu Koh Robert Lo Lawrence Jang Vikram Duvvur Ming Chong Lim Po-Yu Huang Graham Neubig Shuyan Zhou Ruslan Salakhutdinov Daniel Fried evaluation-environment" } }, { "id": "inspect:vqa_rad", "name": "VQA-RAD: Visual Question Answering for Radiology", "slug": "inspect--vqa-rad", "organisation": "Associated paper authors", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vqa_rad", "topics": [ "multimodal" ], "description": "VQA-RAD is the first manually constructed VQA dataset in radiology, where clinicians asked naturally occurring questions about radiology images and provided reference answers. It contains 315 radiology images (head CTs/MRIs, chest X-rays, abdominal CTs) with question-answer pairs spanning 11 question types including modality, plane, organ system, abnormality, and object/condition presence. Questions are either closed-ended (yes/no) or open-ended free-text answers.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vqa_rad/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://doi.org/10.1038/sdata.2018.251", "primary": true }, { "kind": "paper", "label": "Paper", "url": "https://doi.org/10.1038/sdata.2018.251", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vqa_rad/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Flaviagiammarino", "url": "https://huggingface.co/datasets/flaviagiammarino/vqa-rad", "primary": false } ], "resource_count": 4, "paper_url": "https://doi.org/10.1038/sdata.2018.251", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vqa_rad/eval.yaml", "family_title": "VQA-RAD: Visual Question Answering for Radiology", "group": "Multimodal", "version": "3-B", "contributors": [ "MattFisher" ], "tags": [ "Multimodal" ], "task": { "name": "vqa_rad", "dataset_samples": 451, "comment": null, "support_only": false }, "tasks": [ { "name": "vqa_rad", "dataset_samples": 451, "comment": null, "support_only": false } ], "protocol": { "version": "3-B", "dataset_samples": 451, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "flaviagiammarino/vqa-rad", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:vqa_rad", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Associated paper authors", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-linked", "evidence_url": "https://doi.org/10.1038/sdata.2018.251" }, "paper": { "status": "present", "url": "https://doi.org/10.1038/sdata.2018.251", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" }, "papers": [ { "status": "present", "url": "https://doi.org/10.1038/sdata.2018.251", "arxiv_id": null, "title": null, "authors": [], "organizations": [], "source": "upstream-metadata", "metadata_status": "pending" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vqa_rad/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/flaviagiammarino/vqa-rad", "provider": "Flaviagiammarino" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://doi.org/10.1038/sdata.2018.251" }, "code_url": null, "metadata": { "source_search": "Associated paper authors Original paper authors evaluation-task" } }, { "id": "inspect:vstar_bench_attribute_recognition", "name": "Vstar Bench Attribute Recognition", "slug": "inspect--vstar-bench-attribute-recognition", "organisation": "Penghao Wu, Saining Xie", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench", "topics": [ "general-capability" ], "description": "V*Bench is a visual question & answer benchmark that evaluates MLLMs in their ability to process high-resolution and visually crowded images to find and focus on small details.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2312.14135", "primary": true }, { "kind": "paper", "label": "Paper: V*: Guided Visual Search as a Core Mechanism in Multimodal LLMs", "url": "https://arxiv.org/abs/2312.14135", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Craigwu", "url": "https://huggingface.co/datasets/craigwu/vstar_bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2312.14135", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench/eval.yaml", "family_title": "V*Bench: A Visual QA Benchmark with Detailed High-resolution Images", "group": "Multimodal", "version": "2-A", "contributors": [ "bienehito" ], "tags": [ "Multimodal" ], "task": { "name": "vstar_bench_attribute_recognition", "dataset_samples": 115, "comment": null, "support_only": false }, "tasks": [ { "name": "vstar_bench_attribute_recognition", "dataset_samples": 115, "comment": null, "support_only": false }, { "name": "vstar_bench_spatial_relationship_reasoning", "dataset_samples": 76, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 115, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "craigwu/vstar_bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:vstar_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Penghao Wu, Saining Xie", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2312.14135" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2312.14135", "arxiv_id": "2312.14135", "title": "V*: Guided Visual Search as a Core Mechanism in Multimodal LLMs", "authors": [ "Penghao Wu", "Saining Xie" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2312.14135", "arxiv_id": "2312.14135", "title": "V*: Guided Visual Search as a Core Mechanism in Multimodal LLMs", "authors": [ "Penghao Wu", "Saining Xie" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/craigwu/vstar_bench", "provider": "Craigwu" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2312.14135" }, "code_url": null, "metadata": { "source_search": "Penghao Wu, Saining Xie Original paper authors V*: Guided Visual Search as a Core Mechanism in Multimodal LLMs Penghao Wu Saining Xie evaluation-family" } }, { "id": "inspect:vstar_bench_spatial_relationship_reasoning", "name": "Vstar Bench Spatial Relationship Reasoning", "slug": "inspect--vstar-bench-spatial-relationship-reasoning", "organisation": "Penghao Wu, Saining Xie", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench", "topics": [ "general-capability" ], "description": "V*Bench is a visual question & answer benchmark that evaluates MLLMs in their ability to process high-resolution and visually crowded images to find and focus on small details.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2312.14135", "primary": true }, { "kind": "paper", "label": "Paper: V*: Guided Visual Search as a Core Mechanism in Multimodal LLMs", "url": "https://arxiv.org/abs/2312.14135", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Craigwu", "url": "https://huggingface.co/datasets/craigwu/vstar_bench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2312.14135", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench/eval.yaml", "family_title": "V*Bench: A Visual QA Benchmark with Detailed High-resolution Images", "group": "Multimodal", "version": "2-A", "contributors": [ "bienehito" ], "tags": [ "Multimodal" ], "task": { "name": "vstar_bench_spatial_relationship_reasoning", "dataset_samples": 76, "comment": null, "support_only": false }, "tasks": [ { "name": "vstar_bench_attribute_recognition", "dataset_samples": 115, "comment": null, "support_only": false }, { "name": "vstar_bench_spatial_relationship_reasoning", "dataset_samples": 76, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 76, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "craigwu/vstar_bench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:vstar_bench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Penghao Wu, Saining Xie", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2312.14135" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2312.14135", "arxiv_id": "2312.14135", "title": "V*: Guided Visual Search as a Core Mechanism in Multimodal LLMs", "authors": [ "Penghao Wu", "Saining Xie" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2312.14135", "arxiv_id": "2312.14135", "title": "V*: Guided Visual Search as a Core Mechanism in Multimodal LLMs", "authors": [ "Penghao Wu", "Saining Xie" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/vstar_bench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/craigwu/vstar_bench", "provider": "Craigwu" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2312.14135" }, "code_url": null, "metadata": { "source_search": "Penghao Wu, Saining Xie Original paper authors V*: Guided Visual Search as a Core Mechanism in Multimodal LLMs Penghao Wu Saining Xie evaluation-family" } }, { "id": "canonical:webarena", "name": "WebArena", "slug": "canonical--webarena", "organisation": "Carnegie Mellon University and the WebArena paper authors", "source_type": "canonical-source", "source_url": "https://webarena.dev/", "topics": [ "autonomy-agents" ], "description": "Evaluates autonomous agents on realistic tasks across self-hosted web environments.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Web task completion on the benchmark's selected sites and tasks under the stated browser-agent setup.", "does_not_measure": "Performance on the open web, resilience to site changes, safe handling of real accounts, or general workplace automation.", "best_for": "Controlled comparison of browser agents on realistic multi-step web workflows.", "not_sufficient_for": "Claims of reliable or safe deployment on arbitrary live websites.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://webarena.dev", "primary": true }, { "kind": "paper", "label": "Paper: WebArena: A Realistic Web Environment for Building Autonomous Agents", "url": "https://arxiv.org/abs/2307.13854", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/web-arena-x/webarena", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://webarena.dev", "primary": false } ], "resource_count": 4, "source_key": "canonical:webarena", "record_type": "evaluation-environment", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Carnegie Mellon University and the WebArena paper authors", "organization": "Carnegie Mellon University", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://webarena.dev", "organizations": [ "Carnegie Mellon University" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2307.13854", "arxiv_id": "2307.13854", "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents", "authors": [ "Shuyan Zhou", "Frank F. Xu", "Hao Zhu", "Xuhui Zhou", "Robert Lo", "Abishek Sridhar", "Xianyi Cheng", "Tianyue Ou", "Yonatan Bisk", "Daniel Fried", "Uri Alon", "Graham Neubig" ], "organizations": [ "Carnegie Mellon University", "Indian Institute of Technology Madras" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2307.13854", "arxiv_id": "2307.13854", "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents", "authors": [ "Shuyan Zhou", "Frank F. Xu", "Hao Zhu", "Xuhui Zhou", "Robert Lo", "Abishek Sridhar", "Xianyi Cheng", "Tianyue Ou", "Yonatan Bisk", "Daniel Fried", "Uri Alon", "Graham Neubig" ], "organizations": [ "Carnegie Mellon University", "Indian Institute of Technology Madras" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/web-arena-x/webarena", "owner": "WebArena project", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://webarena.dev" }, "paper_url": "https://arxiv.org/abs/2307.13854", "code_url": "https://github.com/web-arena-x/webarena", "metadata": { "source_search": "Carnegie Mellon University and the WebArena paper authors Carnegie Mellon University Evaluation developed by WebArena: A Realistic Web Environment for Building Autonomous Agents Shuyan Zhou Frank F. Xu Hao Zhu Xuhui Zhou Robert Lo Abishek Sridhar Xianyi Cheng Tianyue Ou Yonatan Bisk Daniel Fried Uri Alon Graham Neubig WebArena project evaluation-environment" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Autonomous completion of realistic, long-horizon web tasks on self-hosted websites.", "unit_of_analysis": "One agent trajectory attempting one natural-language task across one or more benchmark websites.", "interaction_type": "Browser interaction using page observations and actions in resettable, self-hosted environments.", "target_outcome": "Functional correctness checked against site state and task-specific evaluators.", "human_participants": "No human subjects. Human performance provides a reference baseline.", "elicitation": "The agent receives a task and browser access under a specified observation/action interface, step limit, and scaffold.", "scoring": "Task-specific functional evaluators inspect final website state rather than relying only on text similarity.", "protocol_dependence": "High: website snapshots, task set, credentials, observation/action interface, and step budget define the result.", "implementation_dependence": "High: environment setup, site state, browser automation, and evaluators can introduce failures.", "limitations": [ "Self-hosted sites improve reproducibility but differ from the changing open web.", "The finite site and task distribution may not generalize to enterprise or personal workflows.", "Success metrics do not fully measure harmful side effects or policy compliance.", "Agent scaffolds and browsing interfaces are major contributors to performance." ], "comparability": "Use identical WebArena release, site snapshots, task IDs, evaluator versions, browser interface, step limit, scaffold, and model version.", "decision_relevance": "Useful for controlled browser-agent capability and regression testing; not sufficient for live-web deployment approval.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2307.13854" }, { "kind": "official", "url": "https://webarena.dev/" }, { "kind": "implementation", "url": "https://github.com/web-arena-x/webarena" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "register:wildclawbench", "name": "WildClawBench", "slug": "register--wildclawbench", "organisation": "Shuangrui Ding, Xuanlang Dai, Long Xing et al.", "source_type": "inspect-register", "source_url": "https://github.com/zytoh0/pinch-wildclawbench-inspect", "topics": [ "autonomy-agents" ], "description": "This is an Inspect AI wrapper of the original WildClawBench implementation. It does not contain its own dataset or scoring logic. The original implementation can be found at https://github.com/internlm/WildClawBench. Users provide a pinned checkout of the original repository, tested here at commit 86d71447413d38f38740a021cb776f64eb396ee0, and the wrapper reports native WildClawBench results through Inspect's scoring interface. WildClawBench contains 60 human-authored bilingual and multimodal long-horizon agent tasks across productivity flow, code intelligence, social interaction, search and retrieval, creative synthesis, and safety alignment. The native harness runs OpenClaw-style agents in Docker and grades with deterministic rule checks, environment-state auditing, and LLM/VLM judges; this wrapper parses native summary*.json or per-task score.json outputs and reports the mean native overall_score across scored tasks with Inspect mean/stderr metrics. It requires Docker, an OpenAI-compatible model endpoint, the pinned Docker base image node:22-bookworm@sha256:c601a46abb4d2ab80a9dc3da208d50d1122642d53f17a101926ace71e5a9bf1c, openclaw@2026.6.10, and the exact Python dependencies pinned in the upstream pyproject.toml and Dockerfile.", "review_status": "catalogued", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/wildclawbench/eval.yaml", "enrichment": "Inspect Evals register metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2605.10912", "primary": true }, { "kind": "paper", "label": "Paper: WildClawBench: A Benchmark for Real-World, Long-Horizon Agent Evaluation", "url": "https://arxiv.org/abs/2605.10912", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/zytoh0/pinch-wildclawbench-inspect", "primary": false }, { "kind": "inspect-registry", "label": "Inspect Evals Register", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/wildclawbench/eval.yaml", "primary": false } ], "paper_title": "WildClawBench: A Benchmark for Real-World, Long-Horizon Agent Evaluation", "paper_url": "https://arxiv.org/abs/2605.10912", "code_url": "https://github.com/zytoh0/pinch-wildclawbench-inspect", "registry_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/wildclawbench/eval.yaml", "documentation_url": null, "dataset_url": null, "tags": [], "contributors": [ "zytoh0" ], "tasks": [ { "name": "wildclawbench", "path": "src/wildclawbench/wildclawbench.py" } ], "protocol": { "implementation_commit": "1acc83dbdc497d966d084d204adcbffe2d1d8aaa", "report_commit": null, "report_timestamp": null, "command": null, "source_comment": null }, "reported_results": [], "reported_result_count": 0, "reported_results_truncated": false, "report_notes": [], "models_reported": [], "metadata_enriched": true, "metadata_enriched_at": "2026-08-30", "resource_count": 4, "source_key": "register:wildclawbench", "record_type": "evaluation-package", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Shuangrui Ding, Xuanlang Dai, Long Xing et al.", "organization": "Shanghai AI Laboratory; Nanyang Technological University; Chinese University of HongKong; Peking University; Shanghai AI Lab", "organizations": [ "Shanghai AI Laboratory", "Nanyang Technological University", "Chinese University of HongKong", "Peking University", "Shanghai AI Lab" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2605.10912" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2605.10912", "arxiv_id": "2605.10912", "title": "WildClawBench: A Benchmark for Real-World, Long-Horizon Agent Evaluation", "authors": [ "Shuangrui Ding", "Xuanlang Dai", "Long Xing", "Shengyuan Ding", "Ziyu Liu", "Yang JingYi", "Penghui Yang", "Zhixiong Zhang", "Xilin Wei", "Xinyu Fang", "Yubo Ma", "Haodong Duan", "Jing Shao", "Jiaqi Wang", "Dahua Lin", "Kai Chen", "Yuhang Zang" ], "organizations": [ "Shanghai AI Laboratory", "Nanyang Technological University", "Chinese University of HongKong", "Peking University", "Shanghai AI Lab" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2605.10912", "arxiv_id": "2605.10912", "title": "WildClawBench: A Benchmark for Real-World, Long-Horizon Agent Evaluation", "authors": [ "Shuangrui Ding", "Xuanlang Dai", "Long Xing", "Shengyuan Ding", "Ziyu Liu", "Yang JingYi", "Penghui Yang", "Zhixiong Zhang", "Xilin Wei", "Xinyu Fang", "Yubo Ma", "Haodong Duan", "Jing Shao", "Jiaqi Wang", "Dahua Lin", "Kai Chen", "Yuhang Zang" ], "organizations": [ "Shanghai AI Laboratory", "Nanyang Technological University", "Chinese University of HongKong", "Peking University", "Shanghai AI Lab" ], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2605.10912", "arxiv_id": "2605.10912", "title": "WildClawBench: A Benchmark for Real-World, Long-Horizon Agent Evaluation", "authors": [ "Shuangrui Ding", "Xuanlang Dai", "Long Xing", "Shengyuan Ding", "Ziyu Liu", "Yang JingYi", "Penghui Yang", "Zhixiong Zhang", "Xilin Wei", "Xinyu Fang", "Yubo Ma", "Haodong Duan", "Jing Shao", "Jiaqi Wang", "Dahua Lin", "Kai Chen", "Yuhang Zang" ], "organizations": [ "Shanghai AI Laboratory", "Nanyang Technological University", "Chinese University of HongKong", "Peking University", "Shanghai AI Lab" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/zytoh0/pinch-wildclawbench-inspect", "owner": "Zytoh0", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "registry_entry", "role_label": "Inspect Evals Register entry", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/register/wildclawbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2605.10912" }, "metadata": { "source_search": "Shuangrui Ding, Xuanlang Dai, Long Xing et al. Shanghai AI Laboratory; Nanyang Technological University; Chinese University of HongKong; Peking University; Shanghai AI Lab Original paper authors WildClawBench: A Benchmark for Real-World, Long-Horizon Agent Evaluation Shuangrui Ding Xuanlang Dai Long Xing Shengyuan Ding Ziyu Liu Yang JingYi Penghui Yang Zhixiong Zhang Xilin Wei Xinyu Fang Yubo Ma Haodong Duan Jing Shao Jiaqi Wang Dahua Lin Kai Chen Yuhang Zang Zytoh0 evaluation-package" } }, { "id": "inspect:winogrande", "name": "WINOGRANDE: An Adversarial Winograd Schema Challenge at Scale", "slug": "inspect--winogrande", "organisation": "Keisuke Sakaguchi, Ronan Le Bras, Chandra Bhagavatula et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/winogrande", "topics": [ "general-capability" ], "description": "Set of 273 expert-crafted pronoun resolution problems originally designed to be unsolvable for statistical models that rely on selectional preferences or word associations.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/winogrande/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/1907.10641", "primary": true }, { "kind": "paper", "label": "Paper: WinoGrande", "url": "https://arxiv.org/abs/1907.10641", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/winogrande/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Allen Institute for AI", "url": "https://huggingface.co/datasets/allenai/winogrande", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/1907.10641", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/winogrande/eval.yaml", "family_title": "WINOGRANDE: An Adversarial Winograd Schema Challenge at Scale", "group": "Reasoning", "version": "2-A", "contributors": [ "xeon27" ], "tags": [ "Reasoning" ], "task": { "name": "winogrande", "dataset_samples": 1267, "comment": null, "support_only": false }, "tasks": [ { "name": "winogrande", "dataset_samples": 1267, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1267, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "allenai/winogrande", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:winogrande", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Keisuke Sakaguchi, Ronan Le Bras, Chandra Bhagavatula et al.", "organization": "Tohoku University; Allen Institute for Artificial Intelligence", "organizations": [ "Tohoku University", "Allen Institute for Artificial Intelligence" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/1907.10641" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/1907.10641", "arxiv_id": "1907.10641", "title": "WinoGrande", "authors": [ "Keisuke Sakaguchi", "Ronan Le Bras", "Chandra Bhagavatula", "Yejin Choi" ], "organizations": [ "Tohoku University", "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/1907.10641", "arxiv_id": "1907.10641", "title": "WinoGrande", "authors": [ "Keisuke Sakaguchi", "Ronan Le Bras", "Chandra Bhagavatula", "Yejin Choi" ], "organizations": [ "Tohoku University", "Allen Institute for Artificial Intelligence" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/winogrande/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/allenai/winogrande", "provider": "Allen Institute for AI" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/1907.10641" }, "code_url": null, "metadata": { "source_search": "Keisuke Sakaguchi, Ronan Le Bras, Chandra Bhagavatula et al. Tohoku University; Allen Institute for Artificial Intelligence Original paper authors WinoGrande Keisuke Sakaguchi Ronan Le Bras Chandra Bhagavatula Yejin Choi evaluation-task" } }, { "id": "canonical:wmdp", "name": "WMDP", "slug": "canonical--wmdp", "organisation": "Center for AI Safety and the WMDP paper authors", "source_type": "canonical-source", "source_url": "https://www.wmdp.ai/", "topics": [ "bio-cbrn", "cyber" ], "description": "Measures hazardous knowledge in biosecurity, chemical security and cybersecurity.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-24", "measures": "Recognition and selection of answers to WMDP's hazardous-knowledge questions under the stated prompt and model configuration.", "does_not_measure": "Operational CBRN or cyber capability, experimental competence, tool use, planning, intent, tacit knowledge, end-to-end harm creation, or deployment risk.", "best_for": "Testing hazardous-knowledge retention and evaluating some unlearning methods while monitoring broad capability side effects.", "not_sufficient_for": "Claims that a model can execute a biological, chemical, or cyber operation or that removing benchmark knowledge eliminates misuse risk.", "evidence_reach": [ "artifact-production" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/centerforaisafety/wmdp", "primary": true }, { "kind": "paper", "label": "Paper: The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "url": "https://arxiv.org/abs/2403.03218", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/centerforaisafety/wmdp", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://www.wmdp.ai", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/centerforaisafety/wmdp", "primary": false } ], "resource_count": 5, "source_key": "canonical:wmdp", "record_type": "evaluation-resource", "topic_provenance": "editorial", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Center for AI Safety and the WMDP paper authors", "organization": "Center for AI Safety", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://github.com/centerforaisafety/wmdp", "organizations": [ "Center for AI Safety" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.03218", "arxiv_id": "2403.03218", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "authors": [ "Nathaniel Li", "Alexander Pan", "Anjali Gopal", "Summer Yue", "Daniel Berrios", "Alice Gatti", "Justin D. Li", "Ann-Kathrin Dombrowski", "Shashwat Goel", "Long Phan", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.03218", "arxiv_id": "2403.03218", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "authors": [ "Nathaniel Li", "Alexander Pan", "Anjali Gopal", "Summer Yue", "Daniel Berrios", "Alice Gatti", "Justin D. Li", "Ann-Kathrin Dombrowski", "Shashwat Goel", "Long Phan", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/centerforaisafety/wmdp", "owner": "Center for AI Safety", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/centerforaisafety/wmdp" }, "paper_url": "https://arxiv.org/abs/2403.03218", "code_url": "https://github.com/centerforaisafety/wmdp", "metadata": { "source_search": "Center for AI Safety and the WMDP paper authors Center for AI Safety Evaluation developed by The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning Nathaniel Li Alexander Pan Anjali Gopal Summer Yue Daniel Berrios Alice Gatti Justin D. Li Ann-Kathrin Dombrowski Shashwat Goel Long Phan Gabriel Mukobi Nathan Helm-Burger Rassin Lababidi Lennart Justen Andrew B. Liu Michael Chen Isabelle Barrass Oliver Zhang Xiaoyuan Zhu Rishub Tamirisa Bhrugu Bharathi Adam Khoja Zhenqi Zhao Ariel Herbert-Voss Cort B. Breuer Samuel Marks Oam Patel Andy Zou Mantas Mazeika Zifan Wang Palash Oswal Weiran Lin Adam A. Hunt Justin Tienken-Harder Kevin Y. Shih Kemper Talley John Guan Russell Kaplan Ian Steneker David Campbell Brad Jokubaitis Alex Levinson Jean Wang William Qian Kallol Krishna Karmakar Steven Basart Stephen Fitz Mindy Levine Ponnurangam Kumaraguru Uday Tupakula Vijay Varadharajan Ruoyu Wang Yan Shoshitaishvili Jimmy Ba Kevin M. Esvelt Alexandr Wang Dan Hendrycks Center for AI Safety evaluation-resource" }, "editorial_review": { "reviewer": "FronteraEval", "reviewed_at": "2026-08-24", "scope": "construct, inference boundary, provenance and source attribution" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-24", "review_confidence": "high", "construct": "Multiple-choice knowledge performance on expert-curated questions in biosecurity, chemical security, and cybersecurity domains judged relevant to malicious use.", "unit_of_analysis": "One multiple-choice answer to one hazardous-knowledge question within a WMDP domain.", "interaction_type": "Closed-form question answering without end-to-end tools, experimentation, or operational execution.", "target_outcome": "Answer accuracy by domain and aggregate benchmark performance, often used to evaluate hazardous knowledge retention or unlearning.", "human_participants": "No human subjects. Domain experts contribute to question selection, review, and hazard relevance.", "elicitation": "The model receives multiple-choice questions under a specified prompting and answer-extraction protocol, usually without operational tools.", "scoring": "Exact multiple-choice accuracy is reported separately for WMDP-Bio, WMDP-Chem, and WMDP-Cyber and may be compared before and after unlearning interventions.", "protocol_dependence": "High: question set, domain composition, prompting, answer extraction, and contamination controls define scores.", "implementation_dependence": "Moderate: tokenizer and answer parsing, model formatting, few-shot choices, and benchmark release can affect accuracy.", "limitations": [ "Multiple-choice recognition is much narrower than operational capability.", "Public questions can be memorized or contaminated through training and evaluation reuse.", "Question difficulty and hazard relevance vary within and across domains.", "Unlearning gains on WMDP may reflect benchmark-specific suppression rather than robust removal of dangerous capability." ], "comparability": "Use the same WMDP version and domain, prompt and few-shot condition, answer parser, model snapshot, and unlearning or elicitation setup; report domains separately.", "decision_relevance": "Useful as one hazardous-knowledge and unlearning diagnostic. It should be combined with operational, tool-using, and expert-elicitation evaluations for dangerous-capability decisions.", "source_basis": [ { "kind": "paper", "url": "https://arxiv.org/abs/2403.03218" }, { "kind": "official", "url": "https://www.wmdp.ai/" }, { "kind": "implementation", "url": "https://github.com/centerforaisafety/wmdp" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:wmdp_bio", "name": "WMDP Bio", "slug": "inspect--wmdp-bio", "organisation": "Nathaniel Li, Alexander Pan, Anjali Gopal et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp", "topics": [ "bio-cbrn" ], "description": "A dataset of 3,668 multiple-choice questions developed by a consortium of academics and technical consultants that serve as a proxy measurement of hazardous knowledge in biosecurity, cybersecurity, and chemical security.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.03218", "primary": true }, { "kind": "paper", "label": "Paper: The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "url": "https://arxiv.org/abs/2403.03218", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Center for AI Safety", "url": "https://huggingface.co/datasets/cais/wmdp", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.03218", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml", "family_title": "WMDP: Measuring and Reducing Malicious Use With Unlearning", "group": "Safeguards", "version": "2-A", "contributors": [ "alexandraabbas" ], "tags": [ "Safeguards" ], "task": { "name": "wmdp_bio", "dataset_samples": 1273, "comment": null, "support_only": false }, "tasks": [ { "name": "wmdp_bio", "dataset_samples": 1273, "comment": null, "support_only": false }, { "name": "wmdp_chem", "dataset_samples": 408, "comment": null, "support_only": false }, { "name": "wmdp_cyber", "dataset_samples": 1987, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1273, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "cais/wmdp", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:wmdp", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Nathaniel Li, Alexander Pan, Anjali Gopal et al.", "organization": "Center for AI Safety; CMU; University of Chicago; TTIC; UC Berkeley", "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.03218" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.03218", "arxiv_id": "2403.03218", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "authors": [ "Nathaniel Li", "Alexander Pan", "Anjali Gopal", "Summer Yue", "Daniel Berrios", "Alice Gatti", "Justin D. Li", "Ann-Kathrin Dombrowski", "Shashwat Goel", "Long Phan", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.03218", "arxiv_id": "2403.03218", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "authors": [ "Nathaniel Li", "Alexander Pan", "Anjali Gopal", "Summer Yue", "Daniel Berrios", "Alice Gatti", "Justin D. Li", "Ann-Kathrin Dombrowski", "Shashwat Goel", "Long Phan", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/cais/wmdp", "provider": "Center for AI Safety" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.03218" }, "code_url": null, "metadata": { "source_search": "Nathaniel Li, Alexander Pan, Anjali Gopal et al. Center for AI Safety; CMU; University of Chicago; TTIC; UC Berkeley Original paper authors The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning Nathaniel Li Alexander Pan Anjali Gopal Summer Yue Daniel Berrios Alice Gatti Justin D. Li Ann-Kathrin Dombrowski Shashwat Goel Long Phan Gabriel Mukobi Nathan Helm-Burger Rassin Lababidi Lennart Justen Andrew B. Liu Michael Chen Isabelle Barrass Oliver Zhang Xiaoyuan Zhu Rishub Tamirisa Bhrugu Bharathi Adam Khoja Zhenqi Zhao Ariel Herbert-Voss Cort B. Breuer Samuel Marks Oam Patel Andy Zou Mantas Mazeika Zifan Wang Palash Oswal Weiran Lin Adam A. Hunt Justin Tienken-Harder Kevin Y. Shih Kemper Talley John Guan Russell Kaplan Ian Steneker David Campbell Brad Jokubaitis Alex Levinson Jean Wang William Qian Kallol Krishna Karmakar Steven Basart Stephen Fitz Mindy Levine Ponnurangam Kumaraguru Uday Tupakula Vijay Varadharajan Ruoyu Wang Yan Shoshitaishvili Jimmy Ba Kevin M. Esvelt Alexandr Wang Dan Hendrycks evaluation-family" } }, { "id": "inspect:wmdp_chem", "name": "WMDP Chem", "slug": "inspect--wmdp-chem", "organisation": "Nathaniel Li, Alexander Pan, Anjali Gopal et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp", "topics": [ "bio-cbrn" ], "description": "A dataset of 3,668 multiple-choice questions developed by a consortium of academics and technical consultants that serve as a proxy measurement of hazardous knowledge in biosecurity, cybersecurity, and chemical security.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.03218", "primary": true }, { "kind": "paper", "label": "Paper: The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "url": "https://arxiv.org/abs/2403.03218", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Center for AI Safety", "url": "https://huggingface.co/datasets/cais/wmdp", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.03218", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml", "family_title": "WMDP: Measuring and Reducing Malicious Use With Unlearning", "group": "Safeguards", "version": "2-A", "contributors": [ "alexandraabbas" ], "tags": [ "Safeguards" ], "task": { "name": "wmdp_chem", "dataset_samples": 408, "comment": null, "support_only": false }, "tasks": [ { "name": "wmdp_bio", "dataset_samples": 1273, "comment": null, "support_only": false }, { "name": "wmdp_chem", "dataset_samples": 408, "comment": null, "support_only": false }, { "name": "wmdp_cyber", "dataset_samples": 1987, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 408, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "cais/wmdp", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:wmdp", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Nathaniel Li, Alexander Pan, Anjali Gopal et al.", "organization": "Center for AI Safety; CMU; University of Chicago; TTIC; UC Berkeley", "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.03218" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.03218", "arxiv_id": "2403.03218", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "authors": [ "Nathaniel Li", "Alexander Pan", "Anjali Gopal", "Summer Yue", "Daniel Berrios", "Alice Gatti", "Justin D. Li", "Ann-Kathrin Dombrowski", "Shashwat Goel", "Long Phan", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.03218", "arxiv_id": "2403.03218", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "authors": [ "Nathaniel Li", "Alexander Pan", "Anjali Gopal", "Summer Yue", "Daniel Berrios", "Alice Gatti", "Justin D. Li", "Ann-Kathrin Dombrowski", "Shashwat Goel", "Long Phan", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/cais/wmdp", "provider": "Center for AI Safety" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.03218" }, "code_url": null, "metadata": { "source_search": "Nathaniel Li, Alexander Pan, Anjali Gopal et al. Center for AI Safety; CMU; University of Chicago; TTIC; UC Berkeley Original paper authors The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning Nathaniel Li Alexander Pan Anjali Gopal Summer Yue Daniel Berrios Alice Gatti Justin D. Li Ann-Kathrin Dombrowski Shashwat Goel Long Phan Gabriel Mukobi Nathan Helm-Burger Rassin Lababidi Lennart Justen Andrew B. Liu Michael Chen Isabelle Barrass Oliver Zhang Xiaoyuan Zhu Rishub Tamirisa Bhrugu Bharathi Adam Khoja Zhenqi Zhao Ariel Herbert-Voss Cort B. Breuer Samuel Marks Oam Patel Andy Zou Mantas Mazeika Zifan Wang Palash Oswal Weiran Lin Adam A. Hunt Justin Tienken-Harder Kevin Y. Shih Kemper Talley John Guan Russell Kaplan Ian Steneker David Campbell Brad Jokubaitis Alex Levinson Jean Wang William Qian Kallol Krishna Karmakar Steven Basart Stephen Fitz Mindy Levine Ponnurangam Kumaraguru Uday Tupakula Vijay Varadharajan Ruoyu Wang Yan Shoshitaishvili Jimmy Ba Kevin M. Esvelt Alexandr Wang Dan Hendrycks evaluation-family" } }, { "id": "inspect:wmdp_cyber", "name": "WMDP Cyber", "slug": "inspect--wmdp-cyber", "organisation": "Nathaniel Li, Alexander Pan, Anjali Gopal et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp", "topics": [ "cyber" ], "description": "A dataset of 3,668 multiple-choice questions developed by a consortium of academics and technical consultants that serve as a proxy measurement of hazardous knowledge in biosecurity, cybersecurity, and chemical security.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2403.03218", "primary": true }, { "kind": "paper", "label": "Paper: The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "url": "https://arxiv.org/abs/2403.03218", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Center for AI Safety", "url": "https://huggingface.co/datasets/cais/wmdp", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2403.03218", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml", "family_title": "WMDP: Measuring and Reducing Malicious Use With Unlearning", "group": "Safeguards", "version": "2-A", "contributors": [ "alexandraabbas" ], "tags": [ "Safeguards" ], "task": { "name": "wmdp_cyber", "dataset_samples": 1987, "comment": null, "support_only": false }, "tasks": [ { "name": "wmdp_bio", "dataset_samples": 1273, "comment": null, "support_only": false }, { "name": "wmdp_chem", "dataset_samples": 408, "comment": null, "support_only": false }, { "name": "wmdp_cyber", "dataset_samples": 1987, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 1987, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "cais/wmdp", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:wmdp", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Nathaniel Li, Alexander Pan, Anjali Gopal et al.", "organization": "Center for AI Safety; CMU; University of Chicago; TTIC; UC Berkeley", "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2403.03218" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.03218", "arxiv_id": "2403.03218", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "authors": [ "Nathaniel Li", "Alexander Pan", "Anjali Gopal", "Summer Yue", "Daniel Berrios", "Alice Gatti", "Justin D. Li", "Ann-Kathrin Dombrowski", "Shashwat Goel", "Long Phan", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.03218", "arxiv_id": "2403.03218", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "authors": [ "Nathaniel Li", "Alexander Pan", "Anjali Gopal", "Summer Yue", "Daniel Berrios", "Alice Gatti", "Justin D. Li", "Ann-Kathrin Dombrowski", "Shashwat Goel", "Long Phan", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang", "Dan Hendrycks" ], "organizations": [ "Center for AI Safety", "CMU", "University of Chicago", "TTIC", "UC Berkeley" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/wmdp/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/cais/wmdp", "provider": "Center for AI Safety" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2403.03218" }, "code_url": null, "metadata": { "source_search": "Nathaniel Li, Alexander Pan, Anjali Gopal et al. Center for AI Safety; CMU; University of Chicago; TTIC; UC Berkeley Original paper authors The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning Nathaniel Li Alexander Pan Anjali Gopal Summer Yue Daniel Berrios Alice Gatti Justin D. Li Ann-Kathrin Dombrowski Shashwat Goel Long Phan Gabriel Mukobi Nathan Helm-Burger Rassin Lababidi Lennart Justen Andrew B. Liu Michael Chen Isabelle Barrass Oliver Zhang Xiaoyuan Zhu Rishub Tamirisa Bhrugu Bharathi Adam Khoja Zhenqi Zhao Ariel Herbert-Voss Cort B. Breuer Samuel Marks Oam Patel Andy Zou Mantas Mazeika Zifan Wang Palash Oswal Weiran Lin Adam A. Hunt Justin Tienken-Harder Kevin Y. Shih Kemper Talley John Guan Russell Kaplan Ian Steneker David Campbell Brad Jokubaitis Alex Levinson Jean Wang William Qian Kallol Krishna Karmakar Steven Basart Stephen Fitz Mindy Levine Ponnurangam Kumaraguru Uday Tupakula Vijay Varadharajan Ruoyu Wang Yan Shoshitaishvili Jimmy Ba Kevin M. Esvelt Alexandr Wang Dan Hendrycks evaluation-family" } }, { "id": "canonical:workarena", "name": "WorkArena", "slug": "canonical--workarena", "organisation": "ServiceNow Research and the WorkArena paper authors", "source_type": "canonical-source", "source_url": "https://github.com/ServiceNow/WorkArena", "topics": [ "autonomy-agents" ], "description": "Evaluates web agents on enterprise knowledge-work tasks.", "review_status": "reviewed", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": "2026-08-27", "measures": "Performance on WorkArena's selected ServiceNow task distribution under the specified instance, agent interface, and budget.", "does_not_measure": "General enterprise productivity, organizational integration, safe access control, user acceptance, or performance across other business software.", "best_for": "Testing browser agents on reproducible enterprise workflows and compositional planning tasks.", "not_sufficient_for": "Procurement claims that an agent can safely automate enterprise work without organization-specific pilots and controls.", "evidence_reach": [ "controlled-model-behaviour" ], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://github.com/ServiceNow/WorkArena", "primary": true }, { "kind": "paper", "label": "Paper: WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?", "url": "https://arxiv.org/abs/2403.07718", "primary": false }, { "kind": "paper", "label": "Paper: WorkArena++: Towards Compositional Planning and Reasoning-based Common Knowledge Work Tasks", "url": "https://arxiv.org/abs/2407.05291", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/ServiceNow/WorkArena", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/ServiceNow/WorkArena", "primary": false } ], "resource_count": 5, "source_key": "canonical:workarena", "record_type": "evaluation-environment", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "ServiceNow Research and the WorkArena paper authors", "organization": "ServiceNow Research", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://github.com/ServiceNow/WorkArena", "organizations": [ "ServiceNow Research" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2403.07718", "arxiv_id": "2403.07718", "title": "WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?", "authors": [ "Alexandre Drouin", "Maxime Gasse", "Massimo Caccia", "Issam H. Laradji", "Manuel Del Verme", "Tom Marty", "Léo Boisvert", "Megh Thakkar", "Quentin Cappart", "David Vazquez", "Nicolas Chapados", "Alexandre Lacoste" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2403.07718", "arxiv_id": "2403.07718", "title": "WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?", "authors": [ "Alexandre Drouin", "Maxime Gasse", "Massimo Caccia", "Issam H. Laradji", "Manuel Del Verme", "Tom Marty", "Léo Boisvert", "Megh Thakkar", "Quentin Cappart", "David Vazquez", "Nicolas Chapados", "Alexandre Lacoste" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2407.05291", "arxiv_id": "2407.05291", "title": "WorkArena++: Towards Compositional Planning and Reasoning-based Common Knowledge Work Tasks", "authors": [ "Léo Boisvert", "Megh Thakkar", "Maxime Gasse", "Massimo Caccia", "Thibault Le Sellier De Chezelles", "Quentin Cappart", "Nicolas Chapados", "Alexandre Lacoste", "Alexandre Drouin" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/ServiceNow/WorkArena", "owner": "ServiceNow Research", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://github.com/ServiceNow/WorkArena" }, "paper_url": "https://arxiv.org/abs/2403.07718", "code_url": "https://github.com/ServiceNow/WorkArena", "metadata": { "source_search": "ServiceNow Research and the WorkArena paper authors ServiceNow Research Evaluation developed by WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks? Alexandre Drouin Maxime Gasse Massimo Caccia Issam H. Laradji Manuel Del Verme Tom Marty Léo Boisvert Megh Thakkar Quentin Cappart David Vazquez Nicolas Chapados Alexandre Lacoste ServiceNow Research evaluation-environment" }, "methodological_review": { "status": "reviewed", "review_type": "documentary", "replication_status": "not-replicated", "reviewed_at": "2026-08-27", "review_confidence": "moderate", "construct": "Completion of common enterprise knowledge-work tasks in a ServiceNow environment.", "unit_of_analysis": "One browser-agent trajectory on a generated or compositional ServiceNow task instance.", "interaction_type": "Browser-based interaction with enterprise forms, lists, knowledge bases, catalogs, dashboards, and workflows.", "target_outcome": "Task-specific validation of the resulting ServiceNow state.", "human_participants": "No human subjects. Human or oracle solutions and performance are used for feasibility and reference.", "elicitation": "The agent receives a natural-language workplace task and operates a configured ServiceNow instance under a browser-agent harness and step limit.", "scoring": "Hand-authored validation functions or task-specific checks against the enterprise application state.", "protocol_dependence": "High: WorkArena version and level, instance data, task distribution, step limit, observation/action mode, and scaffold matter.", "implementation_dependence": "High: ServiceNow version, BrowserGym integration, task validators, and environment state affect outcomes.", "limitations": [ "The environment is limited to ServiceNow and does not represent all enterprise software.", "Generated tasks may not capture ambiguity, exceptions, and social context in real work.", "Task completion does not establish correct authorization or safe handling of sensitive data.", "Atomic and compositional variants should be reported separately." ], "comparability": "Match WorkArena or WorkArena++ release, task level, instance snapshot, validator version, browser interface, step limit, and scaffold.", "decision_relevance": "Useful for enterprise-agent pilots and procurement test design; must be supplemented by organization-specific safety, permissions, and workflow evaluations.", "source_basis": [ { "kind": "paper", "url": "https://proceedings.mlr.press/v235/drouin24a.html" }, { "kind": "paper", "url": "https://arxiv.org/abs/2407.05291" }, { "kind": "implementation", "url": "https://github.com/ServiceNow/WorkArena" } ], "review_scope": "Independent reading of public papers, official project pages, implementations, and versioned evaluation metadata.", "review_limit": "A documentary review can clarify constructs, protocols, scoring, and inference ceilings; it does not reproduce results or establish deployment effects." } }, { "id": "inspect:worldsense", "name": "WorldSense: Grounded Reasoning Benchmark", "slug": "inspect--worldsense", "organisation": "Youssef Benchekroun, Megi Dervishi, Mark Ibrahim et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/worldsense", "topics": [ "general-capability" ], "description": "Measures grounded reasoning over synthetic world descriptions while controlling for dataset bias. Includes three problem types (Infer, Compl, Consist) and two grades (trivial, normal).", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/worldsense/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2311.15930", "primary": true }, { "kind": "paper", "label": "Paper: WorldSense: A Synthetic Benchmark for Grounded Reasoning in Large Language Models", "url": "https://arxiv.org/abs/2311.15930", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/facebookresearch/worldsense", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/worldsense/eval.yaml", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2311.15930", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/worldsense/eval.yaml", "family_title": "WorldSense: Grounded Reasoning Benchmark", "group": "Reasoning", "version": "2-A", "contributors": [ "mjbroerman" ], "tags": [ "Reasoning" ], "task": { "name": "worldsense", "dataset_samples": 87048, "comment": null, "support_only": false }, "tasks": [ { "name": "worldsense", "dataset_samples": 87048, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 87048, "dependency": "worldsense", "requires_internet": true, "sandbox": [] }, "external_assets": [ { "type": "direct_url", "source": "https://github.com/facebookresearch/worldsense/raw/{SHA}/data/", "fetch_method": "download_and_verify", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:worldsense", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Youssef Benchekroun, Megi Dervishi, Mark Ibrahim et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2311.15930" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2311.15930", "arxiv_id": "2311.15930", "title": "WorldSense: A Synthetic Benchmark for Grounded Reasoning in Large Language Models", "authors": [ "Youssef Benchekroun", "Megi Dervishi", "Mark Ibrahim", "Jean-Baptiste Gaya", "Xavier Martinet", "Grégoire Mialon", "Thomas Scialom", "Emmanuel Dupoux", "Dieuwke Hupkes", "Pascal Vincent" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2311.15930", "arxiv_id": "2311.15930", "title": "WorldSense: A Synthetic Benchmark for Grounded Reasoning in Large Language Models", "authors": [ "Youssef Benchekroun", "Megi Dervishi", "Mark Ibrahim", "Jean-Baptiste Gaya", "Xavier Martinet", "Grégoire Mialon", "Thomas Scialom", "Emmanuel Dupoux", "Dieuwke Hupkes", "Pascal Vincent" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2311.15930", "arxiv_id": "2311.15930", "title": "WorldSense: A Synthetic Benchmark for Grounded Reasoning in Large Language Models", "authors": [ "Youssef Benchekroun", "Megi Dervishi", "Mark Ibrahim", "Jean-Baptiste Gaya", "Xavier Martinet", "Grégoire Mialon", "Thomas Scialom", "Emmanuel Dupoux", "Dieuwke Hupkes", "Pascal Vincent" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/facebookresearch/worldsense", "owner": "Meta AI", "relation": "reference_or_upstream_implementation", "confidence": "source-derived" }, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/worldsense/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2311.15930" }, "code_url": "https://github.com/facebookresearch/worldsense", "metadata": { "source_search": "Youssef Benchekroun, Megi Dervishi, Mark Ibrahim et al. Original paper authors WorldSense: A Synthetic Benchmark for Grounded Reasoning in Large Language Models Youssef Benchekroun Megi Dervishi Mark Ibrahim Jean-Baptiste Gaya Xavier Martinet Grégoire Mialon Thomas Scialom Emmanuel Dupoux Dieuwke Hupkes Pascal Vincent Meta AI evaluation-task" } }, { "id": "inspect:writingbench", "name": "WritingBench: A Comprehensive Benchmark for Generative Writing", "slug": "inspect--writingbench", "organisation": "Yuning Wu, Jiahao Mei, Ming Yan et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/writingbench", "topics": [ "general-capability" ], "description": "A comprehensive evaluation benchmark designed to assess large language models' capabilities across diverse writing tasks. The benchmark evaluates models on various writing domains including academic papers, business documents, creative writing, and technical documentation, with multi-dimensional scoring based on domain-specific criteria.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/writingbench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2503.05244", "primary": true }, { "kind": "paper", "label": "Paper: WritingBench: A Comprehensive Benchmark for Generative Writing", "url": "https://arxiv.org/abs/2503.05244", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/writingbench/eval.yaml", "primary": false } ], "resource_count": 3, "paper_url": "https://arxiv.org/abs/2503.05244", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/writingbench/eval.yaml", "family_title": "WritingBench: A Comprehensive Benchmark for Generative Writing", "group": "Writing", "version": "3-A", "contributors": [ "jtv199" ], "tags": [ "Writing" ], "task": { "name": "writingbench", "dataset_samples": 1000, "comment": null, "support_only": false }, "tasks": [ { "name": "writingbench", "dataset_samples": 1000, "comment": null, "support_only": false } ], "protocol": { "version": "3-A", "dataset_samples": 1000, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:writingbench", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Yuning Wu, Jiahao Mei, Ming Yan et al.", "organization": null, "organizations": [], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2503.05244" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2503.05244", "arxiv_id": "2503.05244", "title": "WritingBench: A Comprehensive Benchmark for Generative Writing", "authors": [ "Yuning Wu", "Jiahao Mei", "Ming Yan", "Chenliang Li", "Shaopeng Lai", "Yuran Ren", "Zijia Wang", "Ji Zhang", "Mengyue Wu", "Qin Jin", "Fei Huang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2503.05244", "arxiv_id": "2503.05244", "title": "WritingBench: A Comprehensive Benchmark for Generative Writing", "authors": [ "Yuning Wu", "Jiahao Mei", "Ming Yan", "Chenliang Li", "Shaopeng Lai", "Yuran Ren", "Zijia Wang", "Ji Zhang", "Mengyue Wu", "Qin Jin", "Fei Huang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, { "status": "present", "url": "https://arxiv.org/abs/2503.05244", "arxiv_id": "2503.05244", "title": "WritingBench: A Comprehensive Benchmark for Generative Writing", "authors": [ "Yuning Wu", "Jiahao Mei", "Ming Yan", "Chenliang Li", "Shaopeng Lai", "Yuran Ren", "Zijia Wang", "Ji Zhang", "Mengyue Wu", "Qin Jin", "Fei Huang" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/writingbench/eval.yaml" }, "datasets": [], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2503.05244" }, "code_url": null, "metadata": { "source_search": "Yuning Wu, Jiahao Mei, Ming Yan et al. Original paper authors WritingBench: A Comprehensive Benchmark for Generative Writing Yuning Wu Jiahao Mei Ming Yan Chenliang Li Shaopeng Lai Yuran Ren Zijia Wang Ji Zhang Mengyue Wu Qin Jin Fei Huang evaluation-task" } }, { "id": "inspect:xstest", "name": "XSTest: A benchmark for identifying exaggerated safety behaviours in LLM's", "slug": "inspect--xstest", "organisation": "Paul Röttger, Hannah Rose Kirk, Bertie Vidgen et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/xstest", "topics": [ "safeguards" ], "description": "Dataset with 250 safe prompts across ten prompt types that well-calibrated models should not refuse, and 200 unsafe prompts as contrasts that models, for most applications, should refuse.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/xstest/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2308.01263", "primary": true }, { "kind": "paper", "label": "Paper: XSTest: A Test Suite for Identifying Exaggerated Safety Behaviours in Large Language Models", "url": "https://arxiv.org/abs/2308.01263", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/xstest/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Walledai", "url": "https://huggingface.co/datasets/walledai/XSTest", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2308.01263", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/xstest/eval.yaml", "family_title": "XSTest: A benchmark for identifying exaggerated safety behaviours in LLM's", "group": "Knowledge", "version": "4-A", "contributors": [ "NelsonG-C" ], "tags": [ "Knowledge" ], "task": { "name": "xstest", "dataset_samples": 250, "comment": null, "support_only": false }, "tasks": [ { "name": "xstest", "dataset_samples": 250, "comment": null, "support_only": false } ], "protocol": { "version": "4-A", "dataset_samples": 250, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "walledai/XSTest", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:xstest", "record_type": "evaluation-task", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Paul Röttger, Hannah Rose Kirk, Bertie Vidgen et al.", "organization": "Bocconi University; University of Oxford; The Alan Turing Institute; Politecnico di Torino; Stanford University", "organizations": [ "Bocconi University", "University of Oxford", "The Alan Turing Institute", "Politecnico di Torino", "Stanford University" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2308.01263" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2308.01263", "arxiv_id": "2308.01263", "title": "XSTest: A Test Suite for Identifying Exaggerated Safety Behaviours in Large Language Models", "authors": [ "Paul Röttger", "Hannah Rose Kirk", "Bertie Vidgen", "Giuseppe Attanasio", "Federico Bianchi", "Dirk Hovy" ], "organizations": [ "Bocconi University", "University of Oxford", "The Alan Turing Institute", "Politecnico di Torino", "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2308.01263", "arxiv_id": "2308.01263", "title": "XSTest: A Test Suite for Identifying Exaggerated Safety Behaviours in Large Language Models", "authors": [ "Paul Röttger", "Hannah Rose Kirk", "Bertie Vidgen", "Giuseppe Attanasio", "Federico Bianchi", "Dirk Hovy" ], "organizations": [ "Bocconi University", "University of Oxford", "The Alan Turing Institute", "Politecnico di Torino", "Stanford University" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/xstest/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/walledai/XSTest", "provider": "Walledai" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2308.01263" }, "code_url": null, "metadata": { "source_search": "Paul Röttger, Hannah Rose Kirk, Bertie Vidgen et al. Bocconi University; University of Oxford; The Alan Turing Institute; Politecnico di Torino; Stanford University Original paper authors XSTest: A Test Suite for Identifying Exaggerated Safety Behaviours in Large Language Models Paul Röttger Hannah Rose Kirk Bertie Vidgen Giuseppe Attanasio Federico Bianchi Dirk Hovy evaluation-task" } }, { "id": "inspect:zerobench", "name": "Zerobench", "slug": "inspect--zerobench", "organisation": "Jonathan Roberts, Mohammad Reza Taesiri, Ansh Sharma et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench", "topics": [ "evaluation-integrity" ], "description": "A lightweight visual reasoning benchmark that is (1) Challenging, (2) Lightweight, (3) Diverse, and (4) High-quality.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2502.09696", "primary": true }, { "kind": "paper", "label": "Paper: ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models", "url": "https://arxiv.org/abs/2502.09696", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Jonathan Roberts1", "url": "https://huggingface.co/datasets/jonathan-roberts1/zerobench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2502.09696", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench/eval.yaml", "family_title": "ZeroBench", "group": "Multimodal", "version": "2-A", "contributors": [ "ItsTania" ], "tags": [ "Multimodal" ], "task": { "name": "zerobench", "dataset_samples": 100, "comment": null, "support_only": false }, "tasks": [ { "name": "zerobench", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "zerobench_subquestions", "dataset_samples": 335, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 100, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "jonathan-roberts1/zerobench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:zerobench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jonathan Roberts, Mohammad Reza Taesiri, Ansh Sharma et al.", "organization": "University of Alberta; University of Oxford; The University of Hong Kong; University of Cambridge", "organizations": [ "University of Alberta", "University of Oxford", "The University of Hong Kong", "University of Cambridge" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2502.09696" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2502.09696", "arxiv_id": "2502.09696", "title": "ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models", "authors": [ "Jonathan Roberts", "Mohammad Reza Taesiri", "Ansh Sharma", "Akash Gupta", "Samuel Roberts", "Ioana Croitoru", "Simion-Vlad Bogolin", "Jialu Tang", "Florian Langer", "Vyas Raina", "Vatsal Raina", "Hanyi Xiong", "Vishaal Udandarao", "Jingyi Lu", "Shiyang Chen", "Sam Purkis", "Tianshuo Yan", "Wenye Lin", "Gyungin Shin", "Qiaochu Yang", "Anh Totti Nguyen", "David I. Atkinson", "Aaditya Baranwal", "Alexandru Coca", "Mikah Dang", "Sebastian Dziadzio", "Jakob D. Kunz", "Kaiqu Liang", "Alexander Lo", "Brian Pulfer", "Steven Walton", "Charig Yang", "Kai Han", "Samuel Albanie" ], "organizations": [ "University of Alberta", "University of Oxford", "The University of Hong Kong", "University of Cambridge" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2502.09696", "arxiv_id": "2502.09696", "title": "ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models", "authors": [ "Jonathan Roberts", "Mohammad Reza Taesiri", "Ansh Sharma", "Akash Gupta", "Samuel Roberts", "Ioana Croitoru", "Simion-Vlad Bogolin", "Jialu Tang", "Florian Langer", "Vyas Raina", "Vatsal Raina", "Hanyi Xiong", "Vishaal Udandarao", "Jingyi Lu", "Shiyang Chen", "Sam Purkis", "Tianshuo Yan", "Wenye Lin", "Gyungin Shin", "Qiaochu Yang", "Anh Totti Nguyen", "David I. Atkinson", "Aaditya Baranwal", "Alexandru Coca", "Mikah Dang", "Sebastian Dziadzio", "Jakob D. Kunz", "Kaiqu Liang", "Alexander Lo", "Brian Pulfer", "Steven Walton", "Charig Yang", "Kai Han", "Samuel Albanie" ], "organizations": [ "University of Alberta", "University of Oxford", "The University of Hong Kong", "University of Cambridge" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/jonathan-roberts1/zerobench", "provider": "Jonathan Roberts1" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2502.09696" }, "code_url": null, "metadata": { "source_search": "Jonathan Roberts, Mohammad Reza Taesiri, Ansh Sharma et al. University of Alberta; University of Oxford; The University of Hong Kong; University of Cambridge Original paper authors ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models Jonathan Roberts Mohammad Reza Taesiri Ansh Sharma Akash Gupta Samuel Roberts Ioana Croitoru Simion-Vlad Bogolin Jialu Tang Florian Langer Vyas Raina Vatsal Raina Hanyi Xiong Vishaal Udandarao Jingyi Lu Shiyang Chen Sam Purkis Tianshuo Yan Wenye Lin Gyungin Shin Qiaochu Yang Anh Totti Nguyen David I. Atkinson Aaditya Baranwal Alexandru Coca Mikah Dang Sebastian Dziadzio Jakob D. Kunz Kaiqu Liang Alexander Lo Brian Pulfer Steven Walton Charig Yang Kai Han Samuel Albanie evaluation-family" } }, { "id": "inspect:zerobench_subquestions", "name": "Zerobench Subquestions", "slug": "inspect--zerobench-subquestions", "organisation": "Jonathan Roberts, Mohammad Reza Taesiri, Ansh Sharma et al.", "source_type": "inspect-internal", "source_url": "https://github.com/UKGovernmentBEIS/inspect_evals/tree/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench", "topics": [ "evaluation-integrity" ], "description": "A lightweight visual reasoning benchmark that is (1) Challenging, (2) Lightweight, (3) Diverse, and (4) High-quality.", "review_status": "imported", "code_available": true, "inspect_compatible": true, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": "979bf44681dfdb148021676f9b2be964150bb5ed", "method": "official-source import", "internal_metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench/eval.yaml", "enrichment": "Inspect internal evaluation metadata" }, "links": [ { "kind": "paper_authors", "label": "Original paper authors", "url": "https://arxiv.org/abs/2502.09696", "primary": true }, { "kind": "paper", "label": "Paper: ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models", "url": "https://arxiv.org/abs/2502.09696", "primary": false }, { "kind": "inspect-implementation", "label": "Inspect implementation", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench/eval.yaml", "primary": false }, { "kind": "dataset", "label": "Dataset: Jonathan Roberts1", "url": "https://huggingface.co/datasets/jonathan-roberts1/zerobench", "primary": false } ], "resource_count": 4, "paper_url": "https://arxiv.org/abs/2502.09696", "metadata_url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench/eval.yaml", "family_title": "ZeroBench", "group": "Multimodal", "version": "2-A", "contributors": [ "ItsTania" ], "tags": [ "Multimodal" ], "task": { "name": "zerobench_subquestions", "dataset_samples": 335, "comment": null, "support_only": false }, "tasks": [ { "name": "zerobench", "dataset_samples": 100, "comment": null, "support_only": false }, { "name": "zerobench_subquestions", "dataset_samples": 335, "comment": null, "support_only": false } ], "protocol": { "version": "2-A", "dataset_samples": 335, "dependency": null, "requires_internet": false, "sandbox": [] }, "external_assets": [ { "type": "huggingface", "source": "jonathan-roberts1/zerobench", "fetch_method": "hf_dataset", "state": "pinned", "comment": null } ], "metadata_enriched": true, "metadata_source": "Inspect internal eval.yaml", "metadata_enriched_at": "2026-08-30", "source_key": "inspect-family:zerobench", "record_type": "evaluation-family", "topic_provenance": "heuristic", "source_resolution": { "schema_version": "2.1.0", "status": "paper-only", "method": "associated-paper-metadata", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Jonathan Roberts, Mohammad Reza Taesiri, Ansh Sharma et al.", "organization": "University of Alberta; University of Oxford; The University of Hong Kong; University of Cambridge", "organizations": [ "University of Alberta", "University of Oxford", "The University of Hong Kong", "University of Cambridge" ], "entity_type": "authors", "role": "paper_authors", "role_label": "Original paper authors", "confidence": "paper-metadata", "evidence_url": "https://arxiv.org/abs/2502.09696" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2502.09696", "arxiv_id": "2502.09696", "title": "ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models", "authors": [ "Jonathan Roberts", "Mohammad Reza Taesiri", "Ansh Sharma", "Akash Gupta", "Samuel Roberts", "Ioana Croitoru", "Simion-Vlad Bogolin", "Jialu Tang", "Florian Langer", "Vyas Raina", "Vatsal Raina", "Hanyi Xiong", "Vishaal Udandarao", "Jingyi Lu", "Shiyang Chen", "Sam Purkis", "Tianshuo Yan", "Wenye Lin", "Gyungin Shin", "Qiaochu Yang", "Anh Totti Nguyen", "David I. Atkinson", "Aaditya Baranwal", "Alexandru Coca", "Mikah Dang", "Sebastian Dziadzio", "Jakob D. Kunz", "Kaiqu Liang", "Alexander Lo", "Brian Pulfer", "Steven Walton", "Charig Yang", "Kai Han", "Samuel Albanie" ], "organizations": [ "University of Alberta", "University of Oxford", "The University of Hong Kong", "University of Cambridge" ], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2502.09696", "arxiv_id": "2502.09696", "title": "ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models", "authors": [ "Jonathan Roberts", "Mohammad Reza Taesiri", "Ansh Sharma", "Akash Gupta", "Samuel Roberts", "Ioana Croitoru", "Simion-Vlad Bogolin", "Jialu Tang", "Florian Langer", "Vyas Raina", "Vatsal Raina", "Hanyi Xiong", "Vishaal Udandarao", "Jingyi Lu", "Shiyang Chen", "Sam Purkis", "Tianshuo Yan", "Wenye Lin", "Gyungin Shin", "Qiaochu Yang", "Anh Totti Nguyen", "David I. Atkinson", "Aaditya Baranwal", "Alexandru Coca", "Mikah Dang", "Sebastian Dziadzio", "Jakob D. Kunz", "Kaiqu Liang", "Alexander Lo", "Brian Pulfer", "Steven Walton", "Charig Yang", "Kai Han", "Samuel Albanie" ], "organizations": [ "University of Alberta", "University of Oxford", "The University of Hong Kong", "University of Cambridge" ], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": null, "inspect_provenance": { "role": "maintained_implementation", "role_label": "Inspect implementation", "steward": "UK AI Security Institute", "url": "https://github.com/UKGovernmentBEIS/inspect_evals/blob/979bf44681dfdb148021676f9b2be964150bb5ed/src/inspect_evals/zerobench/eval.yaml" }, "datasets": [ { "url": "https://huggingface.co/datasets/jonathan-roberts1/zerobench", "provider": "Jonathan Roberts1" } ], "organisation_role": "Original paper authors", "attribution_status": "paper-only", "preferred_source": { "label": "Original paper authors", "kind": "paper_authors", "url": "https://arxiv.org/abs/2502.09696" }, "code_url": null, "metadata": { "source_search": "Jonathan Roberts, Mohammad Reza Taesiri, Ansh Sharma et al. University of Alberta; University of Oxford; The University of Hong Kong; University of Cambridge Original paper authors ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models Jonathan Roberts Mohammad Reza Taesiri Ansh Sharma Akash Gupta Samuel Roberts Ioana Croitoru Simion-Vlad Bogolin Jialu Tang Florian Langer Vyas Raina Vatsal Raina Hanyi Xiong Vishaal Udandarao Jingyi Lu Shiyang Chen Sam Purkis Tianshuo Yan Wenye Lin Gyungin Shin Qiaochu Yang Anh Totti Nguyen David I. Atkinson Aaditya Baranwal Alexandru Coca Mikah Dang Sebastian Dziadzio Jakob D. Kunz Kaiqu Liang Alexander Lo Brian Pulfer Steven Walton Charig Yang Kai Han Samuel Albanie evaluation-family" } }, { "id": "canonical:tau-bench", "name": "τ-bench", "slug": "canonical--tau-bench", "organisation": "Sierra Research and the τ-bench paper authors", "source_type": "canonical-source", "source_url": "https://github.com/sierra-research/tau-bench", "topics": [ "autonomy-agents" ], "description": "Evaluates tool-using agents in realistic user-facing service domains.", "review_status": "catalogued", "code_available": false, "inspect_compatible": false, "last_source_check": "2026-08-30", "editorial_reviewed_at": null, "measures": "Not independently assessed by FronteraEval yet.", "does_not_measure": "No inference beyond the upstream source should be made until the protocol is reviewed.", "best_for": "Discovery and source navigation.", "not_sufficient_for": "Substantive capability, safety or policy claims without reading the underlying protocol.", "evidence_reach": [], "provenance": { "source_sha": null, "method": "official-source import" }, "links": [ { "kind": "evaluation_developer", "label": "Evaluation developed by", "url": "https://sierra.ai/resources/research/tau-bench", "primary": true }, { "kind": "paper", "label": "Paper: τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "url": "https://arxiv.org/abs/2406.12045", "primary": false }, { "kind": "implementation", "label": "Reference implementation", "url": "https://github.com/sierra-research/tau-bench", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://github.com/sierra-research/tau-bench", "primary": false }, { "kind": "official", "label": "Official or project page", "url": "https://sierra.ai/resources/research/tau-bench", "primary": false } ], "resource_count": 5, "source_key": "canonical:tau-bench", "record_type": "evaluation-resource", "topic_provenance": "curated", "source_resolution": { "schema_version": "2.1.0", "status": "verified", "method": "curated-primary-sources", "resolved_at": "2026-08-30T08:24:14.203Z", "note": null }, "origin": { "name": "Sierra Research and the τ-bench paper authors", "organization": "Sierra Research", "entity_type": "research_collaboration", "role": "evaluation_developer", "confidence": "verified", "evidence_url": "https://sierra.ai/resources/research/tau-bench", "organizations": [ "Sierra Research" ], "role_label": "Evaluation developed by" }, "paper": { "status": "present", "url": "https://arxiv.org/abs/2406.12045", "arxiv_id": "2406.12045", "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "authors": [ "Shunyu Yao", "Noah Shinn", "Pedram Razavi", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" }, "papers": [ { "status": "present", "url": "https://arxiv.org/abs/2406.12045", "arxiv_id": "2406.12045", "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "authors": [ "Shunyu Yao", "Noah Shinn", "Pedram Razavi", "Karthik Narasimhan" ], "organizations": [], "source": "semantic-scholar", "metadata_status": "resolved" } ], "paper_status": "present", "reference_implementation": { "url": "https://github.com/sierra-research/tau-bench", "owner": "Sierra Research", "relation": "reference_implementation", "confidence": "verified" }, "inspect_provenance": null, "datasets": [], "organisation_role": "Evaluation developed by", "attribution_status": "verified", "preferred_source": { "label": "Evaluation developed by", "kind": "evaluation_developer", "url": "https://sierra.ai/resources/research/tau-bench" }, "paper_url": "https://arxiv.org/abs/2406.12045", "code_url": "https://github.com/sierra-research/tau-bench", "metadata": { "source_search": "Sierra Research and the τ-bench paper authors Sierra Research Evaluation developed by τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains Shunyu Yao Noah Shinn Pedram Razavi Karthik Narasimhan Sierra Research evaluation-resource" } } ], "inspect_source_commit": "979bf44681dfdb148021676f9b2be964150bb5ed", "enriched_at": "2026-08-30T08:25:19.993Z", "source_registry_version": "2.1.0", "source_registry_generated_at": "2026-08-30T08:24:14.203Z" }