Files
ai-taxonomy-literature-anal…/data/labels/o18.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
31 KiB
JSON

{"id": 3803, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Clusters reasoning prefixes to expand only promising paths, cutting self-consistency cost"}], "pattern": "vary-scaffold", "fixes": ["model", "benchmarks"], "varies": ["inference strategy"], "benchmarks": [{"name": "GSM8K", "purpose": "accuracy vs SC", "is_new": false}, {"name": "Math500", "purpose": "accuracy vs SC", "is_new": false}, {"name": "AIME 2024/2025", "purpose": "accuracy vs SC", "is_new": false}, {"name": "GPQA-Diamond", "purpose": "accuracy vs SC", "is_new": false}], "results": [], "workflow": ["sample short prefixes", "cluster prefixes", "expand dominant cluster"], "summary": "Inference-time prefix-consensus method matching self-consistency accuracy with far fewer tokens and lower latency."}
{"id": 12966, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Multi-agent generator and feature evaluators produce difficulty-controlled reading comprehension items"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["single vs multi-agent generation"], "benchmarks": [], "results": [], "workflow": ["generate item", "evaluate features", "revise iteratively"], "summary": "Multi-agent framework for generating reading comprehension items satisfying feature constraints for controlled difficulty."}
{"id": 12706, "in_agent_scope": false, "types": [{"area": "train/data", "role": "method", "why": "Task-profile-guided synthetic data to train cold-start LLM router"}], "pattern": "vary-data", "fixes": ["candidate LLMs"], "varies": ["router training data", "router design"], "benchmarks": [], "results": [], "workflow": ["build task taxonomy", "synthesize QA pairs", "train TRouter"], "summary": "Task-aware LLM router trained on taxonomy-guided synthetic data for cold-start routing."}
{"id": 2166, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "scaffold", "why": "Speculative actions predict likely next API calls with faster models to run in parallel"}], "pattern": "vary-scaffold", "fixes": ["agents", "environments"], "varies": ["execution strategy"], "benchmarks": [], "results": [], "workflow": ["predict next actions", "execute in parallel", "verify and commit"], "summary": "Lossless speculative-execution framework speeding up agents across gaming, e-commerce, web search and OS environments."}
{"id": 12998, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "scaffold", "why": "Dual-level memory with evolution lets mobile GUI agents adapt to app updates"}, {"area": "memory-selfevolve", "role": "scaffold", "why": "Dynamic memory evolution refines stationary and procedural memory"}], "pattern": "vary-scaffold", "fixes": ["foundation model", "AndroidWorld"], "varies": ["memory design"], "benchmarks": [{"name": "AndroidWorld", "purpose": "online evaluation of adaptation", "is_new": false}], "results": [], "workflow": ["store functional semantics", "store procedural intents", "evolve memory"], "summary": "Memory-driven mobile GUI agent adapting to UI changes via stable functional semantics and evolving memory."}
{"id": 11427, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "environment", "why": "LLM clones real websites into executable verifiable synthetic environments"}, {"area": "web-gui", "role": "train-agent", "why": "Agents self-generate tasks and self-evolve in recreated sites"}], "pattern": "fix-agent-vary-env", "fixes": ["training recipe"], "varies": ["training environments"], "benchmarks": [], "results": [], "workflow": ["clone website", "expose Python SDK", "generate verifiable tasks", "train agent"], "summary": "VeriEnv recreates websites as safe, verifiable environments so web agents can self-evolve and generalize."}
{"id": 294, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Graph-based selection and message passing among multiple LLMs"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["multi-agent communication topology"], "benchmarks": [], "results": [], "workflow": ["select agents via model cards", "build edges", "message passing"], "summary": "Graph-of-Agents orchestrates multiple LLMs via node sampling and directed message passing, improving over Mixture-of-Agents."}
{"id": 17777, "in_agent_scope": false, "types": [{"area": "apps/domain", "role": "application", "why": "Pipeline extracting, linking and matching entries across Swedish encyclopedia editions"}], "pattern": "n/a", "fixes": [], "varies": ["pipeline"], "benchmarks": [], "results": [], "workflow": ["extract headwords", "categorize entities", "match editions", "link Wikidata"], "summary": "Pipeline restoring structure of Swedish encyclopedias and linking entries across editions to Wikidata."}
{"id": 2481, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Training-free iterative VLM reasoning-based frame selection for VideoQA"}], "pattern": "vary-scaffold", "fixes": ["VLM"], "varies": ["frame selection"], "benchmarks": [], "results": [], "workflow": ["analyze query with VLM", "select frames iteratively"], "summary": "Training-free adaptive reasoning-based frame selection for video question answering."}
{"id": 5520, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Steerable deep research deciding when to pause for user input"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["interaction control"], "benchmarks": [], "results": [], "workflow": ["plan diversely", "cost-benefit pause decision", "update persona model"], "summary": "SteER enables mid-process user steering in deep research via cost-benefit decisions and live persona modeling."}
{"id": 9961, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Geometry-guided generative graph representation for functional brain connectomes"}], "pattern": "n/a", "fixes": [], "varies": ["representation method"], "benchmarks": [], "results": [], "workflow": ["learn latent geometry", "encode graph"], "summary": "Unsupervised graph-level representation learning for functional brain graphs using latent geometry."}
{"id": 2400, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "environment", "why": "User-assistant simulation with persona user agent rating suggestions"}, {"area": "domain-agents", "role": "benchmark", "why": "New task for proactive personalized assistants"}, {"area": "memory-selfevolve", "role": "train-agent", "why": "ProPerAssistant learns from ratings with retrieval and preference alignment"}], "pattern": "vary-both", "fixes": ["LLM"], "varies": ["simulation environment", "assistant learning"], "benchmarks": [{"name": "ProPerSim", "purpose": "proactive personalized assistant task", "is_new": true}], "results": [], "workflow": ["simulate user", "assistant suggests", "learn from ratings"], "summary": "ProPerSim simulation framework and ProPerAssistant for proactive, personalized assistants learning from user ratings."}
{"id": 8149, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "scaffold", "why": "Distills ReAct GUI agent logic into reusable RPA functions via code synthesis"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["automation method"], "benchmarks": [], "results": [], "workflow": ["translate actions", "synthesize RPA functions", "retrieval-augmented build"], "summary": "AutoRPA converts ReAct GUI agent interactions into robust RPA code for efficient repeated tasks."}
{"id": 14773, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Agentic framework with dynamic graph of thoughts for financial report generation"}, {"area": "multi-agent", "role": "scaffold", "why": "Social collaboration mechanism coordinates agents"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["reasoning topology"], "benchmarks": [], "results": [], "workflow": ["dynamic graph of thoughts", "agent collaboration", "report generation"], "summary": "Cogito generates financial reports using evolving graph-of-thought reasoning and multi-agent collaboration."}
{"id": 11611, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "History-aware backtracking agentic traversal for multimodal graph retrieval"}], "pattern": "vary-scaffold", "fixes": ["corpora"], "varies": ["retrieval traversal"], "benchmarks": [], "results": [], "workflow": ["sequential subgraph retrieval", "backtrack using failures"], "summary": "Agentic multimodal graph retriever that learns from failed traversals through history-aware backtracking."}
{"id": 4058, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "Threat-snapshot b3 benchmark of backbone LLM security in agents"}, {"area": "agent-safety", "role": "attack", "why": "Red-teaming backbone LLMs at isolated agent states"}], "pattern": "fix-env-vary-agent", "fixes": ["threat snapshots"], "varies": ["backbone LLM"], "benchmarks": [{"name": "b3", "purpose": "backbone LLM agent security", "is_new": true}], "results": [], "workflow": ["isolate vulnerable states", "build benchmark", "evaluate LLMs"], "summary": "Threat snapshots and b3 benchmark evaluating how backbone LLM choice affects agent security."}
{"id": 12444, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "Benchmark of research extension tasks for coding agents"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks"], "benchmarks": [{"name": "RExBench", "purpose": "research extension implementation", "is_new": true}], "results": [], "workflow": [], "summary": "RExBench evaluates coding agents on implementing realistic extensions of 12 research papers."}
{"id": 17436, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "Hotel search benchmark balancing quality and efficiency with clarifications"}], "pattern": "fix-agent-vary-env", "fixes": ["agentic search systems"], "varies": ["queries"], "benchmarks": [{"name": "HotelQuEST", "purpose": "agentic search quality and efficiency", "is_new": true}], "results": [], "workflow": [], "summary": "HotelQuEST benchmark of 214 hotel queries evaluating agentic search quality and efficiency with underspecified preferences."}
{"id": 12089, "in_agent_scope": false, "types": [{"area": "safety/security", "role": "attack", "why": "Backdoor keeping clean CoT but steering final answer wrong"}], "pattern": "vary-conditions", "fixes": ["victim LLM"], "varies": ["backdoor attack"], "benchmarks": [], "results": [], "workflow": ["train post-output backdoor", "trigger wrong answer"], "summary": "MirageBD backdoor attack preserving clean reasoning while inducing wrong answers, evading process monitors."}
{"id": 9418, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Co-evolving attacker-defender RL for safety alignment"}, {"area": "agent-safety", "role": "attack", "why": "Attacker agent learns query rewriting vulnerabilities"}], "pattern": "vary-both", "fixes": ["base LLM"], "varies": ["attacker and defender policies"], "benchmarks": [], "results": [], "workflow": ["attacker rewrites queries", "defender refuses", "co-evolve via RL"], "summary": "MAGIC trains attacker and defender LLMs adversarially via multi-agent RL for robust safety alignment."}
{"id": 15041, "in_agent_scope": false, "types": [{"area": "apps/dialogue", "role": "method", "why": "Lite-adaptive stylized QA with cluster-based parameter sharing for many accounts"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["style adaptation"], "benchmarks": [], "results": [], "workflow": ["cluster styles", "share parameters", "contextual QA"], "summary": "WeStar serves stylized contextual question answering across millions of official accounts with lightweight adaptation."}
{"id": 11991, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Tri-view explanations analyzing faithfulness and belief dynamics in strategic games"}, {"area": "multi-agent", "role": "eval-method", "why": "Evidence-anchored explanation artifacts with oracle audits"}], "pattern": "vary-conditions", "fixes": ["LLM agents"], "varies": ["explanation views"], "benchmarks": [], "results": [], "workflow": ["self-reasoning", "belief states", "oracle audit"], "summary": "TriEx framework for analyzing explanation faithfulness of multi-agent LLMs in imperfect-information games."}
{"id": 13480, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Complexity-aware multi-expert routing with LoRA SFT for Chinese medical QA"}], "pattern": "vary-scaffold", "fixes": ["base LLM"], "varies": ["expert routing"], "benchmarks": [], "results": [], "workflow": ["LoRA SFT", "route by complexity", "experts and judge"], "summary": "CAMEC combines medical adaptation and complexity-aware CoT, RAG and KG experts for Chinese medical QA."}
{"id": 14868, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "analysis", "why": "Studies how memory affects chain and tree MLE agents"}, {"area": "code-swe", "role": "scaffold", "why": "Dynamic coding memory reusing debugging experience"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory", "agent architecture"], "benchmarks": [], "results": [], "workflow": ["capture debugging experiences", "integrate into agents"], "summary": "Systematic study showing memory's effect on MLE agents depends on architecture, using a dynamic coding memory."}
{"id": 14729, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Psychology-grounded personality layers plus inner monologue for role-playing consistency"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["personality architecture"], "benchmarks": [], "results": [], "workflow": ["personality layers", "inner monologue", "dual-process generation"], "summary": "PersonaForge improves long-dialogue personality consistency in role-playing agents using psychology-grounded constraints."}
{"id": 14737, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "Multimodal multi-document scientific deep research benchmark"}], "pattern": "fix-agent-vary-env", "fixes": ["MLLMs"], "varies": ["tasks"], "benchmarks": [{"name": "PaperScope", "purpose": "multi-document scientific research", "is_new": true}], "results": [], "workflow": [], "summary": "PaperScope benchmark built on a 2,000-paper knowledge graph for multimodal multi-document agentic research."}
{"id": 16059, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "scaffold", "why": "Failure-aware orchestration of specialized agents for open-source LLMs"}], "pattern": "vary-scaffold", "fixes": ["open-source LLMs"], "varies": ["agent orchestration"], "benchmarks": [], "results": [], "workflow": ["analyze failures", "activate specialized agents"], "summary": "FAMA analyzes failure trajectories and activates tailored agents to improve small LLMs in tool-use benchmarks."}
{"id": 3616, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "Benchmark of multi-step tasks over live MCP servers"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["MCP tasks"], "benchmarks": [{"name": "MCP-Bench", "purpose": "tool-using agents on 28 MCP servers", "is_new": true}], "results": [], "workflow": [], "summary": "MCP-Bench evaluates LLM agents on complex tasks across 28 live MCP servers and 250 tools."}
{"id": 14486, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "train-agent", "why": "SFT and RL train adaptive parallel tool use for code localization"}], "pattern": "fix-env-vary-agent", "fixes": ["SWE-bench Verified"], "varies": ["training"], "benchmarks": [{"name": "SWE-bench Verified", "purpose": "code localization", "is_new": false}], "results": [{"benchmark": "SWE-bench Verified", "system": "FuseSearch-4B", "base_model": null, "setting": "proposed", "metric": "file-level F1", "score": 84.7, "unit": "%"}, {"benchmark": "SWE-bench Verified", "system": "FuseSearch-4B", "base_model": null, "setting": "proposed", "metric": "function-level F1", "score": 56.4, "unit": "%"}], "workflow": ["define tool efficiency", "SFT", "RL"], "summary": "FuseSearch trains small models for efficient adaptive parallel code localization."}
{"id": 4334, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "data", "why": "Exploration-based environment-grounded synthetic task generation for agents"}], "pattern": "vary-data", "fixes": ["training recipe"], "varies": ["task data"], "benchmarks": [], "results": [], "workflow": ["explore environment", "synthesize tasks", "verify tasks"], "summary": "AutoPlay explores interactive environments to synthesize diverse, verifiable tasks for training MLLM agents."}
{"id": 14194, "in_agent_scope": true, "types": [{"area": "search-research", "role": "analysis", "why": "Compares basic, enhanced and agentic RAG approaches empirically"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["RAG approach"], "benchmarks": [], "results": [], "workflow": [], "summary": "Experimental comparison of basic, enhanced and agentic RAG on quality versus cost."}
{"id": 13224, "in_agent_scope": false, "types": [{"area": "align/rlvr", "role": "method", "why": "RL with structure-aware rewards to learn critique-guided outline refinement"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["refinement policy training"], "benchmarks": [{"name": "FreshWiki", "purpose": "outline generation", "is_new": false}, {"name": "WikiOutline", "purpose": "outline generation", "is_new": false}], "results": [], "workflow": ["teacher trajectories", "reasoning chains", "GRPO"], "summary": "Logic-RL learns outline refinement policy via GRPO with structure-aware rewards."}
{"id": 2415, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Multi-agent debate system for evidence-based chemical reaction condition reasoning"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["multi-agent pipeline"], "benchmarks": [], "results": [], "workflow": ["mechanistic grounding", "multi-channel recall", "agentic debate", "rationale aggregation"], "summary": "ChemMAS multi-agent system recommends reaction conditions with evidence-backed rationales."}
{"id": 12631, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "eval-method", "why": "Atomic-criteria MLLM-as-a-judge for fine-grained diagnosis"}, {"area": "eval/bench", "role": "benchmark", "why": "New human-ranked benchmark validating the judge"}], "pattern": "vary-evaluation", "fixes": ["MLLMs"], "varies": ["judge protocol"], "benchmarks": [], "results": [], "workflow": ["decompose into atomic criteria", "verify visual facts"], "summary": "ATOM judge diagnoses MLLM competencies at atomic criteria level and guides self-improvement."}
{"id": 16142, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "analysis", "why": "Compares LLM agents to human survey responses in macro ABMs"}, {"area": "social-sim", "role": "scaffold", "why": "Causal-graph path retrieval aligns agent decisions"}], "pattern": "vary-scaffold", "fixes": ["LLM agents"], "varies": ["causal inference framework"], "benchmarks": [{"name": "Survey of Consumer Expectations", "purpose": "human comparison", "is_new": false}], "results": [], "workflow": ["build causal graph", "path search", "retrieve causal context"], "summary": "Diagnoses micro-level flaws of LLM economic agents and proposes a causal transmission framework to align them."}
{"id": 13005, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "scaffold", "why": "Conflict detection rules improve vector memory quality for embodied planners"}, {"area": "memory-selfevolve", "role": "scaffold", "why": "Detects conflicting memory entries"}], "pattern": "vary-scaffold", "fixes": ["LLM planner"], "varies": ["memory quality rules"], "benchmarks": [], "results": [], "workflow": ["detect conflicts", "correct index", "plan"], "summary": "Conflict Detection Rules fix vector memory data-quality issues, improving embodied LLM planners."}
{"id": 12166, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "RLVR with denser rewards and hints for agentic search"}], "pattern": "fix-env-vary-agent", "fixes": ["benchmarks"], "varies": ["reward density", "training"], "benchmarks": [], "results": [], "workflow": ["sub-goal scaffolding", "pathfinding hints", "dual-agent refinement"], "summary": "InfoFlow raises reward density for agentic search RL via sub-goals, hints and dual-agent refinement."}
{"id": 14031, "in_agent_scope": true, "types": [{"area": "search-research", "role": "application", "why": "Reproducible agentic fact verification on local open search infrastructure"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["retrieval configuration"], "benchmarks": [], "results": [], "workflow": ["local search index", "agentic verification"], "summary": "FactSearch is a reproducible agentic claim-verification system on local open-source search."}
{"id": 12057, "in_agent_scope": false, "types": [{"area": "train/peft", "role": "method", "why": "Fine-tunes adapter on base with instruct-model confidence, merges into instruct model"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["adaptation method"], "benchmarks": [], "results": [], "workflow": ["token-level guidance", "train LoRA", "merge adapter"], "summary": "GIFT adapts instruction-tuned LLMs using instruction-level guidance while preserving instruction-following."}
{"id": 6997, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "analysis", "why": "Open-world shift study showing SFT and RL tool agents degrade"}, {"area": "tool-use", "role": "environment", "why": "Controlled sandbox with four-tier shifts"}], "pattern": "vary-conditions", "fixes": ["trained agents"], "varies": ["distribution shifts"], "benchmarks": [], "results": [], "workflow": [], "summary": "Formalizes OpenAgent and shows SFT/RL tool-use agents are fragile under open-world shifts."}
{"id": 11291, "in_agent_scope": false, "types": [{"area": "efficiency/inference", "role": "method", "why": "Self-speculative decoding using verification-derived critical KV entries"}], "pattern": "n/a", "fixes": ["model"], "varies": ["decoding method"], "benchmarks": [], "results": [], "workflow": ["verify with full KV", "select critical entries", "draft sparsely"], "summary": "Vegas selects KV entries for drafting from verification byproducts, accelerating long-context decoding losslessly."}
{"id": 2945, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "Framework creating open-ended single-cell biology benchmark for co-scientist agents"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks"], "benchmarks": [{"name": "sc-HeurekaBench", "purpose": "single-cell co-scientist", "is_new": true}], "results": [], "workflow": ["extract insights", "generate workflows", "verify findings"], "summary": "HeurekaBench builds exploratory scientific-question benchmarks, instantiated as sc-HeurekaBench for single-cell biology."}
{"id": 16037, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Diversity-aware initialization and confidence-modulated debate"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["debate protocol"], "benchmarks": [], "results": [], "workflow": ["diverse candidate pool", "confidence communication", "debate"], "summary": "Improves multi-agent debate with diverse initial answers and calibrated confidence communication."}
{"id": 4967, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "eval-method", "why": "Standardized evaluation harness and leaderboard for agents"}, {"area": "web-gui", "role": "analysis", "why": "Analyzes models, scaffolds and benchmarks across 21,730 rollouts"}], "pattern": "vary-conditions", "fixes": ["benchmarks"], "varies": ["models", "scaffolds"], "benchmarks": [], "results": [], "workflow": ["parallel harness", "run rollouts", "analyze"], "summary": "HAL provides a standardized agent evaluation harness and analysis across models, scaffolds and nine benchmarks."}
{"id": 17246, "in_agent_scope": false, "types": [{"area": "align/rlvr", "role": "method", "why": "RL with rollout tokens sampled from frozen weak model for intelligibility"}], "pattern": "n/a", "fixes": ["weak model"], "varies": ["training paradigm"], "benchmarks": [], "results": [], "workflow": ["sample weak-model tokens", "train strong model"], "summary": "Tandem training makes strong models' solutions robust to handoff to weaker models via RL."}
{"id": 12098, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "White-box repository-level code reasoning benchmark with mutation and slicing metrics"}], "pattern": "fix-agent-vary-env", "fixes": ["frontier models"], "varies": ["tasks"], "benchmarks": [{"name": "RepoReason", "purpose": "repo-level abductive reasoning", "is_new": true}], "results": [], "workflow": ["mutate execution", "slice programs"], "summary": "RepoReason diagnoses repository-level code reasoning with execution-driven mutation and three metrics."}
{"id": 6007, "in_agent_scope": false, "types": [{"area": "apps/science", "role": "method", "why": "Adaptive topology learning and FiLM-guided distillation for RNA design"}], "pattern": "n/a", "fixes": [], "varies": ["model"], "benchmarks": [], "results": [], "workflow": ["learn adaptive graph", "distill"], "summary": "ATL-FGD improves tertiary structure-based RNA design with learned topology and distillation."}
{"id": 7257, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "Multimodal game development tasks for coding agents"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks"], "benchmarks": [{"name": "GameDevBench", "purpose": "game development agents", "is_new": true}], "results": [], "workflow": [], "summary": "GameDevBench: 333 multimodal game development tasks evaluating agents."}
{"id": 17333, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "eval-method", "why": "Toolkit of 182 Spanish linguistic metrics"}], "pattern": "n/a", "fixes": [], "varies": ["metrics"], "benchmarks": [{"name": "Automated Readability Assessment", "purpose": "usefulness", "is_new": false}], "results": [], "workflow": [], "summary": "Open-source Spanish linguistic analysis toolkit with 182 metrics."}
{"id": 8506, "in_agent_scope": false, "types": [{"area": "train/peft", "role": "method", "why": "Geometry-aware initialization for rank-1 LoRA on MLLMs"}], "pattern": "n/a", "fixes": ["MLLM"], "varies": ["initialization"], "benchmarks": [], "results": [], "workflow": ["estimate modality gap", "align rank-1 direction"], "summary": "Gap-Init aligns rank-1 LoRA with modality-gap axis, stabilizing extremely low-rank PEFT."}
{"id": 11866, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Multi-agent RAG with long-chain training strategy"}], "pattern": "fix-env-vary-agent", "fixes": ["benchmarks"], "varies": ["training strategy"], "benchmarks": [], "results": [], "workflow": ["knowledge induction", "answer reasoning", "train on chains"], "summary": "CoCoA synergizes parametric and retrieved knowledge via multi-agent RAG and long-chain training."}
{"id": 18381, "in_agent_scope": false, "types": [{"area": "apps/science", "role": "analysis", "why": "Figurative language features in Alzheimer's discourse and neural alignment"}], "pattern": "vary-conditions", "fixes": [], "varies": ["feature sets"], "benchmarks": [{"name": "ADReSS", "purpose": "AD classification", "is_new": false}], "results": [], "workflow": ["detect figurative language", "train classifiers"], "summary": "Studies figurative language indicators distinguishing Alzheimer's discourse and their alignment with neural models."}
{"id": 5258, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "Live benchmark for agents predicting future events"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["live tasks"], "benchmarks": [{"name": "FutureX", "purpose": "future prediction", "is_new": true}], "results": [], "workflow": [], "summary": "FutureX is a large live benchmark evaluating LLM agents on future prediction."}
{"id": 16023, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "scaffold", "why": "Dynamic retrieval of tools conditioned on evolving plan"}], "pattern": "vary-scaffold", "fixes": ["LLM backbones"], "varies": ["tool retrieval"], "benchmarks": [], "results": [], "workflow": ["model tool dependencies", "retrieve per plan step"], "summary": "DTDR retrieves tools adaptively using the evolving calling plan for lightweight function calling."}
{"id": 15142, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "RL-trained small LLM prompts larger LLMs in multi-turn collaboration"}], "pattern": "fix-env-vary-agent", "fixes": ["large LLM"], "varies": ["prompting agent"], "benchmarks": [], "results": [], "workflow": ["small LLM prompts", "large LLM reasons", "double-constrained reward"], "summary": "Prompt-R1 trains a small LLM via RL to craft prompts for large LLMs."}
{"id": 18434, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "analysis", "why": "Compares unsupervised, fine-tuned and few-shot LLMs for bilingual dictionary induction"}], "pattern": "vary-conditions", "fixes": ["LLMs"], "varies": ["approach"], "benchmarks": [], "results": [], "workflow": [], "summary": "Comparative study of prompting versus tuning for bilingual dictionary induction in distant languages."}
{"id": 16147, "in_agent_scope": true, "types": [{"area": "science-data", "role": "survey", "why": "Survey of AI agents for science of science"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Survey with taxonomy of SciSci agents as simulations or tools, covering architectures, evaluation and challenges."}
{"id": 14948, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "train-agent", "why": "CoT distillation trains dementia patient simulator with behaviors"}, {"area": "domain-agents", "role": "application", "why": "Expert-guided dementia persona simulation"}], "pattern": "vary-data", "fixes": ["LLM"], "varies": ["persona and training"], "benchmarks": [], "results": [], "workflow": ["build personas", "model nonverbal actions", "distill CoT"], "summary": "DemMA simulates dementia patients with clinical personas and nonverbal actions in a single LLM."}
{"id": 3262, "in_agent_scope": false, "types": [{"area": "train/data", "role": "method", "why": "Learning-free multimodal dataset distillation via CLIP prototypes"}], "pattern": "n/a", "fixes": [], "varies": ["distillation method"], "benchmarks": [], "results": [], "workflow": ["CLIP prototypes", "synthesize data"], "summary": "Learning-free multimodal dataset distillation using prototype-guided synthesis."}
{"id": 7239, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "application", "why": "Multi-agent hypothesis-driven bug detection in consensus protocols"}, {"area": "agent-safety", "role": "attack", "why": "Synthesizes attack scenarios to find protocol bugs"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent framework"], "benchmarks": [{"name": "Raft", "purpose": "target implementation", "is_new": false}, {"name": "EPaxos", "purpose": "target implementation", "is_new": false}, {"name": "HotStuff", "purpose": "target implementation", "is_new": false}], "results": [], "workflow": ["explore states", "synthesize scenarios", "validate findings"], "summary": "Agora multi-agent framework finds logic bugs in production consensus protocol implementations."}