Files
ai-taxonomy-literature-anal…/data/labels/o32.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
33 KiB
JSON

{"id": 8141, "in_agent_scope": false, "types": [{"area": "train/distill", "role": "data", "why": "Spectral step-level weighting in long-CoT distillation prioritizing reasoning steps aligned with consensus gradient subspace"}], "pattern": "vary-data", "fixes": ["student model", "SFT recipe"], "varies": ["step-level gradient weighting"], "benchmarks": [], "results": [], "workflow": ["Decompose gradients by loss subspace", "Identify consensus-aligned steps", "Reweight updates, suppress residual"], "summary": "Gradient analysis of long CoT motivates spectral-guided step-level distillation that prioritizes effective reasoning steps."}
{"id": 16680, "in_agent_scope": true, "types": [{"area": "search-research", "role": "data", "why": "Agentic generator-solver pipeline synthesizes difficulty-controlled deep search QA data with execution feedback"}], "pattern": "vary-data", "fixes": ["search agent training recipe"], "varies": ["training QA data"], "benchmarks": [], "results": [], "workflow": ["Generator proposes QA pairs", "Search agent attempts and gives feedback", "Iterate until target difficulty"], "summary": "SAGE iteratively generates difficulty-controlled deep search QA pairs, improving search agents trained on them."}
{"id": 2109, "in_agent_scope": false, "types": [{"area": "align/rlvr", "role": "data", "why": "Uncertainty-consistency query selection for active learning in RLVR math training"}], "pattern": "vary-data", "fixes": ["RLVR algorithm"], "varies": ["query selection strategy"], "benchmarks": [], "results": [], "workflow": ["Measure uncertainty consistency", "Select informative queries", "Run RLVR on subset"], "summary": "Active-learning query selection guided by uncertainty consistency lets RLVR match performance with fewer annotated queries."}
{"id": 1445, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "Benchmark of 1,002 problems for self-refinement and guided refinement with checklist evaluation"}, {"area": "reason/cot", "role": "analysis", "why": "Measures whether frontier LMs can self-refine across feedback modes"}], "pattern": "fix-agent-vary-env", "fixes": ["evaluated LMs"], "varies": ["tasks", "feedback mode"], "benchmarks": [{"name": "RefineBench", "purpose": "evaluate LM refinement ability", "is_new": true}], "results": [], "workflow": [], "summary": "RefineBench evaluates LM refinement across 11 domains and finds frontier models self-refine poorly."}
{"id": 6252, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "Trains memory agent with tree-guided RL (MoT-GRPO) using dense hindsight credit assignment"}], "pattern": "fix-env-vary-agent", "fixes": ["memory database", "benchmarks"], "varies": ["RL reward density", "memory policy"], "benchmarks": [], "results": [], "workflow": ["Agent manages hierarchical memory", "Build memory operation tree", "Backpropagate dense step rewards"], "summary": "Mem-T memory agent trained with MoT-GRPO turns sparse terminal feedback into dense step-wise supervision."}
{"id": 12530, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "analysis", "why": "Studies whether Middle Chinese helps LLMs model dialect pronunciation via SFT and RL"}, {"area": "multiling/lowres", "role": "benchmark", "why": "Constructs multi-source dataset of Middle Chinese and 12 dialects"}], "pattern": "vary-conditions", "fixes": ["LLMs"], "varies": ["use of Middle Chinese", "training"], "benchmarks": [{"name": "Middle Chinese-dialect dataset", "purpose": "sound rule induction and IPA transcription", "is_new": true}], "results": [], "workflow": ["Build dialect dataset", "LoRA SFT", "GRPO RL"], "summary": "Investigates Middle Chinese as ancestor for dialect rule induction and IPA transcription with LoRA SFT plus RL."}
{"id": 16066, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "analysis", "why": "Controlled pipeline quantifying how webpage visual attributes shape web-agent click decisions"}, {"area": "web-gui", "role": "eval-method", "why": "VAF protocol with Target Click Rate and Target Mention Rate"}], "pattern": "fix-agent-vary-env", "fixes": ["web agents"], "varies": ["visual attributes of page"], "benchmarks": [{"name": "VAF", "purpose": "controlled visual-attribute evaluation", "is_new": true}], "results": [], "workflow": ["Generate visual variants", "Agents browse and click", "Measure click and mention rates"], "summary": "VAF measures how visual attribute variations influence web agent item choice."}
{"id": 4310, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "train-agent", "why": "Self-distills multi-agent social judgments into one MLLM via SCoT data and SCPO preference optimization"}], "pattern": "fix-env-vary-agent", "fixes": ["MMD benchmarks"], "varies": ["training method"], "benchmarks": [], "results": [], "workflow": ["Simulate society of MLLM agents", "Synthesize Social CoT data", "Align with SCPO"], "summary": "Distills collective multi-agent social reasoning into a single MLLM for multimodal misinformation detection."}
{"id": 12823, "in_agent_scope": true, "types": [{"area": "search-research", "role": "data", "why": "WebAggregator pipeline synthesizes compositional aggregation questions for deep research agent fine-tuning"}, {"area": "search-research", "role": "benchmark", "why": "Introduces benchmark for compositional aggregation in deep research"}], "pattern": "vary-data", "fixes": ["SFT recipe"], "varies": ["training data"], "benchmarks": [{"name": "GAIA", "purpose": "evaluate trained agent", "is_new": false}, {"name": "WebWalkerQA", "purpose": "evaluate trained agent", "is_new": false}, {"name": "XBench", "purpose": "evaluate trained agent", "is_new": false}], "results": [], "workflow": ["Explore interconnected knowledge", "Compose complex questions", "Fine-tune agent"], "summary": "WebAggregator synthesizes compositional questions; WebAggregator-32B surpasses GPT-4.1 on deep research benchmarks."}
{"id": 2298, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "environment", "why": "CodeGym converts coding problems into verifiable multi-turn tool-use RL environments"}, {"area": "tool-use", "role": "train-agent", "why": "RL on synthetic environments for generalizable tool use"}], "pattern": "fix-agent-vary-env", "fixes": ["RL recipe"], "varies": ["training environments"], "benchmarks": [{"name": "CodeGym", "purpose": "synthetic tool-use environments", "is_new": true}], "results": [], "workflow": ["Extract atomic functions as tools", "Build interactive tasks", "Run agent RL"], "summary": "CodeGym synthesizes diverse verifiable tool-use environments from coding problems for agent RL."}
{"id": 4987, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "benchmark", "why": "First robot task planning benchmark with vague referring expressions"}, {"area": "embodied-nav", "role": "scaffold", "why": "Proposes approach to overcome vagueness in planning"}], "pattern": "fix-agent-vary-env", "fixes": ["LLM planners"], "varies": ["instruction vagueness"], "benchmarks": [{"name": "REI-Bench", "purpose": "vague instruction robot planning", "is_new": true}], "results": [], "workflow": [], "summary": "REI-Bench shows vague referring expressions degrade LLM robot planning up to 36.9% success drop."}
{"id": 16949, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "survey", "why": "Position paper arguing open-ended generation for LLM social simulation"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Argues LLM social simulation should use open-ended free-form text rather than closed formats."}
{"id": 13106, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Bi-level graph anomaly detection to identify malicious agents in LLM multi-agent systems with explanations"}], "pattern": "fix-agent-vary-env", "fixes": ["MAS threat model"], "varies": ["detection mechanism"], "benchmarks": [], "results": [], "workflow": ["Bi-level agent encoder", "Theme-based anomaly detector", "Token-level score fusion"], "summary": "Explainable fine-grained detector of malicious agents in LLM multi-agent systems."}
{"id": 6304, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Convex Gated Probing and improved audio SSL pipeline yielding BAT"}], "pattern": "n/a", "fixes": ["audio benchmarks"], "varies": ["probing and SSL recipe"], "benchmarks": [{"name": "AudioSet", "purpose": "audio SSL evaluation", "is_new": false}], "results": [], "workflow": ["Introduce CGP probe", "Rework SSL pipeline", "Train BAT"], "summary": "CGP probing guides redesign of audio SSL, producing Better Audio Transformer."}
{"id": 7334, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "benchmark", "why": "MemoryArena gym for agent memory in interdependent multi-session tasks"}], "pattern": "fix-agent-vary-env", "fixes": ["agents with memory"], "varies": ["tasks"], "benchmarks": [{"name": "MemoryArena", "purpose": "multi-session agent memory evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "MemoryArena benchmarks memory-agent-environment loops across web navigation, planning and other multi-session tasks."}
{"id": 9015, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "theory", "why": "Proves aggregation algorithms OW and ISP beat majority voting under assumptions"}, {"area": "multi-agent", "role": "method", "why": "Training-free LLM answer aggregation using higher-order information"}], "pattern": "vary-evaluation", "fixes": ["LLM answers"], "varies": ["aggregation rule"], "benchmarks": [{"name": "UltraFeedback", "purpose": "aggregation evaluation", "is_new": false}, {"name": "MMLU", "purpose": "aggregation evaluation", "is_new": false}, {"name": "ARMMAN", "purpose": "healthcare setting", "is_new": false}], "results": [], "workflow": [], "summary": "Two aggregation algorithms using second-order information provably improve on majority voting for multi-LLM answers."}
{"id": 12806, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "105 C/C++ secure coding tasks reconstructing vulnerability-introducing scenarios"}, {"area": "agent-safety", "role": "benchmark", "why": "Evaluates security of agent-generated code"}], "pattern": "fix-agent-vary-env", "fixes": ["code agents"], "varies": ["tasks"], "benchmarks": [{"name": "SecureVibeBench", "purpose": "secure coding for code agents", "is_new": true}, {"name": "OSS-Fuzz", "purpose": "task source", "is_new": false}], "results": [], "workflow": [], "summary": "SecureVibeBench tests code agents on real vulnerability-introducing scenarios; agents struggle with correct and secure code."}
{"id": 14383, "in_agent_scope": false, "types": [{"area": "arch/diffusion", "role": "method", "why": "Causal concept-guided attention for diffusion LMs using teacher concept graph"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["attention guidance"], "benchmarks": [], "results": [], "workflow": ["Get concept causal graph from teacher", "Guide DLM attention"], "summary": "C^2DLM guides diffusion LM attention with concept-level causal graphs for better reasoning and faster training."}
{"id": 2760, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "828 vision-centric agentic tasks with step-level reasoning evaluation"}, {"area": "tool-use", "role": "eval-method", "why": "Fine-grained step-level evaluation framework"}], "pattern": "fix-agent-vary-env", "fixes": ["agents under test"], "varies": ["tasks"], "benchmarks": [{"name": "Agent-X", "purpose": "vision-centric agentic reasoning", "is_new": true}], "results": [], "workflow": [], "summary": "Agent-X benchmarks multistep multimodal agent reasoning across six environments with step-level evaluation."}
{"id": 15250, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Four-agent collaborative framework for disambiguation in multi-domain translation"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent collaboration"], "benchmarks": [], "results": [], "workflow": ["Allocate domain", "Translate generally", "Disambiguate by domain", "Fuse translations"], "summary": "MACD uses cooperating LLM agents to resolve cross-domain word ambiguity in translation."}
{"id": 2764, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Multi-agent LLM framework synthesizing federated learning systems end-to-end"}], "pattern": "n/a", "fixes": ["LLMs"], "varies": ["FL system design pipeline"], "benchmarks": [], "results": [], "workflow": ["Human-in-loop planning", "Modular code generation", "Automated evaluation and refinement"], "summary": "Helmsman multi-agent framework automates synthesis of federated learning systems from user specs."}
{"id": 3340, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "RL with verifiable meta-reasoning rewards for process-level supervision in long-horizon agents"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["reward design"], "benchmarks": [], "results": [], "workflow": ["Agent tags cognitive steps", "Rule-based meta-reasoning rewards", "End-to-end RL"], "summary": "RLVMR rewards verifiable planning, exploration and reflection behaviors to improve long-horizon agent RL."}
{"id": 10658, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "140 repository-level code performance optimization instances from performance PRs"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["tasks"], "benchmarks": [{"name": "SWE-Perf", "purpose": "repo-level performance optimization", "is_new": true}], "results": [], "workflow": [], "summary": "SWE-Perf is the first benchmark for repository-level code performance optimization."}
{"id": 17491, "in_agent_scope": true, "types": [{"area": "apps/dialogue", "role": "method", "why": "Next Question Suggestion for conversational agents with directional relevance"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "DIRECT introduces next question suggestions to guide users in conversational agent interactions cheaply."}
{"id": 5437, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "498 repository performance optimization tasks on real workloads"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks"], "benchmarks": [{"name": "SWE-fficiency", "purpose": "repo performance optimization", "is_new": true}], "results": [], "workflow": [], "summary": "SWE-fficiency benchmarks agents on speeding up real repositories while preserving correctness."}
{"id": 14775, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "Procedurally generated multi-step programming-by-example inductive reasoning benchmark"}, {"area": "eval/robust", "role": "eval-method", "why": "Contamination-resistant controllable-difficulty generation"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["problem difficulty"], "benchmarks": [{"name": "PBEBench", "purpose": "inductive reasoning", "is_new": true}], "results": [], "workflow": [], "summary": "PBEBench generates controllable-difficulty sound-law-inspired PBE problems for contamination-resistant evaluation."}
{"id": 11977, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "201 mobile tasks across 20 apps with agent-user interaction and MCP tools"}], "pattern": "fix-agent-vary-env", "fixes": ["mobile agents"], "varies": ["tasks, apps"], "benchmarks": [{"name": "MobileWorld", "purpose": "harder mobile agent evaluation", "is_new": true}, {"name": "AndroidWorld", "purpose": "saturated predecessor", "is_new": false}], "results": [], "workflow": [], "summary": "MobileWorld is a harder long-horizon mobile benchmark with user interaction and MCP augmentation."}
{"id": 3058, "in_agent_scope": false, "types": [{"area": "safety/jailbreak", "role": "eval-method", "why": "GuidedEval with case-specific guidelines to fix jailbreak evaluation discrepancies"}, {"area": "safety/jailbreak", "role": "benchmark", "why": "GuidedBench harmful question dataset"}], "pattern": "vary-evaluation", "fixes": ["jailbreak methods"], "varies": ["evaluation guidelines"], "benchmarks": [{"name": "GuidedBench", "purpose": "jailbreak evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "GuidedBench provides guideline-based evaluation for more accurate jailbreak effectiveness measurement."}
{"id": 12912, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Caption-guided asymmetric mixture-of-depths for efficient visual-LLM alignment"}], "pattern": "n/a", "fixes": ["MLLM"], "varies": ["alignment module"], "benchmarks": [], "results": [], "workflow": ["Hybrid query construction", "Asymmetric mixture-of-depths"], "summary": "Cat-MoD speeds multimodal alignment with caption-initialized queries and mixture-of-depths."}
{"id": 14708, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Renders textual memories as images stored as visual notes for compact agent memory"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory representation"], "benchmarks": [], "results": [], "workflow": ["Pre-render memory as images", "Store visual notes", "Retrieve for reasoning"], "summary": "VizoMem compresses agent memory into visual notes for efficient long-horizon reasoning."}
{"id": 14533, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "analysis", "why": "Oracle counterfactual framework measuring criticality of planning, state tracking, long context skills"}, {"area": "planning-decision", "role": "benchmark", "why": "Procedurally generated game-like multi-turn environments"}], "pattern": "vary-conditions", "fixes": ["agents"], "varies": ["oracle skill assistance"], "benchmarks": [{"name": "LUMINA", "purpose": "procedural long-horizon environments", "is_new": true}], "results": [], "workflow": [], "summary": "Oracle interventions reveal which skills matter most for long-horizon multi-turn agents."}
{"id": 11728, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Two-stage section ranking and iterative extraction agent for literature information retrieval"}, {"area": "search-research", "role": "benchmark", "why": "Introduces IntraView task"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent pipeline"], "benchmarks": [{"name": "IntraView", "purpose": "content-grounded literature retrieval", "is_new": true}], "results": [], "workflow": ["Rank relevant sections", "Iteratively extract details"], "summary": "IntrAgent mimics human reading for content-grounded literature information retrieval."}
{"id": 18506, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "method", "why": "Gradient-based connection weights for credit assignment and optimization of LLM multi-agent systems"}], "pattern": "vary-scaffold", "fixes": ["LLM agents"], "varies": ["agent connections"], "benchmarks": [], "results": [], "workflow": ["Model MAS as computational graph", "Compute gradient influence weights", "Optimize connections"], "summary": "GBC attributes credit across agents via gradients to optimize multi-agent systems."}
{"id": 18479, "in_agent_scope": false, "types": [{"area": "interp/mech", "role": "analysis", "why": "Causal tracing with definitional semantic roles in reverse dictionary task"}], "pattern": "vary-conditions", "fixes": ["LMs"], "varies": ["traced components"], "benchmarks": [], "results": [], "workflow": ["Causal tracing", "Overlay semantic roles"], "summary": "DSRA links linguistic structure to mechanisms supporting conceptual interpretation in LMs."}
{"id": 14225, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Path-centric reward shaping for RL training of agentic RAG"}], "pattern": "fix-env-vary-agent", "fixes": ["retrieval env"], "varies": ["reward design"], "benchmarks": [], "results": [], "workflow": ["Path-centric step coverage reward", "Soft scoring of trajectories"], "summary": "Search-P1 shapes dense path rewards to stabilize and speed agentic RAG RL."}
{"id": 3447, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "MCP-based agentic framework unifying RGB and spectral Earth observation tools"}, {"area": "science-data", "role": "benchmark", "why": "Evaluation protocol for EO agents"}], "pattern": "n/a", "fixes": ["MLLMs"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": [], "summary": "Earth-Agent unifies EO data and tools for multi-step quantitative spatiotemporal reasoning."}
{"id": 12949, "in_agent_scope": false, "types": [{"area": "arch/longctx", "role": "method", "why": "Intention-aware KV cache eviction retaining crucial KVs"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["KV eviction"], "benchmarks": [], "results": [], "workflow": ["Estimate intention attention", "Retain crucial KVs"], "summary": "IntentKV compresses KV cache using attention distribution of user intention."}
{"id": 15889, "in_agent_scope": false, "types": [{"area": "reason/code", "role": "benchmark", "why": "Benchmark of combinatorial and exploratory creativity in code generation"}, {"area": "reason/code", "role": "method", "why": "Enhancing creativity via self-evolving challenges"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["tasks"], "benchmarks": [{"name": "CreativeBench", "purpose": "machine creativity in code", "is_new": true}], "results": [], "workflow": [], "summary": "CreativeBench measures and enhances machine creativity in code via reverse engineering and self-play."}
{"id": 1539, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "environment", "why": "Synthesizes scalable web environments and tasks for GUI agent RL"}, {"area": "web-gui", "role": "train-agent", "why": "Decomposed reward RL training for GUI agents"}], "pattern": "vary-both", "fixes": ["base LLM"], "varies": ["environments", "reward"], "benchmarks": [], "results": [], "workflow": ["Synthesize environments", "Generate tasks", "Collect trajectories", "RL with decomposed reward"], "summary": "WebFactory closed-loop pipeline compresses LLM knowledge into grounded web agents."}
{"id": 3977, "in_agent_scope": false, "types": [{"area": "safety/privacy", "role": "defense", "why": "Circuit discovery-based conflict-guided localization for LLM unlearning"}], "pattern": "n/a", "fixes": ["unlearning objective"], "varies": ["localization"], "benchmarks": [], "results": [], "workflow": ["Discover circuits", "Separate forget vs retain nodes", "Targeted unlearning"], "summary": "CLUE localizes forget and retain nodes via circuits for precise LLM unlearning."}
{"id": 15237, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "train-agent", "why": "Agentic RL plus parallel test-time scaling for map-augmented geolocalization"}, {"area": "embodied-nav", "role": "scaffold", "why": "Parallel test-time scaling over candidate paths"}], "pattern": "fix-env-vary-agent", "fixes": ["map tools"], "varies": ["training and TTS"], "benchmarks": [], "results": [], "workflow": ["Agent-in-the-map loop", "Agentic RL", "Parallel test-time scaling"], "summary": "Thinking-with-Map agent uses maps for image geolocalization, trained with RL and parallel TTS."}
{"id": 13410, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "analysis", "why": "Defines Action Boundary Blindness and measures it across agent benchmarks"}, {"area": "planning-decision", "role": "eval-method", "why": "Four automatic metrics ABS, GAR, SVR, BASR"}], "pattern": "fix-agent-vary-env", "fixes": ["seven LLMs"], "varies": ["benchmarks"], "benchmarks": [{"name": "tau-bench", "purpose": "evaluate action boundaries", "is_new": false}, {"name": "WebArena", "purpose": "evaluate action boundaries", "is_new": false}, {"name": "ALFWorld", "purpose": "evaluate action boundaries", "is_new": false}, {"name": "TheAgentCompany", "purpose": "evaluate action boundaries", "is_new": false}, {"name": "OSWorld", "purpose": "evaluate action boundaries", "is_new": false}], "results": [], "workflow": [], "summary": "Formalizes action boundary violations and shows LLM agents score low on new metrics."}
{"id": 5181, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Unified agents for graph construction and retrieval in GraphRAG"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["GraphRAG pipeline"], "benchmarks": [], "results": [], "workflow": ["Schema-bounded extraction agent", "Graph construction", "Agentic retrieval"], "summary": "Youtu-GraphRAG jointly integrates graph construction and retrieval with agents for complex reasoning."}
{"id": 13196, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Hybrid framework externalizing belief inference to probabilistic graph model with LLM interaction"}, {"area": "social-sim", "role": "analysis", "why": "Evaluates reasoning models in Avalon"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["belief inference module"], "benchmarks": [{"name": "Avalon", "purpose": "social deduction game", "is_new": false}], "results": [], "workflow": ["LLM parses language", "Probabilistic model infers beliefs"], "summary": "Graph-informed probabilistic belief model plus LLM matches larger models in Avalon."}
{"id": 14711, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Emotionally stateful agents with PAD representation and emotion-memory coupling"}], "pattern": "vary-scaffold", "fixes": ["base models"], "varies": ["emotion module"], "benchmarks": [], "results": [], "workflow": ["PAD emotion state", "Dual-speed dynamics", "Emotion-memory coupling"], "summary": "Sentipolis adds persistent emotion to social simulation agents, improving continuity with model-dependent gains."}
{"id": 8817, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Studies whether self-organizing LLM teams match best member; finds they hold experts back"}], "pattern": "vary-conditions", "fixes": ["LLMs"], "varies": ["team composition"], "benchmarks": [], "results": [], "workflow": [], "summary": "Self-organizing LLM teams fail to reach strong synergy with their best individual member."}
{"id": 14668, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "Progress-aware RL for multi-turn function calling"}], "pattern": "fix-env-vary-agent", "fixes": ["benchmarks"], "varies": ["RL reward"], "benchmarks": [], "results": [], "workflow": ["Progress awareness summaries", "Planning future actions", "RL training"], "summary": "Progra integrates progress awareness into RL for multi-turn function calling."}
{"id": 17055, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "AndroidBuildBench of 1,019 build failures"}, {"area": "code-swe", "role": "scaffold", "why": "GradleFixer agent with domain tools"}], "pattern": "vary-both", "fixes": ["LLM"], "varies": ["tools", "tasks"], "benchmarks": [{"name": "AndroidBuildBench", "purpose": "Android build repair", "is_new": true}], "results": [], "workflow": ["Inspect Gradle environment", "Apply fixes with tools"], "summary": "Benchmark and tool-equipped agent for Android build repair with 81.4% resolve rate."}
{"id": 8255, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Channel-of-mobile-experts architecture with progressive Expert-FT and Router-FT training"}], "pattern": "fix-env-vary-agent", "fixes": ["mobile benchmarks"], "varies": ["architecture, training"], "benchmarks": [], "results": [], "workflow": ["Expert-FT", "Router-FT", "Progressive training"], "summary": "CoME routes four reasoning-stage experts for hybrid-capability mobile agents."}
{"id": 17829, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "benchmark", "why": "Danish linguistic acceptability benchmark via fourteen corruption functions"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["corruption types"], "benchmarks": [{"name": "DaLA", "purpose": "Danish acceptability", "is_new": true}], "results": [], "workflow": [], "summary": "DaLA builds a broader Danish acceptability benchmark from real-error-based corruptions."}
{"id": 14196, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Multi-agent framework generating teaching content via code"}], "pattern": "n/a", "fixes": ["LLMs"], "varies": ["agent pipeline"], "benchmarks": [], "results": [], "workflow": ["Plan", "Design", "Render via code"], "summary": "TeachMaster generates educational video via collaborative agents using code as intermediate medium."}
{"id": 15350, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "defense", "why": "Latent-signal refusal predicting answerability from hidden activations for Text-to-SQL"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["refusal mechanism"], "benchmarks": [], "results": [], "workflow": ["Read hidden activations", "Predict answerability"], "summary": "LatentRefusal gates unanswerable Text-to-SQL queries using internal activations."}
{"id": 15247, "in_agent_scope": true, "types": [{"area": "search-research", "role": "data", "why": "Multi-hop medical search data synthesis"}, {"area": "domain-agents", "role": "train-agent", "why": "Turn-controlled agentic training and inference for medical DeepResearch"}], "pattern": "vary-data", "fixes": ["base model"], "varies": ["data", "turn control"], "benchmarks": [], "results": [], "workflow": ["Synthesize multi-hop med-search data", "Agentic training", "Turn-controlled inference"], "summary": "DeepMed builds a medical deep research agent with synthesized data and turn control."}
{"id": 637, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Agentic MLLM for Cell Painting mechanism-of-action rationales using experimental context"}], "pattern": "n/a", "fixes": ["MLLM"], "varies": ["agent design"], "benchmarks": [], "results": [], "workflow": [], "summary": "CP-Agent generates interpretable mechanism rationales for cell morphology under chemical perturbations."}
{"id": 12378, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Agent converting geospatial questions into GeoFlow Graph workflows"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["workflow representation"], "benchmarks": [], "results": [], "workflow": ["Parse question to concepts", "Build GeoFlow Graph", "Execute workflow"], "summary": "Spatial-Agent grounds geospatial reasoning in spatial concept transformations."}
{"id": 18383, "in_agent_scope": false, "types": [{"area": "apps/dialogue", "role": "eval-method", "why": "Re-examines LAPDOG evaluation with human and LLM judges beyond surface metrics"}], "pattern": "vary-evaluation", "fixes": ["LAPDOG"], "varies": ["evaluation metrics"], "benchmarks": [], "results": [], "workflow": [], "summary": "Case study showing limits of surface metrics for retrieval-augmented personalized dialogue evaluation."}
{"id": 5099, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "analysis", "why": "ABxLab probes agent shopping choices under price, rating and nudge manipulations"}, {"area": "social-sim", "role": "eval-method", "why": "Framework for controlled choice experiments"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["option attributes", "nudges"], "benchmarks": [{"name": "ABxLab", "purpose": "agent consumer choice", "is_new": true}], "results": [], "workflow": [], "summary": "ABxLab manipulates attributes in a shopping environment to study LLM agent choice."}
{"id": 17811, "in_agent_scope": false, "types": [{"area": "multiling/mt", "role": "benchmark", "why": "Word-level contextual gender annotation guidelines for English-Italian MT"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "ConGA provides linguistically grounded gender annotation guidelines for evaluating gender handling in MT."}
{"id": 11999, "in_agent_scope": false, "types": [{"area": "knowledge/halluc", "role": "analysis", "why": "Assesses epistemic faithfulness of LLM explanations via counterfactuals"}, {"area": "knowledge/halluc", "role": "method", "why": "Attribution guidance to improve faithfulness"}], "pattern": "vary-conditions", "fixes": ["LLMs"], "varies": ["attribution guidance"], "benchmarks": [], "results": [], "workflow": [], "summary": "Faithful Serum uses attribution guidance to close the faithfulness gap in textual explanations."}
{"id": 14922, "in_agent_scope": false, "types": [{"area": "apps/domain", "role": "application", "why": "Probabilistic Quality Graph framework for patent quality assessment and refinement"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "P-QuASAR jointly models patent quality dimensions for assessment and refinement."}