Files
ai-taxonomy-literature-anal…/data/labels/o08.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
34 KiB
JSON

{"id": 15743, "in_agent_scope": true, "types": [{"area": "search-research", "role": "analysis", "why": "Identifies beneficial reasoning behaviors by comparing successful and failed agentic search trajectories"}, {"area": "search-research", "role": "train-agent", "why": "Behavior Priming: SFT on behavior trajectories then RL"}], "pattern": "vary-data", "fixes": ["RL recipe"], "varies": ["SFT behavior data", "reasoning behaviors"], "benchmarks": [], "results": [], "workflow": ["Analyze trajectories", "Collect behavior trajectories", "SFT", "Standard RL"], "summary": "Identifies four beneficial reasoning behaviors in agentic search and proposes Behavior Priming, SFT before RL, to instill them."}
{"id": 3935, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "ReasoningBank memory of reasoning strategies from success and failure, plus memory-aware test-time scaling"}], "pattern": "vary-scaffold", "fixes": ["model weights"], "varies": ["memory design", "test-time scaling"], "benchmarks": [], "results": [], "workflow": ["Distill strategies from experience", "Store in memory", "Retrieve for new tasks", "Memory-aware scaling"], "summary": "Memory framework that curates generalizable reasoning strategies from successes and failures, combined with memory-aware test-time scaling."}
{"id": 1623, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "Mobile agent benchmark with single-path, multi-path, ambiguous and noisy tasks across systems"}], "pattern": "fix-agent-vary-env", "fixes": ["mobile agents under test"], "varies": ["task settings", "noise", "ambiguity"], "benchmarks": [{"name": "SMAN-Bench", "purpose": "evaluate mobile agents on varied task settings", "is_new": true}], "results": [], "workflow": [], "summary": "Benchmark evaluating VLM mobile agents under single-path, multi-path, ambiguous and noisy tasks using slot-based instruction generation."}
{"id": 8398, "in_agent_scope": false, "types": [{"area": "safety/security", "role": "defense", "why": "Gradient-guided localization and masking of critical image tokens against visual prompt injection"}], "pattern": "vary-conditions", "fixes": ["adversarial image threat"], "varies": ["defense mechanism"], "benchmarks": [], "results": [], "workflow": ["Score tokens by hidden-state gradient", "Localize critical tokens", "Mask them"], "summary": "Defense against adversarial visual prompt injection in multimodal LLMs that localizes critical image tokens via gradient norm and masks them."}
{"id": 48, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "theory", "why": "Causal formulation of reasoning tasks as selection mechanisms in latent space"}], "pattern": "n/a", "fixes": [], "varies": ["assumptions"], "benchmarks": [], "results": [], "workflow": [], "summary": "Casts reasoning tasks as a causal selection mechanism to explain difficulty and insights for reflection and self-refinement."}
{"id": 16683, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "Benchmark for GUI agents answering questions on interactive dashboards"}], "pattern": "fix-agent-vary-env", "fixes": ["GUI agents under test"], "varies": ["dashboard tasks"], "benchmarks": [{"name": "DashboardQA", "purpose": "evaluate GUI agents on interactive dashboards", "is_new": true}], "results": [], "workflow": [], "summary": "First benchmark of 292 tasks on 112 interactive dashboards assessing vision-language GUI agents."}
{"id": 16127, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "train-agent", "why": "Self-corrected in-distribution preference pairs via DPO to reduce LVLM hallucination"}], "pattern": "vary-data", "fixes": ["DPO recipe"], "varies": ["preference data source"], "benchmarks": [], "results": [], "workflow": ["Diagnose hallucinations by consensus", "Self-correct", "Build preference pairs", "DPO"], "summary": "AVES-DPO builds in-distribution preference pairs through verified self-correction to mitigate LVLM hallucinations with 5.2k samples."}
{"id": 14828, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "AutoProfiler multi-agent system infers personal attributes from pseudonymous user activity"}], "pattern": "fix-env-vary-agent", "fixes": ["target platforms"], "varies": ["adversarial agent framework"], "benchmarks": [], "results": [], "workflow": ["Retrieve user activity", "Process it", "Extract attributes", "Generate profile"], "summary": "Shows LLM agents can automatically infer sensitive profiles of pseudonymous users, using a four-agent AutoProfiler framework."}
{"id": 12376, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Hierarchical agents reasoning at global, page and element levels for multi-page documents"}, {"area": "domain-agents", "role": "application", "why": "Slide deck and visual document understanding"}], "pattern": "vary-scaffold", "fixes": ["MLLM weights"], "varies": ["agentic framework"], "benchmarks": [], "results": [], "workflow": ["Build multi-level representation", "Activate specialized agents", "Integrate reasoning"], "summary": "SlideAgent hierarchical agentic framework for multi-page visual document understanding with global, page and element agents."}
{"id": 4249, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "train-agent", "why": "RL on LM parameters for ML engineering agents, outperforming larger static models"}], "pattern": "fix-env-vary-agent", "fixes": ["MLE environment"], "varies": ["model weights via RL"], "benchmarks": [], "results": [], "workflow": [], "summary": "Shows RL-trained weaker LMs can outperform larger static models on ML engineering tasks."}
{"id": 14208, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "ProxMO credit assignment for group-based policy optimization in multi-turn agents"}], "pattern": "fix-env-vary-agent", "fixes": ["environment"], "varies": ["credit assignment method"], "benchmarks": [], "results": [], "workflow": ["Success-rate-aware weighting", "Proximity-based soft aggregation", "Policy update"], "summary": "ProxMO improves multi-turn agent RL credit assignment by integrating global context into group-based policy optimization."}
{"id": 3326, "in_agent_scope": false, "types": [{"area": "align/pref", "role": "method", "why": "Token-level importance weighting for DPO via gradient attribution and triplet loss"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["DPO objective"], "benchmarks": [], "results": [], "workflow": ["Weight tokens by gradient and Gaussian prior", "Triplet loss", "DPO"], "summary": "TI-DPO weights token importance with hybrid gradient attribution and uses a triplet loss for finer preference optimization."}
{"id": 9202, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "data", "why": "Agentic problem proposer composing skills to synthesize verifiable training data"}, {"area": "planning-decision", "role": "train-agent", "why": "Trains Agentic-Proposer-4B with MGPO"}], "pattern": "vary-data", "fixes": ["training recipe"], "varies": ["synthetic data"], "benchmarks": [], "results": [], "workflow": ["Select modular skills", "Compose problems with reflection and tools", "Train with MGPO"], "summary": "Agentic Proposing treats problem synthesis as sequential decision making and trains a 4B proposer to generate verifiable reasoning data."}
{"id": 596, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Multi-agent framework for molecular structure elucidation from IR spectra"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": ["Specialized agents interpret IR", "Integrate knowledge", "Elucidate structure"], "summary": "IR-Agent multi-agent framework emulating expert IR analysis to elucidate molecular structures."}
{"id": 17606, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "benchmark", "why": "Human-translated science QA and truthfulness benchmarks in six African languages"}], "pattern": "n/a", "fixes": ["models"], "varies": ["languages", "tasks"], "benchmarks": [{"name": "Uhura-ARC-Easy", "purpose": "science multiple-choice in African languages", "is_new": true}, {"name": "Uhura-TruthfulQA", "purpose": "truthfulness in African languages", "is_new": true}], "results": [], "workflow": [], "summary": "Uhura benchmark of science QA and truthfulness in six African languages, revealing gaps between proprietary and open models."}
{"id": 7861, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "analysis", "why": "Behavioural and probing analysis of goal-directedness of an LLM agent in grid world"}, {"area": "embodied-nav", "role": "eval-method", "why": "Framework combining behavioural evaluation with interpretability"}], "pattern": "vary-conditions", "fixes": ["LLM agent"], "varies": ["grid size", "obstacle density", "goal structure"], "benchmarks": [], "results": [], "workflow": [], "summary": "Framework evaluating goal-directedness of LLM agents in a grid world via behaviour and probing of internal representations."}
{"id": 14372, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "benchmark", "why": "Urdu minimal-pair benchmark of 5,696 pairs across syntactic phenomena"}], "pattern": "n/a", "fixes": ["models"], "varies": ["phenomena"], "benchmarks": [{"name": "UrBLiMP", "purpose": "Urdu linguistic minimal pairs", "is_new": true}], "results": [], "workflow": [], "summary": "UrBLiMP diagnostic suite of Urdu minimal pairs evaluating 21 multilingual LLMs on grammatical competence."}
{"id": 2425, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "Human-annotated paper QA dataset for AI research"}, {"area": "science-data", "role": "data", "why": "ExTrActor synthesizes agent interaction trajectories for training"}], "pattern": "vary-data", "fixes": ["agents"], "varies": ["tasks", "training trajectories"], "benchmarks": [{"name": "M4PQA", "purpose": "paper QA with instance-level evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "M4PQA paper QA benchmark plus ExTrActor, a multi-agent framework to synthesize instruction trajectories."}
{"id": 14276, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "eval-method", "why": "Proxy state-based LLM-driven simulation evaluation without deterministic backend"}, {"area": "tool-use", "role": "environment", "why": "LLM-simulated users and states for multi-turn tool-calling benchmark"}], "pattern": "vary-evaluation", "fixes": ["agents under test"], "varies": ["evaluation protocol"], "benchmarks": [{"name": "tau-bench", "purpose": "prior deterministic baseline", "is_new": false}, {"name": "tau^2-bench", "purpose": "prior deterministic baseline", "is_new": false}, {"name": "AppWorld", "purpose": "prior deterministic baseline", "is_new": false}], "results": [], "workflow": ["Scenario spec", "LLM state tracker infers proxy state", "LLM judges verify"], "summary": "LLM-driven simulation framework with proxy state tracking to evaluate and generate training data for tool-calling agents without deterministic databases."}
{"id": 13199, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "train-agent", "why": "RL with Shapley Information Gain reward for proactive medical questioning"}], "pattern": "fix-env-vary-agent", "fixes": ["environment"], "varies": ["reward", "training pipeline"], "benchmarks": [], "results": [], "workflow": ["MCTS trajectory SFT init", "RL with SIG reward"], "summary": "ProMed trains medical LLMs via RL with Shapley Information Gain reward to ask clinically valuable questions proactively."}
{"id": 16705, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "analysis", "why": "Taxonomy of ambiguity in table QA and test of LLMs detecting it"}], "pattern": "vary-conditions", "fixes": ["methods"], "varies": ["ambiguity types"], "benchmarks": [], "results": [], "workflow": [], "summary": "Outlines types of ambiguity in table question answering and tests LLMs' ability to detect them."}
{"id": 13921, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "scaffold", "why": "LLM-driven incremental map construction with version control and graph repair"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["map construction framework"], "benchmarks": [], "results": [], "workflow": ["Build graph from observations", "Version control history", "Edge Impact Score repair"], "summary": "Framework for LLM incremental topological map construction with version control and minimal-cost repair of inconsistencies."}
{"id": 15053, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "analysis", "why": "Systematic study of experiential knowledge acquisition and activation for tool calling"}, {"area": "tool-use", "role": "scaffold", "why": "Width-expanding parallel sampling with aggregation to activate knowledge"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["knowledge forms", "activation methods"], "benchmarks": [], "results": [], "workflow": [], "summary": "Studies how experiential knowledge affects tool use, finding instance-level knowledge and parallel sampling most effective."}
{"id": 15188, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "benchmark", "why": "AskBench interactive benchmark for clarification and false-premise correction"}, {"area": "social-sim", "role": "train-agent", "why": "Rubric-guided RLVR for targeted clarification"}], "pattern": "vary-both", "fixes": ["QA pairs"], "varies": ["benchmark", "reward rubric"], "benchmarks": [{"name": "AskBench", "purpose": "clarification ability of LLMs", "is_new": true}], "results": [], "workflow": ["Convert QA to multi-turn", "Judge loop simulates user", "Rubric RLVR"], "summary": "AskBench evaluates when and what LLMs ask for clarification, plus rubric-guided RLVR to improve it."}
{"id": 1890, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "A2A-specific security benchmark for multi-agent systems"}, {"area": "agent-safety", "role": "attack", "why": "Six concrete attacks on A2A protocol stages"}], "pattern": "fix-agent-vary-env", "fixes": ["MAS stacks"], "varies": ["attack vectors"], "benchmarks": [{"name": "A2ASecBench", "purpose": "A2A security evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "First security evaluation of agent-to-agent protocols, with taxonomy, six attacks and A2ASecBench benchmark framework."}
{"id": 9820, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Incident response framework detecting, containing and recovering from agent incidents"}], "pattern": "vary-scaffold", "fixes": ["threat model"], "varies": ["protective mechanism"], "benchmarks": [], "results": [], "workflow": ["Detect via semantic checks", "Contain and recover", "Synthesize guardrail rules"], "summary": "AIR is an incident response framework with a DSL integrated into the agent loop to detect, contain and learn from incidents."}
{"id": 12899, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "train-agent", "why": "Evidence-guided dynamic preference optimization for Med-LVLM hallucination"}], "pattern": "vary-data", "fixes": ["Med-LVLM"], "varies": ["preference optimization method"], "benchmarks": [], "results": [], "workflow": [], "summary": "DEPO dynamic evidence-guided preference optimization to align Med-LVLMs with visual evidence and reduce hallucinations."}
{"id": 17807, "in_agent_scope": false, "types": [{"area": "safety/fairness", "role": "benchmark", "why": "Filipino BBQ bias benchmark with over 10,000 prompts"}, {"area": "safety/fairness", "role": "eval-method", "why": "Robust evaluation protocol accounting for response instability"}], "pattern": "vary-evaluation", "fixes": ["models"], "varies": ["protocol"], "benchmarks": [{"name": "FilBBQ", "purpose": "Filipino stereotype bias", "is_new": true}, {"name": "BBQ", "purpose": "source format", "is_new": false}], "results": [], "workflow": [], "summary": "FilBBQ extends BBQ to Filipino context and applies a robust evaluation accounting for response instability."}
{"id": 14307, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Training-free two-stage planned active retrieval and reasoning for multi-hop QA"}], "pattern": "vary-scaffold", "fixes": ["model weights"], "varies": ["retrieval and reasoning workflow"], "benchmarks": [], "results": [], "workflow": ["Breadth-first anchoring", "Depth-first refinement", "Sufficiency control"], "summary": "PAR^2-RAG training-free framework separating evidence coverage from commitment for multi-hop question answering."}
{"id": 2472, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Attention-guided image warping at test time for MLLMs"}], "pattern": "n/a", "fixes": ["model weights"], "varies": ["input image warping"], "benchmarks": [{"name": "TextVQA", "purpose": "evaluate warping", "is_new": false}, {"name": "GQA", "purpose": "evaluate warping", "is_new": false}, {"name": "DocVQA", "purpose": "evaluate warping", "is_new": false}, {"name": "POPE", "purpose": "evaluate warping", "is_new": false}, {"name": "MMMU", "purpose": "evaluate warping", "is_new": false}, {"name": "MIA-Bench", "purpose": "evaluate warping", "is_new": false}, {"name": "MMVP", "purpose": "evaluate warping", "is_new": false}, {"name": "VQAv2", "purpose": "evaluate warping", "is_new": false}], "results": [], "workflow": ["Get cross-modal attention", "Warp image", "Feed MLLM"], "summary": "AttWarp reallocates image resolution to query-relevant regions using MLLM attention, improving fine-grained perception without changing weights."}
{"id": 17452, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "scaffold", "why": "Feedback-guided gradient prompt optimization in embedding space for LLM agents"}], "pattern": "vary-scaffold", "fixes": ["model weights"], "varies": ["prompt optimization"], "benchmarks": [], "results": [], "workflow": ["Feedback error analysis", "Gradient in embedding space", "Decode prompt"], "summary": "CASPER bridges discrete and continuous prompt optimization via feedback-guided gradient descent for workflow automation agents."}
{"id": 3146, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "scaffold", "why": "VLM-driven humanoid agent with instruction compiler and diffusion motion executor"}], "pattern": "vary-scaffold", "fixes": ["VLM"], "varies": ["agent pipeline"], "benchmarks": [], "results": [], "workflow": ["Observe scene", "Compile instructions to parameters via VQA", "Diffusion motion executor"], "summary": "BiBo lets off-the-shelf VLMs control humanoids via an embodied instruction compiler and diffusion-based motion executor."}
{"id": 12160, "in_agent_scope": false, "types": [{"area": "safety/fairness", "role": "benchmark", "why": "Demonstratives human baseline probing embodied cognition and cultural variation"}, {"area": "safety/fairness", "role": "analysis", "why": "Finds LLMs miss proximal-distal contrast and cultural differences"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["language", "demonstratives"], "benchmarks": [], "results": [], "workflow": [], "summary": "Uses demonstratives with human data from English and Chinese speakers to show LLMs fail to capture embodied and cultural cues."}
{"id": 7072, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Prompted constitutional black-box monitors for scheming optimized on synthetic trajectories"}], "pattern": "vary-scaffold", "fixes": ["threat model"], "varies": ["monitor prompts", "synthetic data pipeline"], "benchmarks": [], "results": [], "workflow": ["Generate synthetic trajectories", "Optimize monitor prompts", "Evaluate scheming detection"], "summary": "Constitutional black-box monitors detect scheming in LLM agents, optimized using STRIDE and Gloom synthetic trajectory pipelines."}
{"id": 13587, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "Knowledge-based multi-entity multi-hop VQA benchmark for MLLMs"}], "pattern": "n/a", "fixes": ["16 MLLMs"], "varies": ["knowledge settings"], "benchmarks": [{"name": "M^3-VQA", "purpose": "multi-entity multi-hop VQA", "is_new": true}], "results": [], "workflow": [], "summary": "M^3-VQA benchmark evaluating 16 MLLMs on multi-entity multi-hop knowledge VQA."}
{"id": 14593, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Episodic memory units with MemDecider filter for conversational memory"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory design"], "benchmarks": [], "results": [], "workflow": ["Form episodic memory units", "Filter with MemDecider", "Retrieve for response"], "summary": "EMA organizes dialog into episodic memory units and filters with MemDecider to reduce noise and latency."}
{"id": 12429, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "method", "why": "Multimodal noise-robust ASR using semantic guidance and spectrogram cues"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["ASR method"], "benchmarks": [], "results": [], "workflow": [], "summary": "Speech-MLM reframes noisy ASR as semantics-guided reconstruction combining speech, spectrogram visuals and text variants."}
{"id": 13048, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Rank aggregation credit assignment for cooperative multi-agent RL with multimodal comparisons"}], "pattern": "n/a", "fixes": ["tasks"], "varies": ["credit assignment"], "benchmarks": [], "results": [], "workflow": [], "summary": "MARS-RA reformulates credit assignment as rank aggregation of LMM-generated pairwise contributions for reward shaping."}
{"id": 4135, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Decoding-time dynamic persona importance estimation for role-playing agents"}], "pattern": "vary-scaffold", "fixes": ["model weights"], "varies": ["decoding method"], "benchmarks": [], "results": [], "workflow": ["Estimate persona importance", "Weighted reward-guided decoding"], "summary": "Inference-time persona following via dynamic context-dependent importance estimation integrated in reward-guided decoding."}
{"id": 15466, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "CompassMem event-centric logical memory for agent reasoning"}], "pattern": "vary-scaffold", "fixes": ["model weights"], "varies": ["memory structure"], "benchmarks": [], "results": [], "workflow": ["Build event graph", "Logical relations", "Structure-guided retrieval"], "summary": "CompassMem event-centric memory capturing logical relations among experiences to support long-horizon agent reasoning."}
{"id": 12275, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Co-evolving critic and policy in critique-guided RL"}], "pattern": "fix-env-vary-agent", "fixes": ["environment"], "varies": ["critic", "policy training"], "benchmarks": [], "results": [], "workflow": ["Critic diagnoses trajectory", "Policy refines", "Joint optimization"], "summary": "ECHO jointly optimizes policy and critic in a synchronized co-evolutionary loop to avoid stale feedback."}
{"id": 12329, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "GRPO with Identify-Reason-Verify protocol for digital whiteboard agent"}], "pattern": "fix-env-vary-agent", "fixes": ["environment"], "varies": ["training method"], "benchmarks": [], "results": [{"benchmark": "digital whiteboard tasks", "system": "3B specialized agent", "base_model": null, "setting": "proposed", "metric": "operation accuracy", "score": 81.12, "unit": "%"}], "workflow": ["Distill reasoning schemas", "GRPO with verifiable rewards"], "summary": "Verifiable RL pipeline trains a 3B agent to execute precise spatial operations on digital whiteboards."}
{"id": 14713, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Entropy-based advantage shaping for planning tokens in deep research RL"}], "pattern": "fix-env-vary-agent", "fixes": ["environment"], "varies": ["advantage shaping"], "benchmarks": [], "results": [], "workflow": ["Identify high-entropy planning tokens", "Shape advantage", "End-to-end RL"], "summary": "DeepPlanner shapes token-level advantage with entropy to better optimize planning in deep research agents."}
{"id": 9179, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Instructor model gives step-specific privacy guidance, trained by RL from failure trajectories"}], "pattern": "vary-scaffold", "fixes": ["threat model"], "varies": ["defense paradigm"], "benchmarks": [], "results": [], "workflow": ["Instructor generates guidance", "Agent acts", "RL on failure environments"], "summary": "Contextualized Defense Instructing: an RL-trained instructor proactively guides agents toward privacy-preserving actions."}
{"id": 4258, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Tests whether multi-agent architecture mitigates bias amplification"}, {"area": "multi-agent", "role": "benchmark", "why": "Discrim-Eval-Open open-ended bias benchmark"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["MAS structure", "sensitive attributes"], "benchmarks": [{"name": "Discrim-Eval-Open", "purpose": "system-level bias in MAS", "is_new": true}], "results": [], "workflow": [], "summary": "Measures bias amplification in LLM multi-agent systems with new open-ended benchmark and metrics."}
{"id": 9797, "in_agent_scope": true, "types": [{"area": "science-data", "role": "method", "why": "Automated learning of randomized self-reductions including LLM agent approach"}], "pattern": "n/a", "fixes": ["RSR-Bench"], "varies": ["learning backends"], "benchmarks": [{"name": "RSR-Bench", "purpose": "evaluate RSR discovery", "is_new": true}], "results": [{"benchmark": "RSR-Bench", "system": "linear regression backend", "base_model": null, "setting": "proposed", "metric": "functions with RSR discovered", "score": 43, "unit": "of 80"}], "workflow": [], "summary": "Bitween automates discovery of randomized self-reductions via regression backends and LLM agents."}
{"id": 15126, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "Tree-structured notebook context with entropy-based path evaluation and Data Shadow"}], "pattern": "vary-scaffold", "fixes": ["model weights"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": ["Notebook cells as tree nodes", "Entropy-based path evaluation", "Data Shadow checks"], "summary": "Tree-Notebook data science agent using tree search over notebook cells and Data Shadow to catch silent failures."}
{"id": 4801, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "RL teaching when and how to invoke code interpreter in reasoning"}], "pattern": "fix-env-vary-agent", "fixes": ["code interpreter"], "varies": ["training"], "benchmarks": [], "results": [], "workflow": ["Synthetic cold-start data", "RL with tool rollouts"], "summary": "ReTool trains LLMs with RL to interleave code execution into long-form reasoning."}
{"id": 8670, "in_agent_scope": false, "types": [{"area": "align/pref", "role": "train-agent", "why": "Multi-agent self-play with meta-evaluation to train on non-verifiable tasks"}], "pattern": "n/a", "fixes": ["policy"], "varies": ["training reward signal"], "benchmarks": [], "results": [], "workflow": ["Generate", "Critique", "Meta-evaluate critique usefulness", "Train"], "summary": "CoNL unifies generation, evaluation and meta-evaluation via self-play for non-verifiable tasks."}
{"id": 15695, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "Multi-hop ambiguous question benchmark with 2,209 questions"}], "pattern": "n/a", "fixes": ["models"], "varies": ["ambiguity"], "benchmarks": [{"name": "MARCH", "purpose": "ambiguity in multi-hop QA", "is_new": true}], "results": [], "workflow": [], "summary": "MARCH benchmark of multi-hop ambiguous questions and a proposed approach to address the challenge."}
{"id": 15196, "in_agent_scope": false, "types": [{"area": "knowledge/editing", "role": "method", "why": "Critic-guided multi-agent closed-loop reasoning for multi-hop knowledge editing"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["editing pipeline"], "benchmarks": [], "results": [], "workflow": ["Decompose", "Retrieve edited facts", "Critic verifies", "Revise"], "summary": "CARE uses critic-guided multi-agent verification for multi-hop knowledge editing."}
{"id": 14069, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "eval-method", "why": "Automatic multi-level evaluation of agent behavior at system, trace and node"}], "pattern": "vary-evaluation", "fixes": ["agents under test"], "varies": ["evaluation framework"], "benchmarks": [], "results": [], "workflow": [], "summary": "Agentic CLEAR gives automatic textual insights on agent behavior at system, trace and node levels."}
{"id": 11955, "in_agent_scope": true, "types": [{"area": "science-data", "role": "train-agent", "why": "SFT plus outcome RL for tool-augmented spectral vetting agent"}, {"area": "science-data", "role": "application", "why": "Rare celestial object vetting in LAMOST"}], "pattern": "fix-env-vary-agent", "fixes": ["tasks"], "varies": ["model training"], "benchmarks": [{"name": "LAMOST", "purpose": "rare-object identification tasks", "is_new": false}], "results": [{"benchmark": "LAMOST rare-object tasks", "system": "Spec-o3", "base_model": "7B", "setting": "proposed", "metric": "macro-F1", "score": 76.5, "unit": "%"}], "workflow": ["Cold-start SFT", "Outcome-based RL"], "summary": "Spec-o3 tool-augmented VLM agent for rare celestial object vetting trained with SFT then RL."}
{"id": 8808, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "Benchmark for outcome and process reward models on computer-using agent trajectories"}, {"area": "web-gui", "role": "eval-method", "why": "Reward models as step-wise CUA evaluators"}], "pattern": "fix-agent-vary-env", "fixes": ["reward models under test"], "varies": ["trajectories"], "benchmarks": [{"name": "CUARewardBench", "purpose": "reward models for CUAs", "is_new": true}], "results": [], "workflow": [], "summary": "First benchmark evaluating outcome and process reward models on trajectories from 10 software categories and 7 agent architectures."}
{"id": 13442, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "attack", "why": "TraitBasis steers simulated user traits to stress test agents"}, {"area": "tool-use", "role": "benchmark", "why": "Extends tau-bench with varied user behavior"}], "pattern": "vary-conditions", "fixes": ["agents"], "varies": ["user traits"], "benchmarks": [{"name": "tau-bench", "purpose": "extended with user traits", "is_new": false}], "results": [], "workflow": ["Learn trait directions", "Steer user simulator", "Evaluate agents"], "summary": "TraitBasis learns activation directions of user traits to stress-test conversational agents in an extended tau-bench."}
{"id": 306, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Annotation-free RL for ambiguity-aware QA with AnsF1 reward"}, {"area": "search-research", "role": "data", "why": "Automated pipeline detects ambiguous questions and gathers alternative answers"}], "pattern": "vary-data", "fixes": ["RL framework"], "varies": ["reward", "ambiguity data"], "benchmarks": [{"name": "HotpotQA", "purpose": "multi-hop QA", "is_new": false}, {"name": "MuSiQue", "purpose": "multi-hop QA", "is_new": false}], "results": [], "workflow": ["Detect ambiguous questions", "Gather alternative answers", "RL with AnsF1"], "summary": "A2Search trains QA models with RL to handle multiple valid answers without manual annotation."}
{"id": 12156, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "Unified evaluation framework for agent cascading injection in MAS"}, {"area": "agent-safety", "role": "attack", "why": "Attack surfaces and objectives for compromised agents"}], "pattern": "fix-agent-vary-env", "fixes": ["MAS"], "varies": ["attack surfaces", "objectives"], "benchmarks": [{"name": "ACIArena", "purpose": "MAS robustness to cascading injection", "is_new": true}], "results": [], "workflow": [], "summary": "ACIArena evaluates multi-agent system robustness to cascading injection across attack surfaces and objectives."}
{"id": 15692, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "Capacity-aligned agent distillation to small models"}, {"area": "tool-use", "role": "data", "why": "Selects minimum-NLL teacher trajectory per example"}], "pattern": "vary-data", "fixes": ["teacher"], "varies": ["trajectory selection", "loss weighting"], "benchmarks": [], "results": [], "workflow": ["Select trajectory by NLL", "Weighted supervision", "Distill SLM"], "summary": "SmartAD distills tool-using agent skills to SLMs by selecting compatible trajectories and weighting supervision."}
{"id": 4282, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "Benchmark suite for instruction-guided video editing with 600 source videos"}], "pattern": "n/a", "fixes": ["editing models"], "varies": ["tasks"], "benchmarks": [{"name": "IVEBench", "purpose": "video editing evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "IVEBench: 600 source videos, 8 editing task categories and extended metrics for instruction-guided video editing."}
{"id": 3048, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "scaffold", "why": "Hierarchical agentic diagnostic workflow with guideline RAG for multimodal medical diagnosis"}], "pattern": "vary-scaffold", "fixes": ["VLMs"], "varies": ["workflow"], "benchmarks": [], "results": [], "workflow": ["Guideline-based plan", "Patient-level step reasoning", "Quantitative evidence"], "summary": "MedAgent-Pro agentic workflow for evidence-based multimodal diagnosis with disease-level planning and patient-level reasoning."}