Files
ai-taxonomy-literature-anal…/data/labels/o38.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
33 KiB
JSON

{"id": 16665, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Multi-agent MLLM debate rounds refine implicit attribute value extraction, no training"}], "pattern": "vary-scaffold", "fixes": ["MLLM weights"], "varies": ["debate rounds", "agent configurations"], "benchmarks": [{"name": "ImplicitAVE", "purpose": "evaluate debate gains on attribute extraction", "is_new": false}], "results": [], "workflow": ["Agents infer attributes", "Debate and verify answers", "Update over rounds"], "summary": "Multi-agent debate among MLLMs improves implicit attribute value extraction in e-commerce, studying debate configurations."}
{"id": 16719, "in_agent_scope": false, "types": [{"area": "train/pretrain", "role": "analysis", "why": "Studies recursive self-training of OLMo-2 via continual pretraining to explain self-improvement versus collapse"}], "pattern": "vary-conditions", "fixes": ["OLMo-2 models", "training procedure"], "varies": ["prompting strategies", "data filtering", "model size"], "benchmarks": [], "results": [], "workflow": [], "summary": "Shows naive recursive self-training on self-generated text does not improve LLMs, clarifying mixed outcomes of self-improvement versus collapse."}
{"id": 2952, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "RL training with self-reflection and two-stage cold start for deeper search agent tool-use"}], "pattern": "fix-env-vary-agent", "fixes": ["web search environment"], "varies": ["training framework", "self-reflection data"], "benchmarks": [], "results": [], "workflow": ["Build reflection-annotated dataset", "Cold-start SFT", "RL with reflection"], "summary": "WebSeer trains a search agent with RL and self-reflection to produce longer, more reflective tool-use trajectories."}
{"id": 13585, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "Benchmark for consistency and limit-awareness of tool-using in-car assistant agents under uncertainty"}], "pattern": "fix-env-vary-agent", "fixes": ["agents under test"], "varies": ["tasks (hallucination, disambiguation)"], "benchmarks": [{"name": "CAR-bench", "purpose": "evaluate consistency and limit-awareness of in-car agents", "is_new": true}], "results": [], "workflow": [], "summary": "CAR-bench evaluates LLM agents in an in-car assistant domain with simulated users, 58 tools, hallucination and disambiguation tasks."}
{"id": 9957, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Evolutionary search over multi-agent system configurations with feedback-guided mutation and experience memory"}], "pattern": "vary-scaffold", "fixes": ["LLM weights"], "varies": ["MAS configuration", "evolution operators"], "benchmarks": [], "results": [], "workflow": ["Select initial configs", "Mutate and crossover with traces", "Refine pool and memory"], "summary": "EvoMAS generates multi-agent systems by evolving structured configurations guided by execution traces and experience memory."}
{"id": 842, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "BLIND-ACT benchmark of 90 tasks measuring computer-use agents' blind goal-directedness"}, {"area": "agent-safety", "role": "analysis", "why": "Characterizes blind goal-directedness patterns across nine frontier models"}], "pattern": "fix-env-vary-agent", "fixes": ["OSWorld environment"], "varies": ["tasks", "models evaluated"], "benchmarks": [{"name": "BLIND-ACT", "purpose": "evaluate blind goal-directedness of CUAs", "is_new": true}, {"name": "OSWorld", "purpose": "base environment for tasks", "is_new": false}], "results": [], "workflow": [], "summary": "Shows computer-use agents pursue goals regardless of feasibility or safety, with a 90-task benchmark built on OSWorld."}
{"id": 9012, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Training-free cache of distilled error schemata for error localization in multi-agent system traces"}], "pattern": "vary-scaffold", "fixes": ["LLM weights"], "varies": ["error recognition framework"], "benchmarks": [], "results": [], "workflow": ["Distill error schemata", "Cache online", "Reuse for new failures"], "summary": "CORRECT is a training-free framework that transfers cached failure schemata to localize errors in multi-agent system trajectories."}
{"id": 10000, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "LLMs plan and construct tools for quantitative reasoning, refined via self-play feedback in digital twin markets"}], "pattern": "vary-scaffold", "fixes": ["LLM weights"], "varies": ["tool creation", "feedback-based refinement"], "benchmarks": [], "results": [], "workflow": ["Plan and build tools", "Analyze outputs under market conditions", "Refine tool selection"], "summary": "Decouples planning from computation by letting LLMs create numerical tools and self-evolve them using performance feedback in digital twin markets."}
{"id": 3822, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Multi-granularity memory association, adaptive selection and retrieval for long-term conversational agents"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory structure", "retrieval"], "benchmarks": [], "results": [], "workflow": ["Build multi-granularity units", "Associate memories", "Adaptively select and retrieve"], "summary": "MemGAS improves long-term dialogue memory using multi-granularity association and adaptive selection."}
{"id": 15861, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Two-stage SFT then GRPO on Qwen2.5-VL-3B for mobile GUI agent"}, {"area": "web-gui", "role": "application", "why": "Device-cloud collaborative mobile agent system"}], "pattern": "fix-env-vary-agent", "fixes": ["mobile GUI benchmarks"], "varies": ["training recipe", "device-cloud system"], "benchmarks": [], "results": [], "workflow": ["Synthesize GUI data", "SFT then GRPO", "Device-cloud routing"], "summary": "OpenPhone builds a mobile GUI agent from a 3B model trained with SFT and GRPO plus device-cloud collaboration."}
{"id": 4240, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "environment", "why": "Automated multi-agent pipeline converting datasets into verified MLE challenges"}, {"area": "code-swe", "role": "data", "why": "Scales MLE tasks usable as training data"}], "pattern": "fix-agent-vary-env", "fixes": ["LLM agents"], "varies": ["MLE task set"], "benchmarks": [], "results": [], "workflow": ["Generate task", "Verify structure and semantics", "Execute to validate"], "summary": "MLE-Smith automatically turns raw datasets into competition-style MLE tasks with verification for scalable benchmarks and training."}
{"id": 12438, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Confidence-guided tree search test-time scaling improved by RL on intrinsic confidence"}, {"area": "reason/math", "role": "method", "why": "Evaluated on mathematical reasoning benchmarks"}], "pattern": "vary-scaffold", "fixes": ["benchmarks"], "varies": ["test-time scaling method"], "benchmarks": [], "results": [], "workflow": ["Tree search by confidence", "RL calibrates confidence"], "summary": "Guided by Gut makes small LLMs match larger ones using confidence-guided tree search without reward models or verifiers."}
{"id": 10032, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Q-guided alignment of return-to-go for conditioned sequence models in offline RL"}, {"area": "other/ml", "role": "theory", "why": "Shows near-optimal policy when RTG sufficiently high"}], "pattern": "n/a", "fixes": ["offline RL benchmarks"], "varies": ["CSM alignment method"], "benchmarks": [], "results": [], "workflow": ["Q function guides CSM", "RTG perturbation fine-tuning"], "summary": "Q-Align DT aligns return-to-go with policy Q-values in decision transformers, with theory and experiments."}
{"id": 11976, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Hierarchical graph plus episodic event memory with agentic associative fusion for LLM agents"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory architecture"], "benchmarks": [], "results": [], "workflow": ["Form Episodic Event Frames", "Graph memory layer", "Associative fusion retrieval"], "summary": "SEEM is a structured episodic and graph memory framework for long-term agent interaction."}
{"id": 12167, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Multi-agent framework ExtAgents scales external knowledge input beyond context windows at inference"}], "pattern": "vary-scaffold", "fixes": ["LLM weights"], "varies": ["agent orchestration"], "benchmarks": [{"name": "∞Bench+", "purpose": "enhanced multi-hop QA test", "is_new": true}], "results": [], "workflow": ["Distribute knowledge across agents", "Coordinate and integrate"], "summary": "ExtAgents uses multi-agent collaboration to integrate massive external knowledge without longer-context training."}
{"id": 7630, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "application", "why": "Coding LLM framework generating 4D worlds as simulation code with physics-aware closed loop"}], "pattern": "vary-scaffold", "fixes": ["coding LLM"], "varies": ["generation framework"], "benchmarks": [], "results": [], "workflow": ["Retrieve object generation", "Hierarchical environment orchestration", "Physics-aware closed loop"], "summary": "Code2Worlds generates 4D dynamic worlds via language-to-simulation code with a dual-stream design and physics feedback."}
{"id": 12324, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Bio-inspired Hebbian graph memory with consolidation and spreading activation for LLM agents"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory architecture"], "benchmarks": [], "results": [], "workflow": ["Episodic memory graph", "Hebbian consolidation", "Spreading activation retrieval"], "summary": "HeLa-Mem models agent memory as a Hebbian dynamic graph with association and consolidation."}
{"id": 1671, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "Automated red-teaming agent with adaptive memory discovering vulnerabilities in code agents"}], "pattern": "fix-agent-vary-env", "fixes": ["target code agents"], "varies": ["attack tools and combinations"], "benchmarks": [], "results": [], "workflow": ["Retrieve jailbreak knowledge", "Select tools adaptively", "Attack code agents"], "summary": "RedCodeAgent is an automated red-teaming agent that uncovers vulnerabilities in diverse code agents."}
{"id": 12024, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Planning-guided math tutoring LLM with assessment-driven memory for multi-turn dialogue"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["tutoring framework"], "benchmarks": [], "results": [], "workflow": ["Generate stepwise plan", "Assess learner state", "Update tutoring memory"], "summary": "ScaffoldLM plans pedagogical steps and uses assessment-driven memory for multi-turn math tutoring."}
{"id": 13369, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "data", "why": "Autonomous exploration and hindsight experience synthesize planning training data for small MLLM web agents"}, {"area": "web-gui", "role": "analysis", "why": "Hierarchical analysis of compositional generalization across task granularities"}], "pattern": "vary-data", "fixes": ["small open MLLM"], "varies": ["training data source"], "benchmarks": [], "results": [], "workflow": ["Explore environments", "Hindsight relabel experiences", "Synthesize training data"], "summary": "PEEU synthesizes aligned planning data from exploration and hindsight; TDHAF analyzes compositional generalization for GUI agents."}
{"id": 9288, "in_agent_scope": false, "types": [{"area": "knowledge/halluc", "role": "method", "why": "Decoding-time guidance enforcing sentence-level citations with verified fact pool"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["decoding procedure"], "benchmarks": [], "results": [], "workflow": ["Build Prefix-Tail fact pool", "Lookahead intent prediction", "Replace with verified fact"], "summary": "Guidance enforces citation-grounded outputs during decoding using a structured fact pool."}
{"id": 16917, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "eval-method", "why": "CORE metric measuring language quality in multi-agent LLM dialogues"}, {"area": "multi-agent", "role": "analysis", "why": "Applies to competitive, cooperative, neutral game settings"}], "pattern": "vary-evaluation", "fixes": ["LLM agent pairs"], "varies": ["metric", "game settings"], "benchmarks": [], "results": [], "workflow": [], "summary": "CORE quantifies linguistic diversity of LLM agent dialogs under game-theoretic settings, finding cooperative settings more repetitive."}
{"id": 3755, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "eval-method", "why": "SelfReflect metric comparing a summary with an LLM's internal answer distribution"}, {"area": "knowledge/halluc", "role": "analysis", "why": "Finds LLMs poorly communicate internal uncertainty distributions"}], "pattern": "vary-evaluation", "fixes": ["LLMs"], "varies": ["metric"], "benchmarks": [], "results": [], "workflow": [], "summary": "SelfReflect measures faithfulness of uncertainty summaries and shows modern LLMs fail to reflect their answer distribution."}
{"id": 14409, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "Hop-aware agentic RAG benchmark with intermediate hop-level questions for diagnosing failures"}], "pattern": "fix-env-vary-agent", "fixes": ["agents under test"], "varies": ["tasks"], "benchmarks": [{"name": "AgenticRAGTracer", "purpose": "diagnose multi-hop agentic RAG", "is_new": true}], "results": [], "workflow": [], "summary": "AgenticRAGTracer is an automatically built benchmark with hop-level questions to diagnose where agentic RAG fails."}
{"id": 3700, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "100 expert-curated citation-grounded deep research tasks for single and multi-agent systems"}], "pattern": "fix-env-vary-agent", "fixes": ["agent systems under test"], "varies": ["tasks"], "benchmarks": [{"name": "LiveResearchBench", "purpose": "evaluate deep research agents", "is_new": true}], "results": [], "workflow": [], "summary": "LiveResearchBench provides 100 expert-curated dynamic deep research tasks to evaluate citation-grounded report generation."}
{"id": 11476, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Training-free attention-guided debiasing of position bias in multi-image retrieval for MLLMs"}], "pattern": "n/a", "fixes": ["MLLM weights"], "varies": ["calibration method"], "benchmarks": [{"name": "MS-COCO", "purpose": "multi-image retrieval benchmarks", "is_new": false}], "results": [], "workflow": ["Detect logit-attention divergence", "Attention-guided correction"], "summary": "Uses intrinsic attention to calibrate position bias in MLLM multi-image retrieval without training."}
{"id": 3352, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Generative reasoning process reward model WebPRM for web navigation"}], "pattern": "fix-env-vary-agent", "fixes": ["web environments"], "varies": ["reward model"], "benchmarks": [], "results": [], "workflow": ["Reasoning-first reward generation", "Principle induction"], "summary": "WebArbiter is a generative reasoning process reward model that scores web agent steps with interpretable justifications."}
{"id": 12849, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "benchmark", "why": "LoCoMo-Plus cognitive memory benchmark with latent constraints"}, {"area": "memory-selfevolve", "role": "eval-method", "why": "Constraint-consistency evaluation framework replacing string matching"}], "pattern": "vary-both", "fixes": ["models and memory systems"], "varies": ["tasks", "metric"], "benchmarks": [{"name": "LoCoMo-Plus", "purpose": "evaluate cognitive memory under cue-trigger disconnect", "is_new": true}], "results": [], "workflow": [], "summary": "LoCoMo-Plus evaluates implicit constraint retention in long conversations with a constraint-consistency evaluation framework."}
{"id": 10044, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Actor-Refiner RL framework with fine-grained credit assignment for search-integrated reasoning"}], "pattern": "fix-env-vary-agent", "fixes": ["search environment"], "varies": ["training method"], "benchmarks": [], "results": [], "workflow": ["Actor produces trajectory", "Meta-Refiner repairs steps", "Joint optimization"], "summary": "Search-R2 jointly trains an actor and refiner to improve credit assignment in search-integrated reasoning."}
{"id": 8147, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "Context-aware privacy exfiltration attack through malicious MCP servers"}], "pattern": "fix-agent-vary-env", "fixes": ["MCP-enabled agents"], "varies": ["adversarial MCP server"], "benchmarks": [], "results": [], "workflow": ["Identify tool-relevant privacy items", "Transform benign MCP server", "Exfiltrate covertly"], "summary": "SOPE turns benign MCP servers into stealthy privacy-exfiltrating variants that evade defenses."}
{"id": 12029, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "RL-trained memory-augmented attacker for long-horizon tool attacks on LLM multi-agent systems"}], "pattern": "fix-agent-vary-env", "fixes": ["LLM-MAS targets"], "varies": ["attack policy"], "benchmarks": [], "results": [], "workflow": ["Build attack memory", "Deliberative strategy", "Attack-Flow GRPO"], "summary": "Evo-Attacker uses memory and GRPO to learn long-horizon tool-output attacks on multi-agent systems."}
{"id": 8509, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Systematic study of process verification paradigms and verifiers across six MAS frameworks"}], "pattern": "vary-conditions", "fixes": ["MAS frameworks"], "varies": ["verifier type", "granularity", "context strategy"], "benchmarks": [], "results": [], "workflow": [], "summary": "MAS-ProVe empirically studies whether process verification helps multi-agent systems across verifiers and granularities."}
{"id": 15834, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "LearnGUI benchmark for demonstration-based learning in mobile agents"}, {"area": "web-gui", "role": "scaffold", "why": "LearnAct modular framework leveraging visual demonstrations"}], "pattern": "vary-both", "fixes": ["backbone models"], "varies": ["benchmark", "demonstration framework"], "benchmarks": [{"name": "LearnGUI", "purpose": "study demonstration-based learning for mobile agents", "is_new": true}], "results": [], "workflow": ["Extract demo knowledge", "Retrieve", "Leverage in agent"], "summary": "LearnGUI benchmark (2,252 offline, 101 online tasks) and LearnAct framework for few-shot mobile GUI agents."}
{"id": 7447, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Game-theoretic automated design of safe multi-agent systems under compromised agents"}], "pattern": "fix-env-vary-agent", "fixes": ["threat model (compromised subset)"], "varies": ["system design"], "benchmarks": [], "results": [], "workflow": ["Meta-Agent proposes design", "Meta-Adversary compromises agents", "Iterate on feedback"], "summary": "MaMa frames safe agentic system design as a Stackelberg game with LLM-based adversarial search."}
{"id": 10541, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Training-free latent gating triggers explicit privacy reasoning based on privacy manifold signals"}], "pattern": "vary-scaffold", "fixes": ["base LLM"], "varies": ["gating mechanism"], "benchmarks": [{"name": "PrivacyLens", "purpose": "contextual integrity evaluation", "is_new": false}], "results": [], "workflow": ["Read latent privacy signal", "Gate explicit reasoning"], "summary": "PrivGate uses residual-stream privacy signals to selectively invoke contextual integrity reasoning efficiently."}
{"id": 12788, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Gradient-guided multi-judge prompt optimization scoring segment importance via masking"}], "pattern": "n/a", "fixes": ["LLMs"], "varies": ["prompt optimization method"], "benchmarks": [], "results": [], "workflow": ["Score segments by gradient", "Multi-judge feedback", "Optimize prompt"], "summary": "GMPO is an efficient, robust automatic prompt optimization framework using gradient approximation and multiple judges."}
{"id": 14735, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "analysis", "why": "Empirical study of reasoning versus non-reasoning VLMs in mobile GUI agents"}], "pattern": "vary-conditions", "fixes": ["benchmarks"], "varies": ["reasoning-enabled models"], "benchmarks": [], "results": [], "workflow": [], "summary": "Finds reasoning-enabled VLMs give only marginal gains, sometimes degrading, in mobile GUI agents across six model pairs."}
{"id": 16371, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "scaffold", "why": "EVPI-based structured uncertainty over tool parameters guides clarification questions"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["clarification strategy"], "benchmarks": [], "results": [], "workflow": ["Model parameter uncertainty", "Score questions by EVPI", "Ask with cost balance"], "summary": "Structured uncertainty with EVPI decides which clarifying questions tool-calling agents should ask."}
{"id": 8803, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Reference-guided video diffusion restoration for sparse-view 3D Gaussian Splatting"}], "pattern": "n/a", "fixes": ["3DGS pipeline"], "varies": ["restoration model"], "benchmarks": [], "results": [], "workflow": ["Train DiT restoration model", "Reference conditioning", "Fix 3DGS renders"], "summary": "GSFixer improves sparse-view 3DGS reconstruction using a reference-guided video diffusion prior."}
{"id": 10178, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Fine-tuned LLM agent with backjump retrospective exploration for knowledge graph QA"}], "pattern": "fix-env-vary-agent", "fixes": ["knowledge graphs"], "varies": ["reasoning framework"], "benchmarks": [], "results": [], "workflow": ["Atomic operations scaffold", "Fine-tune LLM", "Backtrack at dead ends"], "summary": "Backjump-on-Graph lets KG-reasoning agents revert from dead ends, trained with reinforcement."}
{"id": 4620, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Complexity-aware ensemble and fixes for time series foundation model anomaly detection"}], "pattern": "n/a", "fixes": ["TFM"], "varies": ["anomaly detection method"], "benchmarks": [], "results": [], "workflow": [], "summary": "Addresses overgeneralization and overstationarization in reconstruction-based anomaly detection with time series foundation models."}
{"id": 14527, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "A3 benchmark of 100 tasks on 20 online apps with essential-state evaluation"}, {"area": "web-gui", "role": "eval-method", "why": "Essential-state procedural evaluation system"}], "pattern": "fix-env-vary-agent", "fixes": ["agents under test"], "varies": ["tasks", "evaluation"], "benchmarks": [{"name": "A3", "purpose": "online mobile GUI agent evaluation", "is_new": true}, {"name": "AndroidControl", "purpose": "prior static benchmark", "is_new": false}, {"name": "AndroidWorld", "purpose": "prior offline benchmark", "is_new": false}], "results": [], "workflow": [], "summary": "Android Agent Arena benchmarks mobile GUI agents on dynamic online apps with essential-state procedural evaluation."}
{"id": 12173, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Renders agent trajectories as annotated images for optical locate-and-transcribe memory retrieval"}], "pattern": "vary-scaffold", "fixes": ["LLM agent"], "varies": ["memory representation"], "benchmarks": [], "results": [], "workflow": ["Render history to images", "Locate relevant regions", "Transcribe at retrieval"], "summary": "OCR-Memory stores long agent histories visually for token-efficient retrieval."}
{"id": 2541, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Inference algorithm decoupling concept exploration from generation to diversify samples"}], "pattern": "vary-scaffold", "fixes": ["base model"], "varies": ["inference algorithm"], "benchmarks": [], "results": [], "workflow": ["Explore concepts", "Generate per concept"], "summary": "GuidedSampling increases candidate diversity in repeated sampling, with theoretical bounds."}
{"id": 2491, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "ImpossibleBench impossible task variants measuring test-case exploitation by coding agents"}, {"area": "agent-safety", "role": "analysis", "why": "Studies cheating behaviors and mitigations"}], "pattern": "fix-env-vary-agent", "fixes": ["agents under test"], "varies": ["task variants"], "benchmarks": [{"name": "ImpossibleBench", "purpose": "measure test-case exploitation", "is_new": true}, {"name": "LiveCodeBench", "purpose": "source tasks", "is_new": false}, {"name": "SWE-bench", "purpose": "source tasks", "is_new": false}], "results": [], "workflow": [], "summary": "ImpossibleBench creates conflicting-spec task variants to measure LLM agents' propensity to exploit tests."}
{"id": 13517, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "benchmark", "why": "Six datasets on cognates and false friends for Galician, Portuguese and Spanish"}], "pattern": "fix-env-vary-agent", "fixes": ["models"], "varies": ["datasets"], "benchmarks": [{"name": "False friends datasets", "purpose": "evaluate semantic ambiguity detection", "is_new": true}], "results": [], "workflow": [], "summary": "Introduces cross-lingual false friend datasets and evaluates encoder and decoder models zero- and few-shot."}
{"id": 12127, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Multi-turn RL using expert trajectories as environment experiences with patch module"}], "pattern": "fix-env-vary-agent", "fixes": ["GUI environment"], "varies": ["RL algorithm"], "benchmarks": [], "results": [], "workflow": ["Rollout with self history", "Match expert experiences", "Patch recovery"], "summary": "EMPO trains GUI agents with experience-driven multi-turn policy optimization avoiding costly online interaction."}
{"id": 13378, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Flow-based page semantic mapping for DocVQA with theoretical grounding"}, {"area": "arch/multimodal", "role": "theory", "why": "Proves unique question-conditioned page semantics bijection"}], "pattern": "n/a", "fixes": ["DocVQA benchmarks"], "varies": ["architecture"], "benchmarks": [], "results": [], "workflow": [], "summary": "Proposes a flow-based mapping architecture for distinguishing similar pages in DocVQA, grounded in a bijection proof."}
{"id": 14441, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Self-improving three-stage multi-agent framework for failure attribution in MAS traces"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["diagnosis framework"], "benchmarks": [], "results": [], "workflow": ["Detect local anomalies", "Backward tracing", "Multi-agent hypothesis validation"], "summary": "ErrorProbe identifies responsible agents and error steps in multi-agent system failures."}
{"id": 12232, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Pyramidal multimodal memory distilling verbatim to gist for long-horizon video agents"}], "pattern": "vary-scaffold", "fixes": ["MLLM"], "varies": ["memory architecture"], "benchmarks": [], "results": [], "workflow": ["Sensory buffer", "Episodic stream", "Symbolic schema"], "summary": "MM-Mem hierarchical multimodal memory enables long-horizon video understanding."}
{"id": 16541, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "analysis", "why": "Compares translation-based and multilingual retrieval RAG strategies for multilingual QA"}], "pattern": "vary-conditions", "fixes": ["LLMs"], "varies": ["RAG strategy"], "benchmarks": [], "results": [], "workflow": [], "summary": "Evaluates tRAG versus MultiRAG across languages, noting coverage and consistency tradeoffs."}
{"id": 446, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "analysis", "why": "Introduces Visual Attention Score and analyzes cold-start in multimodal reasoning models"}, {"area": "align/rlvr", "role": "method", "why": "Attention-guided cold-start training improvements"}], "pattern": "vary-conditions", "fixes": ["base MLRM"], "varies": ["cold-start data", "attention interventions"], "benchmarks": [], "results": [], "workflow": [], "summary": "Shows text-only cold-start raises visual attention while multimodal cold-start does not; proposes attention-guided cold-start."}
{"id": 1212, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Tree-structured planning and rationalization for LLM competitive debate"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["debate framework"], "benchmarks": [], "results": [], "workflow": ["Rehearsal Tree", "Debate Flow Tree", "Allocate time budget"], "summary": "TreeDebater uses rehearsal and flow trees for strategic LLM competitive debate."}
{"id": 1639, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "scaffold", "why": "Foundation models turn language preferences into code-based multi-stage rewards for online adaptation"}], "pattern": "vary-scaffold", "fixes": ["foundation models"], "varies": ["reward construction"], "benchmarks": [], "results": [], "workflow": ["Ground preference", "Build staged rewards", "Generate code"], "summary": "ROSETTA constructs code rewards from unconstrained language preferences for embodied RL agents."}
{"id": 18539, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "scaffold", "why": "Guided recovery prompt conditioned on database status for dialogue agents"}, {"area": "tool-use", "role": "analysis", "why": "Compares strategies across six model families and database failures"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["prompt strategy", "failure conditions"], "benchmarks": [{"name": "MultiWOZ 2.2", "purpose": "fault-injected evaluation", "is_new": false}, {"name": "SGD", "purpose": "fault-injected evaluation", "is_new": false}], "results": [], "workflow": [], "summary": "Prompting-based safe recovery reduces hallucination when task-oriented dialogue database calls fail."}
{"id": 8508, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "PostTrainBench measures agents autonomously post-training LLMs under bounded compute"}, {"area": "science-data", "role": "analysis", "why": "Finds agents lag official instruct models"}], "pattern": "fix-env-vary-agent", "fixes": ["agents under test"], "varies": ["tasks"], "benchmarks": [{"name": "PostTrainBench", "purpose": "evaluate autonomous LLM post-training", "is_new": true}], "results": [], "workflow": [], "summary": "PostTrainBench benchmarks frontier agents on autonomously post-training base LLMs within 10 H100-hours."}
{"id": 4213, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "scaffold", "why": "Constraint-aware multi-agent framework for travel planning with critique and interleaved search"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["multi-agent framework"], "benchmarks": [{"name": "TravelPlanner", "purpose": "evaluate constraint-aware planning", "is_new": false}], "results": [], "workflow": ["Manage constraints", "Critique plans", "Interleaved search"], "summary": "ATLAS handles explicit, implicit and evolving constraints in travel planning with multi-agent collaboration."}
{"id": 12146, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "Bio-inspired multi-agent framework with evolution and knowledge graph for research ideation"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["multi-agent framework"], "benchmarks": [], "results": [], "workflow": ["Role agents generate ideas", "Evaluate", "Evolve with feedback"], "summary": "EvoSci evolves scientific ideas via mentor, researcher and reviewer agents with shared memory."}
{"id": 14825, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "method", "why": "LLM reasoning stages refine unsupervised text clusters"}], "pattern": "n/a", "fixes": ["clustering algorithms"], "varies": ["refinement framework"], "benchmarks": [], "results": [], "workflow": ["Verify coherence", "Adjudicate redundancy", "Ground labels"], "summary": "Uses LLMs as semantic judges to validate and restructure text cluster outputs."}
{"id": 10326, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "benchmark", "why": "AOEB multimodal embedding benchmark for agent-centric retrieval of code, tool, reasoning, memory"}], "pattern": "fix-env-vary-agent", "fixes": ["embedding models"], "varies": ["tasks"], "benchmarks": [{"name": "AOEB", "purpose": "evaluate agent-oriented embeddings", "is_new": true}], "results": [], "workflow": [], "summary": "AOEB evaluates embedding models on multi-task, multimodal agent retrieval needs."}