Files
ai-taxonomy-literature-anal…/data/labels/o01.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
34 KiB
JSON

{"id": 9524, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "train-agent", "why": "Stage-aware RL trains multimodal agent for draft-then-refine poster generation"}], "pattern": "fix-env-vary-agent", "fixes": ["poster generation task"], "varies": ["RL algorithm (stage-aware)", "agentic draft-refine workflow"], "benchmarks": [], "results": [], "workflow": ["Draft poster layout", "Iteratively refine", "Stage-specific RL credit assignment"], "summary": "PosterAgent reformulates poster generation as agentic draft and refinement, trained with Stage-Aware RL for separate credit assignment."}
{"id": 12356, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Proxy agents per member discuss and synthesize consensus deliverables"}], "pattern": "vary-scaffold", "fixes": ["underlying LLM"], "varies": ["multi-agent teamwork workflow"], "benchmarks": [], "results": [], "workflow": ["Create proxy agent per member", "Structured discussion", "Synthesize deliverable and iterate"], "summary": "TeamFusion is a multi-agent system with member proxy agents that discuss and synthesize consensus-oriented deliverables for open-ended teamwork."}
{"id": 15658, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Persona prompt links personalized memory to tool actions in LLM agent"}], "pattern": "vary-scaffold", "fixes": ["LLM weights"], "varies": ["persona prompt", "memory and action modules"], "benchmarks": [], "results": [], "workflow": ["Build episodic and semantic memory", "Persona controls tool actions", "Optimize persona per user"], "summary": "PersonaAgent is a personalized LLM agent framework combining memory and action modules mediated by a per-user persona system prompt."}
{"id": 17286, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Dual-agent framework where Ideator advises implementation agent, training-free"}, {"area": "code-swe", "role": "train-agent", "why": "RL-trains Ideator to generate better ideas"}], "pattern": "vary-scaffold", "fixes": ["MLE-Bench tasks"], "varies": ["ideator agent", "Ideator RL training"], "benchmarks": [{"name": "MLE-Bench", "purpose": "evaluate MLE agents", "is_new": false}], "results": [], "workflow": ["Implementation agent requests help", "Ideator proposes strategy", "RL trains Ideator"], "summary": "MLE-Ideator separates ideation from implementation for MLE agents; RL-trained Qwen3-8B Ideator improves over untrained and surpasses Claude Sonnet 3.5."}
{"id": 56, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "theory", "why": "Theoretical expressivity bounds for multi-agent communication in reasoning"}], "pattern": "n/a", "fixes": [], "varies": ["assumptions on agents and communication"], "benchmarks": [], "results": [], "workflow": ["Define expressivity framework", "Derive agent and communication bounds"], "summary": "Theoretical framework bounding number of agents, communication and speedups for multi-agent reasoning on state tracking, recall and k-hop tasks."}
{"id": 18577, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "ZhoBLiMP Chinese minimal pair benchmark"}, {"area": "eval/judge", "role": "eval-method", "why": "SLLN-LP length-normalized metric"}], "pattern": "vary-evaluation", "fixes": ["LMs under test"], "varies": ["benchmark", "metric"], "benchmarks": [{"name": "ZhoBLiMP", "purpose": "Chinese minimal pair evaluation", "is_new": true}, {"name": "JBLiMP", "purpose": "metric validation", "is_new": false}, {"name": "BLiMP", "purpose": "metric validation", "is_new": false}], "results": [], "workflow": ["Build minimal pair paradigms", "Train Chinese LMs", "Apply SLLN-LP"], "summary": "Introduces ZhoBLiMP and the SLLN-LP metric, studying learning curves of Chinese LMs across tokenizers and sizes."}
{"id": 12559, "in_agent_scope": false, "types": [{"area": "apps/dialogue", "role": "method", "why": "Emotion-attribute retrieval to guide empathetic response generation"}], "pattern": "vary-scaffold", "fixes": ["generator LLM"], "varies": ["retrieval criteria"], "benchmarks": [{"name": "EmpatheticDialogues", "purpose": "evaluate empathetic generation", "is_new": false}], "results": [], "workflow": ["Retrieve by emotion attributes", "Token-level retrieval", "Augment candidates"], "summary": "REG retrieves exemplars by emotion similarity with token-level retrieval and candidate augmentation for empathetic dialogue."}
{"id": 12528, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "Discourse-tree hierarchical retrieval for long document QA"}], "pattern": "vary-scaffold", "fixes": ["datasets"], "varies": ["retrieval method"], "benchmarks": [], "results": [], "workflow": ["Parse discourse trees", "Enhance nodes with LLM", "Hierarchical retrieval"], "summary": "Discourse-aware hierarchical retrieval using RST structure for long-document QA, improving over chunking baselines on four datasets."}
{"id": 11628, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Graph-based step-level credit assignment for agentic RL"}], "pattern": "fix-env-vary-agent", "fixes": ["agentic tasks"], "varies": ["credit assignment algorithm"], "benchmarks": [], "results": [], "workflow": ["Build state-transition graph from rollouts", "Estimate state distance to goal", "Graph-based edge advantage"], "summary": "GraphGPO aggregates rollouts into a state graph to give step-level credit assignment in group-based agentic RL."}
{"id": 14652, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "Multi-agent insight discovery with retrieval, debate and multi-path reasoning"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["multi-agent framework components"], "benchmarks": [], "results": [], "workflow": ["Retrieve external knowledge", "Multi-role debate", "Multi-path reasoning"], "summary": "DataSage is a multi-agent data insight agent using knowledge retrieval, multi-role debate and multi-path reasoning."}
{"id": 5317, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Market-based uncertainty trading among VLM agents with bandit broker"}], "pattern": "vary-scaffold", "fixes": ["VLM agents"], "varies": ["coordination mechanism"], "benchmarks": [{"name": "MMMU", "purpose": "evaluation", "is_new": false}, {"name": "MMBench", "purpose": "evaluation", "is_new": false}, {"name": "MathVision", "purpose": "evaluation", "is_new": false}], "results": [], "workflow": ["Quantify uncertainty types", "Trade uncertainty among agents", "Bandit broker selects collaborators"], "summary": "Agora frames multi-agent VLM coordination as a market for uncertainty with Thompson-sampling broker for cost-efficiency."}
{"id": 3051, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "SFT and RL with tailored rewards to train LLMs for bandit exploration"}, {"area": "planning-decision", "role": "analysis", "why": "Studies exploitation bias in how SFT and RL shape exploration"}], "pattern": "fix-env-vary-agent", "fixes": ["multi-armed bandit task"], "varies": ["SFT vs RL", "reward design"], "benchmarks": [{"name": "multi-armed bandit", "purpose": "exploration training", "is_new": false}], "results": [], "workflow": ["SFT on expert trajectories", "RL with regret and algorithmic rewards", "Evaluate generalization"], "summary": "Trains LLMs for bandit exploration with SFT and RL rewards and examines emergent greedy exploitation bias and generalization."}
{"id": 11950, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "benchmark", "why": "GuideDog dataset and GuideDogQA for BLV navigation guidance"}], "pattern": "vary-evaluation", "fixes": ["MLLMs under test"], "varies": ["dataset"], "benchmarks": [{"name": "GuideDog", "purpose": "BLV guidance dataset", "is_new": true}, {"name": "GuideDogQA", "purpose": "object recognition and depth perception", "is_new": true}], "results": [], "workflow": ["Human-AI annotation pipeline", "Verify descriptions", "Benchmark MLLMs"], "summary": "GuideDog is a 22K image-description dataset for blind and low-vision guidance, plus an 818-sample QA benchmark."}
{"id": 15252, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "benchmark", "why": "DynaSlide benchmark for slide updating via instructions"}, {"area": "domain-agents", "role": "application", "why": "SlideAgent tool-augmented slide update system"}], "pattern": "vary-both", "fixes": ["slide update task"], "varies": ["benchmark", "agent framework"], "benchmarks": [{"name": "DynaSlide", "purpose": "slide update evaluation", "is_new": true}], "results": [], "workflow": ["Parse slides multimodally", "Ground instruction", "Tool-augmented reasoning"], "summary": "Defines dynamic slide update task, releases DynaSlide with 20,036 triples, and proposes SlideAgent."}
{"id": 6250, "in_agent_scope": false, "types": [{"area": "reason/math", "role": "analysis", "why": "Analyzes strategy usage vs executability of human and model solutions"}, {"area": "reason/math", "role": "method", "why": "Selective Strategy Retrieval for guidance"}], "pattern": "vary-conditions", "fixes": ["math problems"], "varies": ["strategy source", "guidance selection"], "benchmarks": [], "results": [], "workflow": ["Compare human and model strategies", "Select executable strategies", "Retrieve as guidance"], "summary": "Shows strategy usage differs from executability and proposes Selective Strategy Retrieval for example-based math guidance."}
{"id": 249, "in_agent_scope": true, "types": [{"area": "search-research", "role": "data", "why": "InfoSeek synthesizes hierarchical constraint satisfaction tasks for agentic search"}], "pattern": "vary-data", "fixes": ["training recipe"], "varies": ["synthetic training data"], "benchmarks": [], "results": [], "workflow": ["Seed expansion (diffusion)", "Retrospection to refine", "Build HCSP tasks"], "summary": "InfoSeek synthesizes agentic search training data modeled as hierarchical constraint satisfaction problems."}
{"id": 12322, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "analysis", "why": "Empirically tests whether VLM agents use world models as tools"}], "pattern": "vary-conditions", "fixes": ["agents"], "varies": ["simulation availability", "enforcement"], "benchmarks": [], "results": [], "workflow": [], "summary": "Finds agents rarely invoke world-model simulation, misuse rollouts, and sometimes degrade; identifies decision and integration bottlenecks."}
{"id": 18176, "in_agent_scope": false, "types": [{"area": "multiling/mt", "role": "defense", "why": "LoRA linguistic-knowledge fine-tuning to reduce gender bias in MT"}], "pattern": "vary-data", "fixes": ["Mistral-7B", "Salamandrata-7B"], "varies": ["fine-tuning with linguistic reasoning"], "benchmarks": [{"name": "MT-GenEval", "purpose": "gender accuracy evaluation", "is_new": false}], "results": [], "workflow": ["Identify gendered referents", "LoRA fine-tune", "Translate"], "summary": "LoRA fine-tuning with linguistic knowledge infusion mitigates gender bias in English-Spanish and Catalan translation."}
{"id": 17783, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "data", "why": "CEFR-annotated WordNet built via LLM similarity"}], "pattern": "vary-data", "fixes": ["classifiers"], "varies": ["annotated resource"], "benchmarks": [{"name": "WordNet", "purpose": "annotated with CEFR", "is_new": false}, {"name": "English Vocabulary Profile", "purpose": "CEFR source", "is_new": false}], "results": [], "workflow": ["LLM matches senses to EVP", "Build corpus", "Train classifiers"], "summary": "Builds CEFR-annotated WordNet using an LLM and validates by training lexical classifiers."}
{"id": 13725, "in_agent_scope": false, "types": [{"area": "apps/science", "role": "survey", "why": "Survey of LLMs for text-guided molecule generation and optimization"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Surveys LLM learning paradigms for molecule generation and optimization organized by validity, synthesizability, property control and diversity."}
{"id": 18560, "in_agent_scope": false, "types": [{"area": "train/peft", "role": "survey", "why": "Survey and guide to fine-tuning LLMs with limited data"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Structured survey of PEFT, domain and cross-lingual adaptation, and preference alignment under data scarcity."}
{"id": 15713, "in_agent_scope": false, "types": [{"area": "align/pref", "role": "method", "why": "Self-play online DPO with reasoning and response dual rewards"}], "pattern": "vary-data", "fixes": ["base LLM"], "varies": ["reward signals for exploration"], "benchmarks": [], "results": [], "workflow": ["Sample responses", "Dual reward: reasoning and quality", "Online DPO update"], "summary": "RGE-DPO uses self-rewarding reasoning rubric plus reward model in online DPO to improve general-domain reasoning."}
{"id": 1401, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "train-agent", "why": "Self-play RL with turn-level advantage for strategic multi-agent games"}], "pattern": "fix-env-vary-agent", "fixes": ["strategic games"], "varies": ["RL advantage estimation"], "benchmarks": [], "results": [], "workflow": ["Self-play in games", "Turn-level advantage", "Agent-specific normalization"], "summary": "MARSHAL trains LLMs with self-play RL in cooperative and competitive games using turn-level advantage and agent-specific normalization."}
{"id": 4041, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "MoSciBench multimodal scientific discovery benchmark"}], "pattern": "fix-env-vary-agent", "fixes": ["agents under test"], "varies": ["tasks"], "benchmarks": [{"name": "MoSciBench", "purpose": "multimodal scientific discovery", "is_new": true}], "results": [], "workflow": ["Construct tasks from studies", "Four-stage pipeline"], "summary": "MoSciBench is a benchmark of 88 end-to-end multimodal data-driven discovery tasks across six domains."}
{"id": 6409, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "scaffold", "why": "Small agent group collaborative deliberation for clinical reasoning"}], "pattern": "vary-scaffold", "fixes": ["small LLMs"], "varies": ["agent group structure"], "benchmarks": [], "results": [], "workflow": ["Distribute reasoning", "Evidence analysis", "Critical audit"], "summary": "Argues and shows a small agent group with deliberation supports clinical reasoning with reliability and lower cost."}
{"id": 3550, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "data", "why": "Compass-Data-22k with annotated feasible actions"}, {"area": "embodied-nav", "role": "train-agent", "why": "Gap-aware hybrid reward for reinforcement fine-tuning"}], "pattern": "fix-env-vary-agent", "fixes": ["navigation task"], "varies": ["training data", "reward"], "benchmarks": [{"name": "Compass-Data-22k", "purpose": "training data", "is_new": true}], "results": [], "workflow": ["Annotate actions with A* distances", "Reinforcement fine-tune LVLM", "Hybrid reward"], "summary": "CompassNav shifts LVLM navigation training from path imitation to decision understanding via a dataset and gap-aware reward."}
{"id": 16072, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "InsightEval expert-curated insight discovery benchmark"}, {"area": "science-data", "role": "analysis", "why": "Investigates shortcomings of InsightBench"}], "pattern": "fix-env-vary-agent", "fixes": ["data agents"], "varies": ["benchmark quality"], "benchmarks": [{"name": "InsightEval", "purpose": "insight discovery", "is_new": true}, {"name": "InsightBench", "purpose": "critiqued baseline", "is_new": false}], "results": [], "workflow": ["Analyze InsightBench flaws", "Define criteria", "Curate benchmark"], "summary": "Identifies flaws in InsightBench and proposes InsightEval, an expert-curated benchmark for data-agent insight discovery."}
{"id": 12659, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Co-evolving proposer and visual critic RL for GUI grounding"}], "pattern": "fix-env-vary-agent", "fixes": ["GUI grounding task"], "varies": ["selection mechanism", "RL training"], "benchmarks": [], "results": [], "workflow": ["Propose candidates", "Critique rendered proposals", "Co-evolve with adaptive curriculum"], "summary": "Replaces static self-consistency with a learned propose-and-critic GUI grounding framework trained by co-evolving RL."}
{"id": 8813, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "environment", "why": "AutoControl Arena synthesizes executable risk test environments"}, {"area": "agent-safety", "role": "benchmark", "why": "X-Bench of 70 scenarios across 7 risk categories"}], "pattern": "fix-agent-vary-env", "fixes": ["frontier models"], "varies": ["environment stress and temptation"], "benchmarks": [{"name": "X-Bench", "purpose": "frontier risk evaluation", "is_new": true}], "results": [{"benchmark": "X-Bench", "system": "frontier models", "base_model": "", "setting": "baseline", "metric": "risk rate", "score": 54.5, "unit": "%"}], "workflow": ["Generate environment with code state", "LLM generative dynamics", "Vary stress and temptation"], "summary": "Automated framework synthesizing executable test environments for agent risk evaluation, with X-Bench revealing risk surges under pressure."}
{"id": 3287, "in_agent_scope": false, "types": [{"area": "efficiency/compress", "role": "method", "why": "Gradient-guided layer pruning during early fine-tuning"}], "pattern": "vary-scaffold", "fixes": ["fine-tuning task"], "varies": ["pruning method"], "benchmarks": [], "results": [], "workflow": ["Accumulate early gradients", "Score layer importance", "Prune and sparsify"], "summary": "GradPruner prunes layers using gradients from early fine-tuning for efficient training and inference."}
{"id": 6269, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "CATArena iterative tournament evaluation of code agents"}, {"area": "code-swe", "role": "eval-method", "why": "Dual metric separating generation from evolutionary potential"}], "pattern": "fix-env-vary-agent", "fixes": ["code agents"], "varies": ["tournament tasks", "metrics"], "benchmarks": [{"name": "CATArena", "purpose": "code agent evolution", "is_new": true}], "results": [], "workflow": ["Multi-turn tournaments", "Self-reflection and peer learning", "Dual metrics"], "summary": "CATArena evaluates evolutionary capability of code agents via iterative tournaments, finding it is uncorrelated with initial proficiency."}
{"id": 7279, "in_agent_scope": false, "types": [{"area": "efficiency/compress", "role": "method", "why": "Robust optimization framework for one-shot pruning"}], "pattern": "vary-scaffold", "fixes": ["pruning baselines"], "varies": ["pruning objective robustness"], "benchmarks": [], "results": [], "workflow": ["Model uncertainty in objective", "Robust optimization", "Apply to existing pruners"], "summary": "RobOP applies robust optimization to one-shot pruning to handle calibration noise, compatible with existing methods."}
{"id": 13662, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "eval-method", "why": "Hypothesis-guided rubric LLM judge from few human evaluations"}], "pattern": "vary-evaluation", "fixes": ["generation systems"], "varies": ["judging framework"], "benchmarks": [], "results": [], "workflow": ["Generate rubrics from human evals", "Score decomposed dimensions", "Combine checklist scores"], "summary": "HypoEval builds detailed rubrics from 30 human evaluations and aggregates dimension scores for judge alignment."}
{"id": 9967, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "benchmark", "why": "HiddenBench 65-task collective reasoning benchmark"}, {"area": "multi-agent", "role": "analysis", "why": "Traces failures of agents reasoning about latent information asymmetry"}], "pattern": "fix-env-vary-agent", "fixes": ["15 frontier LLMs"], "varies": ["tasks"], "benchmarks": [{"name": "HiddenBench", "purpose": "collective reasoning under distributed information", "is_new": true}], "results": [{"benchmark": "HiddenBench", "system": "multi-agent LLMs", "base_model": null, "setting": "baseline", "metric": "accuracy", "score": 30.1, "unit": "%"}, {"benchmark": "HiddenBench", "system": "single agent with full information", "base_model": null, "setting": "baseline", "metric": "accuracy", "score": 80.7, "unit": "%"}], "workflow": ["Hidden Profile tasks", "Evaluate multi-agent LLMs"], "summary": "HiddenBench shows multi-agent LLMs fail to pool distributed information, reaching 30.1% vs 80.7% for single agents with full information."}
{"id": 16548, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "MAPS multilingual benchmark for agent performance and security"}], "pattern": "fix-env-vary-agent", "fixes": ["agentic systems"], "varies": ["languages", "tasks"], "benchmarks": [{"name": "MAPS", "purpose": "multilingual agent performance and security", "is_new": true}], "results": [], "workflow": [], "summary": "MAPS is a multilingual, multi-domain benchmark suite evaluating agentic AI performance and security across languages."}
{"id": 2176, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "environment", "why": "MLE-Live simulated Kaggle community evaluation framework"}, {"area": "code-swe", "role": "scaffold", "why": "CoMind multi-agent system with parallel exploration"}], "pattern": "vary-both", "fixes": ["Kaggle competitions"], "varies": ["community knowledge access", "agent system"], "benchmarks": [{"name": "MLE-Live", "purpose": "community-aware MLE evaluation", "is_new": true}], "results": [], "workflow": ["Integrate external knowledge", "Parallel solution exploration"], "summary": "MLE-Live simulates a Kaggle research community and CoMind multi-agent system leverages it, evaluated on 75 past competitions."}
{"id": 3654, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Plan-answer-refine on knowledge graphs for LLM reasoning"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["KG reasoning workflow"], "benchmarks": [], "results": [], "workflow": ["Plan with structured paths", "Answer", "Refine on graph"], "summary": "PARoG plans, answers and refines on knowledge graphs to mitigate search truncation and entity error amplification."}
{"id": 12126, "in_agent_scope": false, "types": [{"area": "train/continual", "role": "method", "why": "RL-trained agent that re-weights data domains for continual pretraining"}], "pattern": "vary-data", "fixes": ["continual pretraining recipe"], "varies": ["domain mixture"], "benchmarks": [], "results": [], "workflow": ["Collect mixing trajectories", "RL on feedback", "Re-weight domains"], "summary": "Data Mixing Agent learns domain re-weighting via RL for continual pretraining while limiting forgetting."}
{"id": 17800, "in_agent_scope": false, "types": [{"area": "safety/fairness", "role": "benchmark", "why": "EsBBQ and CaBBQ Spanish and Catalan bias QA benchmarks"}], "pattern": "vary-evaluation", "fixes": ["LLMs"], "varies": ["language and context"], "benchmarks": [{"name": "EsBBQ", "purpose": "Spanish bias QA", "is_new": true}, {"name": "CaBBQ", "purpose": "Catalan bias QA", "is_new": true}, {"name": "BBQ", "purpose": "adaptation source", "is_new": false}], "results": [], "workflow": ["Adapt BBQ", "Evaluate LLMs"], "summary": "Spanish and Catalan adaptations of BBQ measuring social bias across 10 categories."}
{"id": 12012, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Training-free multi-agent consistency for text-to-SQL candidate selection"}], "pattern": "vary-scaffold", "fixes": ["SQL generator"], "varies": ["selection method"], "benchmarks": [], "results": [], "workflow": ["Generate candidates", "Dual-paradigm consistency", "Select SQL"], "summary": "DPC is a training-free multi-agent text-to-SQL selection framework closing the generation-selection gap."}
{"id": 17318, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Agent-based FHIR resource synthesis from clinical notes with tools"}], "pattern": "n/a", "fixes": ["general LLM"], "varies": ["domain pipeline"], "benchmarks": [], "results": [], "workflow": ["Agent reads notes", "Execute code and query terminology tools", "Produce FHIR resources"], "summary": "Infherno is an end-to-end LLM agent system with code execution and terminology tools generating FHIR resources from clinical notes."}
{"id": 15414, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Analyzes authority role bias in ChatEval multi-agent evaluation"}], "pattern": "vary-conditions", "fixes": ["ChatEval"], "varies": ["authority role type"], "benchmarks": [{"name": "ChatEval", "purpose": "framework studied", "is_new": false}], "results": [], "workflow": [], "summary": "First systematic analysis of authority bias in multi-agent evaluation; expert and referent roles exert stronger influence than legitimate."}
{"id": 15475, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "benchmark", "why": "NEST KGQA task and NestKGQA dataset"}, {"area": "knowledge/rag", "role": "method", "why": "PyLF logical form and refinement method"}], "pattern": "vary-both", "fixes": ["KG"], "varies": ["task", "method"], "benchmarks": [{"name": "NestKGQA", "purpose": "negative-constrained KGQA", "is_new": true}], "results": [], "workflow": ["Schema-guided matching", "Python logical form", "Self-directed refinement"], "summary": "Introduces negative-constrained KGQA with dataset NestKGQA, PyLF logical form and a schema-guided, self-refining method."}
{"id": 9928, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "JiSi routing and aggregation for LLM collaboration"}], "pattern": "vary-scaffold", "fixes": ["open-source LLMs"], "varies": ["routing and aggregation"], "benchmarks": [], "results": [], "workflow": ["Query-response mixed routing", "Adaptive aggregator selection", "Combine routing and aggregation"], "summary": "JiSi coordinates open-source LLMs through improved routing and aggregation to surpass Gemini-3-Pro."}
{"id": 16375, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Pre-exploration converts ambiguous requests into actionable problem statements"}], "pattern": "vary-scaffold", "fixes": ["SWE agents"], "varies": ["problem statement refinement"], "benchmarks": [], "results": [], "workflow": ["Lightweight codebase pre-exploration", "Refine problem statement", "Run agent"], "summary": "CodeScout refines ambiguous problem statements via pre-exploration without modifying underlying software agents."}
{"id": 14292, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "CI-Work contextual integrity benchmark for enterprise agents"}], "pattern": "fix-env-vary-agent", "fixes": ["frontier models"], "varies": ["tasks"], "benchmarks": [{"name": "CI-Work", "purpose": "enterprise privacy leakage", "is_new": true}], "results": [], "workflow": ["Simulate enterprise workflows", "Measure leakage and utility"], "summary": "CI-Work shows privacy violation rates of 15.8%-50.9% in enterprise agents and a utility-privacy trade-off."}
{"id": 14524, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "DataSciBench with IFC evaluation framework"}], "pattern": "fix-env-vary-agent", "fixes": ["26 models"], "varies": ["tasks"], "benchmarks": [{"name": "DataSciBench", "purpose": "data science evaluation", "is_new": true}], "results": [], "workflow": ["Semi-automated GT generation", "IFC evaluation"], "summary": "DataSciBench is a data science benchmark with curated prompts, semi-automated ground truth and IFC metrics, evaluating 26 models."}
{"id": 14064, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "application", "why": "Self-evolving extraction toolkit with growing knowledge base"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["KB accumulation"], "benchmarks": [], "results": [], "workflow": ["Extract triples", "Populate KB", "Graph reasoning enrichment"], "summary": "DySECT is a toolkit that continually improves information extraction via a self-expanding knowledge base."}
{"id": 16723, "in_agent_scope": false, "types": [{"area": "knowledge/halluc", "role": "method", "why": "Identify-then-Verify prompting for information sufficiency"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["prompting framework"], "benchmarks": [], "results": [], "workflow": ["Hypothesize missing information", "Semantic consensus", "Verify against source"], "summary": "Identify-then-Verify framework assesses QA context sufficiency by reasoning about missing information."}
{"id": 13895, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Online RL with curiosity reward for knowledge-exploring agents"}], "pattern": "fix-env-vary-agent", "fixes": ["structured knowledge benchmarks"], "varies": ["RL reward"], "benchmarks": [], "results": [], "workflow": ["Multi-turn knowledge exploration", "Curiosity-driven reward", "Online RL"], "summary": "KARL trains Qwen2.5-14B agents with curiosity-shaped online RL for multi-turn knowledge-intensive tasks, beating GPT-4o."}
{"id": 2373, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Pixel-level visual self-refine for chart parsing"}], "pattern": "vary-scaffold", "fixes": ["LVLM"], "varies": ["refinement paradigm"], "benchmarks": [], "results": [], "workflow": ["Localize pixel-level outputs", "Visualize and feed back", "Self-correct"], "summary": "Visual Self-Refine lets LVLMs visualize pixel localization outputs to correct chart parsing errors (ChartVSR)."}
{"id": 4621, "in_agent_scope": false, "types": [{"area": "efficiency/inference", "role": "method", "why": "Training-free surprise-guided video token pruning"}], "pattern": "vary-scaffold", "fixes": ["video VLM"], "varies": ["token reduction"], "benchmarks": [], "results": [], "workflow": ["Score token surprise", "Prune predictable tokens", "Aggregate surprise curve"], "summary": "SURGE prunes predictable video tokens via prediction-error surprise, training-free and backbone-agnostic."}
{"id": 14572, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Tree-structured credit assignment for multi-turn agent RL"}], "pattern": "fix-env-vary-agent", "fixes": ["multi-step reasoning tasks"], "varies": ["credit assignment"], "benchmarks": [], "results": [], "workflow": ["Merge trajectories into Cognitive Tree", "Back-propagate rewards", "Step-level advantages"], "summary": "T-STAR merges trajectories into a tree for step-level advantage in agent RL, addressing sparse rewards."}
{"id": 8111, "in_agent_scope": false, "types": [{"area": "efficiency/inference", "role": "method", "why": "Informed routing with feature forecaster for dynamic computation"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["routing policy"], "benchmarks": [], "results": [], "workflow": ["Forecast unit output", "Execute-or-approximate"], "summary": "Lightweight Feature Forecaster enables informed execute-or-approximate routing for token-level computation allocation."}
{"id": 15239, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Soft label and key-extraction CoT defense against instruction backdoors"}], "pattern": "fix-agent-vary-env", "fixes": ["backdoor threat model"], "varies": ["defense mechanism"], "benchmarks": [], "results": [], "workflow": ["Mechanistic analysis", "Soft label", "Key-extraction CoT"], "summary": "SLIP defends LLM agents against instruction backdoors, recovering correct outputs via soft labels and key-extraction CoT."}
{"id": 14081, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "analysis", "why": "Pareto analysis of inference scaling strategies"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["strategy", "configuration"], "benchmarks": [{"name": "MMLU-Pro", "purpose": "reasoning evaluation", "is_new": false}, {"name": "BBH", "purpose": "reasoning evaluation", "is_new": false}], "results": [], "workflow": ["Vary agents and rounds", "Compute Pareto front"], "summary": "Compares self-consistency, refinement, debate and mixture-of-agents across 34 configurations to find Pareto-optimal compute-accuracy methods."}
{"id": 7796, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Online adversarial strategy optimization for social agents"}], "pattern": "vary-scaffold", "fixes": ["social simulation"], "varies": ["strategy adaptation"], "benchmarks": [], "results": [], "workflow": ["Model opponent adaptation", "Online strategy optimization"], "summary": "ALSO is an online strategy optimization framework for LLM social agents in non-stationary multi-agent simulation."}
{"id": 12466, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "Benchmark generator with controllable policy complexity"}, {"area": "tool-use", "role": "train-agent", "why": "Category-aware policy continued pretraining"}], "pattern": "vary-both", "fixes": ["agentic policies"], "varies": ["policy complexity", "training method"], "benchmarks": [], "results": [], "workflow": ["Generate benchmarks by complexity", "Analyze SFT", "Category-aware CPT"], "summary": "Benchmark generator with four complexity levels and a continued-pretraining approach to internalize agent policy documents."}
{"id": 15132, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Aegle graph-based virtual MDT for clinical intake"}], "pattern": "vary-scaffold", "fixes": ["general LLM"], "varies": ["multi-agent pipeline"], "benchmarks": [{"name": "ClinicalBench", "purpose": "evaluation", "is_new": false}], "results": [], "workflow": ["SOAP state", "Orchestrator activates specialists", "Aggregator writes note"], "summary": "Aegle is a multi-agent virtual multidisciplinary team for outpatient consultation with SOAP representation and aggregation."}
{"id": 5818, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "AgentSelect query-to-agent recommendation benchmark"}], "pattern": "vary-evaluation", "fixes": ["recommenders"], "varies": ["queries and agents"], "benchmarks": [{"name": "AgentSelect", "purpose": "agent recommendation", "is_new": true}], "results": [], "workflow": ["Convert evaluation artifacts", "Build interaction data"], "summary": "AgentSelect benchmarks narrative query-to-agent recommendation with 111,179 queries and 107,721 agents."}