Files
ai-taxonomy-literature-anal…/data/labels/o03.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
37 KiB
JSON

{"id": 13666, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "data", "why": "Curates GCA-DS, ~200k multimodal climate QA pairs for GCC states"}, {"area": "domain-agents", "role": "application", "why": "Tool-augmented Gulf Climate Agent for climate analysis"}], "pattern": "vary-data", "fixes": ["general LLMs"], "varies": ["domain data", "tool pipeline"], "benchmarks": [{"name": "GCA-DS", "purpose": "climate QA for GCC", "is_new": true}], "results": [], "workflow": ["Curate GCC climate dataset", "Fine-tune LLM", "Orchestrate geospatial tools"], "summary": "Dataset and tool-augmented agent for GCC climate decision support; fine-tuning and tools improve reliability."}
{"id": 14132, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Lightweight two-stage multi-agent system for patient-oriented discharge summary QA"}, {"area": "domain-agents", "role": "analysis", "why": "Evaluates open and closed LLMs on MeDiSumQA"}], "pattern": "vary-scaffold", "fixes": ["small LLMs"], "varies": ["multi-agent pipeline", "feedback loop"], "benchmarks": [{"name": "MeDiSumQA", "purpose": "patient discharge-summary QA", "is_new": false}], "results": [], "workflow": ["Extract question-relevant evidence", "Simplify answer for patient", "Metric-driven reviewer feedback"], "summary": "Lightweight multi-agent framework for patient-friendly medical QA, with better readability than zero-shot GPT-5."}
{"id": 17043, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Narrative-based working memory built by agentic reasoning offline, coherence-driven retrieval"}], "pattern": "vary-scaffold", "fixes": ["LLM", "benchmark"], "varies": ["memory construction and retrieval"], "benchmarks": [{"name": "LOCOMO", "purpose": "long-term conversational memory", "is_new": false}], "results": [], "workflow": ["Organize fragments into episodic narratives", "Consolidate with momentum", "Semanticize peripheral facts", "Coherence-driven retrieval"], "summary": "Memory framework building coherent narrative memory for conversational agents, near full-context quality with 50% faster response."}
{"id": 16401, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Anchor-GRPO two-stage RL separating first-step planning and execution rewards"}], "pattern": "fix-env-vary-agent", "fixes": ["web search environment", "benchmarks"], "varies": ["RL algorithm", "reward design"], "benchmarks": [{"name": "BrowseComp", "purpose": "web reasoning eval", "is_new": false}, {"name": "BrowseComp-Zh", "purpose": "web reasoning eval", "is_new": false}, {"name": "GAIA", "purpose": "information seeking eval", "is_new": false}, {"name": "XBench-DeepSearch", "purpose": "deep search eval", "is_new": false}], "results": [{"benchmark": "BrowseComp", "system": "WebAnchor-30B", "base_model": null, "setting": "proposed", "metric": "pass@1", "score": 46.0, "unit": "%"}, {"benchmark": "GAIA", "system": "WebAnchor-30B", "base_model": null, "setting": "proposed", "metric": "pass@1", "score": 76.4, "unit": "%"}], "workflow": ["Rubric-reward first-step planning", "Sparse-reward execution alignment"], "summary": "RL framework rewarding first-step plans in long-horizon web agents; WebAnchor-30B reaches 46.0% BrowseComp."}
{"id": 14799, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "train-agent", "why": "RL for single tool-augmented stock trading agent"}, {"area": "planning-decision", "role": "train-agent", "why": "Learns dynamic policy over tool-augmented decision workflow"}], "pattern": "fix-env-vary-agent", "fixes": ["trading data/metrics"], "varies": ["agent training via RL"], "benchmarks": [], "results": [], "workflow": ["Single agent orchestrates tools", "RL on market feedback"], "summary": "End-to-end agentic RL framework for stock trading with a single tool-using agent."}
{"id": 4391, "in_agent_scope": false, "types": [{"area": "align/rlvr", "role": "method", "why": "Difficulty-aware dual-objective reward adapting reasoning length for context QA"}], "pattern": "n/a", "fixes": ["QA benchmarks"], "varies": ["reward", "response length"], "benchmarks": [], "results": [], "workflow": ["Assess difficulty info-theoretically", "Dual-objective length reward"], "summary": "RL method that adapts reasoning length to question difficulty, cutting tokens at competitive accuracy."}
{"id": 12948, "in_agent_scope": true, "types": [{"area": "search-research", "role": "eval-method", "why": "Dynamic automated evaluation with adaptive task escalation for deep research agents"}, {"area": "search-research", "role": "benchmark", "why": "Generates live Information Tree tasks testing deep and wide capability"}], "pattern": "vary-evaluation", "fixes": ["DR agents"], "varies": ["evaluation protocol", "task generation"], "benchmarks": [{"name": "LMSYS Search Arena", "purpose": "human-preference ranking comparison", "is_new": false}], "results": [{"benchmark": "LMSYS Search Arena", "system": "DR-Arena", "base_model": null, "setting": "proposed", "metric": "Spearman correlation", "score": 0.94, "unit": "other"}], "workflow": ["Build Information Trees from web", "Examiner generates tasks", "Adaptive loop escalates difficulty"], "summary": "Automated dynamic evaluation of deep research agents that correlates 0.94 with human-preference leaderboard."}
{"id": 13814, "in_agent_scope": false, "types": [{"area": "safety/security", "role": "method", "why": "Linguistics-aware watermark modulating strength by POS n-gram predictability, logit-free detection"}], "pattern": "n/a", "fixes": ["LLM generation"], "varies": ["watermark strength rule"], "benchmarks": [], "results": [], "workflow": ["Model POS n-gram indeterminacy", "Modulate watermark strength", "Detect without logits"], "summary": "STELA watermark adapting signal to syntactic freedom, enabling public verifiable detection across English, Chinese, Korean."}
{"id": 808, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Three-agent knowledge-graph context construction trained with constrained multi-agent PPO"}, {"area": "planning-decision", "role": "method", "why": "Budget-constrained sequential decision over KG expansion"}], "pattern": "fix-env-vary-agent", "fixes": ["KG QA benchmarks"], "varies": ["context construction policy"], "benchmarks": [{"name": "HotpotQA", "purpose": "multi-hop QA", "is_new": false}, {"name": "MetaQA", "purpose": "KG multi-hop QA", "is_new": false}, {"name": "FactKG", "purpose": "KG fact verification", "is_new": false}], "results": [], "workflow": ["Subgraph Architect expands", "Path Navigator follows paths", "Context Curator selects evidence"], "summary": "Agentic neuro-symbolic KG reasoning optimized with Lagrangian MAPPO under latency and token budgets."}
{"id": 12953, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Unified MLLM with draft-guided localization and self-refinement for inscription restoration"}], "pattern": "n/a", "fixes": ["restoration task"], "varies": ["model design"], "benchmarks": [{"name": "HIRBench", "purpose": "inscription restoration eval", "is_new": true}], "results": [], "workflow": ["Draft-guided damage localization", "Hierarchical self-refinement"], "summary": "Unified MLLM for page-level historical inscription restoration, with new HIRBench and training factory."}
{"id": 14534, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "benchmark", "why": "Code4Detail EMR dataset for ICD coding"}, {"area": "domain-agents", "role": "eval-method", "why": "ToT-ICD agentic tree-exploration evaluation framework"}], "pattern": "vary-both", "fixes": ["LLMs"], "varies": ["coding paradigm", "tasks"], "benchmarks": [{"name": "Code4Detail", "purpose": "ICD coding from EMRs", "is_new": true}], "results": [], "workflow": ["Retrieve similar codes", "Navigate ICD tree path", "Backtrack dynamically"], "summary": "Benchmark of 560 clinical records and agentic ICD-tree evaluation framework for LLM coding."}
{"id": 4326, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Multi-agent GPU kernel optimization with grounded instructions, context management and search"}], "pattern": "vary-scaffold", "fixes": ["LLMs", "KernelBench"], "varies": ["agent framework"], "benchmarks": [{"name": "KernelBench", "purpose": "GPU kernel optimization", "is_new": false}], "results": [], "workflow": ["Multi-agent collaboration", "Profiling feedback", "Strategic search refinement"], "summary": "Agentic multi-agent framework for GPU kernel optimization with up to 16x faster kernels."}
{"id": 15577, "in_agent_scope": false, "types": [{"area": "arch/diffusion", "role": "method", "why": "Training-free post-hoc refinement decoding for masked diffusion LMs"}], "pattern": "n/a", "fixes": ["LLaDA-8B-Instruct"], "varies": ["decoding procedure"], "benchmarks": [{"name": "GSM8K", "purpose": "reasoning accuracy", "is_new": false}], "results": [], "workflow": ["Draft completion", "Score instability", "Window masking and relaxation"], "summary": "Training-free two-phase decoding that unlocks and refines unstable tokens in diffusion LMs."}
{"id": 18601, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "DReaMAD: prior elicitation and perspective diversity in multi-agent debate"}, {"area": "multi-agent", "role": "benchmark", "why": "Introduces MetaNIM Arena"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["debate procedure"], "benchmarks": [{"name": "MetaNIM Arena", "purpose": "belief entrenchment eval", "is_new": true}], "results": [], "workflow": ["Elicit strategic prior knowledge", "Enforce perspective diversity in debate"], "summary": "Mitigates belief entrenchment in multi-agent debate via prior elicitation and diversity."}
{"id": 12959, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "PS-Bench measures intent legitimation in personalized agents"}, {"area": "agent-safety", "role": "defense", "why": "Lightweight detection-reflection mitigation"}], "pattern": "vary-conditions", "fixes": ["agents"], "varies": ["personalization memory"], "benchmarks": [{"name": "PS-Bench", "purpose": "intent legitimation", "is_new": true}], "results": [], "workflow": ["Add benign personal memory", "Measure attack success", "Detect and reflect"], "summary": "Reveals personal memory raises harmful-query compliance in personalized agents; benchmark and mitigation."}
{"id": 15921, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Agentic presentation generation with environment-grounded reflection on rendered slides"}, {"area": "domain-agents", "role": "train-agent", "why": "Fine-tuned DeepPresenter-9B"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["reflection on rendered artifacts"], "benchmarks": [], "results": [], "workflow": ["Plan and research", "Render slides", "Reflect on rendered state", "Revise"], "summary": "Agentic slide generator reflecting on rendered artifacts rather than reasoning traces."}
{"id": 4683, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "analysis", "why": "Shows MLLM verifiers over-validate agent trajectories (agreement bias)"}, {"area": "web-gui", "role": "eval-method", "why": "Self-Grounded Verification improves trajectory verification"}, {"area": "web-gui", "role": "environment", "why": "Updated VisualWebArena/WebArena with better evaluators"}], "pattern": "vary-evaluation", "fixes": ["MLLM verifiers"], "varies": ["verification prompting"], "benchmarks": [{"name": "OSWorld", "purpose": "verification downstream", "is_new": false}, {"name": "VisualWebArena", "purpose": "verification downstream", "is_new": false}, {"name": "WebArena", "purpose": "updated release", "is_new": false}, {"name": "robomimic", "purpose": "robotic verification", "is_new": false}], "results": [], "workflow": ["Elicit self-generated priors", "Evaluate trajectory conditioned on priors"], "summary": "Identifies agreement bias in MLLM verifiers and proposes Self-Grounded Verification, plus improved WebArena releases."}
{"id": 3226, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "ScienceBoard 169 scientific workflow tasks"}, {"area": "science-data", "role": "environment", "why": "Realistic multi-domain environment with professional software"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks", "environments"], "benchmarks": [{"name": "ScienceBoard", "purpose": "scientific computer-use agents", "is_new": true}], "results": [{"benchmark": "ScienceBoard", "system": "best agents", "base_model": null, "setting": "baseline", "metric": "success rate", "score": 15, "unit": "%"}], "workflow": [], "summary": "Environment and 169-task benchmark for computer-using agents in scientific workflows; agents reach about 15%."}
{"id": 2214, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Iterative preference learning with thinking-level DPO for mobile agents"}, {"area": "web-gui", "role": "data", "why": "Instruction evolution generating Q&A from UI screenshots"}], "pattern": "fix-env-vary-agent", "fixes": ["mobile GUI benchmarks"], "varies": ["training data", "preference learning"], "benchmarks": [], "results": [], "workflow": ["Build CoaT-tree by sampling", "Rule-score leaves", "T-DPO pairs"], "summary": "Self-training mobile GUI agents with tree-based thinking-level DPO and evolved instruction data."}
{"id": 3704, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Semantic motion-adaptive control points for dynamic 3D Gaussian splatting"}], "pattern": "n/a", "fixes": ["monocular video reconstruction"], "varies": ["control point allocation"], "benchmarks": [], "results": [], "workflow": ["Patch-token-node correspondences", "Motion-adaptive compression", "Spline trajectories"], "summary": "Allocates 3D Gaussian control points by motion complexity for dynamic scene reconstruction."}
{"id": 17484, "in_agent_scope": false, "types": [{"area": "apps/science", "role": "method", "why": "Inference-time steering vectors for medical QA without fine-tuning"}, {"area": "interp/steering", "role": "method", "why": "Steering vectors guide activations"}], "pattern": "vary-scaffold", "fixes": ["LLM weights"], "varies": ["inference-time steering"], "benchmarks": [], "results": [], "workflow": ["Compute steering vectors", "Steer activations at inference"], "summary": "MedAssess-X steers activations for clinical QA, challenging the need for specialised medical fine-tuning."}
{"id": 8545, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Two-stage SFT and RL for sufficiency-based adaptive reasoning length"}], "pattern": "n/a", "fixes": ["math, code, science benchmarks"], "varies": ["reasoning-length training"], "benchmarks": [], "results": [], "workflow": ["Construct minimal sufficient CoT data", "Fine-tune", "RL with sufficiency rewards"], "summary": "Trains reasoning models to stop at minimal sufficient CoT, improving accuracy and efficiency."}
{"id": 8710, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "environment", "why": "Agentic VLM framework generating simulation-ready indoor scenes"}], "pattern": "fix-agent-vary-env", "fixes": ["robot policy evaluation"], "varies": ["scene generation"], "benchmarks": [], "results": [], "workflow": ["Layout generation", "Furniture placement", "Small object population"], "summary": "Multi-VLM-agent generator of dense physically valid indoor scenes for robot simulation."}
{"id": 15411, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "CogBench 2.1K math questions for cognitive-level alignment in education QA"}], "pattern": "fix-agent-vary-env", "fixes": ["11 LLMs"], "varies": ["tasks"], "benchmarks": [{"name": "CogBench", "purpose": "cognitive alignment in educational QA", "is_new": true}], "results": [], "workflow": [], "summary": "Benchmark measuring whether LLM explanations match student cognitive levels."}
{"id": 12534, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "scaffold", "why": "LogicAgent semiotic-square multi-perspective reasoning with deduction and verification"}, {"area": "planning-decision", "role": "benchmark", "why": "RepublicQA benchmark"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["reasoning framework"], "benchmarks": [{"name": "RepublicQA", "purpose": "semantic-logical reasoning", "is_new": true}], "results": [], "workflow": ["Semiotic square analysis", "Automated deduction", "Reflective verification"], "summary": "Semiotic-guided reasoning agent and new RepublicQA benchmark for semantic-logical reasoning."}
{"id": 12542, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "scaffold", "why": "Chart agent using visual tools to crop, annotate and localize"}], "pattern": "vary-scaffold", "fixes": ["MLLMs"], "varies": ["agent framework"], "benchmarks": [{"name": "ChartBench", "purpose": "chart QA", "is_new": false}, {"name": "ChartX", "purpose": "chart QA", "is_new": false}], "results": [], "workflow": ["Decompose query", "Apply chart vision tools", "Iterate visual reasoning"], "summary": "Tool-augmented multimodal agent reasoning visually in chart space for chart QA."}
{"id": 627, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Simulated society of diverse personas aggregated for behavioral prediction"}, {"area": "multi-agent", "role": "scaffold", "why": "Persona multi-agent aggregation"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["persona crowd"], "benchmarks": [], "results": [], "workflow": ["Instantiate diverse personas", "Each appraises stimulus", "Aggregate opinions"], "summary": "Multi-agent persona crowd simulation improves LLM prediction of human behavioral responses."}
{"id": 14030, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "application", "why": "MASFactory graph-centric framework with Vibe Graphing for building LLM multi-agent systems"}], "pattern": "n/a", "fixes": ["MAS methods"], "varies": ["orchestration framework"], "benchmarks": [], "results": [], "workflow": ["Intent to workflow spec", "Compile to executable graph"], "summary": "Framework for orchestrating multi-agent systems that compiles natural language to graph workflows."}
{"id": 17284, "in_agent_scope": false, "types": [{"area": "reason/math", "role": "analysis", "why": "First systematic study of generative PRMs for table QA"}, {"area": "eval/bench", "role": "analysis", "why": "Step-level vs answer correlation"}], "pattern": "vary-conditions", "fixes": ["PRMs"], "varies": ["PRM types", "data domains"], "benchmarks": [], "results": [], "workflow": [], "summary": "Evaluates generative process reward models on table QA, finding limited generalization and weak step-answer correlation."}
{"id": 13130, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "MirrorCAPTCHA measures web agents solving real-distribution CAPTCHAs"}, {"area": "web-gui", "role": "data", "why": "Synthetic pipeline trains Ovis2-Agent-CAPTCHA-8B"}], "pattern": "fix-env-vary-agent", "fixes": ["agents"], "varies": ["tasks"], "benchmarks": [{"name": "MirrorCAPTCHA", "purpose": "CAPTCHA solving by web agents", "is_new": true}], "results": [], "workflow": ["Cluster CAPTCHA categories", "Simulate encounter frequencies", "Synthesize training data"], "summary": "CAPTCHA benchmark for web agents with new metric and a trained 8B agent beating Gemini-2.5-Pro."}
{"id": 6545, "in_agent_scope": false, "types": [{"area": "arch/attention", "role": "method", "why": "Router regularization matching Dirichlet prior for upcycled MoE expert specialization"}], "pattern": "n/a", "fixes": ["vision-language benchmarks"], "varies": ["router loss"], "benchmarks": [], "results": [], "workflow": ["Define target Dirichlet prior", "Match routing distributions"], "summary": "Dirichlet-prior router loss improving expert specialization in upcycled vision-language MoEs."}
{"id": 164, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "data", "why": "Runtime-free CTF trajectory synthesis from writeups"}, {"area": "agent-safety", "role": "train-agent", "why": "Trains cybersecurity agents Cyber-Zero-32B"}], "pattern": "vary-data", "fixes": ["CTF benchmarks"], "varies": ["training trajectories"], "benchmarks": [{"name": "InterCode-CTF", "purpose": "CTF agent eval", "is_new": false}, {"name": "NYU CTF Bench", "purpose": "CTF agent eval", "is_new": false}, {"name": "Cybench", "purpose": "CTF agent eval", "is_new": false}], "results": [{"benchmark": "InterCode-CTF, NYU CTF Bench, Cybench", "system": "Cyber-Zero", "base_model": "Qwen", "setting": "finetuned", "metric": "absolute gain", "score": 13.1, "unit": "%"}], "workflow": ["Use CTF writeups", "Persona-driven simulation", "Train on trajectories"], "summary": "Synthesizes agent trajectories without runtimes to train cybersecurity agents."}
{"id": 13457, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "scaffold", "why": "Adaptive agent internalizing document context into adapter via online adaptation for document translation"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["context internalization"], "benchmarks": [{"name": "GuoFeng", "purpose": "document MT", "is_new": false}, {"name": "IWSLT2017", "purpose": "document MT", "is_new": false}], "results": [], "workflow": ["Detect discourse discontinuity", "Integrate context into adapter", "Translate"], "summary": "Agentic document translation compiling retrieved constraints into parameters for consistency."}
{"id": 16364, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "Trains small LLM to generate supplements helping frozen larger agent LLM"}], "pattern": "fix-env-vary-agent", "fixes": ["large LLM"], "varies": ["supplement generator"], "benchmarks": [], "results": [], "workflow": ["Train small LLM", "Append supplement to input"], "summary": "Supplement Generation Training boosts agentic tasks without post-training large models."}
{"id": 8782, "in_agent_scope": true, "types": [{"area": "search-research", "role": "data", "why": "Hyper-Search hypergraph multimodal QA generation"}, {"area": "search-research", "role": "train-agent", "why": "Tool-expert trajectories and offline search RL for MM-DeepResearch"}], "pattern": "vary-data", "fixes": ["search tools"], "varies": ["training data", "trajectories"], "benchmarks": [], "results": [], "workflow": ["Generate QA via hypergraph", "Tree-search trajectories", "Agentic RL offline search"], "summary": "Multimodal deep research agent trained on synthesized QA and trajectories with offline search engine."}
{"id": 12026, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Shows KV cache reuse perturbs LLM judge selection in multi-agent systems"}], "pattern": "vary-conditions", "fixes": ["judges"], "varies": ["KV reuse strategies"], "benchmarks": [{"name": "GSM8K", "purpose": "judge consistency", "is_new": false}, {"name": "MMLU", "purpose": "judge consistency", "is_new": false}, {"name": "HumanEval", "purpose": "judge consistency", "is_new": false}], "results": [], "workflow": [], "summary": "Shows KV cache reuse degrades judge consistency by weakening cross-candidate attention."}
{"id": 4947, "in_agent_scope": false, "types": [{"area": "train/data", "role": "data", "why": "Influence-guided rubric optimization for synthetic SFT data"}], "pattern": "vary-data", "fixes": ["target models", "SFT recipe"], "varies": ["rubrics", "synthetic data"], "benchmarks": [], "results": [], "workflow": ["Estimate sample influence", "Adapt rubrics via feedback", "Select data"], "summary": "Optimizes synthetic data rubrics using estimated downstream influence on target models."}
{"id": 16636, "in_agent_scope": false, "types": [{"area": "apps/domain", "role": "benchmark", "why": "TextMineX triple-extraction dataset for mine action"}, {"area": "apps/domain", "role": "eval-method", "why": "Bias-aware LLM-judge evaluation"}], "pattern": "vary-evaluation", "fixes": ["LLMs"], "varies": ["prompts", "dataset"], "benchmarks": [{"name": "TextMineX", "purpose": "mine action knowledge extraction", "is_new": true}], "results": [], "workflow": ["Ontology-guided prompts", "Extract triples"], "summary": "Dataset, bias-aware evaluation and ontology-guided pipeline for mine action knowledge extraction."}
{"id": 9442, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "MADQA 2,250 questions over 800 PDFs"}, {"area": "search-research", "role": "eval-method", "why": "Accuracy-effort trade-off protocol"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks", "protocol"], "benchmarks": [{"name": "MADQA", "purpose": "document collection QA", "is_new": true}], "results": [], "workflow": [], "summary": "Benchmark and effort-aware protocol showing agents rely on brute-force search over documents."}
{"id": 16034, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Co-trains RL agent and LM planner via plan feedback"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["plan extraction", "co-training"], "benchmarks": [], "results": [], "workflow": ["LM generates plans", "RL agent follows plans", "LM refines on feedback"], "summary": "Iterative co-training where language model plans and RL agent execute for instruction following."}
{"id": 6681, "in_agent_scope": false, "types": [{"area": "efficiency/compress", "role": "method", "why": "A* expert pruning for MoEs with Schur-complement bound"}], "pattern": "n/a", "fixes": ["Qwen3-30B-A3B"], "varies": ["pruning algorithm"], "benchmarks": [], "results": [], "workflow": ["Reconstruction-driven subset selection", "A* search"], "summary": "Optimal layerwise MoE expert pruning via A* with Schur-complement heuristic."}
{"id": 14613, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Self-evolving meta-memory teaching LLMs to use memorized knowledge"}], "pattern": "vary-scaffold", "fixes": ["LLM", "memory system"], "varies": ["meta-memory"], "benchmarks": [], "results": [], "workflow": ["Reflect on reasoning", "Update meta-memory"], "summary": "Meta-memory distilled by self-reflection guides memory utilization, over 3.6% gains."}
{"id": 13306, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Nemori predictive episodic and semantic memory distillation"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory formation"], "benchmarks": [], "results": [], "workflow": ["Integrate episodes into narratives", "Distill via prediction error"], "summary": "Adaptive memory distillation deciding what to retain via prediction error."}
{"id": 13273, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Generative process reward model for information-seeking steps"}, {"area": "search-research", "role": "scaffold", "why": "Best-of-n with PRM and trajectory summarization"}], "pattern": "fix-env-vary-agent", "fixes": ["agents"], "varies": ["reward model"], "benchmarks": [{"name": "FRAMES", "purpose": "info seeking", "is_new": false}, {"name": "GAIA", "purpose": "info seeking", "is_new": false}, {"name": "WebWalkerQA", "purpose": "info seeking", "is_new": false}], "results": [], "workflow": ["Score step quality", "Summarize trajectory", "Best-of-n"], "summary": "PRInTS PRM scores and summarizes long information-seeking trajectories to guide agents."}
{"id": 13906, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "NaturalGAIA verifiable long-horizon GUI dataset"}, {"area": "web-gui", "role": "scaffold", "why": "LightManus-Jarvis hierarchical planning and execution"}], "pattern": "vary-both", "fixes": ["agents"], "varies": ["tasks", "framework"], "benchmarks": [{"name": "NaturalGAIA", "purpose": "long-horizon GUI tasks", "is_new": true}], "results": [{"benchmark": "NaturalGAIA", "system": "LightManus-Jarvis", "base_model": null, "setting": "proposed", "metric": "Weighted Pathway Success Rate", "score": 45.6, "unit": "%"}, {"benchmark": "NaturalGAIA", "system": "SOTA baseline", "base_model": null, "setting": "baseline", "metric": "Weighted Pathway Success Rate", "score": 21.1, "unit": "%"}], "workflow": ["Topological planning", "Hybrid visual-structural execution"], "summary": "Verifiable GUI benchmark with hierarchical agent reaching 45.6% weighted pathway success."}
{"id": 16394, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Agentic academic rebuttal pipeline: decompose, retrieve, plan, generate"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["rebuttal pipeline"], "benchmarks": [], "results": [], "workflow": ["Decompose reviews", "Retrieve evidence", "Plan strategy", "Generate"], "summary": "Four-step agentic framework for rebuttals outperforming prior pipelines with an 8B model."}
{"id": 7583, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Two-stage RL agents routing and fusing LLM ensembles"}], "pattern": "n/a", "fixes": ["LLM pool"], "varies": ["ensemble selection"], "benchmarks": [], "results": [], "workflow": ["Decider selects ensemble", "Fusion agent resolves conflicts"], "summary": "RL-Focal selects and fuses LLM ensembles, improving 8.48% over best single LLM."}
{"id": 17051, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Multi-agent course material generation system"}, {"area": "multi-agent", "role": "application", "why": "Role-based educational agent collaboration"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["human involvement modes"], "benchmarks": [], "results": [], "workflow": ["Role-based collaboration", "Generate syllabus slides assessments"], "summary": "Multi-agent framework generating course materials end to end with variable human involvement."}
{"id": 17375, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "benchmark", "why": "Benchmark for language-of-study extraction from ACL papers"}], "pattern": "fix-agent-vary-env", "fixes": ["models"], "varies": ["input text parts"], "benchmarks": [{"name": "ACL Anthology language-of-study benchmark", "purpose": "extraction eval", "is_new": true}], "results": [{"benchmark": "language-of-study benchmark", "system": "GPT-4o mini", "base_model": null, "setting": "baseline", "metric": "exact match", "score": 0.646, "unit": "other"}, {"benchmark": "language-of-study benchmark", "system": "Gemini 2.5 Flash", "base_model": null, "setting": "baseline", "metric": "micro F1", "score": 0.633, "unit": "other"}], "workflow": [], "summary": "First benchmark of 431 papers evaluating LLMs on extracting languages studied."}
{"id": 10144, "in_agent_scope": false, "types": [{"area": "eval/robust", "role": "analysis", "why": "Compares LLM and human verbal uncertainty marker semantics"}, {"area": "eval/robust", "role": "method", "why": "VOCAL learns marker-uncertainty mapping"}], "pattern": "vary-conditions", "fixes": ["LLMs"], "varies": ["uncertainty markers"], "benchmarks": [], "results": [], "workflow": ["Curate human markers", "Fit marker-uncertainty mapping"], "summary": "Shows LLM verbal uncertainty diverges from humans and introduces VOCAL."}
{"id": 6720, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Entropy-guided attention perturbation contrastive decoding for LVLM hallucination"}], "pattern": "n/a", "fixes": ["LVLMs"], "varies": ["decoding"], "benchmarks": [], "results": [], "workflow": ["Perturb visual attention", "Select layers by confidence"], "summary": "Training-free contrastive decoding reducing LVLM hallucination via dynamic attention perturbation."}
{"id": 5192, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "benchmark", "why": "ManBench for Mandela effect in multi-agent systems"}, {"area": "multi-agent", "role": "defense", "why": "Prompt and alignment mitigations"}], "pattern": "vary-conditions", "fixes": ["LLM agents"], "varies": ["interaction protocols"], "benchmarks": [{"name": "ManBench", "purpose": "Mandela effect", "is_new": true}], "results": [], "workflow": [], "summary": "Studies collective false memory in LLM multi-agent systems; mitigation reduces effect 74.40%."}
{"id": 14283, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "method", "why": "Vietnamese fact-checking with semantic evidence retrieval and two-step verdict"}], "pattern": "n/a", "fixes": ["Vietnamese datasets"], "varies": ["pipeline"], "benchmarks": [{"name": "ISE-DSC01", "purpose": "fact verification", "is_new": false}, {"name": "ViWikiFC", "purpose": "fact verification", "is_new": false}], "results": [{"benchmark": "ISE-DSC01", "system": "SemViQA", "base_model": null, "setting": "proposed", "metric": "strict accuracy", "score": 78.97, "unit": "%"}, {"benchmark": "ViWikiFC", "system": "SemViQA", "base_model": null, "setting": "proposed", "metric": "accuracy", "score": 80.82, "unit": "%"}], "workflow": ["Semantic evidence retrieval", "Two-step verdict classification"], "summary": "Vietnamese fact-checking framework balancing accuracy and speed."}
{"id": 18029, "in_agent_scope": false, "types": [{"area": "apps/domain", "role": "benchmark", "why": "ShAnEL-2 multilingual short-answer correction dataset"}, {"area": "knowledge/rag", "role": "analysis", "why": "Compares few-shot and textbook RAG"}], "pattern": "vary-conditions", "fixes": ["GenAI models"], "varies": ["RAG setting"], "benchmarks": [{"name": "ShAnEL-2", "purpose": "answer correction", "is_new": true}], "results": [{"benchmark": "ShAnEL-2", "system": "few-shot RAG", "base_model": null, "setting": "proposed", "metric": "accuracy", "score": 90, "unit": "%"}, {"benchmark": "ShAnEL-2", "system": "few-shot RAG", "base_model": null, "setting": "proposed", "metric": "recall", "score": 95, "unit": "%"}], "workflow": [], "summary": "Dataset benchmarking GenAI and RAG for language-learning answer correction."}
{"id": 13831, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "CxMP minimal-pair benchmark for construction understanding"}], "pattern": "fix-agent-vary-env", "fixes": ["LMs"], "varies": ["constructions"], "benchmarks": [{"name": "CxMP", "purpose": "constructional understanding", "is_new": true}], "results": [], "workflow": [], "summary": "Construction Grammar benchmark showing constructional meaning develops gradually in LMs."}
{"id": 5847, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "train-agent", "why": "RL-trained orchestrator generating difficulty-adaptive communication topology"}, {"area": "code-swe", "role": "scaffold", "why": "Multi-agent code generation topologies"}], "pattern": "vary-scaffold", "fixes": ["code datasets"], "varies": ["topology"], "benchmarks": [], "results": [], "workflow": ["Infer roles and difficulty", "Build layered DAG", "Refine with execution feedback"], "summary": "RL orchestrator builds density-aware agent topologies for code generation, 14.6% pass@1 gain."}
{"id": 16501, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Studies adversarial robustness versus number of agents in sampling-and-voting"}, {"area": "agent-safety", "role": "analysis", "why": "Perturbation robustness"}], "pattern": "vary-conditions", "fixes": ["Agent Forest", "models"], "varies": ["noise type", "agent count"], "benchmarks": [{"name": "GSM8K", "purpose": "math robustness", "is_new": false}, {"name": "MATH", "purpose": "math robustness", "is_new": false}, {"name": "MMLU-Math", "purpose": "math robustness", "is_new": false}, {"name": "MultiArith", "purpose": "math robustness", "is_new": false}], "results": [], "workflow": [], "summary": "More agents help accuracy but robustness gap to typos and noise persists."}
{"id": 15465, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "SFT and RL training of LAMO-3B multi-role GUI agent"}, {"area": "web-gui", "role": "data", "why": "Role-oriented data synthesis"}], "pattern": "fix-env-vary-agent", "fixes": ["GUI benchmarks"], "varies": ["training recipe", "roles"], "benchmarks": [], "results": [], "workflow": ["Role-oriented data synthesis", "SFT distillation", "RL cooperative exploration"], "summary": "Lightweight GUI agent trained for multi-role orchestration, usable as plug-in executor."}
{"id": 2141, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "train-agent", "why": "Retrieve-and-rerank models for issue localization trained on SWELoc"}, {"area": "code-swe", "role": "data", "why": "SWELoc dataset"}], "pattern": "fix-env-vary-agent", "fixes": ["benchmarks"], "varies": ["ranking models", "training data"], "benchmarks": [{"name": "SWE-Bench-Lite", "purpose": "issue localization", "is_new": false}, {"name": "LocBench", "purpose": "issue localization", "is_new": false}, {"name": "SWELoc", "purpose": "training dataset", "is_new": true}], "results": [], "workflow": ["Retrieve candidates", "Rerank code locations"], "summary": "Ranking-based issue localization trained on new dataset, outperforming agent systems."}
{"id": 7098, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "environment", "why": "Gecko stateful simulation environment with feedback for tool calls"}, {"area": "tool-use", "role": "scaffold", "why": "GATS test-time scaling by refining in simulation"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["simulated feedback"], "benchmarks": [{"name": "BFCLv3", "purpose": "tool-call eval", "is_new": false}, {"name": "tau2-bench", "purpose": "tool-call eval", "is_new": false}], "results": [], "workflow": ["Validate tool calls", "Simulate responses", "Judge completion", "Refine"], "summary": "Simulated tool environment gives feedback to refine LLM tool calls before real execution."}