Files
ai-taxonomy-literature-anal…/data/labels/o19.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
31 KiB
JSON

{"id": 12482, "in_agent_scope": false, "types": [{"area": "multiling/mt", "role": "data", "why": "Zero-shot translation-based paraphrase generation with feature-guided diversity to augment low-resource NLP data"}], "pattern": "vary-data", "fixes": ["downstream NLP tasks"], "varies": ["paraphrase data generation"], "benchmarks": [], "results": [], "workflow": ["train multilingual NMT", "zero-shot translate to paraphrase", "feature-guided diversity"], "summary": "Generates diverse paraphrases via zero-shot multilingual NMT with feature guidance for data augmentation."}
{"id": 4986, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "Introduces o3-bench for multimodal reasoning with visual search"}, {"area": "web-gui", "role": "train-agent", "why": "InSight-o3 multi-agent framework trained with RL for visual search"}], "pattern": "vary-both", "fixes": ["base multimodal models"], "varies": ["benchmark tasks", "visual search agent"], "benchmarks": [{"name": "o3-bench", "purpose": "evaluate multimodal reasoning with visual detail", "is_new": true}], "results": [{"benchmark": "o3-bench", "system": "OpenAI o3", "base_model": null, "setting": "baseline", "metric": "accuracy", "score": 42.8, "unit": "%"}], "workflow": ["reasoner agent", "visual search agent", "RL training"], "summary": "Introduces o3-bench and a multi-agent InSight-o3 framework with a generalized visual search agent trained by RL."}
{"id": 14977, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "eval-method", "why": "Dynamic knowledge-driven LLM evaluation via agents and adaptive multi-turn interviews"}], "pattern": "vary-evaluation", "fixes": ["evaluated LLMs"], "varies": ["evaluation protocol"], "benchmarks": [], "results": [], "workflow": ["traverse context graphs", "generate questions", "adaptive multi-turn interview"], "summary": "JudgeAgent is a dynamic evaluation framework using agents with context graphs and difficulty-adaptive interviews."}
{"id": 9710, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "Trains memory-update copilot with multi-turn GRPO for frozen LLM"}], "pattern": "vary-scaffold", "fixes": ["frozen LLM"], "varies": ["memory update policy", "RL reward and advantage"], "benchmarks": [], "results": [], "workflow": ["memory copilot updates memory", "multi-turn GRPO", "turn-wise reward"], "summary": "MemoPilot trains a plug-in memory copilot via multi-turn GRPO to improve test-time learning of a frozen LLM agent."}
{"id": 17681, "in_agent_scope": false, "types": [{"area": "apps/science", "role": "data", "why": "Large Brazilian-Portuguese medical QA corpus"}], "pattern": "n/a", "fixes": ["LLMs"], "varies": ["language-specific medical data"], "benchmarks": [{"name": "MedPT", "purpose": "Brazilian-Portuguese medical QA corpus", "is_new": true}], "results": [], "workflow": [], "summary": "MedPT is a 384k-pair patient-doctor QA corpus for Brazilian Portuguese with curation and annotation."}
{"id": 15521, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Multi-agent MCTS test-time framework for inductive reasoning"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["test-time search method"], "benchmarks": [], "results": [], "workflow": ["multi-agent hypothesis refinement", "MCTS with dual reward", "backtracking and pruning"], "summary": "MATSIR combines multi-agent coordination and MCTS with dual rewards to improve LLM inductive reasoning at test time."}
{"id": 11226, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "benchmark", "why": "VoxParadox adversarial paralinguistic benchmark for audio LLMs"}, {"area": "arch/multimodal", "role": "analysis", "why": "Layer-wise probing of paralinguistic failures"}], "pattern": "vary-conditions", "fixes": ["Audio LLMs"], "varies": ["benchmark tasks", "probing layers"], "benchmarks": [{"name": "VoxParadox", "purpose": "paralinguistic understanding with mismatched transcripts", "is_new": true}], "results": [], "workflow": [], "summary": "Introduces VoxParadox and analyzes why audio LLMs follow text over acoustics, with mitigation."}
{"id": 15209, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Learnable procedural memory repository distilled from trajectories"}], "pattern": "vary-scaffold", "fixes": ["LLM agents"], "varies": ["procedural memory"], "benchmarks": [{"name": "TravelPlanner", "purpose": "show memory improves success", "is_new": false}, {"name": "ALFWorld", "purpose": "show memory improves success", "is_new": false}], "results": [], "workflow": ["distill trajectories", "build memory", "update and deprecate"], "summary": "Memp builds an updatable procedural memory from past trajectories that improves agent success and transfers across models."}
{"id": 12384, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "eval-method", "why": "Agent-driven adaptive question selection for efficient MLLM benchmarking"}], "pattern": "vary-evaluation", "fixes": ["MLLMs"], "varies": ["adaptive evaluation protocol"], "benchmarks": [], "results": [], "workflow": ["decompose abilities", "estimate ability", "select informative questions"], "summary": "AutoJudger adaptively selects informative questions to benchmark MLLMs efficiently."}
{"id": 10741, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "scaffold", "why": "Graph-in-Graph experience memory with GNN for embodied LLM planning"}], "pattern": "vary-scaffold", "fixes": ["LLM planner"], "varies": ["memory and planning structure"], "benchmarks": [], "results": [], "workflow": ["GNN encodes states", "build trace graphs", "retrieve for planning"], "summary": "GiG structures embodied agent memory as graph-in-graph to improve long-horizon LLM planning."}
{"id": 13321, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Fuse scene graphs and commonsense graphs for MLLM VQA"}], "pattern": "vary-scaffold", "fixes": ["MLLMs"], "varies": ["graph retrieval and fusion"], "benchmarks": [], "results": [], "workflow": ["retrieve both graphs", "fuse structured context", "MLLM reasoning"], "summary": "KG-ViP fuses scene and commonsense graphs to improve MLLM visual question answering."}
{"id": 15062, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Memory of clustered failure notes, training-free, with test-time scaling"}], "pattern": "vary-scaffold", "fixes": ["frozen LLM agent"], "varies": ["failure-based memory"], "benchmarks": [], "results": [], "workflow": ["cluster batch failures", "distill mistake notes", "update memory if improved"], "summary": "Mistake Notebook Learning distills batch-clustered failures into notes for training-free agent adaptation."}
{"id": 15490, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "RL agent for KG multi-hop QA with LLM planner guidance and lookahead"}], "pattern": "vary-scaffold", "fixes": ["KG QA setting"], "varies": ["RL policy guidance"], "benchmarks": [], "results": [], "workflow": ["LLM decomposes sub-questions", "stepwise guidance", "lookahead RL policy"], "summary": "PPRR uses LLM-planned sub-questions and structure-aware lookahead to guide RL agents for KGQA."}
{"id": 14, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "Hierarchical RL for tool-integrated mathematical reasoning"}, {"area": "tool-use", "role": "data", "why": "TIRGen multi-agent pipeline builds tool-integrated reasoning data"}], "pattern": "vary-both", "fixes": ["math benchmarks"], "varies": ["training data", "RL strategy", "inference"], "benchmarks": [], "results": [], "workflow": ["TIRGen data", "hierarchical RL", "self-correction inference"], "summary": "THOR builds tool-integrated reasoning data and applies hierarchical RL for math reasoning."}
{"id": 771, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "Principled test-time scaling agent framework for scientific coding"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": ["test-time scaling", "principled search", "scientific coding"], "summary": "SciNav is a principled agent framework for scientific coding tasks using test-time scaling."}
{"id": 9626, "in_agent_scope": false, "types": [{"area": "train/peft", "role": "method", "why": "Meta-learned multi-objective LLM selection with landmark fine-tuning"}], "pattern": "n/a", "fixes": ["candidate LLMs"], "varies": ["selection policy"], "benchmarks": [], "results": [], "workflow": ["landmark fine-tuning", "meta-learn allocation policy", "approximate Pareto front"], "summary": "LAMPS meta-learns a resource allocation policy to find Pareto-optimal LLMs to fine-tune."}
{"id": 13313, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "TPS-Bench with 200 compounding tasks over MCP tools"}], "pattern": "fix-agent-vary-env", "fixes": ["LLM agents"], "varies": ["tasks"], "benchmarks": [{"name": "TPS-Bench", "purpose": "tool planning and scheduling with MCP tools", "is_new": true}], "results": [], "workflow": [], "summary": "TPS-Bench evaluates LLM agents on tool planning and scheduling in compounding MCP tasks."}
{"id": 1191, "in_agent_scope": false, "types": [{"area": "align/rlvr", "role": "train-agent", "why": "RL with case-wise rubrics as rewards for open-ended QA"}, {"area": "eval/judge", "role": "eval-method", "why": "Rubric schema as evaluation signal"}], "pattern": "vary-evaluation", "fixes": ["LLMs"], "varies": ["reward signal"], "benchmarks": [], "results": [], "workflow": ["generate case rubrics", "score answers", "RL training"], "summary": "QuRL uses question-specific rubrics as verifiable rewards for RL on open-ended QA."}
{"id": 9434, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "scaffold", "why": "Sequential Monte Carlo with learned value to steer black-box agents"}], "pattern": "vary-scaffold", "fixes": ["black-box LLM agent"], "varies": ["inference-time sampling"], "benchmarks": [], "results": [], "workflow": ["learn value function", "SMC sampling", "steer agent"], "summary": "Agentic Monte Carlo steers black-box agents at test time via SMC with a learned value function."}
{"id": 8321, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Gradient-free tacit communication with retrieval-based rule memory"}], "pattern": "vary-scaffold", "fixes": ["embodied tasks"], "varies": ["coordination protocol"], "benchmarks": [{"name": "VIKI", "purpose": "cooperation evaluation", "is_new": false}, {"name": "C-WAH", "purpose": "cooperation evaluation", "is_new": false}, {"name": "TDW-MAT", "purpose": "cooperation evaluation", "is_new": false}], "results": [], "workflow": ["value rectification", "identify miscoordination", "tacit rule memory"], "summary": "Contextual Tacit Communication aligns decentralized LLM agents without messages via a retrieval-based rule memory."}
{"id": 16690, "in_agent_scope": false, "types": [{"area": "align/pref", "role": "method", "why": "Conformal-prediction-weighted preferences for DPO/PPO"}], "pattern": "vary-data", "fixes": ["RLHF setup"], "varies": ["preference weighting"], "benchmarks": [], "results": [], "workflow": [], "summary": "CFA weights preference pairs by conformal answer-level reliability for robust alignment."}
{"id": 18072, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "benchmark", "why": "Corpus of authentic, AI-generated and AI-enhanced resumes with baselines"}], "pattern": "n/a", "fixes": ["classifiers"], "varies": ["corpus"], "benchmarks": [{"name": "Resume authenticity corpus", "purpose": "detect AI-generated resumes", "is_new": true}], "results": [], "workflow": [], "summary": "First corpus of authentic, AI-enhanced and AI-generated resumes with classical and neural baselines."}
{"id": 5678, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "benchmark", "why": "CoopEval benchmark of social dilemmas for LLM cooperation"}, {"area": "multi-agent", "role": "analysis", "why": "Compares cooperation mechanisms across games"}], "pattern": "vary-conditions", "fixes": ["LLM agents"], "varies": ["cooperation mechanisms", "games"], "benchmarks": [{"name": "CoopEval", "purpose": "cooperation in social dilemmas", "is_new": true}], "results": [], "workflow": [], "summary": "Compares repetition, reputation, mediators and contracts for sustaining LLM cooperation in social dilemmas."}
{"id": 16573, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "scaffold", "why": "Hierarchical multi-agent planner with map tool modules for geospatial reasoning"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": ["planner decomposes subgoals", "route to modules", "map tools"], "summary": "MapAgent is a hierarchical multi-agent framework with map tools for geospatial reasoning."}
{"id": 13237, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "train-agent", "why": "Multi-agent RL within shared tree search for code generation"}], "pattern": "fix-env-vary-agent", "fixes": ["code benchmarks"], "varies": ["multi-agent RL training"], "benchmarks": [], "results": [], "workflow": ["tree-structured search", "multiple agents collaborate", "RL optimization"], "summary": "MARS^2 trains multiple agents with RL inside a shared tree search for code generation."}
{"id": 11759, "in_agent_scope": false, "types": [{"area": "interp/mech", "role": "analysis", "why": "Sparse crosscoders track linguistic feature emergence over pretraining checkpoints"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["checkpoints"], "benchmarks": [], "results": [], "workflow": ["train crosscoders", "align features", "RelIE metric"], "summary": "Uses sparse crosscoders and RelIE to track emergence of linguistic features during pretraining."}
{"id": 9413, "in_agent_scope": false, "types": [{"area": "apps/science", "role": "train-agent", "why": "ECG reasoning MLLM with protocol-guided data and RL rewards"}, {"area": "apps/science", "role": "data", "why": "Protocol-guided instruction data generation"}], "pattern": "vary-both", "fixes": ["ECG interpretation"], "varies": ["data", "architecture", "RL rewards"], "benchmarks": [], "results": [], "workflow": ["protocol-guided data", "modality dropout", "evidence-reward RL"], "summary": "ECG-R1 is a reasoning ECG MLLM trained with protocol-guided data and RL evidence rewards."}
{"id": 1996, "in_agent_scope": false, "types": [{"area": "arch/diffusion", "role": "method", "why": "Training-free color editing via MM-DiT attention manipulation"}], "pattern": "n/a", "fixes": ["MM-DiT model"], "varies": ["editing method"], "benchmarks": [], "results": [], "workflow": ["manipulate attention maps", "edit value tokens"], "summary": "ColorCtrl performs training-free text-guided color editing by manipulating MM-DiT attention."}
{"id": 16238, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "scaffold", "why": "Multi-agent semantic routing and fusion RAG for medical reasoning"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["RAG framework"], "benchmarks": [], "results": [], "workflow": ["semantic routing", "retrieve per aspect", "agentic fusion"], "summary": "Med-SRAF is a multi-agent RAG framework using semantic routing and agentic fusion for medical reasoning."}
{"id": 15122, "in_agent_scope": false, "types": [{"area": "efficiency/compress", "role": "method", "why": "Joint per-layer quantization bit-width and LoRA rank search"}], "pattern": "n/a", "fixes": ["base LLM"], "varies": ["bit-width and rank allocation"], "benchmarks": [], "results": [], "workflow": ["layer sensitivity", "multi-objective search"], "summary": "QR-Adaptor jointly optimizes per-layer quantization bit-width and LoRA rank for edge fine-tuning."}
{"id": 2625, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Stateful specialist agent teams discovered by evolutionary search and cultivated by experience"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent system design"], "benchmarks": [], "results": [], "workflow": ["evolve specialist archetypes", "cultivate via experience", "retain-then-escalate"], "summary": "ASpec discovers and cultivates stateful specialist agent teams with a retain-then-escalate policy."}
{"id": 14809, "in_agent_scope": false, "types": [{"area": "apps/domain", "role": "benchmark", "why": "EduMARS multimodal rubric-based grading of Chinese K-12 answers"}], "pattern": "fix-agent-vary-env", "fixes": ["VLMs"], "varies": ["benchmark"], "benchmarks": [{"name": "EduMARS", "purpose": "rubric-based grading of student answers", "is_new": true}], "results": [], "workflow": [], "summary": "EduMARS benchmarks VLMs on rubric-aligned grading of 4,500 real Chinese K-12 student answers."}
{"id": 17441, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "environment", "why": "City-scale generative-agent urban behavior simulation"}], "pattern": "fix-agent-vary-env", "fixes": ["LLM agents"], "varies": ["simulation framework"], "benchmarks": [], "results": [], "workflow": ["initialize from survey data", "batched async inference", "low-token communication"], "summary": "MobileCity is a scalable generative-agent framework for city-scale urban behavior simulation."}
{"id": 15180, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "SWE-QA-Pro repository-level code understanding benchmark"}, {"area": "code-swe", "role": "data", "why": "Scalable training recipe for repo understanding"}], "pattern": "vary-both", "fixes": ["agents"], "varies": ["benchmark", "training data"], "benchmarks": [{"name": "SWE-QA-Pro", "purpose": "repository-level code understanding", "is_new": true}], "results": [], "workflow": [], "summary": "SWE-QA-Pro is a long-tail repo-level QA benchmark with a scalable training recipe."}
{"id": 6056, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "ADK letting LLMs self-generate agent topology, tools and memory"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent development kit"], "benchmarks": [], "results": [], "workflow": ["self-generate sub-agents", "create toolkits", "graph memory"], "summary": "OpenSage is an agent development kit where LLMs create their own topology, tools and memory."}
{"id": 13958, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "benchmark", "why": "DixitWorld multi-agent environment and static benchmark for abductive reasoning"}, {"area": "multi-agent", "role": "environment", "why": "DixitArena dynamic multi-agent game"}], "pattern": "fix-agent-vary-env", "fixes": ["VLMs"], "varies": ["evaluation suite"], "benchmarks": [{"name": "DixitWorld", "purpose": "multimodal abductive reasoning", "is_new": true}, {"name": "DixitArena", "purpose": "multi-agent Dixit gameplay", "is_new": true}, {"name": "DixitBench", "purpose": "listener QA", "is_new": true}], "results": [], "workflow": [], "summary": "DixitWorld evaluates VLM abductive reasoning through a multi-agent Dixit arena and static benchmark."}
{"id": 14414, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "train-agent", "why": "Unsupervised multi-agent polling optimization reducing judge self-preference bias"}], "pattern": "vary-evaluation", "fixes": ["judge tasks"], "varies": ["judge optimization"], "benchmarks": [], "results": [], "workflow": ["multi-agent judges", "client-server polling", "unsupervised optimization"], "summary": "Genii mitigates LLM-judge self-preference bias via unsupervised group-based polling optimization."}
{"id": 15176, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "RL-based black-box RAG poisoning attack"}], "pattern": "vary-conditions", "fixes": ["target RAG system"], "varies": ["poisoned documents"], "benchmarks": [], "results": [], "workflow": ["treat RAG as black box", "RL optimize poison documents"], "summary": "RIPRAG uses RL to craft poisoned documents that attack black-box RAG QA systems."}
{"id": 8297, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Preference-RL post-training for safe act-or-refuse tool use"}, {"area": "agent-safety", "role": "train-agent", "why": "Trains agents with plan-check-act loop"}], "pattern": "fix-env-vary-agent", "fixes": ["threat model"], "varies": ["alignment training"], "benchmarks": [], "results": [], "workflow": ["plan", "check safety", "act or refuse", "preference RL"], "summary": "MOSAIC post-trains agents with preference RL to plan, check safety, then act or refuse in tool use."}
{"id": 9009, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "scaffold", "why": "Compile tool plans to verified IR with verification-gated skill learning"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["orchestration framework"], "benchmarks": [], "results": [], "workflow": ["generate IR plan", "optimize and parallelize", "verified skill learning"], "summary": "EvoC2F compiles tool orchestration into a verified IR enabling parallelism and safe continual learning."}
{"id": 7793, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Phase-aware MoE policy for agentic RL"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["policy architecture"], "benchmarks": [], "results": [], "workflow": ["phase-aware routing", "MoE policy", "agentic RL"], "summary": "PA-MoE routes by task phase in an MoE policy to improve agentic RL."}
{"id": 12682, "in_agent_scope": false, "types": [{"area": "apps/domain", "role": "method", "why": "Training-free MCTS-guided personalized distractor generation"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["generation framework"], "benchmarks": [], "results": [], "workflow": ["infer student reasoning", "MCTS reconstruction", "generate distractors"], "summary": "Introduces personalized distractor generation using a training-free MCTS-guided reasoning reconstruction framework."}
{"id": 16409, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Planning agent routes graph tasks across text and visual modalities with GraphRAG"}], "pattern": "vary-scaffold", "fixes": ["VLMs"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": ["GraphRAG retrieval", "planning agent routing", "modality selection"], "summary": "GraphVista uses hierarchical GraphRAG and a planning agent to route graph understanding across modalities."}
{"id": 5957, "in_agent_scope": false, "types": [{"area": "align/rlvr", "role": "data", "why": "Agentic meta-synthesis of logic task families for RLVR"}], "pattern": "vary-data", "fixes": ["RLVR recipe"], "varies": ["synthetic training data"], "benchmarks": [], "results": [], "workflow": ["agents author generator-validator", "validate and refine", "filter tasks"], "summary": "SSLogic evolves verifiable logic task families with LLM agents to scale RLVR data."}
{"id": 7440, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "TadA-Bench million-variant replay benchmark for future-round protein engineering"}], "pattern": "fix-agent-vary-env", "fixes": ["models"], "varies": ["benchmark"], "benchmarks": [{"name": "TadA-Bench", "purpose": "future-round variant ranking", "is_new": true}], "results": [], "workflow": [], "summary": "TadA-Bench replays 31 directed-evolution rounds to test future-round variant ranking for agentic protein engineering."}
{"id": 12748, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Nested browser-action framework for information-seeking agents"}], "pattern": "fix-env-vary-agent", "fixes": ["deep IS benchmarks"], "varies": ["browser action design"], "benchmarks": [], "results": [], "workflow": ["nested browser actions", "decouple control from exploration"], "summary": "NestBrowse gives IS agents a minimal nested browser-action framework for deep web information seeking."}
{"id": 9922, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "GUI-RobustEval measures error recovery"}, {"area": "web-gui", "role": "data", "why": "RoTS synthesizes error-recovery trajectories for fine-tuning"}], "pattern": "vary-both", "fixes": ["GUI agents"], "varies": ["benchmark", "training data"], "benchmarks": [{"name": "GUI-RobustEval", "purpose": "error recovery of GUI agents", "is_new": true}, {"name": "OSWorld", "purpose": "state-of-the-art evaluation", "is_new": false}], "results": [{"benchmark": "OSWorld", "system": "RoTS-32B", "base_model": null, "setting": "finetuned", "metric": "success rate", "score": 47.4, "unit": "%"}], "workflow": ["tree-based error discovery", "synthesize recovery steps", "fine-tune models"], "summary": "Introduces GUI-RobustEval and RoTS trajectory synthesis to train GUI agents that recover from errors."}
{"id": 15423, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "data", "why": "HACHIMI multi-agent student persona generation with 1M persona corpus"}], "pattern": "vary-data", "fixes": ["LLMs"], "varies": ["persona generation"], "benchmarks": [], "results": [], "workflow": ["propose", "validate with neuro-symbolic checks", "revise"], "summary": "HACHIMI generates theory-aligned, quota-controlled student personas via multi-agent propose-validate-revise."}
{"id": 18452, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "method", "why": "Source selection and dual-branch architecture for unseen low-resource varieties"}], "pattern": "n/a", "fixes": ["multilingual models"], "varies": ["generalization method"], "benchmarks": [], "results": [], "workflow": ["TOPPing source selection", "dual-branch VAÇAÍ-Bowl"], "summary": "Two-stage framework using dissimilarity-aware source selection and a dual-branch model to generalize to unseen low-resource varieties."}
{"id": 16363, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "OmniCode with 1794 tasks across four software engineering categories"}], "pattern": "fix-agent-vary-env", "fixes": ["coding agents"], "varies": ["tasks"], "benchmarks": [{"name": "OmniCode", "purpose": "broad software engineering tasks", "is_new": true}, {"name": "SWE-Bench", "purpose": "contrast narrow patch generation", "is_new": false}, {"name": "HumanEval", "purpose": "contrast narrow tasks", "is_new": false}], "results": [], "workflow": [], "summary": "OmniCode benchmarks agents on 1794 tasks: bug fixing, test generation, code review fixing, style fixing."}
{"id": 1569, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "MASS jointly optimizes prompts and topologies of multi-agent systems"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["prompts", "topologies"], "benchmarks": [], "results": [], "workflow": ["block-level prompt optimization", "topology optimization", "workflow-level prompts"], "summary": "MASS automates multi-agent design by interleaving prompt and topology optimization."}
{"id": 3221, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "scaffold", "why": "Dual-scale world models with trajectory frontier and advantage reflection"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["exploration framework"], "benchmarks": [{"name": "Jericho", "purpose": "hard-exploration text games", "is_new": false}], "results": [], "workflow": ["global trajectory frontier", "local advantage reflection"], "summary": "GLoW uses dual-scale world models to guide LLM exploration in Jericho text games."}
{"id": 2621, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "eval-method", "why": "Scene-graph-guided LLM-judge metric for detailed image descriptions"}, {"area": "eval/bench", "role": "benchmark", "why": "DOCENT artwork description dataset"}], "pattern": "vary-evaluation", "fixes": ["VLMs"], "varies": ["metric"], "benchmarks": [{"name": "DOCENT", "purpose": "artwork description evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "PoSh uses scene graphs to guide LLM judges for detailed image description; DOCENT validates it."}
{"id": 9128, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Generative-prior transformer unifying rigid and non-rigid point cloud registration"}], "pattern": "n/a", "fixes": [], "varies": ["registration method"], "benchmarks": [], "results": [], "workflow": [], "summary": "RGGT unifies rigid and non-rigid point cloud registration with a generative-prior-guided transformer."}
{"id": 16055, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Multi-agent data-to-text with orchestrator and guardrail"}, {"area": "apps/dialogue", "role": "benchmark", "why": "New extended DBpedia triple set dataset"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["agent configuration"], "benchmarks": [{"name": "Extended DBpedia triples", "purpose": "long triple set data-to-text", "is_new": true}], "results": [], "workflow": ["order content", "structure text", "surface realisation"], "summary": "Multi-agent LLM pipeline for long triple-set data-to-text in English and Irish with a new dataset."}
{"id": 14429, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "ReflecTool-Bench for self-reflection in tool use"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["benchmark"], "benchmarks": [{"name": "ReflecTool-Bench", "purpose": "reflective tool use", "is_new": true}], "results": [], "workflow": [], "summary": "ReflecTool-Bench evaluates LLM critique and self-reflection on tool-use mistakes across 968 dialogues."}
{"id": 17430, "in_agent_scope": false, "types": [{"area": "apps/domain", "role": "benchmark", "why": "MizanQA Moroccan legal multi-answer MCQ benchmark"}], "pattern": "n/a", "fixes": ["LLMs"], "varies": ["benchmark"], "benchmarks": [{"name": "MizanQA", "purpose": "Moroccan legal QA", "is_new": true}], "results": [], "workflow": [], "summary": "MizanQA is a 1,776-question Moroccan legal multi-answer MCQ benchmark in Arabic."}
{"id": 13381, "in_agent_scope": false, "types": [{"area": "align/rlvr", "role": "train-agent", "why": "Negative experience replay with confidence gating and boundary failure sampling for GRPO"}], "pattern": "fix-env-vary-agent", "fixes": ["RLVR benchmarks"], "varies": ["replay strategy"], "benchmarks": [], "results": [], "workflow": ["mid-confidence gating", "boundary failure sampling", "replay in GRPO"], "summary": "NexGRPO replays gated, boundary-proximal failure trajectories to improve RLVR sample efficiency."}
{"id": 16031, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "DeepSynth-Eval checklist benchmark for survey writing synthesis"}, {"area": "search-research", "role": "eval-method", "why": "Checklist protocol for objective evaluation"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["benchmark", "protocol"], "benchmarks": [{"name": "DeepSynth-Eval", "purpose": "information consolidation in survey writing", "is_new": true}], "results": [], "workflow": [], "summary": "DeepSynth-Eval objectively evaluates post-retrieval synthesis for deep research using checklists on oracle contexts."}
{"id": 12442, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "Knowledge graph from claims with layerwise summarization for biomedical QA"}], "pattern": "n/a", "fixes": ["small LM"], "varies": ["RAG method"], "benchmarks": [], "results": [], "workflow": ["extract propositional claims", "build local KG", "layerwise summarization"], "summary": "CLAIMS builds a local claim knowledge graph and layerwise summaries to improve biomedical RAG QA."}