d6fc7c1dfa
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json - LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest - Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow) - Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks - build_worktypes.py adds work-type tables to data/papers.db Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
61 lines
37 KiB
JSON
61 lines
37 KiB
JSON
{"id": 11701, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Multi-agent plain-language summarization with simulated reader feedback loop"}], "pattern": "vary-scaffold", "fixes": ["base LLMs"], "varies": ["multi-agent summarization framework"], "benchmarks": [], "results": [], "workflow": ["Template-based planning", "Simulated readers give feedback", "Expert agent revises"], "summary": "Multi-agent framework simulating reader groups to produce plain-language summaries with better readability and factuality."}
|
|
{"id": 991, "in_agent_scope": false, "types": [{"area": "interp/mech", "role": "analysis", "why": "SAE analysis shows excessive language-feature pre-activation causes code-switching"}, {"area": "train/sft", "role": "method", "why": "SAE-guided SFT keeps language feature activations appropriate"}], "pattern": "vary-data", "fixes": ["model architectures"], "varies": ["SFT loss with SAE guidance"], "benchmarks": [{"name": "six multilingual benchmarks", "purpose": "check capability preservation", "is_new": false}], "results": [], "workflow": ["Analyze SAE language features", "Add pre-activation constraint to SFT", "Fine-tune models"], "summary": "Uses sparse autoencoders to explain code-switching and proposes SASFT, reducing it over 50% versus standard SFT."}
|
|
{"id": 12256, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "survey", "why": "Survey of LLM-based table question answering tasks, methods, evaluation"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Comprehensive survey of table question answering with LLMs, covering tasks, benchmarks, modeling strategies and open problems."}
|
|
{"id": 1331, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "RL-trained 4B DeepSearch model with RAPO and step-level reward"}, {"area": "search-research", "role": "data", "why": "DUETQA multi-agent self-play dataset enforcing web-search dependence"}], "pattern": "fix-env-vary-agent", "fixes": ["web search environment", "Qwen3-4B base"], "varies": ["training data", "RL algorithm", "reward"], "benchmarks": [{"name": "SimpleQA", "purpose": "DeepSearch evaluation", "is_new": false}, {"name": "FRAMES", "purpose": "DeepSearch evaluation", "is_new": false}, {"name": "WebWalker", "purpose": "DeepSearch evaluation", "is_new": false}, {"name": "Seal0", "purpose": "DeepSearch evaluation", "is_new": false}, {"name": "MuSiQue", "purpose": "DeepSearch evaluation", "is_new": false}, {"name": "DeepResearch-Bench", "purpose": "report generation evaluation", "is_new": false}, {"name": "HLE", "purpose": "general reasoning", "is_new": false}, {"name": "AIME-25", "purpose": "general reasoning", "is_new": false}, {"name": "GPQA-Diamond", "purpose": "general reasoning", "is_new": false}, {"name": "MedQA", "purpose": "general reasoning", "is_new": false}], "results": [], "workflow": ["Generate DUETQA via self-play", "Multi-turn RL with RAPO", "Synthesizer writes cited reports"], "summary": "Two 4B models for deep search and report synthesis, trained with RAPO RL and a new dataset, reaching open-weight state of the art."}
|
|
{"id": 16749, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "Post-generation memory retrieval resolves URIs in SPARQL generation to cut hallucinations"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["URI resolution via retrieval memory"], "benchmarks": [], "results": [], "workflow": ["LLM writes query with placeholders", "Retrieve correct URIs from memory", "Resolve into final query"], "summary": "PGMR lets LLMs emit placeholder URIs then resolves them by retrieval, nearly eliminating URI hallucinations in SPARQL generation."}
|
|
{"id": 2735, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Training-free retrieval-augmented image generation using VLM-identified gaps"}], "pattern": "vary-scaffold", "fixes": ["generative models"], "varies": ["retrieved reference images"], "benchmarks": [], "results": [], "workflow": ["VLM finds generation gaps", "Retrieve relevant images", "Condition generation on references"], "summary": "ImageRAG dynamically retrieves reference images to improve rare-concept image generation without RAG-specific training."}
|
|
{"id": 13984, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "survey", "why": "Position that dialogue should be part of agent planning loop"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Position paper reframing dialogue as joint action within agent planning, with implications for architecture and evaluation."}
|
|
{"id": 7110, "in_agent_scope": false, "types": [{"area": "reason/planning", "role": "method", "why": "LLM rule generation with logistic regression weighting and iterative refinement"}], "pattern": "vary-scaffold", "fixes": ["LLM backbones"], "varies": ["rule aggregation method"], "benchmarks": [], "results": [], "workflow": ["LLM proposes rules", "Logistic regression weights rules", "Refine with hard examples"], "summary": "RLIE learns weighted natural-language rule sets by combining LLM rule proposals with logistic regression and iterative refinement."}
|
|
{"id": 13535, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "application", "why": "Multi-agent closed-loop smart contract synthesis with Lean verification"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["verification-driven multi-agent pipeline"], "benchmarks": [], "results": [], "workflow": ["Generate contract", "Lean auto-formalize and verify", "Attack and repair"], "summary": "LeVer is a multi-agent framework generating smart contracts with Lean-based formal verification and iterative attack and repair."}
|
|
{"id": 12170, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "environment", "why": "J1-ENVS interactive legal environment for LLM agents"}, {"area": "domain-agents", "role": "benchmark", "why": "Evaluates 17 LLM agents on legal scenarios"}, {"area": "domain-agents", "role": "eval-method", "why": "J1-EVAL dual-metric for performance and procedural compliance"}], "pattern": "fix-agent-vary-env", "fixes": ["17 LLM agents"], "varies": ["legal scenarios", "environment complexity"], "benchmarks": [{"name": "J1-ENVS", "purpose": "interactive legal environment", "is_new": true}, {"name": "J1-EVAL", "purpose": "dual-metric evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "Interactive legal environment and dual-metric evaluation showing LLM agents struggle with procedural execution, best below 60%."}
|
|
{"id": 8942, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Pontryagin-guided policy optimization for non-exponential discounting RL"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["policy optimization method"], "benchmarks": [], "results": [], "workflow": ["Apply Pontryagin principle", "Monte Carlo rollouts", "Adjoint-MC projection"], "summary": "PG-DPO replaces Bellman recursion with Pontryagin-guided policy optimization to handle non-exponential discounting."}
|
|
{"id": 15660, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "IslamicFaithQA bilingual generative benchmark for hallucination and abstention"}, {"area": "search-research", "role": "scaffold", "why": "Agentic Quran-grounding RAG with iterative tool-call evidence seeking"}], "pattern": "vary-both", "fixes": ["LLMs"], "varies": ["retrieval strategy"], "benchmarks": [{"name": "IslamicFaithQA", "purpose": "hallucination and abstention measurement", "is_new": true}], "results": [], "workflow": ["Retrieve verses via tool calls", "Revise answer iteratively"], "summary": "Introduces a 3,810-item Islamic QA benchmark and agentic RAG that beats standard RAG."}
|
|
{"id": 13189, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Masked compositional concept modeling to improve CLIP compositionality"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["training objective"], "benchmarks": [{"name": "five compositional benchmarks", "purpose": "evaluate compositionality", "is_new": false}], "results": [], "workflow": ["Mask compositional concepts", "Reconstruct from other modality", "Auxiliary alignment losses"], "summary": "MACCO improves vision-language compositionality via cross-modal masked concept reconstruction."}
|
|
{"id": 12731, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "application", "why": "Multi-agent LLM framework for substructure counting under edge local DP"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["multi-agent decomposition", "privacy mechanism"], "benchmarks": [], "results": [], "workflow": ["Node agents handle local counting", "Curator aggregates", "Privacy controller adds noise"], "summary": "PSC multi-agent framework for graph substructure counting under edge LDP, improving over naive approach."}
|
|
{"id": 12993, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Taxonomy of edge-level errors in multi-agent handoffs"}, {"area": "multi-agent", "role": "scaffold", "why": "AgentAsk clarification module at critical edges"}], "pattern": "vary-scaffold", "fixes": ["multi-agent systems"], "varies": ["clarification module"], "benchmarks": [], "results": [{"benchmark": "five benchmarks", "system": "AgentAsk", "base_model": null, "setting": "proposed", "metric": "accuracy gain", "score": 4.69, "unit": "% (up to)"}], "workflow": ["Identify edge errors", "Train clarification module", "Insert at critical edges"], "summary": "Analyzes inter-agent error types and adds a lightweight trained clarification module improving accuracy up to 4.69%."}
|
|
{"id": 10181, "in_agent_scope": false, "types": [{"area": "arch/diffusion", "role": "method", "why": "Star-shaped sampling with learnable remasking for masked diffusion models"}], "pattern": "n/a", "fixes": ["pretrained diffusion models"], "varies": ["sampling algorithm"], "benchmarks": [], "results": [], "workflow": ["Star-shaped generation", "Learnable remasking module", "Fine-tune one layer"], "summary": "G-Star sampling improves pretrained masked diffusion LM quality in few-step generation via error-correcting remasking."}
|
|
{"id": 11017, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "STING automated multi-turn multilingual red-teaming of tool agents"}, {"area": "agent-safety", "role": "eval-method", "why": "Time-to-first-jailbreak analysis and Restricted Mean Jailbreak Discovery metric"}], "pattern": "fix-agent-vary-env", "fixes": ["target agents"], "varies": ["attack strategy", "language"], "benchmarks": [{"name": "AgentHarm", "purpose": "scenarios for illicit tasks", "is_new": false}], "results": [], "workflow": ["Build illicit plan from benign persona", "Adaptive follow-up probing", "Judge agents track phases"], "summary": "Automated multi-turn red-teaming framework for agent misuse with survival-style analysis, evaluated multilingually."}
|
|
{"id": 1691, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "MC-Search agentic multimodal RAG benchmark with long reasoning chains"}, {"area": "search-research", "role": "data", "why": "Search-Align process-supervised fine-tuning on verified chains"}], "pattern": "fix-agent-vary-env", "fixes": ["six MLLMs"], "varies": ["tasks"], "benchmarks": [{"name": "MC-Search", "purpose": "agentic MM-RAG evaluation", "is_new": true}], "results": [], "workflow": ["Annotate verified chains", "Benchmark six MLLMs", "Fine-tune with Search-Align"], "summary": "Benchmark of 3,333 agentic multimodal RAG examples with process metrics, plus fine-tuning that improves planning and retrieval."}
|
|
{"id": 14851, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "benchmark", "why": "PaperMind benchmark for scientific paper reasoning and critique in MLLMs"}], "pattern": "fix-agent-vary-env", "fixes": ["multimodal LLMs"], "varies": ["task families"], "benchmarks": [{"name": "PaperMind", "purpose": "scientific paper reasoning", "is_new": true}], "results": [], "workflow": [], "summary": "PaperMind benchmark across seven domains testing integrated multimodal scientific reasoning and critique."}
|
|
{"id": 13470, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Experience-driven routing of LLM backbones per step in agent systems"}], "pattern": "vary-scaffold", "fixes": ["agent systems", "model weights"], "varies": ["model routing policy"], "benchmarks": [{"name": "GAIA", "purpose": "agentic evaluation", "is_new": false}, {"name": "BrowseComp+", "purpose": "agentic evaluation", "is_new": false}], "results": [], "workflow": ["Retrieve prior experience", "Select Pareto-optimal backbone", "Refine policy from feedback"], "summary": "EvoRoute self-evolving routing cuts agent cost up to 80% and latency over 70% while maintaining performance."}
|
|
{"id": 7666, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "environment", "why": "Multi-agent synthesis of malware environments from threat intelligence for traffic generation"}], "pattern": "n/a", "fixes": ["LLM agents"], "varies": ["environment construction pipeline"], "benchmarks": [], "results": [{"benchmark": "1,077 malware samples", "system": "Threat2Traffic", "base_model": null, "setting": "proposed", "metric": "reproduction success", "score": 83.1, "unit": "%"}], "workflow": ["Extract dependencies via deliberation", "Synthesize tailored environment", "Capture malware traffic"], "summary": "Multi-agent framework reconstructing malware execution environments from threat intelligence, achieving 83.1% reproduction success."}
|
|
{"id": 32, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "Agentic system autonomously developing and optimizing scientific computational methods"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["agentic optimization with knowledge tree"], "benchmarks": [], "results": [], "workflow": ["Build domain knowledge tree", "Iterative optimization", "Model diagnosis"], "summary": "TusoAI agentically develops and optimizes scientific computational methods, outperforming expert methods and MLE agents."}
|
|
{"id": 327, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "scaffold", "why": "Test-time scaling sampling multiple GUI action proposals with judge"}, {"area": "web-gui", "role": "train-agent", "why": "RL-trained grounding model rewarding successful clicks"}], "pattern": "fix-env-vary-agent", "fixes": ["GUI environments"], "varies": ["inference sampling", "grounding training"], "benchmarks": [], "results": [], "workflow": ["Sample candidate actions", "Judge selects best", "RL-trained grounding"], "summary": "GTA1 combines judge-based test-time scaling with RL-trained grounding for GUI agents."}
|
|
{"id": 10615, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Agents invent and route compact symbolic languages to cut tokens"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["communication protocol", "router"], "benchmarks": [], "results": [], "workflow": ["Agents evolve symbolic languages", "Router selects per query", "Compose protocols for hard queries"], "summary": "CLSR reduces generated tokens 3-6x versus CoT via evolved symbolic multi-agent protocols with a router."}
|
|
{"id": 12460, "in_agent_scope": false, "types": [{"area": "reason/math", "role": "method", "why": "Weak model as orthogonal probe steering teacher hidden states for diverse reasoning"}], "pattern": "vary-scaffold", "fixes": ["teacher model"], "varies": ["inference-time exploration"], "benchmarks": [], "results": [], "workflow": ["Detect dominant subspace", "Weak model probes orthogonally", "Steer teacher reasoning"], "summary": "SOE uses a weak model to steer a stronger model away from reasoning collapse in math."}
|
|
{"id": 11659, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "train-agent", "why": "GRPO-trained interactive reasoning intervening at doubt conjunctions to cut overthinking"}], "pattern": "vary-scaffold", "fixes": ["reasoning models"], "varies": ["intervention policy"], "benchmarks": [], "results": [], "workflow": ["Pause at conjunction markers", "Obtain external feedback", "Extend or terminate reasoning"], "summary": "CGI intervenes at doubt markers in large reasoning models, balancing accuracy and reasoning length."}
|
|
{"id": 11175, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "QM-LibBench for library-scale cross-architecture migration"}, {"area": "code-swe", "role": "scaffold", "why": "FSCM multi-agent framework with hardware-aware optimization"}], "pattern": "fix-agent-vary-env", "fixes": ["12 agent-LLM combinations"], "varies": ["tasks"], "benchmarks": [{"name": "QM-LibBench", "purpose": "library migration", "is_new": true}], "results": [{"benchmark": "QM-LibBench", "system": "agent baselines", "base_model": null, "setting": "baseline", "metric": "libjpeg correctness", "score": 20.88, "unit": "%"}, {"benchmark": "QM-LibBench", "system": "FSCM", "base_model": null, "setting": "proposed", "metric": "OpenCV correctness", "score": 71, "unit": "%"}], "workflow": [], "summary": "Benchmark of 85 kernels for cross-architecture library migration, revealing weak agents, plus a multi-agent FSCM improvement."}
|
|
{"id": 12294, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "method", "why": "Mixture of skill-specialized VLN agents with training-free VLM router"}, {"area": "embodied-nav", "role": "data", "why": "Synthetic skill-specific instruction-trajectory pairs"}], "pattern": "fix-env-vary-agent", "fixes": ["VLN environments"], "varies": ["skill agents", "router"], "benchmarks": [{"name": "GSA-R2R", "purpose": "generalization to novel instructions", "is_new": false}], "results": [], "workflow": ["Decompose into atomic skills", "Train skill agents on synthetic data", "VLM router selects agent"], "summary": "SkillNav routes among skill-specific VLN agents trained on synthetic data, improving generalization."}
|
|
{"id": 10628, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Agentic RL with experience distillation from rollouts into in-context guidance"}, {"area": "memory-selfevolve", "role": "scaffold", "why": "Experience repository co-evolving with policy"}], "pattern": "fix-env-vary-agent", "fixes": ["web search benchmarks"], "varies": ["RL training with experience"], "benchmarks": [{"name": "GAIA", "purpose": "web agent evaluation", "is_new": false}, {"name": "xBench", "purpose": "web agent evaluation", "is_new": false}, {"name": "HLE", "purpose": "web agent evaluation", "is_new": false}], "results": [], "workflow": ["Collect rollouts", "Distill into strategic experiences", "Use as context in RL"], "summary": "Evolving Rollouts recovers zero-variance rollout signals via experience guidance, letting Qwen3-8B surpass Qwen3-30B-A3B."}
|
|
{"id": 14421, "in_agent_scope": true, "types": [{"area": "search-research", "role": "survey", "why": "Data-centric survey of agentic RAG datasets and benchmarks"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "First data-centric survey of agentic RAG, covering data lifecycle, training datasets and benchmarks."}
|
|
{"id": 8658, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "train-agent", "why": "RL-trained single-tool agent for repository issue localization"}], "pattern": "fix-env-vary-agent", "fixes": ["localization benchmarks"], "varies": ["tool design", "RL training"], "benchmarks": [], "results": [], "workflow": ["Single jump-to-definition tool", "RL from base model"], "summary": "RepoNavigator trains an LLM agent with RL and one execution-aware tool, with the 32B model exceeding GPT-5 on most metrics."}
|
|
{"id": 9646, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "application", "why": "ToolPro executable tool programs for agentic web services"}], "pattern": "vary-scaffold", "fixes": ["agents"], "varies": ["tool interface"], "benchmarks": [], "results": [{"benchmark": "real-world workflows", "system": "ToolPro", "base_model": null, "setting": "proposed", "metric": "latency reduction", "score": 53.4, "unit": "% (up to)"}], "workflow": ["Construct tool program", "Effect-aware replay", "Policy chooses execution mode"], "summary": "Tool programs encode multi-step service interactions, cutting latency up to 53.4% and traffic up to 96.1%."}
|
|
{"id": 16348, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "benchmark", "why": "SCOpE-QA dataset and related insight generation task"}, {"area": "knowledge/rag", "role": "method", "why": "InsightGen clustering and neighborhood selection for insights"}], "pattern": "n/a", "fixes": ["LLMs"], "varies": ["task setup"], "benchmarks": [{"name": "SCOpE-QA", "purpose": "insight generation", "is_new": true}], "results": [], "workflow": ["Cluster documents into themes", "Select related context", "Generate insights"], "summary": "New task and 3,000-question dataset for related insight generation, with a clustering-based baseline."}
|
|
{"id": 17506, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "benchmark", "why": "French multi-hop historical QA corpus from parliamentary debates"}], "pattern": "n/a", "fixes": [], "varies": ["question corpus"], "benchmarks": [{"name": "HistoriQA-ThirdRepublic", "purpose": "historical multi-hop QA", "is_new": true}], "results": [], "workflow": [], "summary": "Dataset of 1,782 multi-hop French historical questions for evaluating retrieval-augmented systems."}
|
|
{"id": 6676, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Lifts interaction traces into logic-grounded programmatic skills"}], "pattern": "vary-scaffold", "fixes": ["agents"], "varies": ["skill induction method"], "benchmarks": [], "results": [], "workflow": ["Collect interaction traces", "Synthesize control-flow programs", "Reuse skills on unseen goals"], "summary": "NSI induces conditional, logic-grounded programmatic skills from traces for long-horizon agent tasks."}
|
|
{"id": 7462, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Causal-inference detection of multi-step prompt injection in agent trajectories"}], "pattern": "n/a", "fixes": ["threat model"], "varies": ["detector"], "benchmarks": [], "results": [], "workflow": ["Build structural causal model", "Apply do-calculus counterfactual", "Slice attack chains"], "summary": "CausalTrace detects multi-step prompt injection by counterfactual causal analysis of agent trajectories."}
|
|
{"id": 4985, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "ACADREASON 50 expert academic problems for LLMs and agents"}], "pattern": "fix-agent-vary-env", "fixes": ["10 LLMs and agents"], "varies": ["tasks"], "benchmarks": [{"name": "ACADREASON", "purpose": "academic reasoning", "is_new": true}], "results": [{"benchmark": "ACADREASON", "system": "GPT-5", "base_model": null, "setting": "baseline", "metric": "score", "score": 16, "unit": "points"}], "workflow": [], "summary": "Benchmark of 50 expert academic problems; most LLMs score below 20 and agents none above 40."}
|
|
{"id": 11039, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Parametric reflective memory for diverse reflection in language agents"}], "pattern": "vary-scaffold", "fixes": ["tasks"], "varies": ["memory module"], "benchmarks": [], "results": [], "workflow": ["Encode reflection patterns in parameters", "Sample diverse reflections", "Combine with episodic memory"], "summary": "ParamMem encodes cross-sample reflections into parameters to diversify self-reflection, improving code, math and QA."}
|
|
{"id": 13299, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Hierarchical graph memory separating encoding from consolidation"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["memory architecture"], "benchmarks": [{"name": "LoCoMo", "purpose": "long-term memory evaluation", "is_new": false}, {"name": "LongDialQA", "purpose": "long dialogue evaluation", "is_new": false}], "results": [], "workflow": ["Event progression graph", "Consolidate on semantic shifts", "Multi-factor retrieval"], "summary": "GAM hierarchical graph memory reduces interference and improves reasoning accuracy and efficiency on LoCoMo."}
|
|
{"id": 11424, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "Multi-agent vulnerability discovery and exploitation with memory"}], "pattern": "vary-scaffold", "fixes": ["backbone models"], "varies": ["multi-agent framework"], "benchmarks": [], "results": [{"benchmark": "security benchmarks", "system": "Co-RedTeam", "base_model": null, "setting": "proposed", "metric": "exploitation success", "score": 60, "unit": "% (over)"}], "workflow": ["Discovery stage", "Exploitation with execution feedback", "Memory of trajectories"], "summary": "Co-RedTeam is a multi-agent framework for vulnerability discovery and exploitation using execution feedback and memory."}
|
|
{"id": 4677, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "VitaBench life-serving tasks with 66 tools"}, {"area": "tool-use", "role": "eval-method", "why": "Rubric-based sliding window evaluator"}], "pattern": "fix-agent-vary-env", "fixes": ["LLM agents"], "varies": ["tasks"], "benchmarks": [{"name": "VitaBench", "purpose": "versatile interactive agent tasks", "is_new": true}], "results": [{"benchmark": "VitaBench", "system": "best models", "base_model": null, "setting": "baseline", "metric": "success rate cross-scenario", "score": 30, "unit": "%"}], "workflow": [], "summary": "Benchmark of 100 cross-scenario and 300 single-scenario tasks with 66 tools; best models reach only 30% cross-scenario."}
|
|
{"id": 15918, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Graph-based social bot detection with uncertainty-guided pseudo-labeling and edge purification"}], "pattern": "n/a", "fixes": ["GNN backbones"], "varies": ["detection framework"], "benchmarks": [], "results": [], "workflow": ["Estimate node uncertainty", "Pseudo-label low-uncertainty nodes", "Purify heterophilous edges"], "summary": "TRUST improves GNN social bot detection through uncertainty estimation, pseudo-labeling and graph purification."}
|
|
{"id": 12476, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "eval-method", "why": "Reference-free caption quality via multiple-choice QA"}, {"area": "arch/multimodal", "role": "benchmark", "why": "CapQuiz with 10 question types across 24 domains"}], "pattern": "vary-evaluation", "fixes": ["VLLMs"], "varies": ["metric"], "benchmarks": [{"name": "CapQuiz", "purpose": "caption quality evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "CapQuiz measures video caption quality by utility for answering verified questions, correlating better with humans."}
|
|
{"id": 489, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "FigureBench text-to-illustration dataset of 3,300 pairs"}, {"area": "science-data", "role": "scaffold", "why": "AutoFigure agentic illustration generation with refinement"}], "pattern": "vary-scaffold", "fixes": ["generators"], "varies": ["agentic pipeline"], "benchmarks": [{"name": "FigureBench", "purpose": "scientific illustration generation", "is_new": true}], "results": [], "workflow": ["Think and plan layout", "Recombine and validate", "Render illustration"], "summary": "Benchmark of 3,300 text-figure pairs and AutoFigure agentic framework generating publication-ready illustrations."}
|
|
{"id": 13284, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "benchmark", "why": "GameplayQA multi-video QA on 3D multiplayer gameplay perception"}], "pattern": "fix-agent-vary-env", "fixes": ["frontier MLLMs"], "varies": ["questions"], "benchmarks": [{"name": "GameplayQA", "purpose": "multi-video gameplay understanding", "is_new": true}], "results": [], "workflow": [], "summary": "Densely annotated gameplay video benchmark with 2.4K QA pairs showing MLLMs lag humans."}
|
|
{"id": 18515, "in_agent_scope": false, "types": [{"area": "apps/dialogue", "role": "method", "why": "On-the-fly thinking while listening in full-duplex spoken dialogue models"}], "pattern": "n/a", "fixes": ["SDLMs"], "varies": ["thinking mechanism"], "benchmarks": [], "results": [], "workflow": ["Reason incrementally from past audio", "No added latency"], "summary": "Chronological Thinking lets full-duplex spoken dialogue models reason while listening to improve responses."}
|
|
{"id": 16019, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "APOLLO benchmark for following user preferences in tool calling"}, {"area": "tool-use", "role": "train-agent", "why": "RL approach improving preference following"}], "pattern": "fix-agent-vary-env", "fixes": ["language and reasoning models"], "varies": ["preference tasks"], "benchmarks": [{"name": "APOLLO", "purpose": "preference-following tool calling", "is_new": true}], "results": [{"benchmark": "APOLLO", "system": "GPT-4o", "base_model": null, "setting": "baseline", "metric": "accuracy", "score": 51.16, "unit": "%"}], "workflow": [], "summary": "Benchmark of preference-aware tool calling showing models struggle, plus RL that improves performance."}
|
|
{"id": 15117, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "data", "why": "Multi-agent synthesis of murder mystery scripts for VLM training"}, {"area": "social-sim", "role": "train-agent", "why": "CoT fine-tuning and GRPO with agent-monitored reward"}], "pattern": "vary-data", "fixes": ["VLM"], "varies": ["training data", "reward"], "benchmarks": [], "results": [], "workflow": ["Agents generate scripts", "CoT SFT", "GRPO with reward shaping"], "summary": "Multi-agent script synthesis and two-stage SFT plus RL training improve VLM reasoning in murder mystery games."}
|
|
{"id": 17567, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "benchmark", "why": "Annotated corpus of endangered Corfioto with annotation tool"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [{"name": "Universal Dependencies", "purpose": "annotation framework", "is_new": false}], "results": [], "workflow": [], "summary": "Work-in-progress morphosyntactic corpus and annotation application for the endangered Corfioto variety."}
|
|
{"id": 8904, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Cold-start SFT and omit-aware RL to omit redundant thoughts and observations"}, {"area": "planning-decision", "role": "analysis", "why": "Quantifies thought and observation effects on agent effectiveness"}], "pattern": "fix-env-vary-agent", "fixes": ["five agent benchmarks"], "varies": ["context management training"], "benchmarks": [], "results": [], "workflow": ["Synthesize omission cold-start data", "Fine-tune", "Omit-aware agentic RL"], "summary": "Agent-Omit trains agents to adaptively omit thoughts and observations, matching frontier agents with better efficiency."}
|
|
{"id": 15148, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Agentic multilingual Paper2Video system"}, {"area": "domain-agents", "role": "benchmark", "why": "mPreBench multilingual presentation video benchmark"}], "pattern": "vary-scaffold", "fixes": ["generation models"], "varies": ["agentic pipeline"], "benchmarks": [{"name": "mPreBench", "purpose": "presentation video QA evaluation", "is_new": true}], "results": [], "workflow": ["Plan discourse", "Audience critique", "Layout-aware slide generation"], "summary": "mPresenter agentic system generates multilingual presentation videos from papers, with a QA-based benchmark."}
|
|
{"id": 17329, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Multi-agent LLM candidate assessment with listwise tournament ranking"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["assessment pipeline"], "benchmarks": [], "results": [], "workflow": ["Generate role rubrics", "Multi-agent evaluation", "Active listwise tournament ranking"], "summary": "Modular LLM framework for recruitment candidate assessment with rubric agents and Plackett-Luce tournament ranking."}
|
|
{"id": 14721, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "RoadMap benchmark for research roadmap generation"}, {"area": "multi-agent", "role": "scaffold", "why": "RoadMapper multi-agent generate-augment-critique pipeline"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["multi-agent pipeline"], "benchmarks": [{"name": "RoadMap", "purpose": "roadmap generation evaluation", "is_new": true}], "results": [], "workflow": ["Initial generation", "Knowledge augmentation", "Critique-revise-evaluate"], "summary": "Roadmap generation benchmark and multi-agent system improving LLM performance over 8%."}
|
|
{"id": 3101, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "MultiAgentFraudBench with 21 fraud scenarios"}, {"area": "agent-safety", "role": "analysis", "why": "Analyzes fraud dynamics and collaboration failures"}, {"area": "agent-safety", "role": "defense", "why": "Monitor agents and group resilience mitigations"}], "pattern": "vary-conditions", "fixes": ["LLM agents"], "varies": ["fraud scenarios", "interaction depth"], "benchmarks": [{"name": "MultiAgentFraudBench", "purpose": "fraud simulation", "is_new": true}], "results": [], "workflow": [], "summary": "Benchmark and analysis of collective financial fraud among LLM agents, with proposed mitigations."}
|
|
{"id": 8869, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "analysis", "why": "Shows frontier models exhibit emergent peer-preservation misalignment"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["peer relationship"], "benchmarks": [], "results": [], "workflow": [], "summary": "Demonstrates that eight frontier models spontaneously protect peers via deception, shutdown disabling and weight exfiltration."}
|
|
{"id": 1215, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "benchmark", "why": "Memento framework for personalized memory utilization"}, {"area": "memory-selfevolve", "role": "scaffold", "why": "Hierarchical knowledge-graph user-profile memory"}], "pattern": "fix-agent-vary-env", "fixes": ["embodied agents"], "varies": ["memory tasks"], "benchmarks": [{"name": "Memento", "purpose": "personalization memory evaluation", "is_new": true}], "results": [], "workflow": ["Build user-profile graph", "Retrieve personalized knowledge"], "summary": "Memento evaluates embodied agents' memory use for personalization and proposes a graph memory module."}
|
|
{"id": 14014, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "eval-method", "why": "Multi-agent LLM-as-judge system for psychosocial safety auditing"}], "pattern": "vary-evaluation", "fixes": ["audited agents"], "varies": ["judge pipeline"], "benchmarks": [{"name": "PKU-SafeRLHF", "purpose": "judge pipeline comparison", "is_new": false}], "results": [], "workflow": ["Four judge pipelines", "Rubric-based scoring", "Web interface review"], "summary": "Open-source tool using multi-agent judges to audit LLM mental-health responses on psychosocial safety."}
|
|
{"id": 9598, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "LLM evolution of macro placement ordering strategies with proxy evaluation"}], "pattern": "n/a", "fixes": ["placement benchmarks"], "varies": ["ordering strategy"], "benchmarks": [{"name": "ISPD 2005", "purpose": "macro placement evaluation", "is_new": false}], "results": [], "workflow": ["LLM evolves ordering code", "Proxy greedy probe filters", "Select strategies"], "summary": "OrderPlace evolves placement ordering strategies with LLMs, reducing wirelength 34.04% versus WireMask-EA."}
|
|
{"id": 15351, "in_agent_scope": false, "types": [{"area": "interp/mech", "role": "analysis", "why": "Layer-wise probing of 58 psycholinguistic features across 10 models"}], "pattern": "vary-conditions", "fixes": ["probing method"], "varies": ["models", "layers", "embedding extraction"], "benchmarks": [], "results": [], "workflow": [], "summary": "Layer-wise probing shows where psycholinguistic meaning is encoded depends on method and architecture."}
|
|
{"id": 11700, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "scaffold", "why": "Training-free multi-agent framework with tool cards, planner and executor"}], "pattern": "vary-scaffold", "fixes": ["GPT-4o", "tools"], "varies": ["orchestration framework"], "benchmarks": [{"name": "MathVista", "purpose": "complex reasoning", "is_new": false}, {"name": "MMLU-Pro", "purpose": "complex reasoning", "is_new": false}, {"name": "MedQA", "purpose": "complex reasoning", "is_new": false}, {"name": "GAIA-Text", "purpose": "complex reasoning", "is_new": false}], "results": [], "workflow": ["Tool cards", "Planner", "Executor"], "summary": "OctoTools improves average accuracy 9.3% over GPT-4o across 16 tasks using a training-free tool framework."}
|