d6fc7c1dfa
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json - LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest - Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow) - Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks - build_worktypes.py adds work-type tables to data/papers.db Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
61 lines
37 KiB
JSON
61 lines
37 KiB
JSON
{"id": 5087, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "Hint-free multi-hop deep search benchmark with Wikipedia sandbox"}, {"area": "search-research", "role": "eval-method", "why": "Factorised metrics separating search, knowledge use, refusal"}], "pattern": "vary-both", "fixes": ["25 evaluated models"], "varies": ["questions", "evaluation metrics"], "benchmarks": [{"name": "WebDetective", "purpose": "hint-free multi-hop deep search", "is_new": true}], "results": [], "workflow": ["Hint-free questions", "Controlled Wikipedia sandbox", "Factorised metrics"], "summary": "Benchmark of hint-free multi-hop questions in a traceable sandbox with metrics separating search sufficiency, knowledge utilization and refusal."}
|
|
{"id": 17779, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "analysis", "why": "Tests whether acoustic similarity (ATDS) predicts ASR performance on Yiddish dialects"}], "pattern": "vary-conditions", "fixes": ["XLSR/MMS models"], "varies": ["language/dialect pairs", "layers"], "benchmarks": [], "results": [], "workflow": [], "summary": "Shows acoustic token distribution similarity in speech models predicts post-fine-tuning CER for Yiddish dialect ASR."}
|
|
{"id": 13031, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "survey", "why": "Surveys temporal question answering datasets, tasks and approaches"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Comprehensive survey of temporal question answering covering datasets, methods, RAG and temporal reasoning with LLMs."}
|
|
{"id": 15093, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "Cross-application professional workflow benchmark for computer-use agents"}], "pattern": "fix-agent-vary-env", "fixes": ["evaluated agents"], "varies": ["tasks", "applications"], "benchmarks": [{"name": "WindowsWorld", "purpose": "cross-application GUI tasks", "is_new": true}, {"name": "OSWorld", "purpose": "cited as prior single-app benchmark", "is_new": false}], "results": [], "workflow": ["Multi-agent task generation", "Human review", "Execute in simulated environment"], "summary": "181-task benchmark of multi-application professional GUI workflows; agents score under 21% on multi-application tasks."}
|
|
{"id": 16337, "in_agent_scope": true, "types": [{"area": "science-data", "role": "analysis", "why": "Contamination-aware comparison of multi-agent LLM versus AutoML on tabular classification"}], "pattern": "vary-conditions", "fixes": ["MALLM framework", "AutoML baselines"], "varies": ["pre- vs post-cutoff datasets"], "benchmarks": [{"name": "TABLET", "purpose": "baseline", "is_new": false}, {"name": "TABLLM", "purpose": "baseline", "is_new": false}], "results": [], "workflow": [], "summary": "Finds multi-agent LLM tabular classifier does well on pre-cutoff data but underperforms AutoML on post-cutoff datasets, suggesting memorization."}
|
|
{"id": 17048, "in_agent_scope": false, "types": [{"area": "train/data", "role": "data", "why": "LLM-guided adversarial generation for toxic text augmentation"}], "pattern": "vary-data", "fixes": ["classifiers"], "varies": ["augmentation method"], "benchmarks": [{"name": "hate speech benchmarks", "purpose": "four datasets for augmentation eval", "is_new": false}], "results": [], "workflow": ["LLM neutral exemplars", "Directional adversarial training", "Augment toxic data"], "summary": "ToxiGAN augments toxic text with GAN plus LLM neutral exemplars, improving macro-F1 and hate-F1 on four hate speech benchmarks."}
|
|
{"id": 9276, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "Benchmarks agentic visual query pre-processing for multimodal RAG"}], "pattern": "fix-agent-vary-env", "fixes": ["MLLMs", "MRAG pipelines"], "varies": ["imperfect queries"], "benchmarks": [{"name": "V-QPP-Bench", "purpose": "visual query pre-processing", "is_new": true}], "results": [], "workflow": ["Imperfect visual queries", "MLLM diagnoses and uses tools", "Measure retrieval"], "summary": "V-QPP-Bench with 46,700 imperfect queries evaluates whether MLLMs can diagnose and fix visual queries before multimodal retrieval."}
|
|
{"id": 11994, "in_agent_scope": false, "types": [{"area": "safety/jailbreak", "role": "attack", "why": "Knowledge-graph-guided implicit harmful prompt generation for domain red-teaming"}], "pattern": "vary-conditions", "fixes": ["target LLMs"], "varies": ["harmful prompts"], "benchmarks": [], "results": [], "workflow": ["KG-guided prompt generation", "Obfuscation rewriting", "Build domain datasets"], "summary": "Framework generating domain-specific implicit harmful prompts from knowledge graphs to expose LLM risks in finance and healthcare."}
|
|
{"id": 12862, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Component-specific RL rewards for table image to LaTeX"}], "pattern": "vary-scaffold", "fixes": ["MLLM"], "varies": ["reward design"], "benchmarks": [], "results": [], "workflow": ["Split structure/style/content rewards", "Token-specific backprop"], "summary": "CSPO uses component-specific rewards for RL on table-to-LaTeX generation, plus hierarchical metrics."}
|
|
{"id": 15029, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "Malicious web link manipulation attacks against multi-agent systems"}], "pattern": "vary-conditions", "fixes": ["MAS architectures"], "varies": ["12 attack variants"], "benchmarks": [], "results": [], "workflow": [], "summary": "Proposes Web Fraud Attacks with 12 link-manipulation variants that deceive LLM multi-agent systems into trusting malicious links."}
|
|
{"id": 15219, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Trains MLLMs to learn UI elements in a screen-UI-action loop"}, {"area": "web-gui", "role": "benchmark", "why": "Introduces UI Comprehension benchmark"}], "pattern": "vary-both", "fixes": ["MLLM base"], "varies": ["training paradigm", "UI tasks"], "benchmarks": [{"name": "UI Comprehension-Bench", "purpose": "UI element understanding", "is_new": true}], "results": [], "workflow": ["Locate UI elements", "Learn semantics", "Act in cycle"], "summary": "UILoop reasons through screen, UI elements, action cycle and adds a 26K-sample UI comprehension benchmark."}
|
|
{"id": 18496, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "scaffold", "why": "Concurrent dual-agent spoken dialogue with shared belief state"}], "pattern": "vary-scaffold", "fixes": ["underlying LLMs"], "varies": ["agent concurrency"], "benchmarks": [], "results": [], "workflow": ["Ping extracts intent", "Ponder plans concurrently", "Shared belief state"], "summary": "Dual-agent spoken dialogue where conversation and reasoning agents run concurrently via shared belief state, cutting planning latency."}
|
|
{"id": 14467, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "Earth observation coding benchmark for LLM agents"}, {"area": "science-data", "role": "analysis", "why": "Studies API language, documentation and reflexion effects"}], "pattern": "vary-both", "fixes": ["Google Earth Engine tool"], "varies": ["models", "API language", "documentation", "reflexion"], "benchmarks": [{"name": "UnivEARTH", "purpose": "Earth observation coding questions", "is_new": true}], "results": [{"benchmark": "UnivEARTH", "system": "LLM agents zero-shot", "base_model": null, "setting": "prompted", "metric": "accuracy", "score": 40.0, "unit": "%"}], "workflow": ["Yes/no EO questions", "Agent writes GEE code", "Evaluate accuracy"], "summary": "UnivEARTH benchmark of 408 EO questions; agents reach 40% zero-shot, about 60% with reflexion."}
|
|
{"id": 8763, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "scaffold", "why": "Training-free attention-based GUI grounding fusing attention, OCR and captions"}], "pattern": "vary-scaffold", "fixes": ["MLLM weights"], "varies": ["fusion strategy"], "benchmarks": [], "results": [], "workflow": ["Attention map", "OCR and icon captions", "Consensus-SinglePeak fusion"], "summary": "Trifuse fuses attention, OCR cues and icon captions for GUI grounding without task-specific fine-tuning."}
|
|
{"id": 3877, "in_agent_scope": false, "types": [{"area": "interp/steering", "role": "method", "why": "Attribution-based decoding toward a region of interest"}], "pattern": "vary-scaffold", "fixes": ["model weights"], "varies": ["decoding strategy"], "benchmarks": [], "results": [{"benchmark": "instruction following", "system": "AGD", "base_model": "Llama 3.1", "setting": "proposed", "metric": "overall success rate", "score": 79.1, "unit": "%"}], "workflow": ["Take top token candidates", "Compute attribution to ROI", "Select highest"], "summary": "Attribution-Guided Decoding picks tokens with highest attribution to a user-defined input region, improving instruction following and factuality."}
|
|
{"id": 12749, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "benchmark", "why": "Reasoning camouflaged object segmentation task and datasets"}, {"area": "arch/multimodal", "role": "method", "why": "COSA vision-language segmentation assistant"}], "pattern": "vary-both", "fixes": ["image data"], "varies": ["task", "model"], "benchmarks": [{"name": "CamoQuery", "purpose": "implicit-query camouflaged segmentation", "is_new": true}, {"name": "MCD", "purpose": "multi-instance camouflage dataset", "is_new": true}], "results": [], "workflow": ["Implicit query input", "VL assistant reasons", "Output mask"], "summary": "Introduces LRCOS task, CamoQuery and MCD datasets, and COSA assistant segmenting camouflaged objects from implicit queries."}
|
|
{"id": 11653, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "VeRO-Bench of target agents for harness optimization"}, {"area": "code-swe", "role": "analysis", "why": "Empirical study comparing agent optimizers"}], "pattern": "vary-both", "fixes": ["coding agent optimizers"], "varies": ["target agents", "modifications"], "benchmarks": [{"name": "VeRO-Bench", "purpose": "agent harness optimization", "is_new": true}], "results": [], "workflow": ["Versioned snapshots", "Budgeted evaluation", "Execution traces"], "summary": "VeRO harness and benchmark for coding agents optimizing target agent harnesses, with empirical study of reliable modifications."}
|
|
{"id": 11768, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "RL (SAGE) with skill library for self-improving agents"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["RL training", "skill library use"], "benchmarks": [], "results": [], "workflow": ["Sequential rollout over task chain", "Accumulate skills", "GRPO update"], "summary": "SAGE trains agents with skill-augmented GRPO over task chains so generated skills accumulate and improve later tasks."}
|
|
{"id": 11166, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Self-checking, self-correcting defense for agent memory poisoning"}], "pattern": "vary-scaffold", "fixes": ["agent architecture", "threat model"], "varies": ["defense mechanism"], "benchmarks": [], "results": [], "workflow": ["Check memory context", "Self-correct entries"], "summary": "A-MemGuard is the first defense for LLM agent memory against injected records, making memory self-checking and self-correcting."}
|
|
{"id": 4588, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Generator-implementer-rectifier team that self-generates and rectifies multi-agent systems"}], "pattern": "vary-scaffold", "fixes": ["LLM weights"], "varies": ["MAS generation"], "benchmarks": [], "results": [], "workflow": ["Generator designs MAS", "Implementer builds", "Rectifier fixes at runtime"], "summary": "MAS^2 recursively self-generates multi-agent systems via a tri-agent team that composes and adaptively rectifies target systems."}
|
|
{"id": 14449, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "benchmark", "why": "Benchmark of role-alignment value-conflict dilemmas for role-playing agents"}, {"area": "social-sim", "role": "analysis", "why": "Reveals Role Value Decoupling in mainstream LLMs"}], "pattern": "fix-agent-vary-env", "fixes": ["mainstream LLMs"], "varies": ["dilemma scenarios", "roles"], "benchmarks": [{"name": "RoleCDE", "purpose": "role-alignment cognitive dilemmas", "is_new": true}], "results": [], "workflow": ["Build role profiles", "Generate dilemmas", "Evaluate decisions"], "summary": "RoleCDE benchmark of ~240k dilemma instances shows role-playing agents default to alignment-consistent decisions over role-specific values."}
|
|
{"id": 15108, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Query-memory alignment and coherent chunking for agent memory retrieval"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory retrieval method"], "benchmarks": [{"name": "LoCoMo", "purpose": "long-term memory eval", "is_new": false}, {"name": "LongMemEval", "purpose": "long-term memory eval", "is_new": false}], "results": [], "workflow": ["Chunk by dialogue unit", "Align query and memory", "Multi-grained retrieval"], "summary": "QueryLink aligns queries and memories in a shared semantic space with coherent chunking to improve long-term memory retrieval."}
|
|
{"id": 5503, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "train-agent", "why": "Chain-of-Action trains hierarchical generalist Minecraft agents across action spaces"}, {"area": "embodied-nav", "role": "analysis", "why": "Large-scale study of action spaces in Minecraft"}], "pattern": "fix-env-vary-agent", "fixes": ["Minecraft"], "varies": ["action space", "training mix"], "benchmarks": [{"name": "Minecraft", "purpose": "agent tasks", "is_new": false}], "results": [], "workflow": ["Abstract goal as reasoning step", "Generate executable actions", "Mixed action-space training"], "summary": "Chain-of-Action unifies abstract and low-level actions in one model; no single action space is best, and an all-in-one agent is trained."}
|
|
{"id": 18637, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "JAWS-BENCH jailbreak benchmark across workspace regimes for code agents"}, {"area": "agent-safety", "role": "attack", "why": "Jailbreak attacks on code agents measuring executable harm"}, {"area": "agent-safety", "role": "eval-method", "why": "Hierarchical executable-aware judge framework"}], "pattern": "vary-conditions", "fixes": ["seven LLM backends"], "varies": ["workspace regime", "attack"], "benchmarks": [{"name": "JAWS-BENCH", "purpose": "code agent jailbreaks", "is_new": true}], "results": [{"benchmark": "JAWS-0", "system": "seven LLM backends", "base_model": null, "setting": "baseline", "metric": "compliance", "score": 61, "unit": "%"}], "workflow": ["Empty/single/multi-file workspaces", "Prompt attacks", "Judge compliance to runtime"], "summary": "JAWS-BENCH shows code agents often comply with and execute malicious programs, with attack success rising as workspace context grows."}
|
|
{"id": 14183, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Studies log-probabilities, judge scores and accuracy in two-agent debate"}], "pattern": "vary-conditions", "fixes": ["debate architecture"], "varies": ["domains", "signals"], "benchmarks": [], "results": [], "workflow": ["Constructor and Auditor debate", "LLM judge scores reasoning", "Compare confidence signals"], "summary": "Diagnoses multi-agent debate by relating token log-probabilities, LLM-judge rubric scores and final accuracy across three domains."}
|
|
{"id": 11816, "in_agent_scope": false, "types": [{"area": "safety/fairness", "role": "eval-method", "why": "Sandbox method for sociolinguistic research with demographically split Reddit dataset"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [{"name": "Splits!", "purpose": "sociocultural linguistic Reddit splits", "is_new": true}], "results": [], "workflow": ["Split Reddit by demographics/topic", "Validate by self-identification"], "summary": "Splits! is a demographically and topically split Reddit dataset for flexible sociocultural linguistic investigation."}
|
|
{"id": 18588, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "data", "why": "Expert-annotated parameter-level API graph dataset for tool agents"}, {"area": "tool-use", "role": "train-agent", "why": "Fine-tuning on the dataset helps models produce API graphs"}], "pattern": "vary-data", "fixes": ["LLMs", "API benchmarks"], "varies": ["API graph supervision"], "benchmarks": [{"name": "In-N-Out", "purpose": "API graph dataset", "is_new": true}], "results": [], "workflow": ["Convert docs to API graph", "Use for tool retrieval", "Multi-tool query generation"], "summary": "In-N-Out is an expert-annotated API graph dataset that nearly doubles tool retrieval and query generation performance over documentation alone."}
|
|
{"id": 2270, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "Process-level benchmark for environment configuration agents"}, {"area": "code-swe", "role": "eval-method", "why": "Process-level trajectory assessment beyond end-to-end success"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs and agent frameworks"], "varies": ["tasks with injected README errors"], "benchmarks": [{"name": "EnConda-bench", "purpose": "environment configuration diagnosis", "is_new": true}], "results": [], "workflow": ["Inject README errors", "Validate in Docker", "Assess process stages"], "summary": "EnConda-bench assesses planning, diagnosis, repair and execution in agent environment setup via injected README errors."}
|
|
{"id": 10012, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "environment", "why": "Synthesizes verifiable web environments from finite state machines"}, {"area": "web-gui", "role": "data", "why": "Generates verifiable training trajectories"}], "pattern": "fix-env-vary-agent", "fixes": ["web agents"], "varies": ["synthetic websites"], "benchmarks": [], "results": [], "workflow": ["Model website as FSM", "Coding agents build site", "Programmatic verification"], "summary": "AutoWebWorld synthesizes controllable, programmatically verifiable web environments from FSMs to train Web GUI agents."}
|
|
{"id": 11858, "in_agent_scope": false, "types": [{"area": "interp/mech", "role": "survey", "why": "Position calling for continuous collaborative reviewing to audit mechanistic interpretability"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Position paper calling for a collaborative reviewing platform and guidelines to make mechanistic interpretability findings auditable."}
|
|
{"id": 15290, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "method", "why": "Bi-level routing between single-agent and multi-agent systems by performance and cost"}], "pattern": "vary-scaffold", "fixes": ["LLM backbones"], "varies": ["routing policy"], "benchmarks": [{"name": "MBPP", "purpose": "routing eval", "is_new": false}, {"name": "MATH", "purpose": "routing eval", "is_new": false}], "results": [], "workflow": ["Policy within each system", "Predict performance and cost", "Select regime"], "summary": "BiCSRouter routes queries between single- and multi-agent systems, cutting cost 46% versus GPT-5 ceiling on MBPP."}
|
|
{"id": 14474, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Multi-agent LLM-driven evolutionary genomics data compressor"}], "pattern": "n/a", "fixes": [], "varies": ["compression pipeline"], "benchmarks": [], "results": [], "workflow": ["User layer interface", "Cognitive layer optimizes", "Compression layer executes"], "summary": "AgentGC is an evolutionary LLM multi-agent compressor for genomics data with compression-ratio, throughput and balanced modes."}
|
|
{"id": 16205, "in_agent_scope": false, "types": [{"area": "apps/science", "role": "benchmark", "why": "Bangla medical VQA dataset"}, {"area": "apps/science", "role": "analysis", "why": "Evaluates nine LVLMs under zero-shot, CoT, LoRA"}], "pattern": "vary-conditions", "fixes": ["dataset"], "varies": ["LVLMs", "prompting/fine-tuning"], "benchmarks": [{"name": "BanglaMedVQA", "purpose": "Bangla medical VQA", "is_new": true}], "results": [], "workflow": ["Clinically validate pairs", "Evaluate nine LVLMs"], "summary": "BanglaMedVQA dataset and evaluation of nine LVLMs showing weak fine-grained diagnostic reasoning, improved by LoRA fine-tuning."}
|
|
{"id": 364, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "eval-method", "why": "Promptable multi-round agentic evaluation of manipulation policies"}], "pattern": "vary-evaluation", "fixes": ["manipulation policies"], "varies": ["evaluation framework"], "benchmarks": [], "results": [], "workflow": ["Small-batch rounds", "Adaptive planning", "Generate tasks and evaluators"], "summary": "ManipEvalAgent adaptively evaluates robotic manipulation policies with code-generated tasks and VLM video understanding."}
|
|
{"id": 10146, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Hierarchical multi-agent search for GPU kernel generation"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["search procedure"], "benchmarks": [], "results": [], "workflow": ["Top-down decomposition", "Bottom-up fusion mutation", "Parallel multi-agent search"], "summary": "LEGO uses AND-OR FusionTree multi-agent search for LLM GPU kernel generation, achieving 2.18x-13.48x speedups over PyTorch Eager."}
|
|
{"id": 13503, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "Alternating RL co-optimizing memory retrieval and utilization"}], "pattern": "fix-env-vary-agent", "fixes": ["memory benchmarks"], "varies": ["RL rewards", "training stages"], "benchmarks": [], "results": [], "workflow": ["Optimize retrieval with citation feedback", "Optimize utilization", "Alternate iteratively"], "summary": "MemCoRL alternately trains memory retrieval and utilization with RL for a positive feedback loop."}
|
|
{"id": 12856, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "data", "why": "World-model guided MCTS trajectory synthesis for web agents"}, {"area": "web-gui", "role": "environment", "why": "LLM world model simulates web state transitions"}], "pattern": "vary-data", "fixes": ["web agent training recipe"], "varies": ["trajectory source"], "benchmarks": [{"name": "WebArena", "purpose": "eval", "is_new": false}, {"name": "WebVoyager", "purpose": "eval", "is_new": false}, {"name": "Mind2Web-Online", "purpose": "eval", "is_new": false}], "results": [], "workflow": ["LLM world model simulation", "MCTS exploration", "Train on synthesized trajectories"], "summary": "WebSynthesis synthesizes web trajectories via world-model MCTS; agents trained on them outperform those trained on real data."}
|
|
{"id": 17070, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Shapley attribution to detect hallucinated and biased objects in VLMs"}], "pattern": "vary-scaffold", "fixes": ["four VLMs"], "varies": ["detection method"], "benchmarks": [{"name": "ADE", "purpose": "hallucination eval", "is_new": false}, {"name": "COCO", "purpose": "hallucination eval", "is_new": false}, {"name": "HQH", "purpose": "bias detection", "is_new": true}], "results": [], "workflow": ["Kernel SHAP", "Multi-layer fusion", "Detect objects"], "summary": "SHAPLENS uses Shapley attribution to detect hallucinated and biased objects in VLMs, improving detection accuracy 8-12%."}
|
|
{"id": 16366, "in_agent_scope": true, "types": [{"area": "science-data", "role": "train-agent", "why": "Multi-agent RL where motif agents learn molecular assembly"}], "pattern": "fix-env-vary-agent", "fixes": ["LLM backbone"], "varies": ["training method"], "benchmarks": [], "results": [], "workflow": ["Motif agents negotiate", "Set-based behavioral cloning", "MAPPO topology rewards"], "summary": "MotifAgent trains motif agents sharing an LLM with negotiation and MAPPO to learn molecular assembly rules."}
|
|
{"id": 14006, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Agentic document intelligence pipeline with MCP analytics and compliance validation"}, {"area": "domain-agents", "role": "benchmark", "why": "DocSplit packet segmentation dataset"}], "pattern": "n/a", "fixes": [], "varies": ["pipeline components"], "benchmarks": [{"name": "DocSplit", "purpose": "document packet segmentation", "is_new": true}], "results": [], "workflow": ["Split packets", "Extract structure", "Analytics via MCP", "Validate rules"], "summary": "IDP Accelerator is an end-to-end agentic document system: packet segmentation, extraction, MCP analytics and LLM compliance validation."}
|
|
{"id": 13751, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Curriculum-driven RL with dual-agent co-evolution for super app GUI agents"}, {"area": "web-gui", "role": "data", "why": "Autonomous data synthesis for domain-aware foundation model"}], "pattern": "fix-env-vary-agent", "fixes": ["SuperAPP benchmark"], "varies": ["training data", "RL curriculum"], "benchmarks": [{"name": "SuperAPP", "purpose": "super app GUI eval", "is_new": false}], "results": [], "workflow": ["Synthesize corpora", "Curriculum agent proposes tasks", "RL optimizes actor"], "summary": "GUI0 combines data synthesis and curriculum-agent/actor co-evolution RL, beating Gemini-2.5-Pro and Claude-4-Sonnet on SuperAPP."}
|
|
{"id": 10605, "in_agent_scope": false, "types": [{"area": "efficiency/compress", "role": "method", "why": "Channel-level structural pruning for MoE models"}], "pattern": "vary-scaffold", "fixes": ["DeepSeek and Qwen MoEs"], "varies": ["pruning method"], "benchmarks": [], "results": [], "workflow": ["Attribute channel scores", "Maximize coverage", "Prune with quantization"], "summary": "Structural MoE pruning by channel-score coverage retains accuracy under 25-50% pruning combined with 4-bit quantization."}
|
|
{"id": 14163, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "eval-method", "why": "Label-free estimation of language competence from processing inconsistency"}, {"area": "multiling/lowres", "role": "analysis", "why": "Tests inconsistency hypothesis on 11 LLMs across Turkic languages"}], "pattern": "vary-evaluation", "fixes": ["11 instruction-tuned LLMs"], "varies": ["processing features", "languages"], "benchmarks": [], "results": [], "workflow": ["Extract entropy and tokenizer features", "Correlate with accuracy"], "summary": "Shows variability in generation entropy and tokenization signals predicts language competence without labels across Turkic languages."}
|
|
{"id": 14294, "in_agent_scope": false, "types": [{"area": "train/pretrain", "role": "analysis", "why": "Systematic study of checkpoint merging across pre-training phases"}], "pattern": "vary-conditions", "fixes": ["pretraining runs"], "varies": ["merging methods", "training phase", "scale"], "benchmarks": [], "results": [], "workflow": [], "summary": "Merging checkpoints helps during stable learning rate but gains diminish during decay, linked to reduced checkpoint diversity."}
|
|
{"id": 7989, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "benchmark", "why": "Benchmark of text-to-image prompting proficiency for humans and MLLMs"}, {"area": "eval/judge", "role": "eval-method", "why": "AtelierJudge agentic evaluator for prompt-image pairs"}], "pattern": "fix-agent-vary-env", "fixes": ["T2I backends"], "varies": ["prompters", "tasks"], "benchmarks": [{"name": "AtelierEval", "purpose": "T2I prompting proficiency", "is_new": true}], "results": [{"benchmark": "AtelierEval", "system": "AtelierJudge", "base_model": null, "setting": "proposed", "metric": "Spearman correlation with experts", "score": 0.79, "unit": "correlation"}], "workflow": ["Expert tasks", "Prompters write prompts", "Agentic judge scores"], "summary": "AtelierEval benchmarks 8 MLLMs and 48 humans as T2I prompters with a skill-based agentic judge correlating 0.79 with experts."}
|
|
{"id": 7904, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Hybrid episodic-procedural memory over tool graph for inference"}, {"area": "memory-selfevolve", "role": "train-agent", "why": "Memory reused in training of multi-turn tool-use policies"}], "pattern": "vary-scaffold", "fixes": ["multi-turn environments"], "varies": ["memory strategy"], "benchmarks": [], "results": [], "workflow": ["Build tool graph", "Attach episodic summaries", "Balance recall and procedure"], "summary": "H-EPM builds a tool graph with episodic summaries so multi-turn tool-use agents reuse experience in inference and training."}
|
|
{"id": 13575, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "RL-trained master agent coordinating grounding and vision agents for long video QA"}, {"area": "search-research", "role": "benchmark", "why": "LongTVQA episode-level datasets"}], "pattern": "vary-both", "fixes": ["video QA"], "varies": ["agent design", "RL training"], "benchmarks": [{"name": "LongTVQA", "purpose": "episode-level long-video QA", "is_new": true}, {"name": "LongTVQA+", "purpose": "episode-level long-video QA", "is_new": true}, {"name": "TVQA", "purpose": "source data", "is_new": false}, {"name": "TVQA+", "purpose": "source data", "is_new": false}], "results": [], "workflow": ["Master plans with step limit", "Grounding agent localizes", "Vision agent extracts"], "summary": "Multi-agent long-video QA with RL-trained master agent, grounding and vision agents, outperforming non-agent baselines on LongTVQA."}
|
|
{"id": 17213, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Verification-aware planner with subtask verification functions"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["planning and verification"], "benchmarks": [], "results": [], "workflow": ["Decompose tasks", "Model dependencies", "Encode verification functions"], "summary": "VeriMAP planner decomposes tasks and defines verification functions per subtask, improving multi-agent robustness over baselines."}
|
|
{"id": 15068, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "Iterative instruction-following benchmark with feedback"}, {"area": "eval/bench", "role": "eval-method", "why": "Feedback-driven self-correction protocol"}], "pattern": "vary-evaluation", "fixes": ["LLMs"], "varies": ["feedback turns", "instances"], "benchmarks": [{"name": "Meeseeks", "purpose": "iterative instruction following", "is_new": true}], "results": [], "workflow": ["Identify erroneous parts", "Give feedback", "Iterate"], "summary": "Meeseeks is an automated 700-instance iterative instruction-following benchmark with feedback, showing LLMs vary widely in self-correction."}
|
|
{"id": 15760, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "benchmark", "why": "Benchmark for iterative self-optimization in large search spaces"}, {"area": "planning-decision", "role": "scaffold", "why": "OPT-Agent self-reflection framework"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["tasks"], "benchmarks": [{"name": "OPT-BENCH", "purpose": "iterative self-optimization", "is_new": true}], "results": [], "workflow": ["Receive environment feedback", "Reflect", "Refine solutions"], "summary": "OPT-BENCH combines 20 ML tasks and 10 NP-hard problems to test agent self-improvement, with OPT-Agent framework."}
|
|
{"id": 13163, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Bayesian orchestrator over emotion-strategy agents for small-model negotiation"}, {"area": "social-sim", "role": "benchmark", "why": "Four edge-deployable high-stakes negotiation benchmarks"}], "pattern": "vary-scaffold", "fixes": ["small language models"], "varies": ["orchestration"], "benchmarks": [{"name": "EmoMAS negotiation benchmarks", "purpose": "debt, healthcare negotiation", "is_new": true}], "results": [], "workflow": ["Three specialized agents", "Bayesian fusion", "Update agent reliability"], "summary": "EmoMAS coordinates game-theoretic, RL and psychological agents via Bayesian orchestration for emotional negotiation with SLMs."}
|
|
{"id": 12941, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "Real-estate QA benchmark mixing SQL and API calls"}, {"area": "tool-use", "role": "scaffold", "why": "HIRE-Agent hierarchical baseline"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks"], "benchmarks": [{"name": "ReCoQA", "purpose": "real estate tool-augmented QA", "is_new": true}], "results": [], "workflow": ["Front-end parser", "Planning supervisor", "Execution specialists"], "summary": "ReCoQA has 29,270 real-estate instances with SQL and API supervision plus hierarchical HIRE-Agent baseline."}
|
|
{"id": 1256, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "benchmark", "why": "Urban agent spatiotemporal reasoning benchmark"}, {"area": "domain-agents", "role": "environment", "why": "UAgentEnv interactive city environment"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["tasks", "dimensions"], "benchmarks": [{"name": "USTBench", "purpose": "urban spatiotemporal reasoning", "is_new": true}], "results": [], "workflow": [], "summary": "USTBench evaluates LLM urban agents on understanding, forecasting, planning and reflection in the UAgentEnv city environment."}
|
|
{"id": 14992, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Bandit-guided dynamic strategy selection during inference reasoning"}], "pattern": "vary-scaffold", "fixes": ["LLM weights"], "varies": ["inference strategy"], "benchmarks": [], "results": [], "workflow": ["Evaluate reasoning state", "Bandit picks strategy", "Backtrack or restart"], "summary": "Meta-Reasoner uses contextual bandits to decide whether to backtrack, switch approach or restart during LLM reasoning."}
|
|
{"id": 12670, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Multi-agent dynamic structured knowledge flow for deep research"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["research workflow"], "benchmarks": [{"name": "GAIA", "purpose": "deep research eval", "is_new": false}, {"name": "HLE", "purpose": "deep research eval", "is_new": false}, {"name": "GPQA", "purpose": "scientific eval", "is_new": false}, {"name": "TRQA", "purpose": "scientific eval", "is_new": false}], "results": [], "workflow": ["Plan knowledge flow", "Parallel subtasks", "Adjust from feedback"], "summary": "FlowSearch evolves a structured knowledge flow for parallel, hierarchical deep research, competitive on GAIA, HLE, GPQA, TRQA."}
|
|
{"id": 13088, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "analysis", "why": "Rule-based agents using induced morphemes probe emergent language properties"}], "pattern": "vary-conditions", "fixes": ["emergent language environment"], "varies": ["ablated induction algorithms"], "benchmarks": [], "results": [], "workflow": ["Induce morphemes", "Build phrasebook agents", "Test communication"], "summary": "Phrasebook agents assess induced morphemes in emergent languages and show normalized PMI correlates with communicability better than topographic similarity."}
|
|
{"id": 4377, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "train-agent", "why": "Multi-task GRPO for MLLM agent interacting with SAM"}], "pattern": "fix-env-vary-agent", "fixes": ["SAM"], "varies": ["agent training"], "benchmarks": [], "results": [], "workflow": ["Generate boxes", "Propose refinement points", "Terminate adaptively"], "summary": "SAM-Veteran is an MLLM agent trained with multi-task GRPO to iteratively refine SAM masks for reasoning segmentation."}
|
|
{"id": 5473, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "SandboxEscapeBench CTF benchmark for container escape"}, {"area": "agent-safety", "role": "attack", "why": "Measures LLM capability to exploit sandbox vulnerabilities"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["sandbox vulnerabilities"], "benchmarks": [{"name": "SandboxEscapeBench", "purpose": "container sandbox escape CTF", "is_new": true}], "results": [], "workflow": ["Nested sandbox CTF", "Agent with shell", "Flag capture"], "summary": "SandboxEscapeBench measures whether LLM agents can break out of Docker containers across misconfiguration, privilege and kernel flaws."}
|
|
{"id": 8902, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Entropy-modulated policy gradients for credit assignment in long-horizon agents"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["policy gradient scaling"], "benchmarks": [], "results": [], "workflow": ["Measure step entropy", "Recalibrate gradient", "Use final outcome"], "summary": "EMPG rescales policy gradients by step uncertainty and outcome to fix credit assignment for long-horizon LLM agents."}
|
|
{"id": 16191, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "analysis", "why": "Shows LLM tool-call decisions misalign with human time perception"}, {"area": "tool-use", "role": "benchmark", "why": "TicToc dataset of time-sensitive trajectories"}], "pattern": "vary-conditions", "fixes": ["LLM agents"], "varies": ["elapsed time", "scenarios"], "benchmarks": [{"name": "TicToc", "purpose": "temporal tool-use decisions", "is_new": true}], "results": [], "workflow": ["Build 76 scenarios", "Collect human preferences", "Compare agent decisions"], "summary": "TicToc dataset shows agents are temporally blind, misjudging when to call tools versus answering as elapsed time varies."}
|