d6fc7c1dfa
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json - LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest - Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow) - Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks - build_worktypes.py adds work-type tables to data/papers.db Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
61 lines
32 KiB
JSON
61 lines
32 KiB
JSON
{"id": 3312, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "data", "why": "Synthesizes alternative action decisions to augment 312 human trajectories"}, {"area": "web-gui", "role": "train-agent", "why": "Trains PC Agent-E on enriched trajectories"}], "pattern": "vary-data", "fixes": ["training recipe", "environment"], "varies": ["trajectory data augmentation"], "benchmarks": [{"name": "WindowsAgentArena-V2", "purpose": "evaluate PC Agent-E", "is_new": true}], "results": [], "workflow": ["collect human trajectories", "synthesize alternative actions", "fine-tune model"], "summary": "Trains computer-use agent from 312 human trajectories augmented with Claude-synthesized actions; releases improved WindowsAgentArena-V2."}
|
|
{"id": 15229, "in_agent_scope": false, "types": [{"area": "reason/code", "role": "train-agent", "why": "Prioritizes cross-epoch ambiguous samples for replay in code RL"}], "pattern": "vary-data", "fixes": ["RL algorithm", "benchmarks"], "varies": ["replay data selection and weighting"], "benchmarks": [], "results": [], "workflow": ["track ambiguity across epochs", "replay ambiguous samples", "weight by policy divergence"], "summary": "Plug-and-play replay module for code RL prioritizing ambiguous samples with policy-divergence importance weights, tested on nine LLMs."}
|
|
{"id": 2159, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "Injects visual object-trigger backdoors into VLM embodied agents via SFT and contrastive trigger learning"}], "pattern": "vary-conditions", "fixes": ["agent architecture", "benchmarks"], "varies": ["trigger presence", "training scheme"], "benchmarks": [], "results": [{"benchmark": "embodied agent benchmarks", "system": "BEAT", "base_model": null, "setting": "proposed", "metric": "attack success rate", "score": 80, "unit": "% (up to)"}], "workflow": ["build diverse trigger dataset", "SFT", "contrastive trigger learning"], "summary": "BEAT implants visual backdoors in VLM-based embodied agents using environment objects as triggers, reaching up to 80% attack success."}
|
|
{"id": 5864, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Autoregressive hierarchical policy generating latent subgoal chains for offline GCRL"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["policy architecture"], "benchmarks": [], "results": [], "workflow": ["generate latent subgoals", "condition next subgoal", "output primitive action"], "summary": "Chain-of-Goals Hierarchical Policy generates sequences of latent subgoals autoregressively in one network for long-horizon offline goal-conditioned RL."}
|
|
{"id": 15299, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "Trains agents via RL to edit their own working context as actions"}], "pattern": "fix-env-vary-agent", "fixes": ["tasks"], "varies": ["context management policy", "RL algorithm"], "benchmarks": [], "results": [], "workflow": ["edit context as actions", "train with DCPO", "RL end-to-end"], "summary": "MemAct treats context deletion and insertion as learnable actions trained with Dynamic Context Policy Optimization; 14B matches 16x larger models."}
|
|
{"id": 13202, "in_agent_scope": false, "types": [{"area": "multiling/mt", "role": "benchmark", "why": "Dialectal Arabic-English parallel conversational dataset, 13 countries, 11 domains"}], "pattern": "n/a", "fixes": ["MT systems"], "varies": ["dialects and domains"], "benchmarks": [{"name": "Alexandria", "purpose": "dialectal Arabic MT evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "Community-driven human-translated multi-turn dialectal Arabic-English dataset with city-level metadata and gender configurations."}
|
|
{"id": 16873, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "analysis", "why": "Varies formality, readability, politeness, grammar of queries to measure RAG degradation"}], "pattern": "vary-conditions", "fixes": ["RAG pipeline", "QA datasets"], "varies": ["query linguistic variation", "models"], "benchmarks": [], "results": [], "workflow": [], "summary": "Systematic study showing linguistic query variation harms both retrieval and generation in RAG across two retrievers and nine LLMs."}
|
|
{"id": 17344, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "application", "why": "Moderator-coordinated multi-LLM deliberation interface with argument maps"}], "pattern": "n/a", "fixes": ["general-purpose LLMs"], "varies": ["interface design"], "benchmarks": [], "results": [], "workflow": ["moderator assigns roles", "manage turn-taking", "summarize and map arguments"], "summary": "ARGSBASE is a multi-LLM deliberation interface with a moderator agent, summaries and argument maps; user study shows positive reception."}
|
|
{"id": 16566, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "benchmark", "why": "VQA benchmark for ancient Greek pottery with 67,614 QA pairs"}, {"area": "arch/multimodal", "role": "train-agent", "why": "VaseVL augments SFT with RL using verifiable rewards"}], "pattern": "vary-both", "fixes": ["base MLLM"], "varies": ["training strategy", "dataset"], "benchmarks": [{"name": "VaseVQA", "purpose": "expert-level cultural heritage VQA", "is_new": true}], "results": [], "workflow": ["SFT on domain data", "RL with verifiable rewards"], "summary": "Introduces VaseVQA benchmark of 31,773 images and 67,614 QA pairs and VaseVL trained with SFT plus verifiable-reward RL."}
|
|
{"id": 15990, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "data", "why": "Verifier-based quality control and synthesis of tool-use dialogue data"}, {"area": "tool-use", "role": "benchmark", "why": "Tool-V-Bench for assessing verifier performance"}], "pattern": "vary-data", "fixes": ["training recipe"], "varies": ["data verification"], "benchmarks": [{"name": "Tool-V-Bench", "purpose": "assess verifier performance", "is_new": true}, {"name": "Tool-Verify", "purpose": "curated 3,295-sample dataset", "is_new": true}, {"name": "APIGen-MT", "purpose": "compare dataset quality", "is_new": false}], "results": [], "workflow": ["synthesize trajectories", "verify with Tool-Verifier", "curate dataset"], "summary": "Tool-Verifier-7B filters tool-use trajectories; yields Tool-Verify dataset (3,295 samples) and Tool-V-Bench outperforming APIGen-MT data."}
|
|
{"id": 15850, "in_agent_scope": false, "types": [{"area": "interp/steering", "role": "analysis", "why": "Analyzes answer-to-reasoning attention in thinking LLMs, then proposes training-free steering"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["steering by Self-Reading Quality"], "benchmarks": [], "results": [], "workflow": [], "summary": "Finds correct solutions show forward-drifting, concentrated answer-to-reasoning attention; proposes SRQ-driven training-free steering."}
|
|
{"id": 2902, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "End-to-end RL trains agent to keep constant-size internal state"}, {"area": "search-research", "role": "train-agent", "why": "Applies to multi-hop QA and web shopping agents"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["memory/context mechanism", "RL training"], "benchmarks": [], "results": [], "workflow": ["update compact internal state", "truncate rollouts", "RL training"], "summary": "MEM1 trains agents with RL to consolidate memory and reasoning in a compact state, cutting memory 3.7x while improving 3.5x."}
|
|
{"id": 12999, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "benchmark", "why": "Simulation-free evaluation framework for MLLMs as VLN agents"}, {"area": "embodied-nav", "role": "analysis", "why": "Finds CoT and reflection unexpectedly degrade navigation"}], "pattern": "vary-both", "fixes": ["MLLMs"], "varies": ["agent designs", "navigation tasks"], "benchmarks": [{"name": "VLN-MME", "purpose": "probe MLLMs as zero-shot navigation agents", "is_new": true}], "results": [], "workflow": [], "summary": "VLN-MME evaluates MLLMs as zero-shot navigation agents, revealing poor context awareness and that CoT and reflection can hurt."}
|
|
{"id": 12287, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "SLM-driven tiered memory system with online retrieval and offline consolidation"}], "pattern": "vary-scaffold", "fixes": ["LLM backbone"], "varies": ["memory architecture"], "benchmarks": [{"name": "LoCoMo", "purpose": "memory evaluation vs A-MEM", "is_new": false}], "results": [{"benchmark": "LoCoMo", "system": "LightMem", "base_model": null, "setting": "proposed", "metric": "F1 gain over A-MEM", "score": 2.5, "unit": "F1 points"}], "workflow": ["two-stage retrieval", "online STM/MTM", "offline LTM consolidation"], "summary": "LightMem uses small language models for STM/MTM/LTM memory with bounded retrieval; about 2.5 F1 over A-MEM on LoCoMo."}
|
|
{"id": 6561, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "Jointly trains LLM for memory extraction and management with GRPO"}], "pattern": "fix-env-vary-agent", "fixes": ["benchmarks"], "varies": ["memory extraction training", "reward"], "benchmarks": [], "results": [{"benchmark": "multi-turn interactive tasks", "system": "UMEM", "base_model": null, "setting": "proposed", "metric": "improvement over baselines", "score": 11.49, "unit": "points (up to)"}], "workflow": ["extract memories", "manage memory bank", "GRPO with neighborhood reward"], "summary": "UMEM jointly optimizes memory extraction and management via GRPO with neighborhood-level marginal utility reward, up to 11.49 points gain."}
|
|
{"id": 12547, "in_agent_scope": true, "types": [{"area": "search-research", "role": "environment", "why": "Simulated search environment with verifiable knowledge graph and aligned corpus"}, {"area": "search-research", "role": "train-agent", "why": "SearchGym-RL curriculum RL for search agents"}], "pattern": "vary-both", "fixes": ["agent base models"], "varies": ["training environment", "curriculum"], "benchmarks": [{"name": "ASearcher", "purpose": "baseline comparison", "is_new": false}], "results": [], "workflow": ["build knowledge graph", "generate aligned corpus", "curriculum RL"], "summary": "SearchGym simulates web search with aligned corpus for cheap RL; Qwen2.5-7B trained there beats ASearcher by 10.6% relative."}
|
|
{"id": 15153, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "analysis", "why": "Identifies agentic pressure causing safety-for-utility normative drift in agents"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["pressure conditions"], "benchmarks": [], "results": [], "workflow": [], "summary": "Shows LLM agents sacrifice safety under goal pressure, worsened by reasoning ability; explores pressure isolation as mitigation."}
|
|
{"id": 6537, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "Benchmark for long-horizon attacks on LLM agents, five attack types"}, {"area": "agent-safety", "role": "attack", "why": "Novel attack types such as memory poisoning and objective drifting"}], "pattern": "fix-agent-vary-env", "fixes": ["agents under test"], "varies": ["attack types", "environments"], "benchmarks": [{"name": "AgentLAB", "purpose": "long-horizon attack susceptibility", "is_new": true}], "results": [], "workflow": [], "summary": "AgentLAB offers 644 test cases across 28 environments and five long-horizon attack types; agents remain highly susceptible."}
|
|
{"id": 15567, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "analysis", "why": "Automated causal discovery from micro behaviors to macro emergence in LLM agent simulations"}], "pattern": "n/a", "fixes": ["simulations"], "varies": ["causal discovery method"], "benchmarks": [], "results": [], "workflow": ["form hypotheses as factors", "learn Markov boundary", "counterfactual probing"], "summary": "CAMO converts hypotheses into computable factors and learns causal structure explaining emergent outcomes in LLM agent simulations."}
|
|
{"id": 3125, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "environment", "why": "LLM agent plus physics solver generates complex physical 3D scenes for robotics"}], "pattern": "fix-env-vary-agent", "fixes": ["physics engine"], "varies": ["scene generation framework"], "benchmarks": [], "results": [], "workflow": ["LLM proposes predicates", "solver builds scene", "feedback refines"], "summary": "PhyScensis uses an LLM agent with a physics-engine solver to generate physically plausible complex scenes for robotic data collection."}
|
|
{"id": 481, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "method", "why": "Evolved lightweight coordinator assigns roles to multiple LLMs"}], "pattern": "n/a", "fixes": ["constituent LLMs"], "varies": ["coordinator design"], "benchmarks": [], "results": [], "workflow": ["select LLM each turn", "assign role", "evolution-optimized head"], "summary": "Trinity coordinator (0.6B model plus small head) trained by evolution assigns Thinker, Worker or Verifier roles to LLMs across turns."}
|
|
{"id": 10143, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Self-evolving agent with in-distribution critic for process rewards"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["credit assignment", "policy learning"], "benchmarks": [{"name": "AlfWorld", "purpose": "agent evaluation", "is_new": false}, {"name": "WebShop", "purpose": "agent evaluation", "is_new": false}, {"name": "ScienceWorld", "purpose": "agent evaluation", "is_new": false}], "results": [], "workflow": ["learn IQL critic", "derive process rewards", "behavior-proximal PPO"], "summary": "Q-Evolve learns in-distribution Q critic to derive step-wise process rewards for iterative agent self-improvement."}
|
|
{"id": 9274, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "train-agent", "why": "VLA with parallel action queries trained by SFT and RL for autonomous driving"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["model architecture", "training data"], "benchmarks": [], "results": [], "workflow": ["learnable action queries", "fuse VL features", "SFT then RL"], "summary": "Reasoning-VLA generates action trajectories in parallel from learnable queries; trained on eight consolidated driving datasets."}
|
|
{"id": 5632, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "Bioinformatics agent suite with end-to-end tasks and robustness perturbations"}], "pattern": "fix-agent-vary-env", "fixes": ["agent harnesses"], "varies": ["tasks", "perturbations"], "benchmarks": [{"name": "BioAgent Bench", "purpose": "bioinformatics agent evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "Evaluation suite of curated bioinformatics pipeline tasks; frontier agents finish pipelines but fail under perturbations."}
|
|
{"id": 17674, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "analysis", "why": "Studies genre variation effect on French NER with prompting configurations"}], "pattern": "vary-conditions", "fixes": ["model", "annotation scheme"], "varies": ["genre", "prompting setup"], "benchmarks": [{"name": "NEM.fr", "purpose": "multi-genre French NER", "is_new": false}], "results": [], "workflow": [], "summary": "Evaluates DeepSeek R1 on French NER across 11 genres and six prompt setups, finding genre matters more than prompting."}
|
|
{"id": 295, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Memory-guided agentic workflow synthesis for web search, training-free"}], "pattern": "vary-scaffold", "fixes": ["model weights"], "varies": ["workflow structure", "memory"], "benchmarks": [{"name": "GAIA", "purpose": "web research evaluation", "is_new": false}, {"name": "BrowseComp", "purpose": "web research evaluation", "is_new": false}, {"name": "GPQA", "purpose": "evaluation", "is_new": false}], "results": [], "workflow": ["decompose into sub-goals", "synthesize workflow graph", "distill workflow memory"], "summary": "FlowSearcher decomposes queries into sub-goals with tailored workflow graphs and hierarchical memory, matching RLHF-trained systems without training."}
|
|
{"id": 12860, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "RL-trained LLM agent with graph-native tools for graph learning"}, {"area": "planning-decision", "role": "train-agent", "why": "Graph-conditioned curriculum RL for long-horizon policy"}], "pattern": "fix-env-vary-agent", "fixes": ["benchmarks"], "varies": ["training method", "tools"], "benchmarks": [{"name": "Text-Attributed Graph benchmarks", "purpose": "node classification and link prediction", "is_new": false}], "results": [], "workflow": ["graph-native tool exploration", "search-constrained thinking", "curriculum RL"], "summary": "AgentGL is an RL framework giving LLM agents graph exploration tools; gains up to 17.5% node classification and 28.4% link prediction."}
|
|
{"id": 12219, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "analysis", "why": "Finds evidence tools cause overconfidence while verification tools help calibration"}, {"area": "tool-use", "role": "train-agent", "why": "RL jointly optimizing accuracy and calibration"}], "pattern": "vary-conditions", "fixes": ["agent"], "varies": ["tool type", "reward design"], "benchmarks": [], "results": [], "workflow": [], "summary": "Shows tool type drives verbalized confidence in agents and trains RL agents for better calibration."}
|
|
{"id": 8631, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "957 performance bottlenecks for repository-scale optimization with multi-objective metrics"}], "pattern": "fix-agent-vary-env", "fixes": ["agents under test"], "varies": ["tasks"], "benchmarks": [{"name": "FormulaCode", "purpose": "agentic codebase optimization", "is_new": true}], "results": [], "workflow": [], "summary": "FormulaCode benchmarks agent optimization on 957 real Python performance bottlenecks; repository-scale optimization remains hard."}
|
|
{"id": 4220, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "SFT plus self-evolved DPO for efficient tool-integrated reasoning"}, {"area": "tool-use", "role": "data", "why": "Entropy-guided sampling builds preference pairs"}], "pattern": "vary-data", "fixes": ["benchmarks"], "varies": ["training data", "preference training"], "benchmarks": [], "results": [], "workflow": ["self-evolved sampling", "select preference pairs", "SFT then DPO"], "summary": "Tool-Light builds self-evolved preference data via entropy-guided sampling and trains with SFT then DPO for efficient TIR on 10 datasets."}
|
|
{"id": 14451, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "application", "why": "Multi-agent framework for safety-aware illustrated storybook generation"}], "pattern": "n/a", "fixes": ["generative models"], "varies": ["multi-agent pipeline"], "benchmarks": [], "results": [], "workflow": ["plan narrative", "generate multimodal pages", "verify safety"], "summary": "BookAgent coordinates agents for child-safe illustrated story generation with sequence-level multimodal verification."}
|
|
{"id": 15487, "in_agent_scope": false, "types": [{"area": "interp/mech", "role": "analysis", "why": "Probes and intervenes to show two-stage MCQA computation"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["probing and interventions"], "benchmarks": [], "results": [], "workflow": [], "summary": "Representational and causal analyses show models first decode the winning content then bind it to the output symbol in MCQA."}
|
|
{"id": 12291, "in_agent_scope": true, "types": [{"area": "science-data", "role": "train-agent", "why": "Tool-augmented agent for protein function understanding with interleaved tool-call reasoning"}, {"area": "science-data", "role": "analysis", "why": "Shows text-only RL reasoning is ineffective for proteins"}], "pattern": "fix-env-vary-agent", "fixes": ["benchmarks"], "varies": ["reasoning paradigm"], "benchmarks": [], "results": [], "workflow": [], "summary": "PFUA interleaves tool calls with reasoning for protein function prediction, arguing text-only CoT RL adds no new biological knowledge."}
|
|
{"id": 8432, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Congestion-style agent admission control for KV cache in agentic batch inference"}], "pattern": "n/a", "fixes": ["model"], "varies": ["concurrency control"], "benchmarks": [], "results": [], "workflow": ["monitor cache", "adjust agent admission"], "summary": "CONCUR regulates agent-level concurrency using feedback control to prevent middle-phase KV cache thrashing in agentic batch inference."}
|
|
{"id": 1434, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "18 tasks measuring performance gain and novelty of agent solutions"}, {"area": "science-data", "role": "eval-method", "why": "Introduces performance gain and novelty metrics"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks", "metrics"], "benchmarks": [{"name": "InnoGym", "purpose": "innovation potential of agents", "is_new": true}], "results": [], "workflow": [], "summary": "InnoGym benchmarks AI agent innovation with 18 tasks, measuring performance gain over best known and novelty."}
|
|
{"id": 3656, "in_agent_scope": false, "types": [{"area": "arch/diffusion", "role": "method", "why": "Exploits early answer convergence to speed diffusion LM decoding"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["decoding schedule"], "benchmarks": [{"name": "GSM8K", "purpose": "early answer convergence", "is_new": false}, {"name": "MMLU", "purpose": "early answer convergence", "is_new": false}], "results": [], "workflow": ["detect early convergence", "stop refinement early"], "summary": "Shows diffusion LMs identify correct answers halfway through decoding and uses this for faster inference."}
|
|
{"id": 15471, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Step-wise CoT compression with knowledge guidance for reasoning models"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["compression method"], "benchmarks": [], "results": [], "workflow": ["build contrastive samples", "switch compression mode"], "summary": "STACK compresses chain-of-thought step-wise using stage-specific redundancy and retrieval-augmented guidance."}
|
|
{"id": 3816, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Sequential and parallel test-time scaling with asymmetric verification for deep search"}], "pattern": "vary-scaffold", "fixes": ["model"], "varies": ["test-time scaling strategy"], "benchmarks": [], "results": [], "workflow": [], "summary": "Studies sequential and parallel test-time scaling of deep search agents exploiting easier verification than generation."}
|
|
{"id": 17204, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "analysis", "why": "Large-scale study of LLM agent social dynamics on Chirper.ai"}], "pattern": "vary-conditions", "fixes": ["platform"], "varies": ["analysis dimensions"], "benchmarks": [], "results": [], "workflow": [], "summary": "Analyzes 7M posts from 32K LLM agents on Chirper.ai over a year, finding homophily, influence and toxicity patterns."}
|
|
{"id": 12536, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Multi-agent text2SQL with SQL2Text back-translation validation"}, {"area": "code-swe", "role": "eval-method", "why": "Audits benchmark gold errors with typology"}], "pattern": "vary-scaffold", "fixes": ["model"], "varies": ["validation workflow"], "benchmarks": [], "results": [], "workflow": ["guided generation", "SQL to text", "check alignment"], "summary": "GBV-SQL validates generated SQL by translating back to text; also audits benchmark gold errors."}
|
|
{"id": 13591, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "benchmark", "why": "Multimodal long-term conversational memory benchmark for MLLM agents"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["memory tasks"], "benchmarks": [{"name": "Mem-Gallery", "purpose": "multimodal long-term memory", "is_new": true}], "results": [], "workflow": [], "summary": "Mem-Gallery evaluates preservation, organization and evolution of multimodal memory across multi-session conversations."}
|
|
{"id": 11968, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "Scaffold-aware instruction following in repository coding, 34 environments, 217 tasks"}], "pattern": "fix-agent-vary-env", "fixes": ["models"], "varies": ["scaffold types", "tasks"], "benchmarks": [{"name": "OctoBench", "purpose": "scaffold instruction following", "is_new": true}], "results": [], "workflow": [], "summary": "OctoBench has 34 environments, 217 tasks and 7,098 checklist items; models show a gap between solving and following rules."}
|
|
{"id": 18628, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Context-guided test-time RL for on-device LLMs"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["TTRL sampling with context"], "benchmarks": [], "results": [], "workflow": [], "summary": "CG-TTRL adds in-context guidance to test-time RL sampling to improve pseudo-label accuracy for on-device models."}
|
|
{"id": 11743, "in_agent_scope": false, "types": [{"area": "safety/security", "role": "method", "why": "Anchored sliding window for robust linguistic steganography"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["context window design"], "benchmarks": [], "results": [], "workflow": [], "summary": "ASW anchors prompt and bridge context in the window to make LM steganography robust and imperceptible."}
|
|
{"id": 11924, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "train-agent", "why": "Rollout-guided RL for adaptive pixel-space reasoning in VLMs"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["pixel operation policy"], "benchmarks": [], "results": [], "workflow": [], "summary": "First adaptive pixel reasoning framework for VLMs that decides when pixel-level operations are needed."}
|
|
{"id": 15404, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "method", "why": "Slang-aware tagging framework for Chinese hate speech span extraction"}], "pattern": "n/a", "fixes": ["benchmark"], "varies": ["model modules"], "benchmarks": [{"name": "STATE ToxiCN", "purpose": "fine-grained hate extraction", "is_new": false}], "results": [], "workflow": [], "summary": "Combines slang lexicon fusion and volumetric tagging for Target-Argument-Hateful-Group extraction."}
|
|
{"id": 15418, "in_agent_scope": false, "types": [{"area": "apps/classic", "role": "data", "why": "Memory-guided hard data augmentation for multimodal NER"}], "pattern": "vary-data", "fixes": ["model"], "varies": ["augmentation"], "benchmarks": [], "results": [], "workflow": [], "summary": "Finds model-specific hard data via K-fold and augments it to repair defects in multimodal NER."}
|
|
{"id": 17716, "in_agent_scope": false, "types": [{"area": "align/values", "role": "analysis", "why": "Tests paraphrase wording effects on Yes/No answer consistency"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["question wording"], "benchmarks": [], "results": [], "workflow": [], "summary": "Agreement-seeking paraphrases make many LLMs more likely to answer Yes."}
|
|
{"id": 11873, "in_agent_scope": false, "types": [{"area": "train/data", "role": "method", "why": "Sparse adaptive dependency guidance for LLM tabular data generation"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["generation guidance"], "benchmarks": [], "results": [], "workflow": [], "summary": "SAGE uses mutual-information sparse dependencies and value-aware features for synthetic tabular data."}
|
|
{"id": 14597, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Task-level workflow generation instead of query-level for multi-agent systems"}, {"area": "multi-agent", "role": "analysis", "why": "Shows top-K task-level workflows cover queries"}], "pattern": "vary-scaffold", "fixes": ["models"], "varies": ["workflow generation level"], "benchmarks": [], "results": [], "workflow": [], "summary": "Shows query-level workflows are often unnecessary and proposes low-cost task-level workflow generation."}
|
|
{"id": 9758, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Honeypot memory documents defend against memory extraction attacks"}], "pattern": "fix-agent-vary-env", "fixes": ["threat model"], "varies": ["honeypot defense"], "benchmarks": [], "results": [], "workflow": ["optimize honeypots", "inject into memory", "SPRT detection"], "summary": "MemPot injects optimized trap documents with SPRT detection to defend agent memory against extraction."}
|
|
{"id": 1037, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Multi-agent LLM system for scRNA-seq and spatial transcriptomics analysis"}], "pattern": "n/a", "fixes": ["general LLM"], "varies": ["pipeline"], "benchmarks": [], "results": [], "workflow": [], "summary": "CellAgent automates single-cell analysis through hierarchical multi-agent natural language interaction."}
|
|
{"id": 11307, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "Visual layout memory rendered as image for long-horizon agents, RL trained"}], "pattern": "fix-env-vary-agent", "fixes": ["tasks"], "varies": ["memory representation"], "benchmarks": [], "results": [], "workflow": ["maintain rich-text memory", "render to image", "agent reads image"], "summary": "MemOCR renders structured memory into images with adaptive information density under tight context budgets."}
|
|
{"id": 8096, "in_agent_scope": false, "types": [{"area": "align/values", "role": "method", "why": "Independent value module for stable value guidance"}], "pattern": "n/a", "fixes": ["backbone"], "varies": ["value module"], "benchmarks": [], "results": [], "workflow": [], "summary": "SVGT keeps values in a separate module with explicit behavioral guidance for stable alignment."}
|
|
{"id": 16753, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Step-level advantage assignment via trajectory graph for GRPO agents"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["advantage estimation"], "benchmarks": [], "results": [], "workflow": [], "summary": "SALT assigns step-level advantages using trajectory graphs to improve credit assignment in long-horizon agent RL."}
|
|
{"id": 13759, "in_agent_scope": false, "types": [{"area": "arch/diffusion", "role": "train-agent", "why": "Attention-guided denoising order and optimization for diffusion LMs"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["masking strategy"], "benchmarks": [], "results": [], "workflow": [], "summary": "AGDO uses attention structure to order denoising and weight critical tokens in post-training."}
|
|
{"id": 17887, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "benchmark", "why": "Benchmark for evaluating medical open-ended QA metrics"}, {"area": "eval/judge", "role": "eval-method", "why": "Multiple gold answers to assess NLG metrics"}], "pattern": "vary-evaluation", "fixes": ["systems"], "varies": ["metrics"], "benchmarks": [{"name": "MORQA", "purpose": "evaluate NLG metrics in medical QA", "is_new": true}], "results": [], "workflow": [], "summary": "MORQA is a multilingual benchmark with multiple gold answers for assessing evaluation metrics in medical QA."}
|
|
{"id": 16739, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Prompt optimization for multi-agent systems via MAP inference"}], "pattern": "vary-scaffold", "fixes": ["model"], "varies": ["prompts"], "benchmarks": [], "results": [], "workflow": [], "summary": "MAPRO frames multi-agent prompt optimization as MAP inference addressing search space and credit assignment."}
|
|
{"id": 7502, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "Model organisms resist RL elicitation via exploration hacking"}], "pattern": "vary-conditions", "fixes": ["RL environments"], "varies": ["exploration strategy"], "benchmarks": [], "results": [], "workflow": [], "summary": "Fine-tunes models to underperform strategically and resist RL capability elicitation in biosecurity and AI R&D environments."}
|
|
{"id": 11740, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Reasoning-aligned memory growth and retracing for agentic search"}], "pattern": "vary-scaffold", "fixes": ["model"], "varies": ["memory management"], "benchmarks": [], "results": [], "workflow": ["grow memory", "retrace", "refine"], "summary": "MemSearch-o1 grows fine-grained memory fragments from query seed tokens and refines them to avoid memory dilution."}
|