Files
ai-taxonomy-literature-anal…/data/labels/o14.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
28 KiB
JSON

{"id": 4354, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "Environment Tuning: curriculum RL for multi-turn tool use without expert trajectories"}], "pattern": "vary-data", "fixes": ["agent model", "tool-use benchmarks"], "varies": ["training paradigm", "curriculum"], "benchmarks": [], "results": [], "workflow": ["Structured curriculum", "Environment feedback", "RL from problem instances"], "summary": "Environment Tuning trains tool-use agents via curriculum and environment feedback directly from problem instances, avoiding expert trajectories."}
{"id": 9362, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "scaffold", "why": "Induces LLM workflows via bilevel optimization with textual gradients"}], "pattern": "vary-scaffold", "fixes": ["base LLMs"], "varies": ["workflow structure", "prompts"], "benchmarks": [], "results": [], "workflow": ["Optimize workflow sketch (outer)", "Optimize prompts (inner)", "Textual gradients"], "summary": "FlowBot automatically induces LLM agent workflows by bilevel optimization of workflow sketch and prompts using textual gradients."}
{"id": 7856, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "eval-method", "why": "PULSE framework for human-centric evaluation of agent designs with predicted satisfaction"}], "pattern": "vary-evaluation", "fixes": ["agent designs"], "varies": ["evaluation protocol"], "benchmarks": [], "results": [], "workflow": ["Collect user feedback", "Train satisfaction predictor", "Combine ratings with pseudo-labels"], "summary": "PULSE enables efficient human-centric evaluation of software agents, deployed in a large-scale web platform for case studies."}
{"id": 2713, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "311-task multimodal provenance-aware browsing benchmark"}], "pattern": "fix-agent-vary-env", "fixes": ["agents under test"], "varies": ["tasks"], "benchmarks": [{"name": "MMSearch-Plus", "purpose": "new multimodal browsing benchmark", "is_new": true}], "results": [], "workflow": ["Seed visual-cue questions", "Iterative image-text retrieval", "Agent framework evaluation"], "summary": "MMSearch-Plus is a 311-task benchmark requiring genuine multimodal reasoning for browsing agents, with an agent framework."}
{"id": 13805, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "survey", "why": "Systematizes safety and security threats of computer-using agents"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Survey systematizing safety and security threats to LLM-based computer-using agents."}
{"id": 11767, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "Functionality-grounded e-commerce web agent benchmark including risk evaluation"}], "pattern": "fix-agent-vary-env", "fixes": ["web agents under test"], "varies": ["tasks", "risk metrics"], "benchmarks": [{"name": "Amazon-style e-commerce functionality benchmark", "purpose": "new e-commerce benchmark", "is_new": true}], "results": [], "workflow": [], "summary": "Benchmark for web agents covering broad e-commerce functionality beyond product search and assessing risks from unintended changes."}
{"id": 17163, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "Citation-aware outline-guided adaptive retrieval and synthesis for scientific literature"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["retrieval and synthesis pipeline"], "benchmarks": [], "results": [], "workflow": ["Adaptive retrieval", "Citation-graph reasoning", "Outline-guided synthesis"], "summary": "SciRAG is an open-source RAG framework with adaptive retrieval, citation-aware reasoning and outline-guided synthesis for scientific literature."}
{"id": 12781, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Bi-level expert-to-policy assimilation using RLVR with few expert trajectories"}], "pattern": "fix-env-vary-agent", "fixes": ["OSWorld environment"], "varies": ["training method"], "benchmarks": [{"name": "OSWorld-Verified", "purpose": "evaluate GUI agent", "is_new": false}], "results": [], "workflow": ["Expert trajectories", "Off-policy to on-policy", "RLVR"], "summary": "Improves end-to-end GUI agents by assimilating limited expert trajectories into on-policy RLVR training."}
{"id": 6893, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Speculate-and-verify honeypot detecting multi-turn agent attacks"}], "pattern": "fix-env-vary-agent", "fixes": ["threat model"], "varies": ["defense mechanism"], "benchmarks": [], "results": [], "workflow": ["Predict future behaviors", "Build trajectory tree", "Verify risks"], "summary": "Speculative Safety Honeypot proactively defends agents against multi-turn attacks via multi-agent speculative simulation."}
{"id": 14148, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "analysis", "why": "Trajectory-level confidence analysis for tool-using agents"}], "pattern": "vary-conditions", "fixes": ["tool-using agents"], "varies": ["confidence estimation"], "benchmarks": [], "results": [], "workflow": ["Action-level confidence", "Action-type aggregation", "Temporal pattern discovery"], "summary": "Thesis proposal on trajectory-level uncertainty quantification to detect when tool-using agents are lost."}
{"id": 1696, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Evolutionary test-time learning updating agent prompts/memory between episodes"}, {"area": "embodied-nav", "role": "benchmark", "why": "J-TTL test-time learning benchmark on text games"}], "pattern": "vary-scaffold", "fixes": ["base LLM"], "varies": ["test-time adaptation method"], "benchmarks": [{"name": "Jericho Test-Time Learning (J-TTL)", "purpose": "consecutive-episode learning", "is_new": true}], "results": [], "workflow": ["Play episode", "Evolve agent config", "Repeat"], "summary": "EvoTest evolves agentic systems at test time and introduces J-TTL benchmark for learning across episodes."}
{"id": 15742, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "RL for adaptive search depth in agentic RAG"}], "pattern": "fix-env-vary-agent", "fixes": ["retrieval tools"], "varies": ["training objective"], "benchmarks": [], "results": [], "workflow": ["Analyze search depth", "RL adaptive depth"], "summary": "AutoSearch trains agentic RAG with RL to adapt search depth, balancing accuracy and cost."}
{"id": 15061, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Location preference optimization for GUI agent grounding"}], "pattern": "fix-env-vary-agent", "fixes": ["GUI benchmarks"], "varies": ["training objective"], "benchmarks": [], "results": [], "workflow": ["Locational data preferences", "Preference optimization"], "summary": "LPO optimizes GUI agent interaction using locational preference signals instead of SFT or coarse RL."}
{"id": 2636, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "FuncBenchGen contamination-free synthetic multi-step function calling tasks"}, {"area": "tool-use", "role": "eval-method", "why": "Controlled function-dependency DAG for evaluation"}], "pattern": "fix-agent-vary-env", "fixes": ["models under test"], "varies": ["task complexity", "function count"], "benchmarks": [{"name": "FuncBenchGen", "purpose": "synthetic function-calling generator", "is_new": true}], "results": [], "workflow": ["Hidden function DAG", "Generate tasks", "Stress-test models"], "summary": "FuncBenchGen generates controllable, contamination-free multi-step function-calling tasks from hidden dependency DAGs."}
{"id": 2292, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "data", "why": "Hindsight relabeling of agent trajectories for fine-tuning"}], "pattern": "vary-data", "fixes": ["training recipe"], "varies": ["training data"], "benchmarks": [], "results": [], "workflow": ["Review trajectory", "Relabel achieved goals", "Mask and reweight; fine-tune"], "summary": "Hindsight Supervised Learning relabels agent rollouts with achieved goals to create extra demonstrations."}
{"id": 6282, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "defense", "why": "Calibrated collective oversight with conformal guarantees for agentic systems"}], "pattern": "n/a", "fixes": ["oversight setting"], "varies": ["oversight mechanism"], "benchmarks": [], "results": [], "workflow": ["Aggregate scoring functions", "Conservatism penalty"], "summary": "CCO aggregates auxiliary scorers into a conservatism penalty for scalable oversight with statistical guarantees."}
{"id": 13605, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "benchmark", "why": "ProactiveEval proactive dialogue evaluation with auto-generated data"}, {"area": "social-sim", "role": "eval-method", "why": "Metrics for target planning and dialogue guidance"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs under test"], "varies": ["tasks", "metrics"], "benchmarks": [{"name": "ProactiveEval", "purpose": "proactive dialogue evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "ProactiveEval is a unified framework to evaluate LLM proactive dialogue across domains."}
{"id": 4998, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "environment", "why": "GEM open-source gym with environments for LLM agent RL"}], "pattern": "fix-agent-vary-env", "fixes": ["RL algorithms"], "varies": ["environments"], "benchmarks": [{"name": "GEM", "purpose": "environment suite", "is_new": true}], "results": [], "workflow": ["Standardized env-agent interface", "Vectorized execution"], "summary": "GEM is a Gym-like environment simulator for experience-based training of LLM agents."}
{"id": 6820, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Reconstructs interpretable workflows of black-box agentic systems"}], "pattern": "vary-scaffold", "fixes": ["black-box system"], "varies": ["reconstructed workflow"], "benchmarks": [], "results": [], "workflow": ["Search roles and tools", "Approximate behavior"], "summary": "AgentXRay reconstructs explicit workflows approximating black-box agentic systems via search."}
{"id": 3969, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Large model latent guidance for small LLM reasoning"}], "pattern": "vary-scaffold", "fixes": ["benchmarks"], "varies": ["inference method"], "benchmarks": [], "results": [], "workflow": ["Large model plans", "Compress to latents", "Small model realizes"], "summary": "Latent-Guided Reasoning decouples planning and realization using a large model's latent guidance for small LLMs."}
{"id": 12871, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "HyperCoT structure-guided CoT for n-ary KG completion"}], "pattern": "vary-data", "fixes": ["LLM"], "varies": ["method"], "benchmarks": [], "results": [], "workflow": [], "summary": "HyperCoT guides LLM chain-of-thought with hypergraph structure for n-ary knowledge graph completion."}
{"id": 12973, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "Evolutionary LLM agent framework for automated simulator construction"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["framework"], "benchmarks": [], "results": [], "workflow": ["Static blueprint", "Bi-level optimization", "Strategy playbook"], "summary": "SOCIA-EVO automatically constructs simulators using bi-level optimization and a self-curating strategy playbook."}
{"id": 14273, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "application", "why": "Memory-augmented multimodal agent for phishing URL detection"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["agent pipeline"], "benchmarks": [], "results": [], "workflow": ["Orchestrate tools", "Retrieve episodic memories", "Decide"], "summary": "MemoPhishAgent detects phishing URLs with a memory-augmented multimodal agent, improving recall over baselines."}
{"id": 8287, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "TRAP benchmark of persuasion-based prompt injection for web agents"}, {"area": "agent-safety", "role": "attack", "why": "Persuasion techniques redirect web agent tasks"}], "pattern": "fix-agent-vary-env", "fixes": ["six frontier models"], "varies": ["injection conditions"], "benchmarks": [{"name": "TRAP", "purpose": "persuasion injection on web agents", "is_new": true}], "results": [{"benchmark": "TRAP", "system": "six frontier models", "base_model": null, "setting": "baseline", "metric": "injection susceptibility", "score": 25, "unit": "%"}], "workflow": [], "summary": "TRAP benchmarks how persuasion-based prompt injections misguide web agents."}
{"id": 14238, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Neurosymbolic system for industrial asset maintenance question answering"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["system pipeline"], "benchmarks": [], "results": [], "workflow": ["Telemetry representation", "FMEA knowledge", "Grounded answers"], "summary": "IndustryAssetEQA grounds maintenance assistance in telemetry and FMEA knowledge."}
{"id": 15770, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "application", "why": "Medical multi-agent consultation with context management and self-evolution"}, {"area": "memory-selfevolve", "role": "scaffold", "why": "Structured memory distilling key info and reflection"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["multi-agent framework"], "benchmarks": [], "results": [], "workflow": ["Lead Physician", "Residual context", "Reflect on errors"], "summary": "MDTeamGPT mitigates context collapse in medical multi-agent consultations with structured self-evolving memory."}
{"id": 17747, "in_agent_scope": false, "types": [{"area": "multiling/mt", "role": "method", "why": "Iterative translation-verification-feedback for dialect translation"}, {"area": "multiling/mt", "role": "eval-method", "why": "Dialect fidelity score and target dialect ratio"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["refinement loop", "metrics"], "benchmarks": [], "results": [], "workflow": ["Translate", "Verify with classifier", "Feedback"], "summary": "DIA-REFINE guides LLMs to faithful Korean dialect translation and introduces new dialect metrics."}
{"id": 7358, "in_agent_scope": false, "types": [{"area": "apps/science", "role": "application", "why": "Domain knowledge-guided PPI candidate ranking pipeline"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": ["Interpretability-guided ranking", "Integrate evidence"], "summary": "Proposes PPI candidate ranking with a domain-knowledge-guided pipeline and large-scale evaluation."}
{"id": 12786, "in_agent_scope": false, "types": [{"area": "efficiency/inference", "role": "method", "why": "Training-free loose speculative decoding for Video-LLMs"}], "pattern": "n/a", "fixes": ["Video-LLM"], "varies": ["decoding method"], "benchmarks": [], "results": [], "workflow": ["Identify visual-relevant tokens", "Loose verification"], "summary": "LVSpec accelerates Video-LLM inference with training-free loosely speculative decoding."}
{"id": 5912, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "RL-trained router selecting budget tiers for runtime memory modules"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory routing"], "benchmarks": [], "results": [], "workflow": ["Memory modules with tiers", "Learned router", "Performance-cost control"], "summary": "BudgetMem routes queries across low/mid/high budget memory modules for explicit performance-cost control."}
{"id": 9778, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "train-agent", "why": "Character-centric GRPO for role-aware reasoning"}], "pattern": "fix-env-vary-agent", "fixes": ["role-playing task"], "varies": ["RL objective"], "benchmarks": [], "results": [], "workflow": ["Decouple task and style rewards", "Character-centric GRPO"], "summary": "CRPO adapts GRPO to role-playing agents to preserve character fidelity."}
{"id": 8336, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Agent enabling 2D MLLMs to analyze 3D CT"}], "pattern": "vary-scaffold", "fixes": ["2D MLLMs"], "varies": ["agent pipeline"], "benchmarks": [], "results": [], "workflow": ["Perception tools", "Accumulate evidence", "Clinical understanding"], "summary": "3DMedAgent unifies perception and understanding for 3D CT analysis using 2D MLLMs."}
{"id": 15133, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "train-agent", "why": "Two-phase multi-task SFT and RL recipe for issue resolution"}, {"area": "code-swe", "role": "data", "why": "Three curated datasets for localization, repair, tests"}], "pattern": "fix-env-vary-agent", "fixes": ["SWE-bench"], "varies": ["training recipe"], "benchmarks": [{"name": "SWE-bench", "purpose": "issue resolution evaluation", "is_new": false}], "results": [], "workflow": ["Multi-task SFT", "Execution-feedback RL"], "summary": "SWE-Swiss trains issue resolution via multi-task SFT then RL on code repair."}
{"id": 14945, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "ToolPRMBench step-level evaluation of process reward models for tool agents"}], "pattern": "fix-agent-vary-env", "fixes": ["PRMs under test"], "varies": ["step-level cases"], "benchmarks": [{"name": "ToolPRMBench", "purpose": "PRM evaluation for tool use", "is_new": true}], "results": [], "workflow": ["Convert trajectories", "Step-level test cases"], "summary": "ToolPRMBench evaluates process reward models for tool-using agents on step-level cases."}
{"id": 13795, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "train-agent", "why": "Dual-scale evolutionary policy training for social language games"}], "pattern": "fix-env-vary-agent", "fixes": ["social games"], "varies": ["training algorithm"], "benchmarks": [], "results": [], "workflow": ["Detect impasse", "Evolve policies"], "summary": "DEPT overcomes evolution impasse in self-play RLVR for social language agents."}
{"id": 12738, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "Multi-path multi-agent planning for automated data science"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["workflow planning"], "benchmarks": [], "results": [], "workflow": ["Generate candidate strategies", "Optimization agent refines"], "summary": "SPIO uses multi-agent multi-path planning across data science modules."}
{"id": 13696, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "Learned self-reflection decision to trust or consult external audio perception"}], "pattern": "fix-env-vary-agent", "fixes": ["omni model"], "varies": ["training method"], "benchmarks": [], "results": [], "workflow": ["Reflect", "Decide consult", "Fine-tune"], "summary": "Speech-Hands teaches omni models when to trust themselves versus external audio perception."}
{"id": 18000, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "benchmark", "why": "Paired Rosetta Stone and Match-Up linguistic puzzle corpus"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["puzzle format"], "benchmarks": [{"name": "Rosetta Stone and Match-Up puzzles", "purpose": "paired puzzle corpus", "is_new": true}], "results": [], "workflow": [], "summary": "Paired linguistic puzzle corpus with human and LLM benchmarks."}
{"id": 7128, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Adaptive token refinement for long-tailed LVLM fine-tuning"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "Adaptive token refinement for long-tailed LVLM fine-tuning."}
{"id": 14970, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "train-agent", "why": "Learned agentic workflow design for medical coding"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["workflow design"], "benchmarks": [], "results": [], "workflow": ["Design workflow", "Evaluate", "Refine"], "summary": "MedDCR learns workflow design for medical coding in a closed loop."}
{"id": 12808, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "benchmark", "why": "DynAfford benchmark for dynamic affordance planning"}], "pattern": "fix-agent-vary-env", "fixes": ["embodied agents"], "varies": ["environments"], "benchmarks": [{"name": "DynAfford", "purpose": "dynamic affordance planning", "is_new": true}], "results": [], "workflow": [], "summary": "ADAPT introduces DynAfford for embodied agents handling unspecified affordance constraints."}
{"id": 12577, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "method", "why": "Self-evolving pairwise reasoning story evaluator"}], "pattern": "vary-data", "fixes": ["LLM"], "varies": ["training method"], "benchmarks": [], "results": [], "workflow": ["Self-synthesize data", "Pairwise reasoning"], "summary": "EvolvR trains story evaluators with self-evolving pairwise reasoning."}
{"id": 5221, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Slot-aware adaptation for source-free object detection"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "CGSA applies object-centric slot adaptation to source-free object detection."}
{"id": 14235, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "scaffold", "why": "Knowledge-evolving browser agent with human feedback"}], "pattern": "vary-scaffold", "fixes": ["VLM"], "varies": ["knowledge adaptation"], "benchmarks": [], "results": [], "workflow": ["Human feedback", "Knowledge adaptation"], "summary": "ColorBrowserAgent evolves knowledge for robust long-horizon web automation."}
{"id": 7619, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "analysis", "why": "Language preference in multilingual RAG citation"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["document language"], "benchmarks": [], "results": [], "workflow": [], "summary": "Controlled study showing multilingual RAG models prefer English sources."}
{"id": 2434, "in_agent_scope": false, "types": [{"area": "train/distill", "role": "method", "why": "KL-guided layer selection for hybrid attention distillation"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": ["Score layer importance", "Select layers", "Distill"], "summary": "Layer selection recipe for distilling Transformers into hybrid linear-attention models."}
{"id": 5990, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "TableAgent-Bench multi-turn agentic TableQA"}], "pattern": "fix-agent-vary-env", "fixes": ["LLM agents"], "varies": ["tasks"], "benchmarks": [{"name": "TableAgent-Bench", "purpose": "agentic TableQA", "is_new": true}], "results": [], "workflow": [], "summary": "TableAgent-Bench evaluates agentic multi-turn table QA with 1,310 dialogues."}
{"id": 18299, "in_agent_scope": true, "types": [{"area": "search-research", "role": "analysis", "why": "Controlled study of budget-constrained agentic search"}], "pattern": "vary-conditions", "fixes": ["models"], "varies": ["search depth", "retrieval"], "benchmarks": [], "results": [], "workflow": [], "summary": "Measures accuracy and cost impact of design decisions in budget-constrained agentic search."}
{"id": 15629, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Probe and Verify agent for text-to-SQL"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": ["Probe database", "Rule-based verify", "Refine SQL"], "summary": "PV-SQL probes databases and verifies rule-based checklists for text-to-SQL."}
{"id": 8744, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "scaffold", "why": "Sub-scale collaboration decoupling exploration from exploitation"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["framework"], "benchmarks": [], "results": [], "workflow": ["Small model explores", "LLM exploits"], "summary": "SCOUT decouples exploration using small models for unseen nonlinguistic tasks."}
{"id": 18395, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "eval-method", "why": "Grammar-book guided probing for Luxembourgish"}], "pattern": "vary-evaluation", "fixes": ["LLMs"], "varies": ["protocol"], "benchmarks": [], "results": [], "workflow": [], "summary": "Grammar-book guided evaluation of LLM grammar in Luxembourgish."}
{"id": 14824, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "Bayesian active learning with GPs for passage retrieval"}], "pattern": "n/a", "fixes": [], "varies": [], "benchmarks": [], "results": [], "workflow": [], "summary": "BAGEL propagates sparse LLM relevance signals via Gaussian processes for retrieval."}
{"id": 13323, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Observation-token supervision for agentic RL"}], "pattern": "fix-env-vary-agent", "fixes": ["environment"], "varies": ["RL objective"], "benchmarks": [], "results": [], "workflow": ["Advantage on observations"], "summary": "SOAR supervises observation tokens in agentic RL."}
{"id": 6415, "in_agent_scope": false, "types": [{"area": "interp/icl", "role": "analysis", "why": "Inference dynamics under contextual repetition"}], "pattern": "vary-conditions", "fixes": ["LLMs"], "varies": ["context"], "benchmarks": [], "results": [], "workflow": [], "summary": "Theoretical and empirical study of contextual influence on LLM inference dynamics."}
{"id": 1404, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "environment", "why": "Co-Gym framework for human-agent collaboration"}, {"area": "social-sim", "role": "benchmark", "why": "Evaluation of collaborative agents"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["task environments"], "benchmarks": [{"name": "Collaborative Gym", "purpose": "human-agent collaboration", "is_new": true}], "results": [], "workflow": [], "summary": "Co-Gym is an open framework for developing and evaluating human-agent collaboration."}
{"id": 11581, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "ABC-Bench agentic biosecurity capabilities"}, {"area": "agent-safety", "role": "benchmark", "why": "Dual-use biology tasks"}], "pattern": "fix-agent-vary-env", "fixes": ["LLM agents"], "varies": ["tasks"], "benchmarks": [{"name": "ABC-Bench", "purpose": "biosecurity agent tasks", "is_new": true}], "results": [], "workflow": [], "summary": "ABC-Bench measures agentic biosecurity-relevant capabilities."}
{"id": 16434, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "analysis", "why": "Safety risks of experience-driven self-evolving agents"}, {"area": "agent-safety", "role": "analysis", "why": "Benign experience degrades safety"}], "pattern": "vary-conditions", "fixes": ["agents"], "varies": ["experience"], "benchmarks": [], "results": [], "workflow": [], "summary": "Shows self-evolving agents' accumulated experience can compromise safety."}
{"id": 3261, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "SCUBA Salesforce CRM computer-use benchmark"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks"], "benchmarks": [{"name": "SCUBA", "purpose": "CRM computer use", "is_new": true}], "results": [], "workflow": [], "summary": "SCUBA benchmarks computer-use agents on 300 Salesforce CRM tasks."}
{"id": 15021, "in_agent_scope": false, "types": [{"area": "align/values", "role": "defense", "why": "Self-reflection token improves large reasoning model safety"}], "pattern": "n/a", "fixes": ["threat model"], "varies": ["alignment"], "benchmarks": [], "results": [], "workflow": ["Self-Reflection token", "Reflect during generation"], "summary": "Self-Reflection tokens let reasoning models introspect for safety."}
{"id": 8458, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "SP-Mind spatial proteomics analysis agent"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent pipeline"], "benchmarks": [{"name": "SP-Mind benchmark", "purpose": "benchmarking", "is_new": true}], "results": [], "workflow": ["Natural language query", "Tools", "Analysis"], "summary": "SP-Mind is an autonomous agent for spatial proteomics analysis."}