Files
ai-taxonomy-literature-anal…/data/labels/o28.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
30 KiB
JSON

{"id": 14823, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "Role-specialized agents summarize, extract, reason over retrieved evidence, then synthesize answer"}], "pattern": "vary-scaffold", "fixes": ["LLM", "retrieved contexts"], "varies": ["generation pipeline structure"], "benchmarks": [], "results": [], "workflow": ["summarize evidence", "extract evidence", "reason", "synthesize answer"], "summary": "Multi-agent synthesis for RAG that handles noisy retrieved contexts with role-specialized agents."}
{"id": 12834, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "Uses dynamic information flow to partition knowledge and route federated retrieval"}], "pattern": "vary-scaffold", "fixes": ["LLM", "knowledge sources"], "varies": ["retrieval routing method"], "benchmarks": [], "results": [], "workflow": ["probe information flow", "infer query intents", "align partitions", "retrieve"], "summary": "Federated retrieval framework using LLM dynamic information flow to align multi-prototype knowledge partitions."}
{"id": 11422, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "method", "why": "Multi-order message passing among LLM agents to capture multi-hop dependencies"}], "pattern": "vary-scaffold", "fixes": ["LLM", "tasks"], "varies": ["inter-agent communication scheme"], "benchmarks": [], "results": [], "workflow": ["capture multi-hop dependencies", "structure messages", "aggregate"], "summary": "Communication scheme for LLM multi-agent systems capturing multi-hop dependencies instead of first-order concatenation."}
{"id": 10622, "in_agent_scope": false, "types": [{"area": "reason/code", "role": "method", "why": "LLM-guided adaptive per-object heap abstractions for JavaScript abstract interpretation"}], "pattern": "vary-scaffold", "fixes": ["static analysis framework"], "varies": ["heap abstraction strategy"], "benchmarks": [], "results": [], "workflow": ["LLM proposes abstraction", "abstract interpretation"], "summary": "Language-model-guided static analysis with adaptive heap abstractions for JavaScript."}
{"id": 9153, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Bayesian sparse-reward RL for MLLM image-editing agents avoiding proxy reward bias"}], "pattern": "vary-both", "fixes": ["image-editing tool environment"], "varies": ["reward modeling", "RL training"], "benchmarks": [], "results": [], "workflow": ["model sparse reward", "RL train editing agent"], "summary": "Reward-centric RL framework for training tool-using image-editing agents with Bayesian sparse rewards."}
{"id": 14730, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "Trains KGQA agents with visibility-grounded supervision from SPARQL programs"}, {"area": "tool-use", "role": "data", "why": "Converts expert SPARQL into local-view action supervision"}], "pattern": "vary-data", "fixes": ["Freebase KG tools"], "varies": ["supervision data", "agent weights"], "benchmarks": [{"name": "Freebase", "purpose": "KGQA environment", "is_new": false}], "results": [], "workflow": ["convert SPARQL to actions", "ground in visible KG", "train agent"], "summary": "Trains faithful KG agents using supervision derived from SPARQL with visibility grounding."}
{"id": 12312, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "train-agent", "why": "Online RL for Android agents sampling multiple actions per emulator state"}], "pattern": "fix-env-vary-agent", "fixes": ["Android emulator"], "varies": ["RL training paradigm"], "benchmarks": [], "results": [], "workflow": ["sample multiple actions per state", "update policy"], "summary": "Online RL framework letting Android agents use Single State Multiple Actions for better sample efficiency."}
{"id": 2570, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Agentic RL algorithm with entropy-based adaptive rollouts at tool-call steps"}], "pattern": "fix-env-vary-agent", "fixes": ["tool environments"], "varies": ["RL algorithm"], "benchmarks": [], "results": [], "workflow": ["detect post-tool uncertainty", "branch rollouts", "optimize policy"], "summary": "ARPO: agentic RL algorithm that explores fine-grained multi-turn tool-call steps."}
{"id": 2084, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Discovers hierarchical orchestrator/sub-agent SWE agent structures via bandit optimization"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent hierarchy design"], "benchmarks": [], "results": [], "workflow": ["decompose into sub-agents", "bandit search structures"], "summary": "Bandit-optimized discovery of hierarchical SWE agents with specialized sub-agents."}
{"id": 9860, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "scaffold", "why": "Cache grafting of validated function skeletons for fast robust code-policy synthesis"}], "pattern": "vary-scaffold", "fixes": ["CodeLLM"], "varies": ["decoding procedure"], "benchmarks": [], "results": [], "workflow": ["library of code skeletons", "graft cache", "generate policy"], "summary": "FCGraft speeds and stabilizes embodied code-policy generation using validated function-level skeleton caches."}
{"id": 11761, "in_agent_scope": false, "types": [{"area": "arch/diffusion", "role": "method", "why": "Progressive diffusion for text, timing and phoneme controllable audio generation"}], "pattern": "vary-data", "fixes": ["diffusion framework"], "varies": ["training stages", "data construction"], "benchmarks": [], "results": [], "workflow": ["construct data", "progressive multi-task diffusion"], "summary": "Progressive diffusion approach for controllable text-to-audio generation with timing and intelligibility."}
{"id": 7935, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "data", "why": "Pipeline synthesizes verified terminal trajectories from Dockerized environments"}, {"area": "code-swe", "role": "environment", "why": "Builds Dockerized execution environments from repositories"}], "pattern": "vary-data", "fixes": ["training recipe"], "varies": ["terminal trajectory data"], "benchmarks": [], "results": [], "workflow": ["filter repos", "build Docker envs", "generate tasks", "collect trajectories"], "summary": "TerminalTraj generates large-scale executable, verifiable terminal agent trajectories from Docker environments."}
{"id": 13403, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "Iterative LLM-SLM framework jointly improving KB completion and KBQA"}], "pattern": "vary-scaffold", "fixes": ["KB"], "varies": ["joint framework"], "benchmarks": [], "results": [], "workflow": ["LLM reasoning", "SLM scoring", "iterate"], "summary": "JCQL combines LLMs and SLMs to jointly solve KB completion and question answering."}
{"id": 10370, "in_agent_scope": false, "types": [{"area": "align/rlvr", "role": "analysis", "why": "Studies how far internally generated supervision matches oracle training in self-evolution"}], "pattern": "vary-conditions", "fixes": ["base model", "unlabeled prompts"], "varies": ["self-evolution strategy"], "benchmarks": [{"name": "Knights and Knaves", "purpose": "logical reasoning testbed", "is_new": false}], "results": [], "workflow": [], "summary": "Analyzes generalization gap of four self-evolution strategies versus oracle supervision on reasoning."}
{"id": 6970, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Agentic framework generating academic illustrations with retrieval, planning, rendering, critique"}, {"area": "science-data", "role": "benchmark", "why": "Introduces PaperBananaBench of 292 methodology diagram cases"}], "pattern": "vary-scaffold", "fixes": ["VLMs"], "varies": ["agent pipeline"], "benchmarks": [{"name": "PaperBananaBench", "purpose": "evaluate illustration generation", "is_new": true}], "results": [], "workflow": ["retrieve references", "plan content", "render", "self-critique"], "summary": "PaperBanana agentic pipeline for publication-ready academic illustrations plus a 292-case benchmark."}
{"id": 16312, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Godel agent repairs policy via experience abstraction and minimal code patches"}], "pattern": "vary-scaffold", "fixes": ["small language model"], "varies": ["policy patches"], "benchmarks": [], "results": [], "workflow": ["analyze failure", "form strategy", "abstract", "patch policy"], "summary": "Polaris: self-improving agent for small models that turns failures into small persistent policy patches."}
{"id": 4283, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "environment", "why": "Real-Time Reasoning Gym for agents in evolving environments"}, {"area": "planning-decision", "role": "scaffold", "why": "Compares reactive and planning agent paradigms under time pressure"}], "pattern": "vary-both", "fixes": ["language models"], "varies": ["environment dynamics", "agent paradigm"], "benchmarks": [{"name": "Real-Time Reasoning Gym", "purpose": "evaluate timely agent reasoning", "is_new": true}], "results": [], "workflow": [], "summary": "Formulates real-time reasoning and builds a gym comparing reactive and planning LLM agents."}
{"id": 11399, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "Benchmark for long-horizon repository generation from scratch from requirements document"}], "pattern": "fix-agent-vary-env", "fixes": ["coding agents"], "varies": ["tasks"], "benchmarks": [{"name": "NL2Repo-Bench", "purpose": "long-horizon repo generation", "is_new": true}], "results": [], "workflow": [], "summary": "NL2Repo-Bench evaluates coding agents building full repositories from a natural-language spec."}
{"id": 10213, "in_agent_scope": true, "types": [{"area": "science-data", "role": "train-agent", "why": "Stateful RL trains file-reasoning agent in persistent sandbox"}, {"area": "science-data", "role": "benchmark", "why": "Introduces 400+ question file reasoning benchmark"}], "pattern": "fix-env-vary-agent", "fixes": ["file sandbox"], "varies": ["RL training"], "benchmarks": [{"name": "FIRE file reasoning benchmark", "purpose": "evaluate file reasoning", "is_new": true}], "results": [], "workflow": ["data pipeline", "stateful file execution", "RL train"], "summary": "FIRE trains agents with stateful RL to answer questions over raw XLSX/PDF/DOCX/PPTX files and releases a benchmark."}
{"id": 14240, "in_agent_scope": false, "types": [{"area": "efficiency/inference", "role": "method", "why": "Hardware-in-the-loop architecture search for low-latency on-device LLMs"}], "pattern": "n/a", "fixes": ["mobile runtime"], "varies": ["model architecture"], "benchmarks": [], "results": [], "workflow": ["latency-guided search", "attention skipping"], "summary": "Methodology for designing latency-optimized on-device LLMs deployable on standard mobile runtimes."}
{"id": 8686, "in_agent_scope": false, "types": [{"area": "train/peft", "role": "method", "why": "Confidence-weighted probe samples to plan LoRA rank allocation"}], "pattern": "n/a", "fixes": ["LoRA"], "varies": ["rank allocation"], "benchmarks": [], "results": [], "workflow": ["reweight probe samples", "estimate module importance", "allocate ranks"], "summary": "CSPLoRA plans LoRA rank allocation using uncertainty-weighted module importance."}
{"id": 1909, "in_agent_scope": false, "types": [{"area": "apps/science", "role": "benchmark", "why": "Large chest X-ray VQA dataset with localization and tags derived from MIMIC-CXR"}], "pattern": "fix-agent-vary-env", "fixes": ["models"], "varies": ["dataset"], "benchmarks": [{"name": "MIMIC-Ext-CXR-QBA", "purpose": "new CXR VQA dataset", "is_new": true}, {"name": "MIMIC-CXR", "purpose": "source data", "is_new": false}], "results": [], "workflow": ["generate from scene graphs"], "summary": "CXR-QBA: 42M-pair chest X-ray VQA dataset with bounding boxes and structured tags."}
{"id": 15177, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "AST-guided adaptive memory for repository-level iterative code generation"}, {"area": "code-swe", "role": "scaffold", "why": "Memory manages repo context across interactions"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory representation"], "benchmarks": [], "results": [], "workflow": ["AST memory", "dynamic update"], "summary": "CodeMEM manages repository context with AST-based memory for iterative code generation."}
{"id": 16227, "in_agent_scope": true, "types": [{"area": "embodied-nav", "role": "scaffold", "why": "Estimate-Verify-Update mechanism to correct belief inertia"}, {"area": "embodied-nav", "role": "analysis", "why": "Probing analysis characterizes belief inertia"}], "pattern": "vary-scaffold", "fixes": ["LLM agents", "embodied tasks"], "varies": ["belief intervention mechanism"], "benchmarks": [], "results": [], "workflow": ["estimate", "verify", "update beliefs"], "summary": "Identifies belief inertia in embodied LLM agents and proposes EVU active intervention."}
{"id": 2326, "in_agent_scope": true, "types": [{"area": "search-research", "role": "eval-method", "why": "Audit framework with eight dimensions for citations and evidence in deep research"}, {"area": "search-research", "role": "analysis", "why": "Audits popular public deep research systems"}], "pattern": "vary-evaluation", "fixes": ["deep research systems"], "varies": ["audit metrics"], "benchmarks": [], "results": [], "workflow": ["decompose statements", "build citation matrices"], "summary": "DeepTRACE audits deep research and generative search systems across eight measurable reliability dimensions."}
{"id": 13608, "in_agent_scope": true, "types": [{"area": "search-research", "role": "scaffold", "why": "Agentic multimodal search with incremental synthesis for long-form reports"}, {"area": "search-research", "role": "benchmark", "why": "Introduces multimodal long-form generation task"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": ["multimodal search", "filter", "checklist synthesis"], "summary": "Deep-Reporter agentic framework producing grounded multimodal long-form reports."}
{"id": 7767, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Hypothesis-experiment-validation loop finds causal dependencies for agent decisions"}], "pattern": "vary-scaffold", "fixes": ["LLM agents"], "varies": ["causal verification loop"], "benchmarks": [], "results": [], "workflow": ["generate hypotheses", "experiment", "validate"], "summary": "Cycle-of-Science lets agents verify causal hypotheses counterfactually to improve decisions."}
{"id": 16889, "in_agent_scope": false, "types": [{"area": "knowledge/halluc", "role": "method", "why": "Finds activation direction for unanswerability detection in extractive QA"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["detection method"], "benchmarks": [], "results": [], "workflow": ["find direction via activation addition", "project and classify"], "summary": "Linear activation direction detects unanswerable questions in extractive QA."}
{"id": 5426, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "benchmark", "why": "Benchmark and synthesis framework for GUI grounding in multi-window desktops"}, {"area": "web-gui", "role": "environment", "why": "Parametrically generates cluttered desktop scenarios"}], "pattern": "fix-agent-vary-env", "fixes": ["MLLMs"], "varies": ["window occlusion", "layout density"], "benchmarks": [{"name": "WinDeskGround", "purpose": "GUI grounding robustness", "is_new": true}], "results": [], "workflow": [], "summary": "WinDeskGround benchmarks GUI grounding robustness under occlusion and clutter in desktop environments."}
{"id": 14285, "in_agent_scope": true, "types": [{"area": "search-research", "role": "data", "why": "Multi-agent synthesis of long-horizon e-commerce tool-use trajectories"}, {"area": "domain-agents", "role": "train-agent", "why": "Trains shopping research agents on distilled trajectories"}], "pattern": "vary-data", "fixes": ["training recipe"], "varies": ["synthetic trajectories"], "benchmarks": [], "results": [], "workflow": ["infer intents", "supervisor orchestrates", "distill trajectories", "train"], "summary": "ProductResearch synthesizes trajectories via multi-agent collaboration to train e-commerce deep research agents."}
{"id": 15024, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "scaffold", "why": "Deliberative multi-agent framework with tip retrieval, pre-execution alignment, reflection for mobile GUI"}], "pattern": "vary-scaffold", "fixes": ["MLLM"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": ["retrieve tips", "align thought-action", "execute", "reflect"], "summary": "D-Artemis is a training-free deliberative mobile GUI agent with pre-execution checks and reflection."}
{"id": 6386, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Bandit-based prompt optimization for multi-agent systems using GNNs"}], "pattern": "vary-scaffold", "fixes": ["MAS topology", "LLM"], "varies": ["agent prompts"], "benchmarks": [], "results": [], "workflow": ["model prompt coupling with GNN", "bandit search prompts"], "summary": "MASPOB optimizes prompts in fixed-topology multi-agent systems sample-efficiently."}
{"id": 17035, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "benchmark", "why": "Benchmark for end-to-end document-style form filling"}, {"area": "web-gui", "role": "benchmark", "why": "Evaluates VLM agents on form completion"}], "pattern": "fix-agent-vary-env", "fixes": ["baseline VLMs"], "varies": ["form datasets"], "benchmarks": [{"name": "FormGym", "purpose": "form filling", "is_new": true}], "results": [], "workflow": [], "summary": "FormGym benchmarks end-to-end form filling by agents using four datasets."}
{"id": 12951, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Predictive context management adjusting memory granularity for next action"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["context management"], "benchmarks": [], "results": [], "workflow": ["predict relevance", "adjust granularity", "build context"], "summary": "PACE reframes context management for long-horizon agents as next-step prediction."}
{"id": 1269, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "Operational safety benchmark for refusing off-topic queries in agentic use cases"}, {"area": "agent-safety", "role": "analysis", "why": "Evaluates six model families"}], "pattern": "fix-agent-vary-env", "fixes": ["models"], "varies": ["queries"], "benchmarks": [{"name": "OffTopicEval", "purpose": "operational safety", "is_new": true}], "results": [], "workflow": [], "summary": "OffTopicEval measures whether LLM agents appropriately accept or refuse queries for their purpose."}
{"id": 7820, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "Framework for evaluating tool-using agent robustness under noise"}], "pattern": "vary-conditions", "fixes": ["agents"], "varies": ["noise conditions"], "benchmarks": [{"name": "AgentNoiseBench", "purpose": "robustness under noise", "is_new": true}], "results": [], "workflow": [], "summary": "AgentNoiseBench evaluates robustness of tool-using agents to noisy environments."}
{"id": 16882, "in_agent_scope": false, "types": [{"area": "train/sft", "role": "analysis", "why": "Studies input design principles for sample-efficient linguistic rule induction"}], "pattern": "vary-conditions", "fixes": ["lightweight models"], "varies": ["input design principles"], "benchmarks": [], "results": [], "workflow": [], "summary": "Examines analogical structure, contrast and minimal cues for efficient learning of verb alternation rules."}
{"id": 6229, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Execution-based agent searches discriminative test inputs to re-rank candidate solutions"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["verification procedure"], "benchmarks": [], "results": [], "workflow": ["generate inputs", "execute candidates", "rerank"], "summary": "Agentic Verifier actively finds discriminative tests for test-time scaling in competitive coding."}
{"id": 1761, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "Agent uses code tools to analyze data and iteratively optimize equations"}, {"area": "science-data", "role": "train-agent", "why": "Trains with RL for equation discovery"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent framework"], "benchmarks": [], "results": [], "workflow": ["analyze data", "implement equation", "evaluate", "optimize"], "summary": "SR-Scientist turns the LLM into an autonomous equation-discovery agent with data-analysis tools."}
{"id": 6646, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Hierarchical Mamba-based network for image-to-point-cloud registration"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["registration architecture"], "benchmarks": [], "results": [], "workflow": ["focus-sweep hierarchy"], "summary": "FS-I2P registration network with focus-sweep paradigm and SSM interaction."}
{"id": 13549, "in_agent_scope": false, "types": [{"area": "safety/privacy", "role": "method", "why": "Training-free inference-time unlearning using entropy-guided probe"}], "pattern": "n/a", "fixes": ["LLM weights"], "varies": ["decoding procedure"], "benchmarks": [], "results": [], "workflow": ["probe queries", "entropy-guided decoding"], "summary": "SEGUE unlearns facts at inference time without parameter updates."}
{"id": 15840, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "Long-horizon CLI programming benchmark with 20 curated tasks"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks"], "benchmarks": [{"name": "LongCLI-Bench", "purpose": "long-horizon CLI tasks", "is_new": true}], "results": [], "workflow": [], "summary": "LongCLI-Bench evaluates agentic programming on long-horizon command-line tasks."}
{"id": 16001, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "environment", "why": "Multi-agent society simulation grounded in affect control and social theory"}, {"area": "multi-agent", "role": "analysis", "why": "Studies emergent relational order and authority stratification"}], "pattern": "vary-conditions", "fixes": ["LLM agents"], "varies": ["social environment"], "benchmarks": [], "results": [], "workflow": [], "summary": "CAREB-MAS simulates LLM agent societies showing emergent relational order and authority stratification."}
{"id": 15013, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "scaffold", "why": "Lookahead meta-adaptive exploration integrating tool execution and reasoning"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["inference exploration"], "benchmarks": [], "results": [], "workflow": ["lookahead", "tool execution", "adaptive reasoning"], "summary": "MAXS gives tool-using agents lookahead exploration for stable, efficient reasoning."}
{"id": 10632, "in_agent_scope": true, "types": [{"area": "science-data", "role": "scaffold", "why": "Agent dynamically selects among single-step retrosynthesis models during search"}], "pattern": "vary-scaffold", "fixes": ["retrosynthesis models"], "varies": ["transition model selection"], "benchmarks": [], "results": [], "workflow": ["assess molecule state", "select model", "search"], "summary": "RetrOrchestrator orchestrates single-step models to navigate retrosynthesis transition space."}
{"id": 15036, "in_agent_scope": false, "types": [{"area": "eval/judge", "role": "eval-method", "why": "Domain-adapted claim decomposition and verification for medical long-form factuality"}], "pattern": "vary-evaluation", "fixes": ["LLM outputs"], "varies": ["metric"], "benchmarks": [], "results": [], "workflow": ["decompose claims", "verify"], "summary": "MedScore evaluates factuality of long-form medical answers via domain-adapted claim decomposition."}
{"id": 16480, "in_agent_scope": false, "types": [{"area": "safety/jailbreak", "role": "attack", "why": "White-box inference-time jailbreak via sparse logit editing"}], "pattern": "vary-conditions", "fixes": ["target LLM"], "varies": ["adversarial logit edits"], "benchmarks": [], "results": [], "workflow": ["dictionary of penalties", "edit logits"], "summary": "SIBI jailbreaks LLMs by editing logits with a tokenizer-aligned dictionary, no gradients."}
{"id": 9138, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "scaffold", "why": "Multi-agent world-model planning for LLM heuristic design over entailment graph"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["heuristic search framework"], "benchmarks": [], "results": [], "workflow": ["entailment graph memory", "plan derivation", "evaluate"], "summary": "PathWise formulates automated heuristic design as sequential planning with a world model."}
{"id": 12756, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Human-in-the-loop agentic pipeline for autoformalising physics into Lean"}, {"area": "science-data", "role": "data", "why": "Builds formal physics dataset"}], "pattern": "vary-scaffold", "fixes": ["LLMs"], "varies": ["pipeline"], "benchmarks": [], "results": [], "workflow": ["agentic code generation", "expert review"], "summary": "FormalScience helps experts autoformalise scientific reasoning in Lean."}
{"id": 13211, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Multi-agent orchestrator and collaborators for project-based learning"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["agent roles"], "benchmarks": [], "results": [], "workflow": ["orchestrator scaffolds", "collaborators assist"], "summary": "SimPBL multi-agent framework supporting project-based learning."}
{"id": 8093, "in_agent_scope": false, "types": [{"area": "safety/security", "role": "method", "why": "Streaming latent-dynamics head for real-time risk detection during generation"}], "pattern": "n/a", "fixes": ["LM"], "varies": ["guardrail"], "benchmarks": [], "results": [], "workflow": ["read hidden states", "detect risk"], "summary": "PlugGuard detects harmful generation in streaming using hidden-state dynamics."}
{"id": 16317, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Preference learning on verified critical steps for agent post-training"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["training objective"], "benchmarks": [], "results": [], "workflow": ["find critical steps", "verify flips", "preference optimize"], "summary": "CSO trains agents on verified critical decision steps for credit assignment."}
{"id": 16714, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Multimodal dialogue model with plan reasoning and plan-based retrieval for instructional videos"}], "pattern": "n/a", "fixes": ["benchmarks"], "varies": ["model"], "benchmarks": [], "results": [], "workflow": ["plan reasoning", "plan retrieval"], "summary": "VIGiA grounded dialogue model for instructional video guidance."}
{"id": 9070, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Trains agents to adapt cognitive depth per step"}], "pattern": "fix-env-vary-agent", "fixes": ["environments"], "varies": ["cognitive level policy"], "benchmarks": [], "results": [], "workflow": ["four cognitive levels", "train router"], "summary": "CogRouter trains LLM agents to adapt reasoning depth at each step."}
{"id": 678, "in_agent_scope": true, "types": [{"area": "science-data", "role": "benchmark", "why": "End-to-end web-based data science benchmark of 870 tasks"}, {"area": "web-gui", "role": "benchmark", "why": "Requires web interaction"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks"], "benchmarks": [{"name": "WebDS", "purpose": "web data science", "is_new": true}], "results": [], "workflow": [], "summary": "WebDS benchmarks agents on 870 end-to-end web-based data science tasks."}
{"id": 10745, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "benchmark", "why": "186-task benchmark for vulnerability of agent-generated code"}, {"area": "code-swe", "role": "analysis", "why": "Evaluates 12 coding agent settings"}], "pattern": "fix-agent-vary-env", "fixes": ["coding agents"], "varies": ["tasks"], "benchmarks": [{"name": "SUSVIBES", "purpose": "code security", "is_new": true}], "results": [], "workflow": [], "summary": "SUSVIBES shows coding agents often produce insecure code on real feature requests."}
{"id": 2519, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Self-improving coding agent guided by clade-level metaproductivity metric"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["self-modification selection"], "benchmarks": [], "results": [], "workflow": ["evaluate descendants", "select modifications"], "summary": "Huxley-Godel Machine guides self-modification using descendants' aggregated performance."}
{"id": 823, "in_agent_scope": false, "types": [{"area": "eval/bench", "role": "eval-method", "why": "Dynamic agent protocol generates, verifies, and solves problems for evaluating reasoning"}], "pattern": "vary-evaluation", "fixes": ["LLMs"], "varies": ["evaluation protocol"], "benchmarks": [], "results": [], "workflow": ["teacher generates", "orchestrator verifies", "student solves"], "summary": "Agent-centric dynamic benchmarking protocol replacing static datasets."}
{"id": 6383, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "analysis", "why": "Empirical study of RFT generalization across difficulty and environments"}], "pattern": "vary-conditions", "fixes": ["RFT method"], "varies": ["environments", "difficulty"], "benchmarks": [], "results": [], "workflow": [], "summary": "Systematic study of whether reinforcement fine-tuning generalizes for LLM agents."}
{"id": 1974, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Large-scale end-to-end RL for long-horizon search agent without commercial APIs"}], "pattern": "fix-env-vary-agent", "fixes": ["search tools"], "varies": ["RL training"], "benchmarks": [], "results": [], "workflow": ["RL train QwQ-32B"], "summary": "ASearcher: search agent trained purely by RL for long-horizon complex search."}