Files
ai-taxonomy-literature-anal…/data/labels/o06.jsonl
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

61 lines
29 KiB
JSON

{"id": 10359, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "benchmark", "why": "Cooperative game benchmark with diverse simulated player profiles for human-agent collaboration"}, {"area": "multi-agent", "role": "train-agent", "why": "Collaborative agentic RL with hybrid task and affective reward"}], "pattern": "vary-both", "fixes": ["cooperative games"], "varies": ["player profiles", "training paradigm"], "benchmarks": [{"name": "CollabBench", "purpose": "evaluate collaborative agents", "is_new": true}, {"name": "CWAH-MultiPlayer", "purpose": "extended multiplayer environment", "is_new": true}, {"name": "Cook-MultiPlayer", "purpose": "extended multiplayer environment", "is_new": true}], "results": [], "workflow": ["Simulate player profiles", "Agentic rollouts", "Hybrid reward RL"], "summary": "CollabBench benchmarks and trains LLM agents collaborating with diverse simulated players in cooperative games."}
{"id": 12282, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "train-agent", "why": "RL-trained Memory Manager and Answer Agent for memory operations"}], "pattern": "fix-env-vary-agent", "fixes": ["benchmark", "memory bank"], "varies": ["RL-trained memory policy"], "benchmarks": [], "results": [], "workflow": ["Memory Manager edits memory", "Answer Agent retrieves", "Outcome-based RL"], "summary": "Memory-R1 uses RL to train agents to add, update, delete and use external memory."}
{"id": 8906, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "End-to-end RL for multi-turn agentic GraphRAG over hypergraph"}], "pattern": "fix-env-vary-agent", "fixes": ["QA benchmarks"], "varies": ["retrieval policy", "reward"], "benchmarks": [], "results": [], "workflow": ["Build knowledge hypergraph", "Multi-turn retrieval", "End-to-end reward RL"], "summary": "Graph-R1 is an agentic GraphRAG framework trained end-to-end with RL on knowledge hypergraphs."}
{"id": 1315, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Key-value cache refinement to reduce VLM visual hallucination"}], "pattern": "n/a", "fixes": ["VLM"], "varies": ["KV refinement"], "benchmarks": [], "results": [], "workflow": ["Analyze key scatter", "Refine KV cache"], "summary": "DSCR refines key-value cache using depth and spatial cues to mitigate visual hallucination in VLMs."}
{"id": 17437, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Agentic system extracting schema-guided tables from financial filings"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["agentic extraction pipeline"], "benchmarks": [], "results": [], "workflow": ["Detect tables", "Schema-guided extraction", "Learn from feedback"], "summary": "TASER is a continuously learning agentic system extracting normalized holdings from multi-page financial tables."}
{"id": 14640, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Two-stage multi-agent debate for entity alignment between knowledge graphs"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["debate workflow"], "benchmarks": [], "results": [], "workflow": ["Refine candidate set", "Debate alignment decision"], "summary": "AgentEA uses two-stage multi-agent debate to improve candidate reliability and decisions in knowledge-graph entity alignment."}
{"id": 6933, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "eval-method", "why": "Guideline-grounded verifier for clinical agent decisions with calibrated signals"}, {"area": "agent-safety", "role": "defense", "why": "Verification of high-stakes agent decisions"}], "pattern": "vary-evaluation", "fixes": ["agent decisions"], "varies": ["verifier design"], "benchmarks": [], "results": [], "workflow": ["Rate guideline alignment", "Aggregate ratings", "Calibrate correctness"], "summary": "GLEAN compiles expert guidelines into calibrated step-wise verification signals for high-stakes agent decisions."}
{"id": 9318, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "benchmark", "why": "516-task user-in-the-loop benchmark of diverse agent-user interactions on AppWorld"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["tasks", "user interactions"], "benchmarks": [{"name": "AppWorld-UL", "purpose": "agent-user interaction evaluation", "is_new": true}, {"name": "AppWorld", "purpose": "base framework", "is_new": false}], "results": [], "workflow": [], "summary": "AppWorld-UL benchmarks tool-use agents on 516 tasks requiring clarification, confirmation and infeasibility handling."}
{"id": 14662, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Decoding strategy eliciting CoT paths for general QA"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["decoding"], "benchmarks": [], "results": [], "workflow": ["Fibonacci branching", "Split reasoning and answer", "Score paths"], "summary": "GCoT-decoding extends prompt-free CoT decoding to general QA using branching and error backtracking."}
{"id": 14997, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "LLM-augmented episodic RL with semantic representations and selective retrieval"}], "pattern": "n/a", "fixes": ["RL tasks"], "varies": ["episodic control architecture"], "benchmarks": [], "results": [], "workflow": ["LLM semantic augmenter", "Critical state recognizer", "Retrieve experiences"], "summary": "Agentic Episodic Control integrates LLM semantic augmentation and critical-state retrieval into episodic RL."}
{"id": 17018, "in_agent_scope": true, "types": [{"area": "domain-agents", "role": "application", "why": "Dialogue agents assisting meta-reviewers in document-grounded decision-making"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["dialogue agent design"], "benchmarks": [], "results": [], "workflow": [], "summary": "Explores dialogue agents assisting meta-reviewers by framing meta-reviewing as document-grounded dialogue."}
{"id": 12408, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "train-agent", "why": "Distill multi-agent debate into one LLM via two-stage fine-tuning"}], "pattern": "fix-env-vary-agent", "fixes": ["benchmarks"], "varies": ["training pipeline"], "benchmarks": [], "results": [], "workflow": ["Learn debate structure", "Internalize via reward schedule"], "summary": "Latent Agents internalize multi-agent debate into a single model, matching debate accuracy with up to 93% fewer tokens."}
{"id": 15877, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Graph-based optimization of multi-agent system prompts, tools and workflows"}, {"area": "memory-selfevolve", "role": "scaffold", "why": "Optimizer learns from experience to improve its strategy"}], "pattern": "vary-scaffold", "fixes": ["LLM weights"], "varies": ["MAS configuration"], "benchmarks": [], "results": [], "workflow": ["Model MAS as graph", "Optimize nodes", "Learn optimization strategy"], "summary": "TPGO models multi-agent systems as textual parameter graphs and learns to evolve them."}
{"id": 7384, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "analysis", "why": "Empirical study of multi-agent architecture effects on attack resistance and performance"}], "pattern": "vary-conditions", "fixes": ["attacks", "web agents"], "varies": ["MAS architecture"], "benchmarks": [], "results": [], "workflow": [], "summary": "Empirical study showing how multi-agent coordination design shapes the trade-off between task performance and attack resistance."}
{"id": 4779, "in_agent_scope": false, "types": [{"area": "reason/cot", "role": "method", "why": "Tool-integrated verification lets small LMs verify under test-time scaling"}], "pattern": "n/a", "fixes": ["small LMs"], "varies": ["verifier with tools"], "benchmarks": [], "results": [], "workflow": ["Filter candidates", "Tool-integrated verification"], "summary": "T1 filters candidates then uses tools for verification so small LMs can verify numerics and facts."}
{"id": 7790, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "train-agent", "why": "Information-gain turn-level reward for multi-turn GRPO"}, {"area": "social-sim", "role": "train-agent", "why": "User-centric agents resolving underspecified requests"}], "pattern": "fix-env-vary-agent", "fixes": ["interaction tasks"], "varies": ["credit assignment reward"], "benchmarks": [], "results": [], "workflow": ["Measure information gain", "Turn-level credit", "Policy optimization"], "summary": "InfoPO rewards interaction turns by information gain to improve multi-turn RL for user-centric agents."}
{"id": 9142, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Hierarchical task-aware composition for continual offline RL"}], "pattern": "n/a", "fixes": ["CORL benchmarks"], "varies": ["knowledge composition"], "benchmarks": [], "results": [], "workflow": ["Dual-level task encoding", "Hierarchical composition"], "summary": "HTAC balances plasticity and stability in continual offline RL via dual-level task encoding."}
{"id": 12947, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "train-agent", "why": "Train agents to communicate via latent hidden states with compression"}], "pattern": "fix-env-vary-agent", "fixes": ["tasks"], "varies": ["communication medium"], "benchmarks": [], "results": [], "workflow": ["Send hidden states", "Compress latent message"], "summary": "Interlat lets LLM agents communicate directly through compressed last hidden states instead of text."}
{"id": 7445, "in_agent_scope": true, "types": [{"area": "search-research", "role": "train-agent", "why": "Jointly optimize planner and executors in dynamic agentic RAG"}], "pattern": "fix-env-vary-agent", "fixes": ["RAG tasks"], "varies": ["joint optimization"], "benchmarks": [], "results": [], "workflow": ["Planner orchestrates", "Executors adapt", "Joint optimization"], "summary": "JADE jointly optimizes planner and executors in agentic RAG to close strategic-operational mismatch."}
{"id": 4339, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "benchmark", "why": "Gistify task: produce minimal single file reproducing codebase functionality"}], "pattern": "fix-agent-vary-env", "fixes": ["coding LLMs"], "varies": ["codebase tasks"], "benchmarks": [{"name": "Gistify", "purpose": "codebase understanding evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "Gistify is a codebase-level task requiring coding LLMs to generate a minimal file reproducing an entrypoint's output."}
{"id": 14793, "in_agent_scope": false, "types": [{"area": "safety/security", "role": "method", "why": "Training-free in-context audio deepfake detection with pairwise comparison"}], "pattern": "n/a", "fixes": ["audio LMs"], "varies": ["ICL prompting"], "benchmarks": [], "results": [], "workflow": ["Pairwise comparative reasoning", "Filter hallucinations", "Route with detector"], "summary": "ICLAD uses comparison-guided in-context learning with audio language models for generalizable deepfake detection."}
{"id": 14420, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "scaffold", "why": "Self-adaptive hierarchical planning with variable granularity"}], "pattern": "vary-scaffold", "fixes": ["LLM", "environments"], "varies": ["planning granularity"], "benchmarks": [], "results": [], "workflow": ["Coarse plan", "Adaptive refinement"], "summary": "AdaPlan-H adaptively chooses planning granularity from coarse to fine for LLM agents."}
{"id": 5498, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "application", "why": "Coding-agent system translating decision problems into executable optimization code"}], "pattern": "vary-scaffold", "fixes": ["coding agents"], "varies": ["coordination patterns"], "benchmarks": [], "results": [], "workflow": ["Generate code", "Asymmetric validation", "Repair"], "summary": "NEMO uses autonomous coding agents with validation and repair to turn natural-language problems into optimization code."}
{"id": 6603, "in_agent_scope": true, "types": [{"area": "tool-use", "role": "train-agent", "why": "SFT/RL with Plackett-Luce ranking for dynamic tool selection"}, {"area": "tool-use", "role": "data", "why": "Builds 200k dataset for tool selection"}], "pattern": "fix-env-vary-agent", "fixes": ["reasoning tasks"], "varies": ["training for dynamic toolsets"], "benchmarks": [], "results": [], "workflow": ["SFT and RL stabilization", "Ranking-based tool selection"], "summary": "AutoTool trains agents to select tools dynamically during reasoning, using a 200k dataset."}
{"id": 1764, "in_agent_scope": false, "types": [{"area": "arch/multimodal", "role": "method", "why": "Uncertainty-aware LLM-integrated VQA with calibrated knowledge exchange"}], "pattern": "n/a", "fixes": ["VQA"], "varies": ["integration model"], "benchmarks": [], "results": [], "workflow": [], "summary": "Uni-VQA integrates LLMs into VQA with calibrated uncertainty for cost-effective reliable answers."}
{"id": 18553, "in_agent_scope": false, "types": [{"area": "multiling/lowres", "role": "benchmark", "why": "Multilingual minimal pairs benchmark across 101 languages"}], "pattern": "n/a", "fixes": ["LLMs"], "varies": ["languages"], "benchmarks": [{"name": "MultiBLiMP 1.0", "purpose": "multilingual grammatical evaluation", "is_new": true}], "results": [], "workflow": [], "summary": "MultiBLiMP 1.0 is a 101-language benchmark of subject-verb agreement minimal pairs, exposing low-resource shortcomings."}
{"id": 12458, "in_agent_scope": false, "types": [{"area": "align/pref", "role": "method", "why": "Checklist feedback with discriminative variance weighting as reward"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["feedback form"], "benchmarks": [], "results": [], "workflow": ["Generate checklists", "DIVA weighting", "Scalar reward"], "summary": "DIVA decomposes AI feedback into prompt-specific checklists aggregated into rewards for RL and test-time use."}
{"id": 17223, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Dual temporal and semantic tree memory with mode controller"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory architecture"], "benchmarks": [], "results": [], "workflow": ["Build two trees", "Mode controller", "Hybrid retrieval"], "summary": "H-Mem stores conversational facts in temporal and semantic trees with adaptive hybrid retrieval."}
{"id": 4734, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "train-agent", "why": "Reasoning-retrieval reward model and role-playing agent training"}], "pattern": "fix-env-vary-agent", "fixes": ["role-play dialogue"], "varies": ["reward modeling"], "benchmarks": [], "results": [], "workflow": ["Reason", "Retrieve character context", "Reward"], "summary": "R4 equips reward model and role-playing agent with nested reasoning and retrieval for persona-faithful dialogue."}
{"id": 8875, "in_agent_scope": false, "types": [{"area": "train/pretrain", "role": "method", "why": "Curriculum-guided progressive layer stacking for efficient pretraining"}], "pattern": "n/a", "fixes": ["100M scale"], "varies": ["layer growth", "curriculum"], "benchmarks": [{"name": "PIQA", "purpose": "QA evaluation", "is_new": false}, {"name": "ARC", "purpose": "QA evaluation", "is_new": false}], "results": [], "workflow": [], "summary": "CGLS synchronizes data difficulty with progressive layer stacking for compute-efficient pretraining."}
{"id": 2204, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Multimodal agent with entity-centric long-term memory"}, {"area": "embodied-nav", "role": "benchmark", "why": "M3-Bench long-video QA from robot perspective"}], "pattern": "vary-both", "fixes": ["video QA"], "varies": ["memory design"], "benchmarks": [{"name": "M3-Bench", "purpose": "memory reasoning in videos", "is_new": true}], "results": [], "workflow": ["Process audio-video", "Build memory", "Multi-turn retrieval"], "summary": "M3-Agent builds multimodal episodic and semantic memory; M3-Bench evaluates memory-based reasoning on long videos."}
{"id": 351, "in_agent_scope": false, "types": [{"area": "efficiency/inference", "role": "method", "why": "Entropy-guided hidden-state output length prediction"}], "pattern": "n/a", "fixes": ["serving"], "varies": ["length predictor"], "benchmarks": [], "results": [], "workflow": ["Entropy-guided pooling", "Progressive prediction"], "summary": "Reuses hidden states and token entropy for lightweight LLM output length prediction in serving and RL sampling."}
{"id": 14443, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "environment", "why": "Unified memory operation language and specification"}, {"area": "memory-selfevolve", "role": "benchmark", "why": "Executable evaluation of memory operations"}], "pattern": "n/a", "fixes": ["agents"], "varies": ["memory operations"], "benchmarks": [], "results": [], "workflow": [], "summary": "Text2Mem is a unified language standardizing translation of natural-language instructions into executable memory operations."}
{"id": 15007, "in_agent_scope": false, "types": [{"area": "knowledge/rag", "role": "method", "why": "Generative LLM node importance estimation in knowledge graphs"}], "pattern": "n/a", "fixes": ["KGs"], "varies": ["method"], "benchmarks": [], "results": [], "workflow": [], "summary": "GenNIE uses LLMs with topological information to generate node importance scores in KGs."}
{"id": 16003, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "Four-agent system for foreign key detection in databases"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["multi-agent pipeline"], "benchmarks": [], "results": [], "workflow": ["Profile and prune candidates", "Validate FK pairs", "Resolve consistency"], "summary": "LLM-FK coordinates specialized agents to detect foreign keys in large complex databases."}
{"id": 13285, "in_agent_scope": true, "types": [{"area": "search-research", "role": "benchmark", "why": "Evolving benchmark for fact-checking deep research reports"}, {"area": "search-research", "role": "eval-method", "why": "Audit-then-Score revisable labels"}], "pattern": "vary-evaluation", "fixes": ["verifiers"], "varies": ["benchmark labels"], "benchmarks": [{"name": "DeepFact", "purpose": "fact-checker evaluation on reports", "is_new": true}], "results": [{"benchmark": "hidden known-answer claims", "system": "unassisted experts", "base_model": null, "setting": "baseline", "metric": "accuracy", "score": 60.8, "unit": "%"}], "workflow": [], "summary": "DeepFact co-evolves benchmark and agents via Audit-then-Score for claim-level factuality of deep research reports."}
{"id": 16875, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Hierarchical multi-level memory architecture for LLM agents"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory organization"], "benchmarks": [], "results": [], "workflow": ["Multi-level memory", "Efficient retrieval"], "summary": "H-MEM organizes agent memory hierarchically for efficient long-term retrieval and reasoning."}
{"id": 13455, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "Manipulates social dynamics to show representative agent accuracy degrades"}, {"area": "agent-safety", "role": "analysis", "why": "Social conformity and persuasion as vulnerabilities"}], "pattern": "vary-conditions", "fixes": ["LLM agents"], "varies": ["adversaries", "expertise", "argument style"], "benchmarks": [], "results": [], "workflow": [], "summary": "Shows social conformity, dominant speakers and persuasion degrade decision accuracy of representative LLM agents."}
{"id": 14591, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Context-Value-Action architecture grounded in human values for behavior simulation"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["architecture"], "benchmarks": [], "results": [], "workflow": ["Context", "Value inference", "Action"], "summary": "CVA reduces value polarization in simulated agents using Schwartz values, evaluated against empirical ground truth."}
{"id": 11266, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "analysis", "why": "276 model pairs across eight games relating representational similarity to cooperation"}], "pattern": "vary-conditions", "fixes": ["games"], "varies": ["model pairs"], "benchmarks": [], "results": [], "workflow": [], "summary": "Pairs of LLMs with more similar representations cooperate more but show less novelty."}
{"id": 12147, "in_agent_scope": true, "types": [{"area": "search-research", "role": "eval-method", "why": "Agentic metrics and taxonomy for evaluating deep research reports"}], "pattern": "vary-evaluation", "fixes": ["deep research agents"], "varies": ["metrics"], "benchmarks": [], "results": [], "workflow": [], "summary": "DREAM introduces tool-using agentic evaluators for deep research reports to avoid surface-fluency bias."}
{"id": 15865, "in_agent_scope": true, "types": [{"area": "memory-selfevolve", "role": "scaffold", "why": "Hierarchical memory with post-execution critique for self-improvement"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["memory module"], "benchmarks": [], "results": [], "workflow": ["Execute sub-task", "Critique logs", "Store knowledge"], "summary": "MUSE lets agents distill execution logs into reusable knowledge for self-evolution."}
{"id": 15895, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "benchmark", "why": "Benchmark of multi-agent collaboration under privacy constraints"}, {"area": "agent-safety", "role": "analysis", "why": "Privacy constraints degrade coordination"}], "pattern": "fix-agent-vary-env", "fixes": ["agents"], "varies": ["privacy constraints"], "benchmarks": [{"name": "PAC-Bench", "purpose": "collaboration under privacy", "is_new": true}], "results": [], "workflow": [], "summary": "PAC-Bench evaluates multi-agent collaboration under privacy constraints and diagnoses coordination failures."}
{"id": 12072, "in_agent_scope": true, "types": [{"area": "multi-agent", "role": "scaffold", "why": "Markov state-aware soft evolution of agent topology"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["topology evolution"], "benchmarks": [], "results": [], "workflow": [], "summary": "AgentRevive avoids hard pruning so temporarily failing agents can recover in multi-agent discussion."}
{"id": 4875, "in_agent_scope": false, "types": [{"area": "other/ml", "role": "method", "why": "Action-guided attention for video action anticipation"}], "pattern": "n/a", "fixes": ["video"], "varies": ["attention"], "benchmarks": [], "results": [], "workflow": [], "summary": "Action-Guided Attention replaces pixel dot-product attention to anticipate future actions in videos."}
{"id": 14755, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Stable persona-driven simulation framework for multi-turn dialogue synthesis"}, {"area": "social-sim", "role": "data", "why": "Synthetic dialogue generation"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["simulation framework"], "benchmarks": [], "results": [], "workflow": [], "summary": "SPASM reduces persona drift and echoing in LLM-LLM simulated multi-turn dialogues."}
{"id": 14924, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "scaffold", "why": "Hierarchical state abstraction of GUI history"}], "pattern": "vary-scaffold", "fixes": ["GUI agent"], "varies": ["history representation"], "benchmarks": [], "results": [], "workflow": ["Abstract steps", "Refine contexts", "Induce knowledge"], "summary": "HiSA organizes GUI agent histories into hierarchical abstractions instead of raw logs."}
{"id": 17467, "in_agent_scope": true, "types": [{"area": "agent-safety", "role": "attack", "why": "Assertion-conditioned compliance vulnerability in multi-turn tool-calling agents"}, {"area": "tool-use", "role": "analysis", "why": "Probes function-calling models beyond BFCL"}], "pattern": "vary-conditions", "fixes": ["tool-calling models"], "varies": ["adversarial assertions"], "benchmarks": [{"name": "BFCL", "purpose": "existing benchmark context", "is_new": false}], "results": [], "workflow": [], "summary": "Reveals a provenance-aware vulnerability where multi-turn tool-calling LLMs comply with assertions."}
{"id": 11967, "in_agent_scope": false, "types": [{"area": "multiling/mt", "role": "analysis", "why": "Systematic study of document-level LLM refinement for literary translation"}], "pattern": "vary-conditions", "fixes": ["six LLMs"], "varies": ["refinement granularity", "strategy"], "benchmarks": [], "results": [], "workflow": [], "summary": "Finds document-level MT plus segment-level refinement works best across six LLMs and seven language pairs."}
{"id": 13733, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "benchmark", "why": "Quantitative evaluation of LLM agents simulating real shopping behavior"}, {"area": "social-sim", "role": "analysis", "why": "Tests step-wise human behavior fidelity"}], "pattern": "fix-agent-vary-env", "fixes": ["LLMs"], "varies": ["real behavior data"], "benchmarks": [], "results": [], "workflow": [], "summary": "First large-scale quantitative evaluation of LLM agents simulating multi-turn online shopping behavior."}
{"id": 16225, "in_agent_scope": false, "types": [{"area": "multiling/mt", "role": "method", "why": "Element-oriented controllable translation with structured reasoning"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["MT framework"], "benchmarks": [], "results": [], "workflow": [], "summary": "CEMT detects translation-specific elements then translates with structured reasoning for controllability."}
{"id": 6589, "in_agent_scope": false, "types": [{"area": "eval/robust", "role": "eval-method", "why": "Bias-reduced MCQ evaluation protocol with uniform labels"}], "pattern": "vary-evaluation", "fixes": ["LLMs"], "varies": ["labels"], "benchmarks": [{"name": "NonsenseQA", "purpose": "synthetic bias probe", "is_new": true}], "results": [], "workflow": [], "summary": "Shows MCQ label-position bias with NonsenseQA and proposes uniform unordered labels evaluation."}
{"id": 5816, "in_agent_scope": true, "types": [{"area": "planning-decision", "role": "eval-method", "why": "Twelve-metric reliability profile for agents across four dimensions"}], "pattern": "vary-evaluation", "fixes": ["agents"], "varies": ["metrics"], "benchmarks": [], "results": [], "workflow": [], "summary": "Proposes holistic agent reliability metrics beyond accuracy: consistency, robustness, predictability and error severity."}
{"id": 220, "in_agent_scope": false, "types": [{"area": "safety/jailbreak", "role": "defense", "why": "Multi-agent RL jointly training safeguard and responder to reduce unsafe and overrefusal"}], "pattern": "n/a", "fixes": ["threat model"], "varies": ["training"], "benchmarks": [], "results": [], "workflow": [], "summary": "WaltzRL jointly trains conversation and feedback agents to balance harmlessness and helpfulness."}
{"id": 9857, "in_agent_scope": true, "types": [{"area": "social-sim", "role": "scaffold", "why": "Ontology-guided multi-agent reasoning for cultural alignment"}], "pattern": "vary-scaffold", "fixes": ["LLM"], "varies": ["reasoning framework"], "benchmarks": [], "results": [], "workflow": [], "summary": "OG-MAR builds cultural ontology from WVS and uses multi-agent reasoning for culturally aligned outputs."}
{"id": 11881, "in_agent_scope": true, "types": [{"area": "science-data", "role": "analysis", "why": "Predicts ML solution preference without execution"}, {"area": "science-data", "role": "benchmark", "why": "Corpus of 18,438 pairwise comparisons"}], "pattern": "vary-conditions", "fixes": ["LLMs"], "varies": ["prediction vs execution"], "benchmarks": [{"name": "Data-centric Solution Preference", "purpose": "pairwise solution comparison", "is_new": true}], "results": [], "workflow": [], "summary": "Formalizes Data-centric Solution Preference to let ML agents predict outcomes without execution."}
{"id": 7607, "in_agent_scope": true, "types": [{"area": "science-data", "role": "application", "why": "LLM agent operating Rosetta for protein design"}], "pattern": "n/a", "fixes": ["LLM"], "varies": ["structured environment"], "benchmarks": [], "results": [], "workflow": [], "summary": "Agent Rosetta pairs an LLM agent with a structured Rosetta environment for protein design."}
{"id": 7419, "in_agent_scope": true, "types": [{"area": "web-gui", "role": "data", "why": "Tree-structured verifiable GUI trajectory collection"}, {"area": "web-gui", "role": "train-agent", "why": "Scales GUI planning"}], "pattern": "vary-data", "fixes": ["GUI agents"], "varies": ["trajectory data"], "benchmarks": [], "results": [], "workflow": [], "summary": "TreeCUA organizes GUI trajectories as trees to cut redundant exploration and scale computer-use agents."}
{"id": 15823, "in_agent_scope": true, "types": [{"area": "code-swe", "role": "scaffold", "why": "Training-free interactive SPARQL refinement"}], "pattern": "vary-scaffold", "fixes": ["generators"], "varies": ["refinement pipeline"], "benchmarks": [], "results": [], "workflow": [], "summary": "InteracSPARQL refines NL2SPARQL queries interactively with natural language explanations."}
{"id": 9637, "in_agent_scope": false, "types": [{"area": "reason/math", "role": "benchmark", "why": "Dafny translation of miniF2F for auto-active theorem proving"}], "pattern": "n/a", "fixes": ["LLMs"], "varies": ["verifier"], "benchmarks": [{"name": "miniF2F-Dafny", "purpose": "auto-active proving", "is_new": true}, {"name": "miniF2F", "purpose": "source benchmark", "is_new": false}], "results": [], "workflow": [], "summary": "miniF2F-Dafny translates miniF2F into Dafny to evaluate LLM theorem proving."}