{ "roles": { "benchmark": { "label": "New benchmark / task suite", "fixes": [ "agents/models under test" ], "varies": [ "tasks", "environments" ], "desc": "Introduces a new evaluation set or task suite and measures existing models/agents on it. The contribution is the yardstick; the systems being measured are held fixed.", "pattern": "fix-agent-vary-env" }, "environment": { "label": "New environment / simulator / world model", "fixes": [ "agent" ], "varies": [ "environment" ], "desc": "Builds or synthesizes an interactive environment, simulator, sandbox or world model (often for scalable training or evaluation). The environment is the thing being changed.", "pattern": "fix-agent-vary-env" }, "train-agent": { "label": "Train / fine-tune the model (SFT, RL, preference, self-evolution)", "fixes": [ "environment", "benchmark" ], "varies": [ "model weights", "training algorithm or reward" ], "desc": "Changes the model itself with fine-tuning, RL, preference or self-improvement training, evaluated in a fixed environment or benchmark.", "pattern": "fix-env-vary-model" }, "data": { "label": "Data / trajectory synthesis or selection", "fixes": [ "environment", "training recipe" ], "varies": [ "training data" ], "desc": "Constructs, filters or selects training data (tasks, trajectories, queries) so that an otherwise standard training recipe works better.", "pattern": "vary-data" }, "scaffold": { "label": "Training-free design (prompting, workflow, planning, memory, test-time compute)", "fixes": [ "model weights", "environment" ], "varies": [ "scaffold", "inference procedure" ], "desc": "Keeps the underlying model frozen and changes how it is orchestrated: prompts, workflows, planning, reflection, memory, tool routing or test-time search.", "pattern": "fix-model-vary-scaffold" }, "analysis": { "label": "Empirical analysis / failure study", "fixes": [ "methods" ], "varies": [ "conditions", "models", "settings" ], "desc": "Measures or explains behaviour of existing models/agents under controlled variations to find limits, failure modes or scaling trends, without proposing a new system.", "pattern": "vary-conditions" }, "eval-method": { "label": "Evaluation methodology / metrics / judges", "fixes": [ "systems under test" ], "varies": [ "evaluation protocol", "metric" ], "desc": "Proposes a new way to measure (metric, judge, protocol, contamination control) rather than a new task set.", "pattern": "vary-evaluation" }, "attack": { "label": "Attack / red-teaming / vulnerability discovery", "fixes": [ "agent" ], "varies": [ "adversarial input or condition" ], "desc": "Probes an agent or model for safety, security or privacy failures with attacks or automated red-teaming.", "pattern": "fix-agent-vary-condition" }, "defense": { "label": "Defense / monitoring / guardrails / alignment", "fixes": [ "threat model" ], "varies": [ "protective mechanism" ], "desc": "Adds a protective mechanism (monitor, filter, alignment training, unlearning) and checks it against a fixed threat model.", "pattern": "fix-threat-vary-defense" }, "survey": { "label": "Survey / position / tutorial", "fixes": [], "varies": [], "desc": "Organizes or argues about a field instead of running a new experiment.", "pattern": "n/a" }, "application": { "label": "Domain application / end-to-end system", "fixes": [ "general-purpose model" ], "varies": [ "domain pipeline" ], "desc": "Packages models and tools into a system for a specific domain or product and evaluates it there.", "pattern": "vary-pipeline" }, "theory": { "label": "Theory / formal analysis", "fixes": [], "varies": [ "assumptions" ], "desc": "Proves properties or provides formal analysis of a learning or decision procedure.", "pattern": "vary-assumptions" }, "method": { "label": "General method (not specific to agents)", "fixes": [ "benchmark" ], "varies": [ "method" ], "desc": "A new technique evaluated on standard benchmarks, used when the paper does not fit a finer role.", "pattern": "vary-method" } }, "agent_areas": { "web-gui": "Web, browser, GUI, mobile and computer-use agents; GUI grounding", "search-research": "Deep research, agentic search, information-seeking, search-augmented agents", "code-swe": "Coding agents, SWE-bench style tasks, ML-engineering agents, text-to-SQL agents", "tool-use": "Tool calling, function calling, MCP, API use, tool-integrated reasoning", "multi-agent": "Multi-agent coordination, debate, team structure, communication, agent societies", "memory-selfevolve": "Agent memory, experience reuse, reflection, self-evolving or lifelong agents", "planning-decision": "Planning, exploration, sequential decision making, long-horizon tasks, credit assignment, agentic RL algorithms", "embodied-nav": "Embodied agents, navigation, robotics, games, world-model use", "science-data": "Scientific-discovery and data-analysis / data-science agents", "social-sim": "Role-play, user simulators, persona, negotiation, economic and social simulation", "domain-agents": "Domain agents (health, finance, legal, education, recommendation, CAD, design, etc.)", "agent-safety": "Safety, security, privacy and trustworthiness of agents" }, "general_areas": { "arch/attention": "Attention, SSMs, MoE", "arch/diffusion": "Diffusion / non-autoregressive LMs", "arch/tokenization": "Tokenization & embeddings", "arch/multimodal": "Multimodal & vision-language models", "arch/longctx": "Long context & memory", "train/pretrain": "Pretraining & scaling laws", "train/data": "Data curation & synthetic data", "train/sft": "Instruction tuning & SFT", "train/peft": "Parameter-efficient fine-tuning", "train/continual": "Continual learning & model merging", "train/distill": "Distillation & small models", "align/pref": "Preference optimization & reward models", "align/rlvr": "RL for reasoning (RLVR/GRPO)", "align/values": "Values, sycophancy, pluralistic alignment", "reason/cot": "Chain-of-thought & test-time compute", "reason/math": "Math & formal reasoning", "reason/code": "Code generation (non-agentic)", "reason/planning": "Logic, commonsense, planning (non-agentic)", "knowledge/rag": "Retrieval-augmented generation", "knowledge/halluc": "Hallucination & factuality", "knowledge/editing": "Knowledge editing & probing", "eval/bench": "Benchmarks & datasets (non-agentic)", "eval/judge": "LLM-as-a-judge & eval methodology", "eval/robust": "Robustness, contamination, calibration", "interp/mech": "Mechanistic interpretability", "interp/steering": "Representation analysis & steering", "interp/icl": "In-context learning & emergent abilities", "efficiency/compress": "Quantization & pruning", "efficiency/inference": "Inference acceleration", "efficiency/training": "Efficient training", "safety/jailbreak": "Jailbreaks & red-teaming (models)", "safety/security": "Security, backdoors, watermarking (models)", "safety/privacy": "Privacy, memorization, unlearning", "safety/fairness": "Bias, fairness, culture", "multiling/multi": "Multilingual LLMs", "multiling/lowres": "Low-resource languages", "multiling/mt": "Machine translation", "apps/science": "Biomedical & scientific applications", "apps/domain": "Legal, finance, education, social science", "apps/dialogue": "Dialogue & generation", "apps/classic": "Classical NLP tasks", "other/ml": "Not language-model related (vision, RL theory, etc.)" } }