d6fc7c1dfa
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json - LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest - Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow) - Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks - build_worktypes.py adds work-type tables to data/papers.db Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
208 lines
7.7 KiB
JSON
208 lines
7.7 KiB
JSON
{
|
|
"roles": {
|
|
"benchmark": {
|
|
"label": "New benchmark / task suite",
|
|
"fixes": [
|
|
"agents/models under test"
|
|
],
|
|
"varies": [
|
|
"tasks",
|
|
"environments"
|
|
],
|
|
"desc": "Introduces a new evaluation set or task suite and measures existing models/agents on it. The contribution is the yardstick; the systems being measured are held fixed.",
|
|
"pattern": "fix-agent-vary-env"
|
|
},
|
|
"environment": {
|
|
"label": "New environment / simulator / world model",
|
|
"fixes": [
|
|
"agent"
|
|
],
|
|
"varies": [
|
|
"environment"
|
|
],
|
|
"desc": "Builds or synthesizes an interactive environment, simulator, sandbox or world model (often for scalable training or evaluation). The environment is the thing being changed.",
|
|
"pattern": "fix-agent-vary-env"
|
|
},
|
|
"train-agent": {
|
|
"label": "Train / fine-tune the model (SFT, RL, preference, self-evolution)",
|
|
"fixes": [
|
|
"environment",
|
|
"benchmark"
|
|
],
|
|
"varies": [
|
|
"model weights",
|
|
"training algorithm or reward"
|
|
],
|
|
"desc": "Changes the model itself with fine-tuning, RL, preference or self-improvement training, evaluated in a fixed environment or benchmark.",
|
|
"pattern": "fix-env-vary-model"
|
|
},
|
|
"data": {
|
|
"label": "Data / trajectory synthesis or selection",
|
|
"fixes": [
|
|
"environment",
|
|
"training recipe"
|
|
],
|
|
"varies": [
|
|
"training data"
|
|
],
|
|
"desc": "Constructs, filters or selects training data (tasks, trajectories, queries) so that an otherwise standard training recipe works better.",
|
|
"pattern": "vary-data"
|
|
},
|
|
"scaffold": {
|
|
"label": "Training-free design (prompting, workflow, planning, memory, test-time compute)",
|
|
"fixes": [
|
|
"model weights",
|
|
"environment"
|
|
],
|
|
"varies": [
|
|
"scaffold",
|
|
"inference procedure"
|
|
],
|
|
"desc": "Keeps the underlying model frozen and changes how it is orchestrated: prompts, workflows, planning, reflection, memory, tool routing or test-time search.",
|
|
"pattern": "fix-model-vary-scaffold"
|
|
},
|
|
"analysis": {
|
|
"label": "Empirical analysis / failure study",
|
|
"fixes": [
|
|
"methods"
|
|
],
|
|
"varies": [
|
|
"conditions",
|
|
"models",
|
|
"settings"
|
|
],
|
|
"desc": "Measures or explains behaviour of existing models/agents under controlled variations to find limits, failure modes or scaling trends, without proposing a new system.",
|
|
"pattern": "vary-conditions"
|
|
},
|
|
"eval-method": {
|
|
"label": "Evaluation methodology / metrics / judges",
|
|
"fixes": [
|
|
"systems under test"
|
|
],
|
|
"varies": [
|
|
"evaluation protocol",
|
|
"metric"
|
|
],
|
|
"desc": "Proposes a new way to measure (metric, judge, protocol, contamination control) rather than a new task set.",
|
|
"pattern": "vary-evaluation"
|
|
},
|
|
"attack": {
|
|
"label": "Attack / red-teaming / vulnerability discovery",
|
|
"fixes": [
|
|
"agent"
|
|
],
|
|
"varies": [
|
|
"adversarial input or condition"
|
|
],
|
|
"desc": "Probes an agent or model for safety, security or privacy failures with attacks or automated red-teaming.",
|
|
"pattern": "fix-agent-vary-condition"
|
|
},
|
|
"defense": {
|
|
"label": "Defense / monitoring / guardrails / alignment",
|
|
"fixes": [
|
|
"threat model"
|
|
],
|
|
"varies": [
|
|
"protective mechanism"
|
|
],
|
|
"desc": "Adds a protective mechanism (monitor, filter, alignment training, unlearning) and checks it against a fixed threat model.",
|
|
"pattern": "fix-threat-vary-defense"
|
|
},
|
|
"survey": {
|
|
"label": "Survey / position / tutorial",
|
|
"fixes": [],
|
|
"varies": [],
|
|
"desc": "Organizes or argues about a field instead of running a new experiment.",
|
|
"pattern": "n/a"
|
|
},
|
|
"application": {
|
|
"label": "Domain application / end-to-end system",
|
|
"fixes": [
|
|
"general-purpose model"
|
|
],
|
|
"varies": [
|
|
"domain pipeline"
|
|
],
|
|
"desc": "Packages models and tools into a system for a specific domain or product and evaluates it there.",
|
|
"pattern": "vary-pipeline"
|
|
},
|
|
"theory": {
|
|
"label": "Theory / formal analysis",
|
|
"fixes": [],
|
|
"varies": [
|
|
"assumptions"
|
|
],
|
|
"desc": "Proves properties or provides formal analysis of a learning or decision procedure.",
|
|
"pattern": "vary-assumptions"
|
|
},
|
|
"method": {
|
|
"label": "General method (not specific to agents)",
|
|
"fixes": [
|
|
"benchmark"
|
|
],
|
|
"varies": [
|
|
"method"
|
|
],
|
|
"desc": "A new technique evaluated on standard benchmarks, used when the paper does not fit a finer role.",
|
|
"pattern": "vary-method"
|
|
}
|
|
},
|
|
"agent_areas": {
|
|
"web-gui": "Web, browser, GUI, mobile and computer-use agents; GUI grounding",
|
|
"search-research": "Deep research, agentic search, information-seeking, search-augmented agents",
|
|
"code-swe": "Coding agents, SWE-bench style tasks, ML-engineering agents, text-to-SQL agents",
|
|
"tool-use": "Tool calling, function calling, MCP, API use, tool-integrated reasoning",
|
|
"multi-agent": "Multi-agent coordination, debate, team structure, communication, agent societies",
|
|
"memory-selfevolve": "Agent memory, experience reuse, reflection, self-evolving or lifelong agents",
|
|
"planning-decision": "Planning, exploration, sequential decision making, long-horizon tasks, credit assignment, agentic RL algorithms",
|
|
"embodied-nav": "Embodied agents, navigation, robotics, games, world-model use",
|
|
"science-data": "Scientific-discovery and data-analysis / data-science agents",
|
|
"social-sim": "Role-play, user simulators, persona, negotiation, economic and social simulation",
|
|
"domain-agents": "Domain agents (health, finance, legal, education, recommendation, CAD, design, etc.)",
|
|
"agent-safety": "Safety, security, privacy and trustworthiness of agents"
|
|
},
|
|
"general_areas": {
|
|
"arch/attention": "Attention, SSMs, MoE",
|
|
"arch/diffusion": "Diffusion / non-autoregressive LMs",
|
|
"arch/tokenization": "Tokenization & embeddings",
|
|
"arch/multimodal": "Multimodal & vision-language models",
|
|
"arch/longctx": "Long context & memory",
|
|
"train/pretrain": "Pretraining & scaling laws",
|
|
"train/data": "Data curation & synthetic data",
|
|
"train/sft": "Instruction tuning & SFT",
|
|
"train/peft": "Parameter-efficient fine-tuning",
|
|
"train/continual": "Continual learning & model merging",
|
|
"train/distill": "Distillation & small models",
|
|
"align/pref": "Preference optimization & reward models",
|
|
"align/rlvr": "RL for reasoning (RLVR/GRPO)",
|
|
"align/values": "Values, sycophancy, pluralistic alignment",
|
|
"reason/cot": "Chain-of-thought & test-time compute",
|
|
"reason/math": "Math & formal reasoning",
|
|
"reason/code": "Code generation (non-agentic)",
|
|
"reason/planning": "Logic, commonsense, planning (non-agentic)",
|
|
"knowledge/rag": "Retrieval-augmented generation",
|
|
"knowledge/halluc": "Hallucination & factuality",
|
|
"knowledge/editing": "Knowledge editing & probing",
|
|
"eval/bench": "Benchmarks & datasets (non-agentic)",
|
|
"eval/judge": "LLM-as-a-judge & eval methodology",
|
|
"eval/robust": "Robustness, contamination, calibration",
|
|
"interp/mech": "Mechanistic interpretability",
|
|
"interp/steering": "Representation analysis & steering",
|
|
"interp/icl": "In-context learning & emergent abilities",
|
|
"efficiency/compress": "Quantization & pruning",
|
|
"efficiency/inference": "Inference acceleration",
|
|
"efficiency/training": "Efficient training",
|
|
"safety/jailbreak": "Jailbreaks & red-teaming (models)",
|
|
"safety/security": "Security, backdoors, watermarking (models)",
|
|
"safety/privacy": "Privacy, memorization, unlearning",
|
|
"safety/fairness": "Bias, fairness, culture",
|
|
"multiling/multi": "Multilingual LLMs",
|
|
"multiling/lowres": "Low-resource languages",
|
|
"multiling/mt": "Machine translation",
|
|
"apps/science": "Biomedical & scientific applications",
|
|
"apps/domain": "Legal, finance, education, social science",
|
|
"apps/dialogue": "Dialogue & generation",
|
|
"apps/classic": "Classical NLP tasks",
|
|
"other/ml": "Not language-model related (vision, RL theory, etc.)"
|
|
}
|
|
} |