Files
ai-taxonomy-literature-anal…/scripts/worktypes.json
T
zzyang d6fc7c1dfa Add types-of-work classification, benchmarks and score pages
- Work type = area x role (fixed vs varied), vocabulary in scripts/worktypes.json
- LLM-read labels for 2,678 agent-related papers (data/labels), rule labels for the rest
- Per-type synthesis notes (why-same, sub-variants, benchmarks, workflow)
- Pages: /work-types matrix and detail (timeline, benchmarks, workflow, score charts), /benchmarks
- build_worktypes.py adds work-type tables to data/papers.db

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:55:46 +00:00

208 lines
7.7 KiB
JSON

{
"roles": {
"benchmark": {
"label": "New benchmark / task suite",
"fixes": [
"agents/models under test"
],
"varies": [
"tasks",
"environments"
],
"desc": "Introduces a new evaluation set or task suite and measures existing models/agents on it. The contribution is the yardstick; the systems being measured are held fixed.",
"pattern": "fix-agent-vary-env"
},
"environment": {
"label": "New environment / simulator / world model",
"fixes": [
"agent"
],
"varies": [
"environment"
],
"desc": "Builds or synthesizes an interactive environment, simulator, sandbox or world model (often for scalable training or evaluation). The environment is the thing being changed.",
"pattern": "fix-agent-vary-env"
},
"train-agent": {
"label": "Train / fine-tune the model (SFT, RL, preference, self-evolution)",
"fixes": [
"environment",
"benchmark"
],
"varies": [
"model weights",
"training algorithm or reward"
],
"desc": "Changes the model itself with fine-tuning, RL, preference or self-improvement training, evaluated in a fixed environment or benchmark.",
"pattern": "fix-env-vary-model"
},
"data": {
"label": "Data / trajectory synthesis or selection",
"fixes": [
"environment",
"training recipe"
],
"varies": [
"training data"
],
"desc": "Constructs, filters or selects training data (tasks, trajectories, queries) so that an otherwise standard training recipe works better.",
"pattern": "vary-data"
},
"scaffold": {
"label": "Training-free design (prompting, workflow, planning, memory, test-time compute)",
"fixes": [
"model weights",
"environment"
],
"varies": [
"scaffold",
"inference procedure"
],
"desc": "Keeps the underlying model frozen and changes how it is orchestrated: prompts, workflows, planning, reflection, memory, tool routing or test-time search.",
"pattern": "fix-model-vary-scaffold"
},
"analysis": {
"label": "Empirical analysis / failure study",
"fixes": [
"methods"
],
"varies": [
"conditions",
"models",
"settings"
],
"desc": "Measures or explains behaviour of existing models/agents under controlled variations to find limits, failure modes or scaling trends, without proposing a new system.",
"pattern": "vary-conditions"
},
"eval-method": {
"label": "Evaluation methodology / metrics / judges",
"fixes": [
"systems under test"
],
"varies": [
"evaluation protocol",
"metric"
],
"desc": "Proposes a new way to measure (metric, judge, protocol, contamination control) rather than a new task set.",
"pattern": "vary-evaluation"
},
"attack": {
"label": "Attack / red-teaming / vulnerability discovery",
"fixes": [
"agent"
],
"varies": [
"adversarial input or condition"
],
"desc": "Probes an agent or model for safety, security or privacy failures with attacks or automated red-teaming.",
"pattern": "fix-agent-vary-condition"
},
"defense": {
"label": "Defense / monitoring / guardrails / alignment",
"fixes": [
"threat model"
],
"varies": [
"protective mechanism"
],
"desc": "Adds a protective mechanism (monitor, filter, alignment training, unlearning) and checks it against a fixed threat model.",
"pattern": "fix-threat-vary-defense"
},
"survey": {
"label": "Survey / position / tutorial",
"fixes": [],
"varies": [],
"desc": "Organizes or argues about a field instead of running a new experiment.",
"pattern": "n/a"
},
"application": {
"label": "Domain application / end-to-end system",
"fixes": [
"general-purpose model"
],
"varies": [
"domain pipeline"
],
"desc": "Packages models and tools into a system for a specific domain or product and evaluates it there.",
"pattern": "vary-pipeline"
},
"theory": {
"label": "Theory / formal analysis",
"fixes": [],
"varies": [
"assumptions"
],
"desc": "Proves properties or provides formal analysis of a learning or decision procedure.",
"pattern": "vary-assumptions"
},
"method": {
"label": "General method (not specific to agents)",
"fixes": [
"benchmark"
],
"varies": [
"method"
],
"desc": "A new technique evaluated on standard benchmarks, used when the paper does not fit a finer role.",
"pattern": "vary-method"
}
},
"agent_areas": {
"web-gui": "Web, browser, GUI, mobile and computer-use agents; GUI grounding",
"search-research": "Deep research, agentic search, information-seeking, search-augmented agents",
"code-swe": "Coding agents, SWE-bench style tasks, ML-engineering agents, text-to-SQL agents",
"tool-use": "Tool calling, function calling, MCP, API use, tool-integrated reasoning",
"multi-agent": "Multi-agent coordination, debate, team structure, communication, agent societies",
"memory-selfevolve": "Agent memory, experience reuse, reflection, self-evolving or lifelong agents",
"planning-decision": "Planning, exploration, sequential decision making, long-horizon tasks, credit assignment, agentic RL algorithms",
"embodied-nav": "Embodied agents, navigation, robotics, games, world-model use",
"science-data": "Scientific-discovery and data-analysis / data-science agents",
"social-sim": "Role-play, user simulators, persona, negotiation, economic and social simulation",
"domain-agents": "Domain agents (health, finance, legal, education, recommendation, CAD, design, etc.)",
"agent-safety": "Safety, security, privacy and trustworthiness of agents"
},
"general_areas": {
"arch/attention": "Attention, SSMs, MoE",
"arch/diffusion": "Diffusion / non-autoregressive LMs",
"arch/tokenization": "Tokenization & embeddings",
"arch/multimodal": "Multimodal & vision-language models",
"arch/longctx": "Long context & memory",
"train/pretrain": "Pretraining & scaling laws",
"train/data": "Data curation & synthetic data",
"train/sft": "Instruction tuning & SFT",
"train/peft": "Parameter-efficient fine-tuning",
"train/continual": "Continual learning & model merging",
"train/distill": "Distillation & small models",
"align/pref": "Preference optimization & reward models",
"align/rlvr": "RL for reasoning (RLVR/GRPO)",
"align/values": "Values, sycophancy, pluralistic alignment",
"reason/cot": "Chain-of-thought & test-time compute",
"reason/math": "Math & formal reasoning",
"reason/code": "Code generation (non-agentic)",
"reason/planning": "Logic, commonsense, planning (non-agentic)",
"knowledge/rag": "Retrieval-augmented generation",
"knowledge/halluc": "Hallucination & factuality",
"knowledge/editing": "Knowledge editing & probing",
"eval/bench": "Benchmarks & datasets (non-agentic)",
"eval/judge": "LLM-as-a-judge & eval methodology",
"eval/robust": "Robustness, contamination, calibration",
"interp/mech": "Mechanistic interpretability",
"interp/steering": "Representation analysis & steering",
"interp/icl": "In-context learning & emergent abilities",
"efficiency/compress": "Quantization & pruning",
"efficiency/inference": "Inference acceleration",
"efficiency/training": "Efficient training",
"safety/jailbreak": "Jailbreaks & red-teaming (models)",
"safety/security": "Security, backdoors, watermarking (models)",
"safety/privacy": "Privacy, memorization, unlearning",
"safety/fairness": "Bias, fairness, culture",
"multiling/multi": "Multilingual LLMs",
"multiling/lowres": "Low-resource languages",
"multiling/mt": "Machine translation",
"apps/science": "Biomedical & scientific applications",
"apps/domain": "Legal, finance, education, social science",
"apps/dialogue": "Dialogue & generation",
"apps/classic": "Classical NLP tasks",
"other/ml": "Not language-model related (vision, RL theory, etc.)"
}
}