{
 "title": "Controlling Untrusted AI: A Survey",
 "thesis": "Deployed AI agents must meet their operator's requirements: be correct, compliant, in scope, within budget, and safe. They cannot be trusted to meet them on their own. On the attack side the survey distinguishes three tiers: benign loss of control (the agent makes mistakes and no one is attacking), malicious external attacks (an outsider redirects the agent through what it reads, calls, or remembers), and insider threats (a person with partial access to data, weights, pipelines, credentials, or oversight, or the model itself pursuing objectives of its own). On the defence side it separates what is implemented inside the executing model (train, steer) from what is implemented outside it (constrain, observe, intervene), with verification testing either; the survey's position is that guarantees come from outside the model, while inside-the-model measures lower the base rate the outside layer must catch. Every mechanism is graded by a two-level test (does it still work if the agent is indifferent; does it still work if the agent, or an insider, is trying to defeat it and knows how it works) and by how its effectiveness interacts with the attacker's capabilities and incentives. The survey covers both sides across all five requirements, asks of every mechanism which requirement it serves, who can apply it, what it assumes, what it costs, and where it has evidence, and treats the adversarial agent as the stress test rather than the premise.",
 "readers": [
  "Researchers inside one mechanism family (agent security: injection defences, isolation, information-flow control; guardrails; monitoring; control evaluation; formal enforcement; post-training; steering) who want the other families on one map with the same five questions answered.",
  "Teams deploying agents with real credentials who must choose and combine mechanisms for a requirement and a threat model.",
  "Evaluators, auditors, and policy staff who need to know what evidence a control claim rests on and which mechanisms satisfy which governance requirements (human oversight, logging, risk management, robustness)."
 ],
 "generated": "2026-10-19",
 "structure": "f91f5efaa064",
 "requirements": [
  {
   "id": "correct",
   "label": "Correct"
  },
  {
   "id": "compliant",
   "label": "Compliant"
  },
  {
   "id": "in-scope",
   "label": "In scope"
  },
  {
   "id": "within-budget",
   "label": "Within budget"
  },
  {
   "id": "safe",
   "label": "Safe"
  }
 ],
 "failure_sources": [
  {
   "id": "benign",
   "label": "Benign",
   "color": "#D95358",
   "blurb": "The agent goes out of control with no attacker present."
  },
  {
   "id": "external",
   "label": "External",
   "color": "#E09A3B",
   "blurb": "An outsider redirects the agent through what it reads, calls, or remembers."
  },
  {
   "id": "insider",
   "label": "Insider",
   "color": "#A37CAD",
   "blurb": "Someone with legitimate partial access contributes to loss of control, whether a person (data, weights, pipeline, credentials, oversight) or the model itself."
  }
 ],
 "operations": [
  {
   "id": "failures",
   "label": "Failure Modes of Deployed Agents",
   "section": "s2",
   "color": "#D95358",
   "lever": null
  },
  {
   "id": "constrain",
   "label": "Constrain",
   "section": "s4",
   "color": "#E27553",
   "lever": "System-Level Control"
  },
  {
   "id": "monitor",
   "label": "Monitor",
   "section": "s5",
   "color": "#598DD2",
   "lever": "System-Level Control"
  },
  {
   "id": "intervene",
   "label": "Intervene",
   "section": "s6",
   "color": "#E09A3B",
   "lever": "System-Level Control"
  },
  {
   "id": "steer",
   "label": "Steer",
   "section": "s7",
   "color": "#65C2B9",
   "lever": "Model-Level Control"
  },
  {
   "id": "train",
   "label": "Train",
   "section": "s7",
   "color": "#65C2B9",
   "lever": "Model-Level Control"
  },
  {
   "id": "evaluate",
   "label": "Evaluate",
   "section": "s8",
   "color": "#62A16E",
   "lever": "Evaluation and Assurance"
  },
  {
   "id": "deploy",
   "label": "Deploy",
   "section": "s9",
   "color": "#A37CAD",
   "lever": "Evaluation and Assurance"
  }
 ],
 "parts": [
  {
   "title": "Foundations",
   "sections": [
    "s1",
    "s2",
    "s3"
   ]
  },
  {
   "title": "System-Level Control",
   "sections": [
    "s4",
    "s5",
    "s6"
   ]
  },
  {
   "title": "Model-Level Control",
   "sections": [
    "s7"
   ]
  },
  {
   "title": "Evaluation and Assurance",
   "sections": [
    "s8",
    "s9",
    "s10"
   ]
  }
 ],
 "sections": [
  {
   "id": "s1",
   "number": 1,
   "title": "Introduction",
   "slug": "introduction",
   "operations": [],
   "color": "#7D7D7D",
   "part": 0,
   "subsections": [
    {
     "id": "s1_1",
     "number": "1.1",
     "title": "The Control Problem",
     "slug": "the-control-problem",
     "counts": {
      "primary": 25,
      "selected": 0,
      "cross": 52,
      "grey": 3
     }
    },
    {
     "id": "s1_2",
     "number": "1.2",
     "title": "Scope",
     "slug": "scope",
     "counts": {
      "primary": 41,
      "selected": 2,
      "cross": 12,
      "grey": 2
     }
    },
    {
     "id": "s1_3",
     "number": "1.3",
     "title": "Methodology",
     "slug": "methodology",
     "counts": {
      "primary": 0,
      "selected": 0,
      "cross": 0,
      "grey": 0
     }
    },
    {
     "id": "s1_4",
     "number": "1.4",
     "title": "Contributions",
     "slug": "contributions",
     "counts": {
      "primary": 0,
      "selected": 0,
      "cross": 0,
      "grey": 0
     }
    }
   ]
  },
  {
   "id": "s2",
   "number": 2,
   "title": "Failure Modes of Deployed Agents",
   "slug": "failures",
   "operations": [
    "failures"
   ],
   "color": "#D95358",
   "part": 0,
   "subsections": [
    {
     "id": "s2_1",
     "number": "2.1",
     "title": "Benign Failures",
     "slug": "benign-failures",
     "counts": {
      "primary": 20,
      "selected": 1,
      "cross": 14,
      "grey": 5
     }
    },
    {
     "id": "s2_2",
     "number": "2.2",
     "title": "External Attacks",
     "slug": "external-attacks",
     "counts": {
      "primary": 95,
      "selected": 5,
      "cross": 37,
      "grey": 5
     }
    },
    {
     "id": "s2_3",
     "number": "2.3",
     "title": "Insider Threats",
     "slug": "insider-threats",
     "counts": {
      "primary": 75,
      "selected": 9,
      "cross": 52,
      "grey": 6
     }
    }
   ]
  },
  {
   "id": "s3",
   "number": 3,
   "title": "Preliminaries",
   "slug": "preliminaries",
   "operations": [],
   "color": "#7D7D7D",
   "part": 0,
   "subsections": [
    {
     "id": "s3_1",
     "number": "3.1",
     "title": "Threat Model",
     "slug": "threat-model",
     "counts": {
      "primary": 27,
      "selected": 8,
      "cross": 66,
      "grey": 11
     }
    },
    {
     "id": "s3_2",
     "number": "3.2",
     "title": "Control Protocols",
     "slug": "control-protocols",
     "counts": {
      "primary": 27,
      "selected": 8,
      "cross": 40,
      "grey": 13
     }
    },
    {
     "id": "s3_3",
     "number": "3.3",
     "title": "Assessment Criteria",
     "slug": "assessment-criteria",
     "counts": {
      "primary": 0,
      "selected": 0,
      "cross": 0,
      "grey": 2
     }
    }
   ]
  },
  {
   "id": "s4",
   "number": 4,
   "title": "Action Constraints",
   "slug": "constrain",
   "operations": [
    "constrain"
   ],
   "color": "#E27553",
   "part": 1,
   "subsections": [
    {
     "id": "s4_1",
     "number": "4.1",
     "title": "Confinement",
     "slug": "confinement",
     "counts": {
      "primary": 56,
      "selected": 6,
      "cross": 21,
      "grey": 1
     }
    },
    {
     "id": "s4_2",
     "number": "4.2",
     "title": "Formal Enforcement",
     "slug": "formal-enforcement",
     "counts": {
      "primary": 60,
      "selected": 5,
      "cross": 18,
      "grey": 0
     }
    },
    {
     "id": "s4_3",
     "number": "4.3",
     "title": "Information-Flow Control",
     "slug": "information-flow-control",
     "counts": {
      "primary": 33,
      "selected": 4,
      "cross": 0,
      "grey": 3
     }
    },
    {
     "id": "s4_4",
     "number": "4.4",
     "title": "Delegated Authority",
     "slug": "delegated-authority",
     "counts": {
      "primary": 2,
      "selected": 2,
      "cross": 0,
      "grey": 0
     }
    }
   ]
  },
  {
   "id": "s5",
   "number": 5,
   "title": "Monitoring",
   "slug": "monitor",
   "operations": [
    "monitor"
   ],
   "color": "#598DD2",
   "part": 1,
   "subsections": [
    {
     "id": "s5_1",
     "number": "5.1",
     "title": "Automated Monitors",
     "slug": "automated-monitors",
     "counts": {
      "primary": 197,
      "selected": 24,
      "cross": 38,
      "grey": 4
     }
    },
    {
     "id": "s5_2",
     "number": "5.2",
     "title": "Untrusted Monitors",
     "slug": "untrusted-monitors",
     "counts": {
      "primary": 17,
      "selected": 9,
      "cross": 17,
      "grey": 0
     }
    },
    {
     "id": "s5_3",
     "number": "5.3",
     "title": "Human Oversight",
     "slug": "human-oversight",
     "counts": {
      "primary": 12,
      "selected": 3,
      "cross": 0,
      "grey": 2
     }
    }
   ]
  },
  {
   "id": "s6",
   "number": 6,
   "title": "Intervention",
   "slug": "intervene",
   "operations": [
    "intervene"
   ],
   "color": "#E09A3B",
   "part": 1,
   "subsections": [
    {
     "id": "s6_1",
     "number": "6.1",
     "title": "Approval Gates",
     "slug": "approval-gates",
     "counts": {
      "primary": 61,
      "selected": 8,
      "cross": 25,
      "grey": 8
     }
    },
    {
     "id": "s6_2",
     "number": "6.2",
     "title": "Resampling",
     "slug": "resampling",
     "counts": {
      "primary": 4,
      "selected": 3,
      "cross": 4,
      "grey": 7
     }
    },
    {
     "id": "s6_3",
     "number": "6.3",
     "title": "Shutdown",
     "slug": "shutdown",
     "counts": {
      "primary": 23,
      "selected": 6,
      "cross": 8,
      "grey": 9
     }
    }
   ]
  },
  {
   "id": "s7",
   "number": 7,
   "title": "Model-Level Methods",
   "slug": "model",
   "operations": [
    "steer",
    "train"
   ],
   "color": "#65C2B9",
   "part": 2,
   "subsections": [
    {
     "id": "s7_1",
     "number": "7.1",
     "title": "Inference-Time Steering",
     "slug": "inference-time-steering",
     "counts": {
      "primary": 223,
      "selected": 18,
      "cross": 47,
      "grey": 11
     }
    },
    {
     "id": "s7_2",
     "number": "7.2",
     "title": "Training-Time Methods",
     "slug": "training-time-methods",
     "counts": {
      "primary": 356,
      "selected": 20,
      "cross": 31,
      "grey": 4
     }
    }
   ]
  },
  {
   "id": "s8",
   "number": 8,
   "title": "Evaluation Methodology",
   "slug": "evaluate",
   "operations": [
    "evaluate"
   ],
   "color": "#62A16E",
   "part": 3,
   "subsections": [
    {
     "id": "s8_1",
     "number": "8.1",
     "title": "Benchmarks",
     "slug": "benchmarks",
     "counts": {
      "primary": 137,
      "selected": 10,
      "cross": 14,
      "grey": 14
     }
    },
    {
     "id": "s8_2",
     "number": "8.2",
     "title": "Control Evaluations",
     "slug": "control-evaluations",
     "counts": {
      "primary": 37,
      "selected": 11,
      "cross": 21,
      "grey": 9
     }
    },
    {
     "id": "s8_3",
     "number": "8.3",
     "title": "Model Auditing",
     "slug": "model-auditing",
     "counts": {
      "primary": 37,
      "selected": 8,
      "cross": 12,
      "grey": 3
     }
    }
   ]
  },
  {
   "id": "s9",
   "number": 9,
   "title": "Composition and Assurance",
   "slug": "deploy",
   "operations": [
    "deploy"
   ],
   "color": "#A37CAD",
   "part": 3,
   "subsections": [
    {
     "id": "s9_1",
     "number": "9.1",
     "title": "Reference Architectures",
     "slug": "reference-architectures",
     "counts": {
      "primary": 58,
      "selected": 8,
      "cross": 45,
      "grey": 3
     }
    },
    {
     "id": "s9_2",
     "number": "9.2",
     "title": "Safety Cases",
     "slug": "safety-cases",
     "counts": {
      "primary": 72,
      "selected": 13,
      "cross": 12,
      "grey": 30
     }
    }
   ]
  },
  {
   "id": "s10",
   "number": 10,
   "title": "Outlook",
   "slug": "outlook",
   "operations": [],
   "color": "#7D7D7D",
   "part": 3,
   "subsections": [
    {
     "id": "s10_1",
     "number": "10.1",
     "title": "Open Problems",
     "slug": "open-problems",
     "counts": {
      "primary": 92,
      "selected": 4,
      "cross": 86,
      "grey": 6
     }
    },
    {
     "id": "s10_2",
     "number": "10.2",
     "title": "Conclusion",
     "slug": "conclusion",
     "counts": {
      "primary": 1,
      "selected": 0,
      "cross": 23,
      "grey": 3
     }
    }
   ]
  }
 ]
}