{ "updated": "2026-09-29", "sources": { "agenthazard": { "title": "AgentHazard", "url": "https://arxiv.org/abs/2604.02947" }, "vera": { "title": "VERA", "url": "https://arxiv.org/abs/2607.01793" }, "braveguard": { "title": "BraveGuard", "url": "https://arxiv.org/abs/2606.01166" }, "hazardauditor": { "title": "HazardAuditor", "url": "https://arxiv.org/abs/2609.15134" }, "adaguard": { "title": "AdaGuard", "url": "https://arxiv.org/abs/2609.34241" } }, "tables": [ { "id": "agenthazard", "work": "agenthazard", "title": { "en": "AgentHazard · agent risk exposure", "zh": "AgentHazard · 智能体风险暴露" }, "table": "2", "dimensions": [ "OpenClaw", "Claude Code", "IFlow" ], "columns": [ "asr", "harm" ], "rows": [ [ "Qwen2.5-72B-Inst.", "OpenClaw", [ 24.1, 2.07 ] ], [ "Kimi-K2.5", "OpenClaw", [ 50.0, 4.28 ] ], [ "Qwen3-32B", "OpenClaw", [ 59.18, 5.45 ] ], [ "Qwen3-VL-235B-Inst.", "OpenClaw", [ 63.0, 5.45 ] ], [ "Qwen2.5-Coder-32B-Inst.", "OpenClaw", [ 64.06, 5.46 ] ], [ "GLM-4.6", "OpenClaw", [ 70.8, 6.15 ] ], [ "Kimi-K2", "OpenClaw", [ 71.1, 5.82 ] ], [ "Qwen2.5-72B-Inst.", "Claude Code", [ 20.4, 1.69 ] ], [ "Kimi-K2", "Claude Code", [ 24.6, 2.08 ] ], [ "Qwen2.5-Coder-32B-Inst.", "Claude Code", [ 57.8, 4.58 ] ], [ "Qwen3-VL-235B-Inst.", "Claude Code", [ 65.6, 5.45 ] ], [ "Qwen3-Coder", "Claude Code", [ 73.63, 5.95 ] ], [ "Kimi-K2.5", "Claude Code", [ 78.3, 6.47 ] ], [ "GLM-4.6", "Claude Code", [ 82.9, 7.05 ] ], [ "Qwen2.5-Coder-32B-Inst.", "IFlow", [ 74.7, 6.06 ] ] ], "note": { "en": "Full-trajectory (round_all), LLM-as-Judge evaluation. ASR measures successful attacks; lower is safer. Harmfulness is on a 0–10 scale. Backend and framework jointly define each configuration.", "zh": "完整轨迹（round_all），使用 LLM 裁判。ASR 为攻击成功率，越低表示风险越小；危害分数范围为 0–10。框架和后端模型共同定义评估配置。" }, "direction": "lower" }, { "id": "vera", "work": "vera", "title": { "en": "VERA · execution outcomes", "zh": "VERA · 执行结果" }, "table": "IV", "dimensions": [ "Single-channel", "Multi-channel", "Benign" ], "columns": [ "esr" ], "rows": [ [ "Claude Code", "Single-channel", [ 95.2 ] ], [ "Codex", "Single-channel", [ 91.1 ] ], [ "OpenClaw", "Single-channel", [ 82.8 ] ], [ "Hermes", "Single-channel", [ 93.4 ] ], [ "Claude Code", "Multi-channel", [ 93.1 ] ], [ "Codex", "Multi-channel", [ 95.8 ] ], [ "OpenClaw", "Multi-channel", [ 89.1 ] ], [ "Hermes", "Multi-channel", [ 97.8 ] ], [ "Claude Code", "Benign", [ 80.1 ] ], [ "Codex", "Benign", [ 69.1 ] ], [ "OpenClaw", "Benign", [ 58 ] ], [ "Hermes", "Benign", [ 74.8 ] ] ], "note": { "en": "ESR is the fraction of attempted runs satisfying the setting-specific executable predicate without infrastructure failure. For attack modes it confirms the harmful outcome; for benign runs it confirms legitimate task completion. Compare within a mode, not across meanings.", "zh": "ESR 是在无基础设施故障的前提下满足对应可执行判据的运行比例。攻击模式验证危害结果；正常模式验证合法任务完成。不同模式的成功含义不同。" }, "direction": "context" }, { "id": "braveguard", "work": "braveguard", "title": { "en": "BraveGuard · trajectory detection", "zh": "BraveGuard · 轨迹风险检测" }, "table": "1", "dimensions": [ "GPT-5.5", "Claude Sonnet 4.6", "Gemini 3.1 Pro", "Qwen3-235B-A22B" ], "columns": [ "accuracy", "recall", "f1" ], "rows": [ [ "GPT-5.4", "GPT-5.5", [ 74.14, 85.13, 83.0 ] ], [ "GPT-5.4", "Claude Sonnet 4.6", [ 73.38, 82.89, 81.58 ] ], [ "GPT-5.4", "Gemini 3.1 Pro", [ 84.41, 88.11, 91.3 ] ], [ "GPT-5.4", "Qwen3-235B-A22B", [ 80.61, 85.9, 88.44 ] ], [ "Claude Sonnet 4.6", "GPT-5.5", [ 57.79, 53.33, 65.2 ] ], [ "Claude Sonnet 4.6", "Claude Sonnet 4.6", [ 63.12, 56.15, 68.4 ] ], [ "Claude Sonnet 4.6", "Gemini 3.1 Pro", [ 76.43, 77.87, 85.97 ] ], [ "Claude Sonnet 4.6", "Qwen3-235B-A22B", [ 69.2, 66.08, 78.74 ] ], [ "Gemini 3.1 Flash", "GPT-5.5", [ 57.41, 53.33, 65.0 ] ], [ "Gemini 3.1 Flash", "Claude Sonnet 4.6", [ 59.7, 50.8, 64.19 ] ], [ "Gemini 3.1 Flash", "Gemini 3.1 Pro", [ 77.95, 79.51, 87.0 ] ], [ "Gemini 3.1 Flash", "Qwen3-235B-A22B", [ 71.86, 71.37, 81.41 ] ], [ "Qwen3.5 Flash", "GPT-5.5", [ 33.59, 13.92, 23.68 ] ], [ "Qwen3.5 Flash", "Claude Sonnet 4.6", [ 40.68, 18.18, 30.36 ] ], [ "Qwen3.5 Flash", "Gemini 3.1 Pro", [ 50.95, 47.13, 64.07 ] ], [ "Qwen3.5 Flash", "Qwen3-235B-A22B", [ 36.88, 26.87, 42.36 ] ], [ "General LLM judges Avg.", "GPT-5.5", [ 55.73, 51.43, 59.22 ] ], [ "General LLM judges Avg.", "Claude Sonnet 4.6", [ 59.22, 52.0, 61.13 ] ], [ "General LLM judges Avg.", "Gemini 3.1 Pro", [ 72.44, 73.16, 82.08 ] ], [ "General LLM judges Avg.", "Qwen3-235B-A22B", [ 64.64, 62.55, 72.74 ] ], [ "LlamaGuard3-8B", "GPT-5.5", [ 27.0, 3.59, 6.8 ] ], [ "LlamaGuard3-8B", "Claude Sonnet 4.6", [ 29.28, 0.53, 1.06 ] ], [ "LlamaGuard3-8B", "Gemini 3.1 Pro", [ 13.69, 6.97, 13.03 ] ], [ "LlamaGuard3-8B", "Qwen3-235B-A22B", [ 16.35, 3.08, 5.98 ] ], [ "Qwen3-Guard-8B", "GPT-5.5", [ 26.24, 1.03, 2.02 ] ], [ "Qwen3-Guard-8B", "Claude Sonnet 4.6", [ 29.66, 1.07, 2.12 ] ], [ "Qwen3-Guard-8B", "Gemini 3.1 Pro", [ 11.03, 4.1, 7.87 ] ], [ "Qwen3-Guard-8B", "Qwen3-235B-A22B", [ 15.76, 2.97, 5.54 ] ], [ "Qwen3-Guard-4B", "GPT-5.5", [ 26.62, 1.03, 2.03 ] ], [ "Qwen3-Guard-4B", "Claude Sonnet 4.6", [ 30.04, 1.6, 3.16 ] ], [ "Qwen3-Guard-4B", "Gemini 3.1 Pro", [ 8.37, 1.23, 2.43 ] ], [ "Qwen3-Guard-4B", "Qwen3-235B-A22B", [ 15.59, 2.2, 4.31 ] ], [ "NemoGuard", "GPT-5.5", [ 28.94, 2.73, 4.21 ] ], [ "NemoGuard", "Claude Sonnet 4.6", [ 28.52, 11.76, 18.97 ] ], [ "NemoGuard", "Gemini 3.1 Pro", [ 12.17, 5.74, 10.81 ] ], [ "NemoGuard", "Qwen3-235B-A22B", [ 15.21, 3.96, 7.47 ] ], [ "YuFeng-XGuard", "GPT-5.5", [ 33.46, 13.33, 22.91 ] ], [ "YuFeng-XGuard", "Claude Sonnet 4.6", [ 38.78, 17.11, 28.44 ] ], [ "YuFeng-XGuard", "Gemini 3.1 Pro", [ 27.38, 21.72, 35.69 ] ], [ "YuFeng-XGuard", "Qwen3-235B-A22B", [ 27.38, 16.3, 27.92 ] ], [ "AgentDoG-Llama3.1-8B", "GPT-5.5", [ 64.26, 58.97, 70.99 ] ], [ "AgentDoG-Llama3.1-8B", "Claude Sonnet 4.6", [ 66.16, 58.82, 71.2 ] ], [ "AgentDoG-Llama3.1-8B", "Gemini 3.1 Pro", [ 80.99, 82.38, 88.94 ] ], [ "AgentDoG-Llama3.1-8B", "Qwen3-235B-A22B", [ 66.92, 63.88, 76.92 ] ], [ "AgentDoG-Qwen2.5-7B", "GPT-5.5", [ 65.02, 60.51, 71.95 ] ], [ "AgentDoG-Qwen2.5-7B", "Claude Sonnet 4.6", [ 61.98, 54.55, 67.11 ] ], [ "AgentDoG-Qwen2.5-7B", "Gemini 3.1 Pro", [ 79.09, 80.74, 87.75 ] ], [ "AgentDoG-Qwen2.5-7B", "Qwen3-235B-A22B", [ 65.02, 60.79, 75.0 ] ], [ "Off-the-shelf guard models Avg.", "GPT-5.5", [ 38.79, 20.17, 25.84 ] ], [ "Off-the-shelf guard models Avg.", "Claude Sonnet 4.6", [ 40.63, 20.78, 27.44 ] ], [ "Off-the-shelf guard models Avg.", "Gemini 3.1 Pro", [ 33.25, 28.98, 35.22 ] ], [ "Off-the-shelf guard models Avg.", "Qwen3-235B-A22B", [ 31.75, 21.88, 29.02 ] ], [ "BraveGuard-Llama-Guard-8B", "GPT-5.5", [ 82.51, 92.82, 88.73 ] ], [ "BraveGuard-Llama-Guard-8B", "Claude Sonnet 4.6", [ 82.89, 82.89, 87.32 ] ], [ "BraveGuard-Llama-Guard-8B", "Gemini 3.1 Pro", [ 87.83, 90.57, 93.25 ] ], [ "BraveGuard-Llama-Guard-8B", "Qwen3-235B-A22B", [ 89.35, 90.75, 93.64 ] ], [ "BraveGuard-Qwen3-Guard-8B", "GPT-5.5", [ 83.65, 91.28, 89.22 ] ], [ "BraveGuard-Qwen3-Guard-8B", "Claude Sonnet 4.6", [ 81.37, 80.21, 85.96 ] ], [ "BraveGuard-Qwen3-Guard-8B", "Gemini 3.1 Pro", [ 90.49, 91.8, 94.71 ] ], [ "BraveGuard-Qwen3-Guard-8B", "Qwen3-235B-A22B", [ 87.45, 87.67, 92.34 ] ], [ "BraveGuard-Qwen3-Guard-4B", "GPT-5.5", [ 80.99, 88.72, 87.37 ] ], [ "BraveGuard-Qwen3-Guard-4B", "Claude Sonnet 4.6", [ 80.61, 82.35, 85.79 ] ], [ "BraveGuard-Qwen3-Guard-4B", "Gemini 3.1 Pro", [ 90.87, 92.21, 94.94 ] ], [ "BraveGuard-Qwen3-Guard-4B", "Qwen3-235B-A22B", [ 90.49, 91.19, 94.31 ] ], [ "BraveGuard-trained guard models Avg.", "GPT-5.5", [ 82.38, 90.94, 88.44 ] ], [ "BraveGuard-trained guard models Avg.", "Claude Sonnet 4.6", [ 81.62, 81.82, 86.36 ] ], [ "BraveGuard-trained guard models Avg.", "Gemini 3.1 Pro", [ 89.73, 91.53, 94.3 ] ], [ "BraveGuard-trained guard models Avg.", "Qwen3-235B-A22B", [ 89.1, 89.87, 93.43 ] ] ], "note": { "en": "AgentHazard-Strongest; trajectories generated by OpenClaw 3.11. Every judge sees the same trajectories within a backend. Unsafe is the positive class. “Avg.” rows are model-group means, not individual models.", "zh": "AgentHazard-Strongest；轨迹由 OpenClaw 3.11 生成。同一后端内的检测器评估相同轨迹，不安全轨迹为正类。“Avg.” 行是模型组均值，并非单个模型。" }, "direction": "higher" }, { "id": "hazardauditor", "work": "hazardauditor", "title": { "en": "HazardAuditor · cross-agent auditing", "zh": "HazardAuditor · 跨框架安全审计" }, "table": "1", "dimensions": [ "Claude Code", "Codex", "Hermes", "OpenClaw" ], "columns": [ "accuracy", "macro_f1", "macro_recall", "macro_precision" ], "rows": [ [ "LlamaGuard3-8B", "Claude Code", [ 58.5, 49.87, 58.5, 77.32 ] ], [ "LlamaGuard3-8B", "Codex", [ 64.0, 58.97, 64.0, 77.45 ] ], [ "LlamaGuard3-8B", "Hermes", [ 52.0, 39.12, 52.0, 63.02 ] ], [ "LlamaGuard3-8B", "OpenClaw", [ 57.5, 49.16, 57.5, 71.81 ] ], [ "Qwen3Guard-Gen-8B", "Claude Code", [ 61.5, 55.33, 61.5, 78.9 ] ], [ "Qwen3Guard-Gen-8B", "Codex", [ 65.0, 61.24, 65.0, 80.86 ] ], [ "Qwen3Guard-Gen-8B", "Hermes", [ 56.0, 47.55, 56.0, 74.35 ] ], [ "Qwen3Guard-Gen-8B", "OpenClaw", [ 55.0, 43.69, 55.0, 76.46 ] ], [ "NemoGuard", "Claude Code", [ 54.0, 52.79, 54.0, 54.46 ] ], [ "NemoGuard", "Codex", [ 56.0, 55.56, 56.0, 56.25 ] ], [ "NemoGuard", "Hermes", [ 46.0, 45.99, 46.0, 46.0 ] ], [ "NemoGuard", "OpenClaw", [ 47.0, 47.0, 47.0, 47.0 ] ], [ "YuFeng-XGuard", "Claude Code", [ 75.0, 73.96, 75.0, 79.76 ] ], [ "YuFeng-XGuard", "Codex", [ 87.0, 86.97, 87.0, 87.37 ] ], [ "YuFeng-XGuard", "Hermes", [ 65.5, 64.14, 65.5, 68.28 ] ], [ "YuFeng-XGuard", "OpenClaw", [ 65.0, 61.79, 65.0, 72.6 ] ], [ "Claude-Sonnet-4.6", "Claude Code", [ 88.5, 88.72, 88.5, 89.19 ] ], [ "Claude-Sonnet-4.6", "Codex", [ 94.0, 94.0, 94.0, 94.07 ] ], [ "Claude-Sonnet-4.6", "Hermes", [ 82.0, 81.64, 82.0, 84.72 ] ], [ "Claude-Sonnet-4.6", "OpenClaw", [ 78.0, 76.99, 78.0, 84.0 ] ], [ "GPT-5.2", "Claude Code", [ 75.0, 73.6, 75.0, 81.71 ] ], [ "GPT-5.2", "Codex", [ 78.5, 77.56, 78.5, 84.26 ] ], [ "GPT-5.2", "Hermes", [ 72.0, 70.71, 72.0, 76.71 ] ], [ "GPT-5.2", "OpenClaw", [ 72.0, 70.71, 72.0, 76.71 ] ], [ "Gemini-3-Flash", "Claude Code", [ 74.5, 72.87, 74.5, 82.24 ] ], [ "Gemini-3-Flash", "Codex", [ 82.0, 81.53, 82.0, 85.65 ] ], [ "Gemini-3-Flash", "Hermes", [ 68.5, 65.27, 68.5, 79.47 ] ], [ "Gemini-3-Flash", "OpenClaw", [ 66.5, 62.27, 66.5, 79.94 ] ], [ "Jev", "Claude Code", [ 87.5, 86.03, 87.5, 89.23 ] ], [ "Jev", "Codex", [ 93.0, 92.99, 93.0, 93.15 ] ], [ "Jev", "Hermes", [ 78.5, 77.74, 78.5, 83.02 ] ], [ "Jev", "OpenClaw", [ 71.0, 69.91, 71.0, 74.54 ] ], [ "Laya", "Claude Code", [ 50.5, 50.44, 50.5, 50.5 ] ], [ "Laya", "Codex", [ 54.0, 52.63, 54.0, 54.52 ] ], [ "Laya", "Hermes", [ 49.0, 48.48, 49.0, 48.96 ] ], [ "Laya", "OpenClaw", [ 45.5, 44.63, 45.5, 45.2 ] ], [ "Laya-Typed-Decisions", "Claude Code", [ 55.0, 52.49, 55.0, 56.34 ] ], [ "Laya-Typed-Decisions", "Codex", [ 57.0, 53.05, 57.0, 60.55 ] ], [ "Laya-Typed-Decisions", "Hermes", [ 52.0, 48.52, 52.0, 52.74 ] ], [ "Laya-Typed-Decisions", "OpenClaw", [ 53.0, 51.43, 53.0, 53.45 ] ], [ "AgentDoG-Qwen2.5-8B", "Claude Code", [ 59.5, 57.13, 59.5, 62.19 ] ], [ "AgentDoG-Qwen2.5-8B", "Codex", [ 57.0, 50.6, 57.0, 64.53 ] ], [ "AgentDoG-Qwen2.5-8B", "Hermes", [ 62.5, 61.69, 62.5, 63.65 ] ], [ "AgentDoG-Qwen2.5-8B", "OpenClaw", [ 61.0, 60.43, 61.0, 61.67 ] ], [ "AgentDoG1.5-4B", "Claude Code", [ 19.0, 30.26, 19.0, 74.84 ] ], [ "AgentDoG1.5-4B", "Codex", [ 21.0, 31.62, 21.0, 78.48 ] ], [ "AgentDoG1.5-4B", "Hermes", [ 14.0, 23.82, 14.0, 79.9 ] ], [ "AgentDoG1.5-4B", "OpenClaw", [ 10.5, 18.34, 10.5, 72.38 ] ], [ "BraveGuard-Qwen3-Guard-8B", "Claude Code", [ 81.5, 80.98, 81.5, 85.35 ] ], [ "BraveGuard-Qwen3-Guard-8B", "Codex", [ 91.5, 91.47, 91.5, 92.01 ] ], [ "BraveGuard-Qwen3-Guard-8B", "Hermes", [ 77.0, 76.23, 77.0, 81.02 ] ], [ "BraveGuard-Qwen3-Guard-8B", "OpenClaw", [ 71.0, 68.72, 71.0, 79.64 ] ], [ "HAZARDAUDITOR-SFT", "Claude Code", [ 82.0, 81.59, 82.0, 85.16 ] ], [ "HAZARDAUDITOR-SFT", "Codex", [ 91.0, 90.97, 91.0, 91.6 ] ], [ "HAZARDAUDITOR-SFT", "Hermes", [ 75.0, 74.69, 75.0, 76.27 ] ], [ "HAZARDAUDITOR-SFT", "OpenClaw", [ 74.0, 73.4, 74.0, 76.37 ] ], [ "HAZARDAUDITOR", "Claude Code", [ 94.0, 94.0, 94.0, 94.07 ] ], [ "HAZARDAUDITOR", "Codex", [ 95.5, 95.5, 95.5, 95.61 ] ], [ "HAZARDAUDITOR", "Hermes", [ 86.5, 86.42, 86.5, 87.34 ] ], [ "HAZARDAUDITOR", "OpenClaw", [ 87.5, 87.46, 87.5, 87.96 ] ] ], "note": { "en": "CUA-EXEC balanced diagnostic: 100 safe + 100 unsafe trajectories per framework. All entries are transcribed from the same table in the reported evaluation. Macro metrics average both classes.", "zh": "CUA-EXEC 平衡诊断集：每个框架含 100 条安全与 100 条不安全轨迹。全部数值来自报告的同一张表；宏平均指标对两个类别取平均。" }, "direction": "higher" }, { "id": "adaguard", "work": "adaguard", "title": { "en": "AdaGuard · policy-conditioned detection", "zh": "AdaGuard · 策略条件下的违规检测" }, "table": "1", "dimensions": [ "AdaptiveSafety", "DynaBench" ], "columns": [ "accuracy", "precision", "recall", "f1" ], "rows": [ [ "Qwen3Guard-Gen-4B", "AdaptiveSafety", [ 59.0, 74.46, 27.4, 40.06 ] ], [ "Qwen3Guard-Gen-4B", "DynaBench", [ 51.57, 83.33, 1.87, 3.66 ] ], [ "Qwen3Guard-Gen-8B", "AdaptiveSafety", [ 58.9, 75.72, 26.2, 38.93 ] ], [ "Qwen3Guard-Gen-8B", "DynaBench", [ 51.93, 100.0, 2.25, 4.4 ] ], [ "YuFeng-XGuard-Reason-8B", "AdaptiveSafety", [ 57.3, 66.82, 29.0, 40.45 ] ], [ "YuFeng-XGuard-Reason-8B", "DynaBench", [ 50.09, 48.36, 22.1, 30.33 ] ], [ "DynaGuard-4B", "AdaptiveSafety", [ 67.3, 67.76, 66.0, 66.87 ] ], [ "DynaGuard-4B", "DynaBench", [ 74.22, 77.25, 67.42, 72.0 ] ], [ "DynaGuard-8B", "AdaptiveSafety", [ 64.1, 61.97, 73.0, 67.03 ] ], [ "DynaGuard-8B", "DynaBench", [ 80.11, 88.04, 68.91, 77.31 ] ], [ "Laya", "AdaptiveSafety", [ 54.4, 54.98, 48.6, 51.59 ] ], [ "Laya", "DynaBench", [ 50.46, 42.86, 2.25, 4.27 ] ], [ "Laya-Typed-Decisions", "AdaptiveSafety", [ 52.3, 51.26, 93.6, 66.24 ] ], [ "Laya-Typed-Decisions", "DynaBench", [ 50.09, 38.89, 2.62, 4.91 ] ], [ "AdaGuard-0.6B", "AdaptiveSafety", [ 82.6, 90.55, 72.8, 80.71 ] ], [ "AdaGuard-0.6B", "DynaBench", [ 51.38, 50.37, 76.4, 60.71 ] ], [ "AdaGuard-4B", "AdaptiveSafety", [ 89.3, 95.38, 82.6, 88.53 ] ], [ "AdaGuard-4B", "DynaBench", [ 71.82, 70.65, 73.03, 71.82 ] ], [ "AdaGuard-8B", "AdaptiveSafety", [ 89.5, 96.25, 82.2, 88.67 ] ], [ "AdaGuard-8B", "DynaBench", [ 76.8, 77.22, 74.91, 76.05 ] ] ], "note": { "en": "Binary assessment of local models. AdaptiveSafety: 1,000 test examples; DynaBench: 543. Violations are positive; invalid outputs count as errors. Transfer performance differs from in-domain performance.", "zh": "本地模型二分类评估。AdaptiveSafety 测试集 1,000 条，DynaBench 543 条。违规为正类；无效输出计错。分布外迁移效果与域内表现不同。" }, "direction": "higher" }, { "id": "rules", "work": "adaguard", "title": { "en": "AdaGuard · identifying violated rules", "zh": "AdaGuard · 违规规则识别" }, "table": "2", "dimensions": [ "AdaptiveSafety", "DynaBench" ], "columns": [ "exact", "rule_f1" ], "rows": [ [ "Laya", "AdaptiveSafety", [ 31.8, 6.27 ] ], [ "Laya", "DynaBench", [ 49.91, 3.38 ] ], [ "Laya-Typed-Decisions", "AdaptiveSafety", [ 8.5, 8.98 ] ], [ "Laya-Typed-Decisions", "DynaBench", [ 49.36, 2.05 ] ], [ "AdaGuard-0.6B", "AdaptiveSafety", [ 68.1, 59.28 ] ], [ "AdaGuard-0.6B", "DynaBench", [ 44.94, 50.5 ] ], [ "AdaGuard-4B", "AdaptiveSafety", [ 76.6, 71.5 ] ], [ "AdaGuard-4B", "DynaBench", [ 63.72, 54.99 ] ], [ "AdaGuard-8B", "AdaptiveSafety", [ 77.1, 74.05 ] ], [ "AdaGuard-8B", "DynaBench", [ 70.72, 63.42 ] ] ], "note": { "en": "Exact is complete-set accuracy; Rule-F1 is micro-F1 over rule identifiers. These are distinct from binary violation detection. Binary-only models are excluded in the paper.", "zh": "Exact 为完整违规集合准确率；Rule-F1 为规则标识符的微平均 F1。两者不同于二分类违规检测；原表不包含仅支持二分类的模型。" }, "direction": "higher" }, { "id": "ada-api", "work": "adaguard", "title": { "en": "AdaGuard · API comparison", "zh": "AdaGuard · API 对比" }, "table": "A1", "dimensions": [ "AdaptiveSafety", "DynaBench" ], "columns": [ "accuracy", "precision", "recall", "f1" ], "rows": [ [ "GPT-5.2", "AdaptiveSafety", [ 76.1, 69.28, 93.8, 79.69 ] ], [ "GPT-5.2", "DynaBench", [ 83.06, 88.89, 74.91, 81.3 ] ], [ "Gemini-3.1-Flash-Lite", "AdaptiveSafety", [ 72.1, 75.52, 65.4, 70.1 ] ], [ "Gemini-3.1-Flash-Lite", "DynaBench", [ 80.85, 80.07, 81.27, 80.67 ] ], [ "Jev", "AdaptiveSafety", [ 82.4, 79.03, 88.2, 83.36 ] ], [ "Jev", "DynaBench", [ 83.61, 93.2, 71.91, 81.18 ] ], [ "Qwen3Guard-Gen-4B", "AdaptiveSafety", [ 59.0, 74.46, 27.4, 40.06 ] ], [ "Qwen3Guard-Gen-4B", "DynaBench", [ 51.57, 83.33, 1.87, 3.66 ] ], [ "Qwen3Guard-Gen-8B", "AdaptiveSafety", [ 58.9, 75.72, 26.2, 38.93 ] ], [ "Qwen3Guard-Gen-8B", "DynaBench", [ 51.93, 100.0, 2.25, 4.4 ] ], [ "YuFeng-XGuard-Reason-8B", "AdaptiveSafety", [ 57.3, 66.82, 29.0, 40.45 ] ], [ "YuFeng-XGuard-Reason-8B", "DynaBench", [ 50.09, 48.36, 22.1, 30.33 ] ], [ "DynaGuard-4B", "AdaptiveSafety", [ 67.3, 67.76, 66.0, 66.87 ] ], [ "DynaGuard-4B", "DynaBench", [ 74.22, 77.25, 67.42, 72.0 ] ], [ "DynaGuard-8B", "AdaptiveSafety", [ 64.1, 61.97, 73.0, 67.03 ] ], [ "DynaGuard-8B", "DynaBench", [ 80.11, 88.04, 68.91, 77.31 ] ], [ "Laya", "AdaptiveSafety", [ 54.4, 54.98, 48.6, 51.59 ] ], [ "Laya", "DynaBench", [ 50.46, 42.86, 2.25, 4.27 ] ], [ "Laya-Typed-Decisions", "AdaptiveSafety", [ 52.3, 51.26, 93.6, 66.24 ] ], [ "Laya-Typed-Decisions", "DynaBench", [ 50.09, 38.89, 2.62, 4.91 ] ], [ "AdaGuard-0.6B", "AdaptiveSafety", [ 82.6, 90.55, 72.8, 80.71 ] ], [ "AdaGuard-0.6B", "DynaBench", [ 51.38, 50.37, 76.4, 60.71 ] ], [ "AdaGuard-4B", "AdaptiveSafety", [ 89.3, 95.38, 82.6, 88.53 ] ], [ "AdaGuard-4B", "DynaBench", [ 71.82, 70.65, 73.03, 71.82 ] ], [ "AdaGuard-8B", "AdaptiveSafety", [ 89.5, 96.25, 82.2, 88.67 ] ], [ "AdaGuard-8B", "DynaBench", [ 76.8, 77.22, 74.91, 76.05 ] ] ], "note": { "en": "Same evaluation examples, different interfaces and inference budgets. Jev returns one probability per rule; threshold ≥ 0.5 selects that rule. AdaGuard emits explanations and ordered rule identifiers. Evaluated Jev version: typesafe/jev-1.13-20260917; server-side context handling is unknown. Invalid outputs count as errors.", "zh": "评估样本相同，但接口与推理预算不同。Jev 对每条规则返回概率，阈值 ≥ 0.5 时选中该规则；AdaGuard 输出解释与有序规则标识。评估的 Jev 版本为 typesafe/jev-1.13-20260917，服务端上下文处理方式未知。无效输出计错。" }, "direction": "higher" }, { "id": "ada-api-rules", "work": "adaguard", "title": { "en": "AdaGuard · rule identification", "zh": "AdaGuard · 规则识别对比" }, "table": "A2", "dimensions": [ "AdaptiveSafety", "DynaBench" ], "columns": [ "exact", "rule_precision", "rule_recall", "rule_f1" ], "rows": [ [ "GPT-5.2", "AdaptiveSafety", [ 52.7, 44.17, 78.18, 56.45 ] ], [ "GPT-5.2", "DynaBench", [ 82.87, 88.5, 74.91, 81.14 ] ], [ "Gemini-3.1-Flash-Lite", "AdaptiveSafety", [ 61.0, 64.8, 42.31, 51.2 ] ], [ "Gemini-3.1-Flash-Lite", "DynaBench", [ 80.85, 80.07, 81.27, 80.67 ] ], [ "Jev", "AdaptiveSafety", [ 57.6, 39.7, 74.82, 51.88 ] ], [ "Jev", "DynaBench", [ 80.85, 86.82, 71.54, 78.44 ] ], [ "Laya", "AdaptiveSafety", [ 31.8, 3.34, 50.07, 6.27 ] ], [ "Laya", "DynaBench", [ 49.91, 6.82, 2.25, 3.38 ] ], [ "Laya-Typed-Decisions", "AdaptiveSafety", [ 8.5, 4.86, 59.59, 8.98 ] ], [ "Laya-Typed-Decisions", "DynaBench", [ 49.36, 12.0, 1.12, 2.05 ] ], [ "AdaGuard-0.6B", "AdaptiveSafety", [ 68.1, 71.73, 50.51, 59.28 ] ], [ "AdaGuard-0.6B", "DynaBench", [ 44.94, 40.78, 66.29, 50.5 ] ], [ "AdaGuard-4B", "AdaptiveSafety", [ 76.6, 81.73, 63.54, 71.5 ] ], [ "AdaGuard-4B", "DynaBench", [ 63.72, 51.64, 58.8, 54.99 ] ], [ "AdaGuard-8B", "AdaptiveSafety", [ 77.1, 85.01, 65.59, 74.05 ] ], [ "AdaGuard-8B", "DynaBench", [ 70.72, 61.11, 65.92, 63.42 ] ] ], "note": { "en": "Same evaluation examples, different interfaces and inference budgets. Jev returns one probability per rule; threshold ≥ 0.5 selects that rule. AdaGuard emits explanations and ordered rule identifiers. Evaluated Jev version: typesafe/jev-1.13-20260917; server-side context handling is unknown. Invalid outputs count as errors.", "zh": "评估样本相同，但接口与推理预算不同。Jev 对每条规则返回概率，阈值 ≥ 0.5 时选中该规则；AdaGuard 输出解释与有序规则标识。评估的 Jev 版本为 typesafe/jev-1.13-20260917，服务端上下文处理方式未知。无效输出计错。" }, "direction": "higher" }, { "id": "ha-agenthazard", "work": "hazardauditor", "title": { "en": "HazardAuditor · AgentHazard transfer", "zh": "HazardAuditor · AgentHazard 迁移评估" }, "table": "2", "dimensions": [ "GPT-5.5", "Claude Sonnet 4.6", "Gemini 3.1 Pro", "Qwen3-235B-A22B" ], "columns": [ "accuracy", "recall", "f1" ], "rows": [ [ "GPT-5.4", "GPT-5.5", [ 74.14, 85.13, 83.0 ] ], [ "GPT-5.4", "Claude Sonnet 4.6", [ 73.38, 82.89, 81.58 ] ], [ "GPT-5.4", "Gemini 3.1 Pro", [ 84.41, 88.11, 91.3 ] ], [ "GPT-5.4", "Qwen3-235B-A22B", [ 80.61, 85.9, 88.44 ] ], [ "Claude Sonnet 4.6", "GPT-5.5", [ 57.79, 53.33, 65.2 ] ], [ "Claude Sonnet 4.6", "Claude Sonnet 4.6", [ 63.12, 56.15, 68.4 ] ], [ "Claude Sonnet 4.6", "Gemini 3.1 Pro", [ 76.43, 77.87, 85.97 ] ], [ "Claude Sonnet 4.6", "Qwen3-235B-A22B", [ 69.2, 66.08, 78.74 ] ], [ "Gemini 3.1 Flash", "GPT-5.5", [ 57.41, 53.33, 65.0 ] ], [ "Gemini 3.1 Flash", "Claude Sonnet 4.6", [ 59.7, 50.8, 64.19 ] ], [ "Gemini 3.1 Flash", "Gemini 3.1 Pro", [ 77.95, 79.51, 87.0 ] ], [ "Gemini 3.1 Flash", "Qwen3-235B-A22B", [ 71.86, 71.37, 81.41 ] ], [ "Jev", "GPT-5.5", [ 34.22, 16.41, 27.0 ] ], [ "Jev", "Claude Sonnet 4.6", [ 36.5, 12.3, 21.6 ] ], [ "Jev", "Gemini 3.1 Pro", [ 40.3, 36.89, 53.41 ] ], [ "Jev", "Qwen3-235B-A22B", [ 33.46, 23.35, 37.72 ] ], [ "Laya", "GPT-5.5", [ 69.58, 79.49, 79.49 ] ], [ "Laya", "Claude Sonnet 4.6", [ 59.7, 72.19, 71.81 ] ], [ "Laya", "Gemini 3.1 Pro", [ 77.19, 81.15, 86.84 ] ], [ "Laya", "Qwen3-235B-A22B", [ 57.41, 62.56, 71.72 ] ], [ "Laya-Typed-Decisions", "GPT-5.5", [ 53.23, 51.79, 62.15 ] ], [ "Laya-Typed-Decisions", "Claude Sonnet 4.6", [ 49.43, 43.32, 54.92 ] ], [ "Laya-Typed-Decisions", "Gemini 3.1 Pro", [ 53.61, 54.92, 68.72 ] ], [ "Laya-Typed-Decisions", "Qwen3-235B-A22B", [ 44.11, 42.73, 56.89 ] ], [ "LlamaGuard3-8B", "GPT-5.5", [ 27.0, 3.59, 6.8 ] ], [ "LlamaGuard3-8B", "Claude Sonnet 4.6", [ 29.28, 0.53, 1.06 ] ], [ "LlamaGuard3-8B", "Gemini 3.1 Pro", [ 13.69, 6.97, 13.03 ] ], [ "LlamaGuard3-8B", "Qwen3-235B-A22B", [ 16.35, 3.08, 5.98 ] ], [ "Qwen3-Guard-8B", "GPT-5.5", [ 26.24, 1.03, 2.02 ] ], [ "Qwen3-Guard-8B", "Claude Sonnet 4.6", [ 29.66, 1.07, 2.12 ] ], [ "Qwen3-Guard-8B", "Gemini 3.1 Pro", [ 11.03, 4.1, 7.87 ] ], [ "Qwen3-Guard-8B", "Qwen3-235B-A22B", [ 15.76, 2.97, 5.54 ] ], [ "NemoGuard", "GPT-5.5", [ 28.94, 2.73, 4.21 ] ], [ "NemoGuard", "Claude Sonnet 4.6", [ 28.52, 11.76, 18.97 ] ], [ "NemoGuard", "Gemini 3.1 Pro", [ 12.17, 5.74, 10.81 ] ], [ "NemoGuard", "Qwen3-235B-A22B", [ 15.21, 3.96, 7.47 ] ], [ "YuFeng-XGuard", "GPT-5.5", [ 33.46, 13.33, 22.91 ] ], [ "YuFeng-XGuard", "Claude Sonnet 4.6", [ 38.78, 17.11, 28.44 ] ], [ "YuFeng-XGuard", "Gemini 3.1 Pro", [ 27.38, 21.72, 35.69 ] ], [ "YuFeng-XGuard", "Qwen3-235B-A22B", [ 27.38, 16.3, 27.92 ] ], [ "AgentDoG-Llama3.1-8B", "GPT-5.5", [ 64.26, 58.97, 70.99 ] ], [ "AgentDoG-Llama3.1-8B", "Claude Sonnet 4.6", [ 66.16, 58.82, 71.2 ] ], [ "AgentDoG-Llama3.1-8B", "Gemini 3.1 Pro", [ 80.99, 82.38, 88.94 ] ], [ "AgentDoG-Llama3.1-8B", "Qwen3-235B-A22B", [ 66.92, 63.88, 76.92 ] ], [ "AgentDoG-Qwen2.5-7B", "GPT-5.5", [ 65.02, 60.51, 71.95 ] ], [ "AgentDoG-Qwen2.5-7B", "Claude Sonnet 4.6", [ 61.98, 54.55, 67.11 ] ], [ "AgentDoG-Qwen2.5-7B", "Gemini 3.1 Pro", [ 79.09, 80.74, 87.75 ] ], [ "AgentDoG-Qwen2.5-7B", "Qwen3-235B-A22B", [ 65.02, 60.79, 75.0 ] ], [ "BraveGuard-Qwen3-Guard-8B", "GPT-5.5", [ 83.65, 91.28, 89.22 ] ], [ "BraveGuard-Qwen3-Guard-8B", "Claude Sonnet 4.6", [ 81.37, 80.21, 85.96 ] ], [ "BraveGuard-Qwen3-Guard-8B", "Gemini 3.1 Pro", [ 90.49, 91.8, 94.71 ] ], [ "BraveGuard-Qwen3-Guard-8B", "Qwen3-235B-A22B", [ 87.45, 87.67, 92.34 ] ], [ "HAZARDAUDITOR-SFT", "GPT-5.5", [ 82.51, 82.07, 82.24 ] ], [ "HAZARDAUDITOR-SFT", "Claude Sonnet 4.6", [ 82.63, 82.51, 83.03 ] ], [ "HAZARDAUDITOR-SFT", "Gemini 3.1 Pro", [ 92.78, 94.04, 93.28 ] ], [ "HAZARDAUDITOR-SFT", "Qwen3-235B-A22B", [ 88.63, 91.63, 92.3 ] ], [ "HAZARDAUDITOR", "GPT-5.5", [ 84.03, 84.75, 84.31 ] ], [ "HAZARDAUDITOR", "Claude Sonnet 4.6", [ 85.55, 87.44, 85.98 ] ], [ "HAZARDAUDITOR", "Gemini 3.1 Pro", [ 90.87, 94.24, 95.07 ] ], [ "HAZARDAUDITOR", "Qwen3-235B-A22B", [ 89.73, 92.26, 92.51 ] ] ], "note": { "en": "AgentHazard trajectory evaluation. Methods are compared within the same agent backend. Unsafe is the positive class. Results describe the recorded evaluation, not a fresh run or the current Jev service.", "zh": "AgentHazard 轨迹评估，检测器在同一智能体后端内比较。不安全轨迹为正类。数值对应记录中的评估，并非本次重新运行或当前 Jev 服务的表现。" }, "direction": "higher" } ], "statistics": [ { "work": "agenthazard", "table": "1", "name": "AgentHazard", "items": [ [ "2,653", "instances" ], [ "10", "risk_categories" ], [ "10", "attack_strategies" ], [ "10,000+", "trajectories" ], [ "11.55", "mean_steps" ], [ "3", "frameworks" ] ] }, { "work": "vera", "table": "§ V.A–B", "name": "VERA-Bench", "items": [ [ "1,600", "cases" ], [ "124", "risk_categories" ], [ "77", "attack_methods" ], [ "30", "environments" ], [ "72", "mcp_tools" ], [ "3", "execution_modes" ] ] }, { "work": "braveguard", "table": "4", "name": "BraveGuard", "items": [ [ "7,308", "tasks" ], [ "28", "risk_categories" ], [ "32", "attack_methods" ], [ "3.36", "mean_steps" ] ] }, { "work": "hazardauditor", "table": "1", "name": "CUA-EXEC", "items": [ [ "4", "frameworks" ], [ "100 + 100", "safe_unsafe_per_framework" ], [ "800", "total_derived" ] ] }, { "work": "adaguard", "table": "Abstract", "name": "AdaptiveSafety", "items": [ [ "10,939", "train_examples" ], [ "1,000", "test_examples" ], [ "1–100", "rules_per_policy" ], [ "0.6B / 4B / 8B", "model_sizes" ] ] } ] }