{ "kind": "phase-4-harness-validation-score", "scoredAt": "2026-08-27T17:11:00.168095Z", "model": "gemini-3.7-flash", "head": "53544114ec40c9c86520e2279160d09b4524a8b4", "calls": { "sentinel": 1, "questions": 314, "total": 315 }, "guard": { "perCallUsd": 0.02, "totalSpendCapUsd": 7.0, "worstCaseUsd": 6.3 }, "tokenUsage": { "promptTokens": 928546, "candidateTokens": 19472, "sentinelPromptTokens": 2970, "sentinelCandidateTokens": 65 }, "cost": { "source": "token-implied estimate using rates recorded before execution; API returned tokens, not currency", "estimatedUsd": 0.7694295, "includesSentinel": true, "conservativeGuardUsd": 6.3 }, "latencyMs": 2583703, "sentinelLatencyMs": 7316, "transportErrors": [], "untypedQuestionIds": [], "knownDefect": { "detected": 14, "denominator": 17, "recall": 0.8235294117647058, "labels": [ { "corpusEntryId": "defect-2024-dec-31-en-dec31-pre-collage", "findingId": "dec31-manmohan-collage", "candidateQuestionIds": [ "2024-dec-31-en-uncovered-277ee492b085-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "defect-2024-dec-31-hi-dec31-pre-collage", "findingId": "dec31-manmohan-collage", "candidateQuestionIds": [ "2024-dec-31-hi-uncovered-277ee492b085-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "defect-2025-apr-30-en-apr30-pre-srinagar-strip", "findingId": "apr30-srinagar-strip", "candidateQuestionIds": [ "2025-apr-30-en-uncovered-277ee492b085-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "defect-2025-apr-30-hi-apr30-pre-srinagar-strip", "findingId": "apr30-srinagar-strip", "candidateQuestionIds": [ "2025-apr-30-hi-uncovered-277ee492b085-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "defect-2025-jun-30-en-jun30-pre-statistics-panel", "findingId": "jun30-statistics-panel", "candidateQuestionIds": [ "2025-jun-30-en-uncovered-277ee492b085-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "defect-2025-jun-30-hi-jun30-pre-statistics-panel", "findingId": "jun30-statistics-panel", "candidateQuestionIds": [ "2025-jun-30-hi-uncovered-277ee492b085-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "defect-2025-mar-15-en-mar15-pre-chamar-card", "findingId": "mar15-chamar-studio", "candidateQuestionIds": [ "2025-mar-15-en-uncovered-006cb4669769-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "defect-2025-mar-15-hi-mar15-pre-chamar-card", "findingId": "mar15-chamar-studio", "candidateQuestionIds": [ "2025-mar-15-hi-uncovered-006cb4669769-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "defect-sheet3-2024-nov-28-hi-pre-repair", "findingId": "sheet3-row-9-crop", "candidateQuestionIds": [ "2024-nov-28-hi-alignment-0a99b3eb9f65-face-or-main-action-outside-crop", "2024-nov-28-hi-alignment-4e837ef51dc4-face-or-main-action-outside-crop", "2024-nov-28-hi-alignment-98662dc18f90-face-or-main-action-outside-crop", "2024-nov-28-hi-alignment-ea2cbf4e5ca4-face-or-main-action-outside-crop", "2024-nov-28-hi-alignment-f9d3f8ee72e4-face-or-main-action-outside-crop" ], "detected": false }, { "corpusEntryId": "defect-sheet3-2025-aug-31-en-pre-repair", "findingId": "sheet3-row-17-missing-image", "candidateQuestionIds": [ "2025-aug-31-en-uncovered-1dae7a5f358e-graphic-present-absent", "2025-aug-31-en-uncovered-ff3d5de8c07b-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "defect-sheet3-2025-aug-31-hi-pre-repair", "findingId": "sheet3-row-18-missing-image", "candidateQuestionIds": [ "2025-aug-31-hi-uncovered-1dae7a5f358e-graphic-present-absent", "2025-aug-31-hi-uncovered-ff3d5de8c07b-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "outstanding-sheet3-2024-dec-31-en", "findingId": "sheet3-row-2-crop", "candidateQuestionIds": [ "2024-dec-31-en-alignment-605174b83ffc-face-or-main-action-outside-crop", "2024-dec-31-en-alignment-8fb159d954cb-face-or-main-action-outside-crop" ], "detected": false }, { "corpusEntryId": "outstanding-sheet3-2024-dec-31-hi", "findingId": "sheet3-row-3-crop", "candidateQuestionIds": [ "2024-dec-31-hi-alignment-31ea4f89cc6f-face-or-main-action-outside-crop", "2024-dec-31-hi-alignment-87f1f1de9432-face-or-main-action-outside-crop" ], "detected": false }, { "corpusEntryId": "outstanding-sheet3-2025-apr-30-en", "findingId": "sheet3-row-13-missing-image", "candidateQuestionIds": [ "2025-apr-30-en-uncovered-508a3d78f8c3-graphic-present-absent", "2025-apr-30-en-uncovered-bc5607772758-graphic-present-absent", "2025-apr-30-en-uncovered-e92c27b12702-graphic-present-absent", "2025-apr-30-en-uncovered-006cb4669769-graphic-present-absent", "2025-apr-30-en-uncovered-8572d1cde671-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "outstanding-sheet3-2025-apr-30-hi", "findingId": "sheet3-row-14-missing-image", "candidateQuestionIds": [ "2025-apr-30-hi-uncovered-508a3d78f8c3-graphic-present-absent", "2025-apr-30-hi-uncovered-bc5607772758-graphic-present-absent", "2025-apr-30-hi-uncovered-e92c27b12702-graphic-present-absent", "2025-apr-30-hi-uncovered-006cb4669769-graphic-present-absent", "2025-apr-30-hi-uncovered-8572d1cde671-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "outstanding-sheet3-2026-may-31-en", "findingId": "sheet3-row-32-missing-image", "candidateQuestionIds": [ "2026-may-31-en-uncovered-e92c27b12702-graphic-present-absent" ], "detected": true }, { "corpusEntryId": "outstanding-sheet3-2026-may-31-hi", "findingId": "sheet3-row-31-missing-image", "candidateQuestionIds": [ "2026-may-31-hi-uncovered-e92c27b12702-graphic-present-absent" ], "detected": true } ], "excluded": [ { "corpusEntryId": "defect-sheet3-2025-aug-31-en-pre-repair", "findingId": "sheet3-row-17-text-coverage", "classification": "DETERMINISTIC_TEXT_COVERAGE", "coverageState": "EXCLUDED", "coverageRoute": "Phase 1 deterministic text coverage; excluded from vision recall." }, { "corpusEntryId": "defect-sheet3-2025-aug-31-hi-pre-repair", "findingId": "sheet3-row-18-text-coverage", "classification": "DETERMINISTIC_TEXT_COVERAGE", "coverageState": "EXCLUDED", "coverageRoute": "Phase 1 deterministic text coverage; excluded from vision recall." }, { "corpusEntryId": "outstanding-sheet3-2024-sep-24-en", "findingId": "sheet3-row-12-relative-size", "classification": "RELATIVE_SIZING_MISMATCH", "coverageState": "EXCLUDED", "coverageRoute": "Relative-sizing vision question only if the generated evidence resolves a candidate; otherwise human contact sheet. No matching generated question exists; fail closed to human contact-sheet review." }, { "corpusEntryId": "outstanding-sheet3-2025-mar-15-en", "findingId": "sheet3-row-30-crop", "classification": "FACE_OR_MAIN_ACTION_OUTSIDE_CROP", "coverageState": "EXCLUDED", "coverageRoute": "Phase 4 vision recall where mapped source geometry supplies candidates; otherwise human contact sheet. No matching generated question exists; fail closed to human contact-sheet review." }, { "corpusEntryId": "outstanding-sheet3-2025-mar-15-hi", "findingId": "sheet3-row-29-crop", "classification": "FACE_OR_MAIN_ACTION_OUTSIDE_CROP", "coverageState": "EXCLUDED", "coverageRoute": "Phase 4 vision recall where mapped source geometry supplies candidates; otherwise human contact sheet. No matching generated question exists; fail closed to human contact-sheet review." } ] }, "cleanControls": { "falsePositivePackages": 13, "denominator": 13, "falsePositiveRate": 1.0, "packages": [ { "corpusEntryId": "clean-sheet3-2024-dec-12-en", "sheetRowIds": [ 4, 15 ], "questionCount": 17, "positiveQuestionIds": [ "2024-dec-12-en-uncovered-1960f61c8a37-graphic-present-absent", "2024-dec-12-en-uncovered-f5adbb34f106-graphic-present-absent", "2024-dec-12-en-uncovered-67e0a96e9c9f-graphic-present-absent", "2024-dec-12-en-alignment-3cb3d80826e8-relative-sizing-mismatch", "2024-dec-12-en-uncovered-3f989558d173-graphic-present-absent", "2024-dec-12-en-uncovered-b41408882606-graphic-present-absent", "2024-dec-12-en-uncovered-d0d179d16f6e-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2024-dec-12-hi", "sheetRowIds": [ 5, 16 ], "questionCount": 17, "positiveQuestionIds": [ "2024-dec-12-hi-uncovered-1960f61c8a37-graphic-present-absent", "2024-dec-12-hi-uncovered-f5adbb34f106-graphic-present-absent", "2024-dec-12-hi-uncovered-67e0a96e9c9f-graphic-present-absent", "2024-dec-12-hi-alignment-50f8d929a6fb-relative-sizing-mismatch", "2024-dec-12-hi-uncovered-53e2a36e779e-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2024-nov-11-en", "sheetRowIds": [ 6 ], "questionCount": 3, "positiveQuestionIds": [ "2024-nov-11-en-uncovered-277ee492b085-graphic-present-absent", "2024-nov-11-en-uncovered-e92c27b12702-graphic-present-absent", "2024-nov-11-en-uncovered-006cb4669769-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2024-nov-11-hi", "sheetRowIds": [ 7 ], "questionCount": 3, "positiveQuestionIds": [ "2024-nov-11-hi-uncovered-277ee492b085-graphic-present-absent", "2024-nov-11-hi-uncovered-e92c27b12702-graphic-present-absent", "2024-nov-11-hi-uncovered-006cb4669769-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2024-nov-28-en", "sheetRowIds": [ 8 ], "questionCount": 7, "positiveQuestionIds": [ "2024-nov-28-en-uncovered-277ee492b085-graphic-present-absent", "2024-nov-28-en-uncovered-a5f5f5d80226-graphic-present-absent", "2024-nov-28-en-uncovered-170184b7dc83-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2024-oct-11-en", "sheetRowIds": [ 10 ], "questionCount": 3, "positiveQuestionIds": [ "2024-oct-11-en-uncovered-e92c27b12702-graphic-present-absent", "2024-oct-11-en-uncovered-006cb4669769-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2024-oct-11-hi", "sheetRowIds": [ 11 ], "questionCount": 3, "positiveQuestionIds": [ "2024-oct-11-hi-uncovered-277ee492b085-graphic-present-absent", "2024-oct-11-hi-uncovered-e92c27b12702-graphic-present-absent", "2024-oct-11-hi-uncovered-006cb4669769-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2025-dec-23-en", "sheetRowIds": [ 19 ], "questionCount": 5, "positiveQuestionIds": [ "2025-dec-23-en-uncovered-277ee492b085-graphic-present-absent", "2025-dec-23-en-uncovered-e92c27b12702-graphic-present-absent", "2025-dec-23-en-uncovered-006cb4669769-graphic-present-absent", "2025-dec-23-en-uncovered-8572d1cde671-graphic-present-absent", "2025-dec-23-en-uncovered-a162f7dec97f-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2025-dec-23-hi", "sheetRowIds": [ 20 ], "questionCount": 5, "positiveQuestionIds": [ "2025-dec-23-hi-uncovered-277ee492b085-graphic-present-absent", "2025-dec-23-hi-uncovered-e92c27b12702-graphic-present-absent", "2025-dec-23-hi-uncovered-006cb4669769-graphic-present-absent", "2025-dec-23-hi-uncovered-8572d1cde671-graphic-present-absent", "2025-dec-23-hi-uncovered-a162f7dec97f-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2025-feb-15-en", "sheetRowIds": [ 21 ], "questionCount": 3, "positiveQuestionIds": [ "2025-feb-15-en-uncovered-277ee492b085-graphic-present-absent", "2025-feb-15-en-uncovered-e92c27b12702-graphic-present-absent", "2025-feb-15-en-uncovered-006cb4669769-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2025-feb-15-hi", "sheetRowIds": [ 22 ], "questionCount": 3, "positiveQuestionIds": [ "2025-feb-15-hi-uncovered-277ee492b085-graphic-present-absent", "2025-feb-15-hi-uncovered-e92c27b12702-graphic-present-absent", "2025-feb-15-hi-uncovered-006cb4669769-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2025-jul-15-en", "sheetRowIds": [ 23 ], "questionCount": 3, "positiveQuestionIds": [ "2025-jul-15-en-uncovered-277ee492b085-graphic-present-absent", "2025-jul-15-en-uncovered-e92c27b12702-graphic-present-absent", "2025-jul-15-en-uncovered-006cb4669769-graphic-present-absent" ], "falsePositive": true }, { "corpusEntryId": "clean-sheet3-2025-jul-15-hi", "sheetRowIds": [ 24 ], "questionCount": 3, "positiveQuestionIds": [ "2025-jul-15-hi-uncovered-277ee492b085-graphic-present-absent", "2025-jul-15-hi-uncovered-e92c27b12702-graphic-present-absent", "2025-jul-15-hi-uncovered-006cb4669769-graphic-present-absent" ], "falsePositive": true } ] }, "repairedControls": [ { "corpusEntryId": "repaired-sheet3-2024-nov-28-hi", "positiveQuestionIds": [ "2024-nov-28-hi-uncovered-277ee492b085-graphic-present-absent", "2024-nov-28-hi-alignment-764e81c515ee-face-or-main-action-outside-crop", "2024-nov-28-hi-alignment-764e81c515ee-relative-sizing-mismatch", "2024-nov-28-hi-uncovered-6a824ee2821d-graphic-present-absent", "2024-nov-28-hi-alignment-0a99b3eb9f65-face-or-main-action-outside-crop", "2024-nov-28-hi-alignment-0a99b3eb9f65-relative-sizing-mismatch", "2024-nov-28-hi-alignment-98662dc18f90-face-or-main-action-outside-crop", "2024-nov-28-hi-alignment-ea2cbf4e5ca4-face-or-main-action-outside-crop", "2024-nov-28-hi-alignment-f9d3f8ee72e4-face-or-main-action-outside-crop", "2024-nov-28-hi-uncovered-1ea13624bb4c-graphic-present-absent", "2024-nov-28-hi-uncovered-bd3957c6aa62-graphic-present-absent", "2024-nov-28-hi-uncovered-ed71e257611a-graphic-present-absent", "2024-nov-28-hi-uncovered-f4d47f571795-graphic-present-absent" ] }, { "corpusEntryId": "repaired-sheet3-2025-aug-31-en", "positiveQuestionIds": [ "2025-aug-31-en-uncovered-1dae7a5f358e-graphic-present-absent", "2025-aug-31-en-uncovered-ff3d5de8c07b-graphic-present-absent", "2025-aug-31-en-uncovered-a6e5ad168f04-graphic-present-absent", "2025-aug-31-en-alignment-43df07f806c2-relative-sizing-mismatch", "2025-aug-31-en-uncovered-9d1d75b5c8d4-graphic-present-absent", "2025-aug-31-en-uncovered-8572d1cde671-graphic-present-absent", "2025-aug-31-en-uncovered-f482d5cd7e6f-graphic-present-absent", "2025-aug-31-en-uncovered-f63bf8ec0627-graphic-present-absent" ] }, { "corpusEntryId": "repaired-sheet3-2025-aug-31-hi", "positiveQuestionIds": [ "2025-aug-31-hi-uncovered-1dae7a5f358e-graphic-present-absent", "2025-aug-31-hi-uncovered-ff3d5de8c07b-graphic-present-absent", "2025-aug-31-hi-uncovered-a6e5ad168f04-graphic-present-absent", "2025-aug-31-hi-alignment-a6bbc2701b60-relative-sizing-mismatch", "2025-aug-31-hi-uncovered-9d1d75b5c8d4-graphic-present-absent", "2025-aug-31-hi-uncovered-8572d1cde671-graphic-present-absent", "2025-aug-31-hi-uncovered-7d4dd4a3688a-graphic-present-absent", "2025-aug-31-hi-uncovered-f482d5cd7e6f-graphic-present-absent", "2025-aug-31-hi-uncovered-f63bf8ec0627-graphic-present-absent" ] } ], "preRegisteredDecisionRule": { "requiredRecall": 1.0, "maximumCleanFalsePositivePackages": 1, "requiresAllTypedNoTransportErrors": true }, "decision": "UNTRUSTED_MANUAL_REVIEW_REQUIRED", "rawResponsesPath": "tools/newsletter-builder/design-planner-pilot/review/fidelity/phase4/paid-20260827-flash-v2/full/raw-responses.jsonl", "rawResponsesSha256": "87a8e0b7091afdd3313a3530f2f1230d2b009af4866077022ca558b9e0a2b5b5" }