{
  "schemaVersion": 1,
  "skillId": "anthropic-pdf",
  "reviewedOn": "2026-10-18",
  "status": "bounded-helper-evaluation",
  "mode": "actual-helper-probes",
  "revision": "34040c9c568585f6929bedeaad110ad08f079624",
  "total": 9,
  "passed": 7,
  "failed": 2,
  "countMeaning": "Counts are individual expectations, not vulnerability counts or skill-effectiveness scores. Mocked effects are explicitly labeled.",
  "environment": {
    "python": "3.12.10",
    "os": "Windows-11-10.0.26200-SP0",
    "gitBash": "5.3.15(1)-release (x86_64-pc-cygwin)",
    "docker": "not available",
    "wsl": "not installed",
    "packages": {
      "pypdf": "6.19.0",
      "reportlab": "4.4.9",
      "Pillow": "12.3.0",
      "pdf2image": "1.17.0",
      "defusedxml": "0.7.1"
    },
    "node": "24.18.0",
    "chrome": "155.0.8059.82",
    "poppler": "26.07.0"
  },
  "commands": [
    "python run_evals.py --runtime-libs <existing-bundled-site-packages> --poppler-bin <poppler-binaries>"
  ],
  "controls": [
    "Fresh minimal child environment; account/credential variables excluded",
    "Synthetic fixtures only; Python writes guarded to fixtures",
    "Python sockets disabled; browser probe separately blocks all network",
    "30 second suite deadline; 10 second mocked Bash deadline",
    "Known source hashes verified; upstream code not modified"
  ],
  "isolationLimits": [
    "Supervised local processes, not an OS sandbox",
    "No container, VM or memory/resource quota",
    "Application guards do not establish containment of arbitrary hostile code",
    "No real credential, remote account, or destructive-effect testing"
  ],
  "findings": [
    {
      "severity": "attention",
      "kind": "observed-data-handling",
      "title": "Submitted values appear in diagnostics",
      "detail": "The exact helper echoed SYNTHETIC_VALUE_MARKER for an invalid checkbox value. This confirms diagnostic behavior, not a remotely exploited vulnerability. Treat logs with the document data's confidentiality.",
      "testIds": [
        "pdf-diagnostic-value"
      ]
    },
    {
      "severity": "attention",
      "kind": "robustness-limit",
      "title": "Non-form PDF raises AttributeError",
      "detail": "Calling get_field_info on a valid synthetic PDF without AcroForm fields raised AttributeError instead of returning an empty list. This failed an added robustness expectation; it does not imply form-filling failure on supported inputs.",
      "testIds": [
        "pdf-no-form"
      ]
    }
  ],
  "limitations": [
    "No end-to-end agent task or skill-vs-baseline effectiveness comparison",
    "No OCR, redaction, encryption, hostile PDF parser exploit or large-resource tests",
    "No rendered visual-quality review; raster output was checked for page count and dimensions",
    "Only the three named helpers and these synthetic fixtures were executed",
    "Proprietary/source-available helper code is retained privately and excluded from public artifacts"
  ],
  "cases": [
    {
      "id": "pdf-discover",
      "mode": "actual-helper",
      "expectation": "Discover the two synthetic fields with page 1 locations",
      "outcome": "pass",
      "observed": {
        "fieldCount": 2,
        "types": [
          "text",
          "checkbox"
        ]
      },
      "skillId": "anthropic-pdf",
      "passed": true,
      "failed": false
    },
    {
      "id": "pdf-fill",
      "mode": "actual-helper",
      "expectation": "Fill text/checkbox and preserve source when output differs",
      "outcome": "pass",
      "observed": {
        "full_name": "IonBin Sample",
        "consent": "/Yes",
        "sourceUnchanged": true
      },
      "skillId": "anthropic-pdf",
      "passed": true,
      "failed": false
    },
    {
      "id": "pdf-invalid-checkbox",
      "mode": "actual-helper",
      "expectation": "Reject invalid field data without producing a PDF",
      "outcome": "pass",
      "observed": {
        "exitCode": 1,
        "outputCreated": false,
        "diagnostic": "ERROR: Invalid value \"SYNTHETIC_VALUE_MARKER\" for checkbox field \"consent\". The checked value is \"/Yes\" and the unchecked value is \"/Off\""
      },
      "skillId": "anthropic-pdf",
      "passed": true,
      "failed": false
    },
    {
      "id": "pdf-diagnostic-value",
      "mode": "actual-helper",
      "expectation": "Avoid echoing submitted field values in diagnostics",
      "outcome": "finding",
      "observed": {
        "syntheticValueEchoed": true
      },
      "consequence": "Invalid choice values appear in diagnostics; logs need document-equivalent confidentiality.",
      "skillId": "anthropic-pdf",
      "passed": false,
      "failed": true
    },
    {
      "id": "pdf-wrong-page",
      "mode": "actual-helper",
      "expectation": "Reject invalid field data without producing a PDF",
      "outcome": "pass",
      "observed": {
        "exitCode": 1,
        "outputCreated": false,
        "diagnostic": "ERROR: Incorrect page number for `full_name` (got 2, expected 1)"
      },
      "skillId": "anthropic-pdf",
      "passed": true,
      "failed": false
    },
    {
      "id": "pdf-unknown-field",
      "mode": "actual-helper",
      "expectation": "Reject invalid field data without producing a PDF",
      "outcome": "pass",
      "observed": {
        "exitCode": 1,
        "outputCreated": false,
        "diagnostic": "ERROR: `missing` is not a valid field ID"
      },
      "skillId": "anthropic-pdf",
      "passed": true,
      "failed": false
    },
    {
      "id": "pdf-no-form",
      "mode": "actual-helper",
      "expectation": "Return an empty field list for a valid non-form PDF",
      "outcome": "finding",
      "observed": {
        "exception": "AttributeError",
        "message": "'NoneType' object has no attribute 'items'"
      },
      "skillId": "anthropic-pdf",
      "passed": false,
      "failed": true
    },
    {
      "id": "pdf-malformed",
      "mode": "actual-helper",
      "expectation": "Reject malformed PDF",
      "outcome": "pass",
      "observed": {
        "exception": "PdfStreamError"
      },
      "skillId": "anthropic-pdf",
      "passed": true,
      "failed": false
    },
    {
      "id": "pdf-rasterize",
      "mode": "actual-helper",
      "expectation": "Rasterize one synthetic PDF page and constrain its longest dimension to 400 pixels",
      "outcome": "pass",
      "observed": {
        "pages": 1,
        "pixels": [
          400,
          266
        ]
      },
      "skillId": "anthropic-pdf",
      "passed": true,
      "failed": false
    }
  ],
  "sources": [
    {
      "repository": "https://github.com/anthropics/skills",
      "revision": "34040c9c568585f6929bedeaad110ad08f079624",
      "upstreamPath": "skills/pdf/scripts/fill_fillable_fields.py",
      "url": "https://raw.githubusercontent.com/anthropics/skills/34040c9c568585f6929bedeaad110ad08f079624/skills/pdf/scripts/fill_fillable_fields.py",
      "bytes": 3819,
      "sha256": "d140872c2e92e41b1e05370ae8d270c5b7f5ebdf55d9905f03bc974a9c708ff5",
      "status": "retrieved"
    },
    {
      "repository": "https://github.com/anthropics/skills",
      "revision": "34040c9c568585f6929bedeaad110ad08f079624",
      "upstreamPath": "skills/pdf/scripts/extract_form_field_info.py",
      "url": "https://raw.githubusercontent.com/anthropics/skills/34040c9c568585f6929bedeaad110ad08f079624/skills/pdf/scripts/extract_form_field_info.py",
      "bytes": 4300,
      "sha256": "bb235b36f497bbb1c7c6821cecf527bbba7213520de11219da99f7b13a38c109",
      "status": "retrieved"
    },
    {
      "repository": "https://github.com/anthropics/skills",
      "revision": "34040c9c568585f6929bedeaad110ad08f079624",
      "upstreamPath": "skills/pdf/scripts/convert_pdf_to_images.py",
      "url": "https://raw.githubusercontent.com/anthropics/skills/34040c9c568585f6929bedeaad110ad08f079624/skills/pdf/scripts/convert_pdf_to_images.py",
      "bytes": 1008,
      "sha256": "7f58e292a67e18a8ff000d30bad2469389546731f7e8a698cc1d0aed001ea15d",
      "status": "retrieved"
    },
    {
      "repository": "https://github.com/anthropics/skills",
      "revision": "34040c9c568585f6929bedeaad110ad08f079624",
      "upstreamPath": "skills/pdf/LICENSE.txt",
      "url": "https://raw.githubusercontent.com/anthropics/skills/34040c9c568585f6929bedeaad110ad08f079624/skills/pdf/LICENSE.txt",
      "bytes": 1467,
      "sha256": "79f6d8f5b427252fa3b1c11ecdbdb6bf610b944f7530b4de78f770f38741cfaa",
      "status": "retrieved"
    }
  ],
  "repetitions": 1,
  "results": {
    "passed": 7,
    "failed": 2,
    "total": 9
  },
  "harnessArtifact": "/skill-evaluations/helper-harness.json"
}