{
  "schemaVersion": 1,
  "skillId": "openai-playwright",
  "reviewedOn": "2026-10-18",
  "status": "bounded-helper-evaluation",
  "mode": "mocked-wrapper-dispatch-only",
  "revision": "49f948faa9258a0c61caceaf225e179651397431",
  "total": 6,
  "passed": 6,
  "failed": 0,
  "countMeaning": "Counts are individual expectations, not vulnerability counts or skill-effectiveness scores. Mocked effects are explicitly labeled.",
  "environment": {
    "python": "3.12.10",
    "os": "Windows-11-10.0.26200-SP0",
    "gitBash": "5.3.15(1)-release (x86_64-pc-cygwin)",
    "docker": "not available",
    "wsl": "not installed",
    "packages": {
      "pypdf": "6.19.0",
      "reportlab": "4.4.9",
      "Pillow": "12.3.0",
      "pdf2image": "1.17.0",
      "defusedxml": "0.7.1"
    },
    "node": "24.18.0",
    "chrome": "155.0.8059.82",
    "poppler": "26.07.0"
  },
  "commands": [
    "python run_evals.py --runtime-libs <existing-bundled-site-packages> --poppler-bin <poppler-binaries>"
  ],
  "controls": [
    "Fresh minimal child environment; account/credential variables excluded",
    "Synthetic fixtures only; Python writes guarded to fixtures",
    "Python sockets disabled; browser probe separately blocks all network",
    "30 second suite deadline; 10 second mocked Bash deadline",
    "Known source hashes verified; upstream code not modified"
  ],
  "isolationLimits": [
    "Supervised local processes, not an OS sandbox",
    "No container, VM or memory/resource quota",
    "Application guards do not establish containment of arbitrary hostile code",
    "No real credential, remote account, or destructive-effect testing"
  ],
  "findings": [],
  "limitations": [
    "Five wrapper dispatch checks used a fake npx, plus one Git Bash environment check",
    "No real npx, npm package, Playwright CLI or target browser workflow was executed",
    "No install/compatibility, live-site, account, session-security or artifact-confidentiality claim follows",
    "The wrapper still delegates to an unpinned npm package; these checks do not evaluate that package",
    "This is the historical catalog revision, not the current successor plugin"
  ],
  "cases": [
    {
      "id": "playwright-bash-environment",
      "mode": "environment-check",
      "expectation": "Start installed Git Bash successfully",
      "outcome": "pass",
      "observed": {
        "version": "GNU bash, version 5.3.15(1)-release (x86_64-pc-cygwin)"
      },
      "skillId": "openai-playwright",
      "passed": true,
      "failed": false
    },
    {
      "id": "playwright-nominal",
      "mode": "mocked-dispatch",
      "expectation": "Pass expected CLI arguments literally to a fake npx; never install or launch CLI",
      "outcome": "pass",
      "observed": {
        "exitCode": 0,
        "arguments": [
          "--yes",
          "--package",
          "@playwright/cli",
          "playwright-cli",
          "open",
          "https://example.invalid/"
        ],
        "realNpxExecuted": false,
        "realBrowserExecuted": false
      },
      "skillId": "openai-playwright",
      "passed": true,
      "failed": false
    },
    {
      "id": "playwright-environment-session",
      "mode": "mocked-dispatch",
      "expectation": "Pass expected CLI arguments literally to a fake npx; never install or launch CLI",
      "outcome": "pass",
      "observed": {
        "exitCode": 0,
        "arguments": [
          "--yes",
          "--package",
          "@playwright/cli",
          "playwright-cli",
          "--session",
          "fixture-session",
          "snapshot"
        ],
        "realNpxExecuted": false,
        "realBrowserExecuted": false
      },
      "skillId": "openai-playwright",
      "passed": true,
      "failed": false
    },
    {
      "id": "playwright-explicit-session",
      "mode": "mocked-dispatch",
      "expectation": "Pass expected CLI arguments literally to a fake npx; never install or launch CLI",
      "outcome": "pass",
      "observed": {
        "exitCode": 0,
        "arguments": [
          "--yes",
          "--package",
          "@playwright/cli",
          "playwright-cli",
          "--session=explicit",
          "snapshot"
        ],
        "realNpxExecuted": false,
        "realBrowserExecuted": false
      },
      "skillId": "openai-playwright",
      "passed": true,
      "failed": false
    },
    {
      "id": "playwright-literal-argument",
      "mode": "mocked-dispatch",
      "expectation": "Pass expected CLI arguments literally to a fake npx; never install or launch CLI",
      "outcome": "pass",
      "observed": {
        "exitCode": 0,
        "arguments": [
          "--yes",
          "--package",
          "@playwright/cli",
          "playwright-cli",
          "fill",
          "$(touch SHOULD_NOT_EXIST); space value"
        ],
        "realNpxExecuted": false,
        "realBrowserExecuted": false
      },
      "skillId": "openai-playwright",
      "passed": true,
      "failed": false
    },
    {
      "id": "playwright-no-npx",
      "mode": "mocked-dispatch",
      "expectation": "Fail clearly when npx is absent",
      "outcome": "pass",
      "observed": {
        "exitCode": 1,
        "diagnostic": "Error: npx is required but not found on PATH."
      },
      "skillId": "openai-playwright",
      "passed": true,
      "failed": false
    }
  ],
  "sources": [
    {
      "repository": "https://github.com/openai/skills",
      "revision": "49f948faa9258a0c61caceaf225e179651397431",
      "upstreamPath": "skills/.curated/playwright/scripts/playwright_cli.sh",
      "url": "https://raw.githubusercontent.com/openai/skills/49f948faa9258a0c61caceaf225e179651397431/skills/.curated/playwright/scripts/playwright_cli.sh",
      "bytes": 526,
      "sha256": "aa3fdff5d0e4556177f4dfd5f04117e772aa54f94b6a2e34b6c0edf629c6b9b5",
      "status": "retrieved"
    },
    {
      "repository": "https://github.com/openai/skills",
      "revision": "49f948faa9258a0c61caceaf225e179651397431",
      "upstreamPath": "skills/.curated/playwright/LICENSE.txt",
      "url": "https://raw.githubusercontent.com/openai/skills/49f948faa9258a0c61caceaf225e179651397431/skills/.curated/playwright/LICENSE.txt",
      "bytes": 11552,
      "sha256": "9a7110fc2d2f964038e5dc49128f908f29f47a574c961cba16085914e879cbda",
      "status": "retrieved"
    }
  ],
  "repetitions": 1,
  "results": {
    "passed": 6,
    "failed": 0,
    "total": 6
  },
  "harnessArtifact": "/skill-evaluations/helper-harness.json"
}