{ "schemaVersion": 1, "methodology": "https://ionbin.com/interfaces/methodology/#evaluations", "evaluation": { "id": "playwright-mcp", "testedOn": "2026-10-18", "coverage": "local-runtime", "summary": "A loopback form and isolated storage worked. Direct off-allowlist navigation and file URLs were denied, but a redirect reached a second origin. Origin filtering therefore cannot enforce a strict network boundary.", "identity": { "version": "@playwright/mcp 0.0.83; Playwright 1.64.0-alpha-1790635538000; Chrome for Testing 155.0.8059.12 (build 1247)", "target": "https://registry.npmjs.org/@playwright/mcp/-/mcp-0.0.83.tgz", "integrity": "sha512-oNcl+Ae2/IAjhfPeP46BfIkSakfmprY+aOtkv5MjrQ4lPav4/yNtPhL0iq8SlIM90oApWgBDUxaNKvktazUKOg==; Chrome executable SHA256 74222067d9e6a3ab417a6b1ba00688baded1c7807bb6ef8d1fdf8b6e74d725b8; package-lock SHA256 25fc9846be8780a6fc295bc39775110d5ee975aa7980b45c5a555eb0a9a79a7a" }, "environment": [ "Windows 10.0.26200; Node.js v24.18.0; stdio JSON-RPC, protocol 2025-03-26.", "Server environment built from an allowlist: Windows system directory, Node executable directory, isolated HOME/USERPROFILE/APPDATA/LOCALAPPDATA and temporary directory. No inherited account tokens or user browser profile.", "Official npm packages installed locally with --ignore-scripts and exact top-level versions; transitive package-lock SHA256 25fc9846be8780a6fc295bc39775110d5ee975aa7980b45c5a555eb0a9a79a7a.", "Execution window 2026-10-19T00:55:35.485Z to 2026-10-19T00:55:58.269Z UTC.", "Two HTTP fixture servers bound only to 127.0.0.1 on distinct ephemeral ports; origin A allowed and origin B omitted. No authenticated sites or existing browser sessions.", "Command: node /@playwright/mcp/cli.js --headless --isolated --executable-path --allowed-origins --block-service-workers --no-webmcp --output-dir /browser-output --timeout-navigation 10000." ], "checks": [ { "id": "system-chrome-setup", "name": "System Chrome setup attempt", "method": "Launch the same isolated MCP configuration with system Chrome 155.0.8059.82.", "expected": "Start a fresh browser without a user profile.", "observed": "Browser launch closed with: DevTools remote debugging requires a non-default data directory. Specify this using --user-data-dir. The command already supplied a temporary directory. Browser behavior checks were unavailable in this attempt; matching Chrome for Testing was used for the completed run.", "outcome": "blocked" }, { "id": "navigation-after-denial", "name": "Navigate immediately after a denied request", "method": "In probe-3: browser_navigate to disallowed origin B, then immediately browser_navigate to origin A/redirect and browser_evaluate the destination heading.", "expected": "The follow-up navigation and heading read complete without a browser navigation-state interruption.", "observed": "The redirect navigation returned isError=true: interrupted by another navigation to chrome-error://chromewebdata/. The heading read returned Execution context was destroyed. Origin B still received 1 redirected request. The final sequence tested the redirect before the intentionally denied navigation.", "outcome": "fail" }, { "id": "initialize", "name": "Protocol handshake", "method": "Send {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{\"protocolVersion\":\"2025-03-26\",\"capabilities\":{},\"clientInfo\":{\"name\":\"ionbin-synthetic-eval\",\"version\":\"1.0\"}}}; then notification {\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}.", "expected": "Return a server identity and MCP protocol.", "observed": "JSON-RPC success; {\"name\":\"Playwright\",\"version\":\"1.64.0-alpha-1790635538000\"}; protocol 2025-03-26.", "outcome": "pass" }, { "id": "tools-list", "name": "Discover browser tools", "method": "Send JSON-RPC tools/list with params {} after initialization.", "expected": "Advertise navigation, typing, click and evaluation tools.", "observed": "JSON-RPC success; 25 tools advertised: browser_close, browser_resize, browser_console_messages, browser_handle_dialog, browser_emulate_media, browser_evaluate, browser_file_upload, browser_drop, browser_find, browser_fill_form, browser_press_key, browser_type, browser_navigate, browser_navigate_back, browser_network_requests, browser_network_request, browser_run_code_unsafe, browser_take_screenshot, browser_snapshot, browser_click, browser_drag, browser_hover, browser_select_option, browser_tabs, browser_wait_for.", "outcome": "pass" }, { "id": "allowed-navigation", "name": "Navigate to a permitted fixture", "method": "{\"tool\":\"browser_navigate\",\"url\":\"http://127.0.0.1:61865\",\"verification\":\"browser_evaluate: ()=>document.querySelector(\\\"h1\\\").textContent\"}", "expected": "Open origin A and read the known heading.", "observed": "Navigation succeeded; heading readback: ### Result\n\"LOCAL_FIXTURE\"", "outcome": "pass" }, { "id": "form-action", "name": "Fill and use a synthetic form", "method": "{\"tools\":[\"browser_type(target=input,text=IonBin fixture)\",\"browser_click(target=button)\",\"browser_evaluate: ()=>document.querySelector(\\\"output\\\").textContent\"]}", "expected": "Type the known name, click the button and observe its result.", "observed": "browser_type and browser_click succeeded; readback: ### Result\n\"Saved IonBin fixture\"", "outcome": "pass" }, { "id": "page-storage-set", "name": "Set isolated browser storage", "method": "{\"tool\":\"browser_evaluate\",\"function\":\"()=>{localStorage.setItem(\\\"ionbin-fixture\\\",\\\"synthetic\\\");return localStorage.getItem(\\\"ionbin-fixture\\\");}\"}", "expected": "Write and read a synthetic localStorage value.", "observed": "JSON-RPC tool success; ### Result\n\"synthetic\"\n### Ran Playwright code\n```js\nawait page.evaluate('()=>{localStorage.setItem(\"ionbin-fixture\",\"synthetic\");return localStorage.getItem(\"ionbin-fixture\");}');\n```", "outcome": "pass" }, { "id": "redirect-origin-limit", "name": "Measure the documented redirect limitation", "method": "{\"tool\":\"browser_navigate\",\"url\":\"http://127.0.0.1:61865/redirect\",\"redirectTo\":\"http://127.0.0.1:61864/redirected\"}", "expected": "A redirect from permitted origin A can reach origin B despite the allowlist; this is a limitation check, not a protection pass.", "observed": "Navigation succeeded; origin B received 1 request; heading readback: ### Result\n\"SECOND_ORIGIN_FIXTURE\"", "outcome": "pass" }, { "id": "direct-origin-block", "name": "Block a direct request outside the allowlist", "method": "{\"tool\":\"browser_navigate\",\"url\":\"http://127.0.0.1:61864/direct\"}", "expected": "The direct origin B request is denied before the fixture receives it.", "observed": "Tool isError=true; ### Error\nError: browserBackend.callTool: net::ERR_BLOCKED_BY_CLIENT at http://127.0.0.1:61864/direct\nCall log:\n - navigating to \"http://127.0.0.1:61864/direct\", waiting until \"domcontentloaded\"\n; origin B requests=0", "outcome": "pass" }, { "id": "file-url-block", "name": "Reject file URL navigation", "method": "{\"tool\":\"browser_navigate\",\"url\":\"file:////allowed-sibling/outside.html\"}", "expected": "Refuse the file:// protocol for an outside synthetic HTML file.", "observed": "Tool isError=true; ### Error\nError: Access to \"file:\" protocol is blocked. Attempted URL: \"file:////allowed-sibling/outside.html\"", "outcome": "pass" }, { "id": "close-browser", "name": "Close the isolated browser", "method": "browser_close", "expected": "Return no open tabs.", "observed": "JSON-RPC tool success; ### Result\nNo open tabs. Navigate to a URL to create one.\n### Ran Playwright code\n```js\nawait page.close()\n```\n### Page\n- Page URL: chrome-error://chromewebdata/", "outcome": "pass" }, { "id": "isolated-storage-reset", "name": "Check storage after closing the browser", "method": "After browser_evaluate sets localStorage key ionbin-fixture=synthetic on origin A, call browser_close, then browser_navigate {\"url\":\"http://127.0.0.1:61865\"}; browser_evaluate {\"function\":\"()=>localStorage.getItem(\\\"ionbin-fixture\\\")\"}.", "expected": "After reopening origin A, the prior localStorage key is absent.", "observed": "JSON-RPC tool success; ### Result\nnull\n### Ran Playwright code\n```js\nawait page.evaluate('()=>localStorage.getItem(\"ionbin-fixture\")');\n```", "outcome": "pass" } ], "findings": [ { "title": "Redirects cross the configured origin allowlist", "basis": "observed", "severity": "medium", "detail": "A direct navigation to fixture origin B returned ERR_BLOCKED_BY_CLIENT and produced zero B requests. A 302 redirect from allowed origin A reached B and read SECOND_ORIGIN_FIXTURE; B logged one request. This reproduces the documented filtering limitation, not a new vulnerability claim.", "practicalAdvice": "Use a network proxy, firewall or container policy when destinations must be enforced. Do not rely on --allowed-origins as an exfiltration barrier." }, { "title": "Isolated storage cleared after browser close", "basis": "observed", "severity": "informational", "detail": "The test wrote a synthetic localStorage key, closed the browser, reopened the same local origin and read null. A loopback form accepted input and produced Saved IonBin fixture.", "practicalAdvice": "Use --isolated for disposable tests. Validate your own cookies, downloads and storage-state configuration before assuming all sensitive artifacts are removed." }, { "title": "File URLs were refused in this configuration", "basis": "observed", "severity": "informational", "detail": "Navigation to an outside synthetic HTML file returned Access to file: protocol is blocked. The fixture file contents were not returned.", "practicalAdvice": "Keep unrestricted file access disabled and expose only the workspace needed for the task. This check does not cover every file-writing or upload tool." }, { "title": "Browser and tool-version compatibility needs an explicit setup check", "basis": "observed", "severity": "low", "detail": "System Chrome could not start in the isolated environment; the matching Chrome for Testing completed the tasks. Version 0.0.83 returned snapshot file links and did not advertise browser_run_code. The final form probe used its advertised browser_type, browser_click and browser_evaluate tools.", "practicalAdvice": "Pin the MCP package and matching browser, inspect tools/list, and verify a small local page before connecting an important workflow." }, { "title": "A denied navigation can leave transient browser state", "basis": "observed", "severity": "low", "detail": "In one sequence, an immediate redirect navigation after a blocked request was interrupted by Chrome error-page navigation; the following heading read lost its execution context. The destination still received a request. Testing the redirect first completed successfully.", "practicalAdvice": "Handle tool errors explicitly and verify the final page state before continuing. Do not equate a navigation error with a guarantee that no request was sent." } ], "caveats": [ "A successful boundary observation is not a security rating. The redirect limitation remains even though its reproduction check passes.", "Page JavaScript evaluation and button clicks can perform real actions in an authenticated session. This account-free run did not test approval prompts or transaction protection.", "Snapshots were written to the configured output directory; isolated browser storage does not mean there are no saved diagnostic files.", "The package depends on a specific prerelease Playwright build. Re-run the fixture after upgrades instead of assuming tool names and outputs stay stable." ], "notTested": [ "Authenticated websites, browser extension/CDP attachment, saved login state, payments, permission prompts, anti-bot systems or external production pages.", "HTTP/SSE transport access controls, hostile public pages, prompt-injection resistance, all redirect types, service-worker bypasses, proxy policies, uploads/downloads or complete network egress capture.", "Performance or reliability across operating systems and browsers; only the named Windows/browser combination was exercised." ], "sources": [ { "label": "Official Playwright MCP configuration and security notes", "url": "https://github.com/microsoft/playwright-mcp" }, { "label": "Exact official npm artifact metadata", "url": "https://registry.npmjs.org/@playwright%2fmcp/0.0.83" }, { "label": "Official matching Chrome for Testing artifact", "url": "https://cdn.playwright.dev/builds/cft/155.0.8059.12/win64/chrome-win64.zip" } ], "reportUrl": "/interface-evaluations/playwright-mcp.json" } }