{
  "schemaVersion": 1,
  "methodology": "https://ionbin.com/interfaces/methodology/#evaluations",
  "evaluation": {
    "id": "codex-cli",
    "testedOn": "2026-10-18",
    "coverage": "local-runtime",
    "summary": "The tested alpha CLI rejected an unknown strict-config field and an invalid sandbox value, and stored/read a synthetic MCP definition. No model session or sandboxed command was run.",
    "identity": {
      "version": "0.162.0-alpha.17.2",
      "target": "Codex desktop-bundled Windows CLI executable",
      "integrity": "SHA-256 e98c516f1c8935789aabe9122ee7e05ec4c3efd1ba6acf179242e031c495b99e of the installed executable; this identifies the tested file, not an independent provenance audit."
    },
    "environment": [
      "Windows 11 x64; Python 3.12.10 harness. Raw timestamps UTC.",
      "Fresh per-tool home/config/cache; child environment constructed from OS/runtime paths, explicit isolation variables and no inherited credentials. Synthetic fixtures only; no user account sign-in.",
      "Explicit CODEX_HOME under the fixture; no shared daemon/model session was started."
    ],
    "checks": [
      {
        "id": "codex-cli-version",
        "name": "Installed version",
        "method": "<user-home>\\AppData\\Local\\OpenAI\\Codex\\bin\\9614f1d9b40d9c4f\\codex.exe --version",
        "expected": "Print the executable version.",
        "observed": "Exit 0. codex-cli 0.162.0-alpha.17.2. The initial version probe warned that the fresh CODEX_HOME directory did not exist; the harness created it before subsequent checks.",
        "outcome": "pass"
      },
      {
        "id": "codex-cli-auth-empty",
        "name": "Empty authentication configuration",
        "method": "codex login status",
        "expected": "Report not logged in.",
        "observed": "Exit 1. Not logged in",
        "outcome": "pass"
      },
      {
        "id": "codex-cli-strict-exec",
        "name": "Strict configuration validation",
        "method": "codex --strict-config -c ionbin_unknown_setting=true --no-daemon exec --skip-git-repo-check \"Return fixture.\"",
        "expected": "Reject the unknown ionbin_unknown_setting override before a model session.",
        "observed": "Exit 1. Error loading config.toml: unknown configuration field `ionbin_unknown_setting` in -c/--config override",
        "outcome": "pass"
      },
      {
        "id": "codex-cli-invalid-sandbox",
        "name": "Invalid sandbox mode",
        "method": "codex exec --sandbox invalid \"Return fixture.\"",
        "expected": "Reject a value outside the supported sandbox modes.",
        "observed": "Exit 2. error: invalid value 'invalid' for '--sandbox <SANDBOX_MODE>'\n  [possible values: read-only, workspace-write, danger-full-access]\n\nFor more information, try '--help'.",
        "outcome": "pass"
      },
      {
        "id": "codex-cli-mcp-get",
        "name": "Synthetic MCP configuration round trip",
        "method": "codex mcp get ionbin-fixture --json",
        "expected": "Read the definition previously added by codex mcp add ionbin-fixture --url http://127.0.0.1:1/mcp.",
        "observed": "Exit 0. JSON reported name=ionbin-fixture, enabled=true, transport.type=streamable_http and URL=http://127.0.0.1:1/mcp; no server connection was tested.",
        "outcome": "pass"
      },
      {
        "id": "codex-cli-strict-config",
        "name": "Strict flag on features subcommand",
        "method": "codex --strict-config -c ionbin_unknown_setting=true features list",
        "expected": "Determine whether features supports the global strict-config flag.",
        "observed": "Exit 1. --strict-config is not supported for codex features. Coverage limitation: use the exec validation path tested separately.",
        "outcome": "pass"
      }
    ],
    "findings": [
      {
        "title": "Configuration validation is command-specific",
        "basis": "observed",
        "severity": "informational",
        "detail": "The exec path rejected an unknown configuration override; the features subcommand instead rejected the strict-config flag itself. A successful configuration listing alone is not a strict validation check.",
        "practicalAdvice": "Validate the same launch path used by automation, and keep configuration checks tied to the CLI version."
      },
      {
        "title": "Stored MCP configuration is not a connection test",
        "basis": "observed",
        "severity": "informational",
        "detail": "Adding and reading the localhost MCP definition succeeded without credentials or a listening server. Enabled=true reports configuration state, not server reachability or safety.",
        "practicalAdvice": "Review the server command/URL and credentials, then separately test connection and tool permissions in a disposable workspace."
      }
    ],
    "caveats": [
      "This is a desktop-bundled alpha build; results do not establish behavior of a different stable release.",
      "Sandbox-name validation does not demonstrate filesystem or network enforcement."
    ],
    "notTested": [
      "Authenticated model requests, generated edits, command approvals and sandbox boundary enforcement.",
      "MCP connection/authentication/tool calls, plugin trust, prompt injection resistance and cloud sessions."
    ],
    "sources": [
      {
        "label": "Official documentation",
        "url": "https://learn.chatgpt.com/docs/codex/cli"
      }
    ],
    "reportUrl": "/interface-evaluations/codex-cli.json"
  }
}