{ "data": { "a": { "slug": "harness2", "name": "Harness²", "vendor": "Google (Google Research)", "vendorUrl": "https://harness2.github.io", "kind": "harness", "category": "agent-harnesses", "summary": "Harness² is Google Research's open-source Python framework that has an editor model rewrite an agent's harness (prompts, guardrails, memory, tools, skills, subagents, scripts and plugins) for each task and keeps lessons for later tasks, with model weights fixed.", "url": "https://www.anchorterminal.com/tools/harness2", "markdownUrl": "https://www.anchorterminal.com/tools/harness2.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/harness2.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/harness2.json", "repo": "https://github.com/google-research/harness2", "license": "Apache-2.0. Bundled benchmark runners keep their upstream licences (Apache-2.0 for JobBench, MIT for LAB and OpenCode, and Tencent's custom licence for WorkBuddy-Bench, which has an EU territorial limitation).", "transports": [], "packages": [], "auth": "none", "authNotes": "No Harness² account or key. Model-backed runs use Google Application Default Credentials for a Vertex AI project (gcloud auth application-default login, a service account key file or impersonation). The optional Codex bridge generates its own local access key.", "pricing": "free", "pricingNotes": "Free under Apache-2.0, with nothing to buy. Model-backed runs bill every executor, editor and judge call to the person's Vertex AI project or other configured provider, and each refinement step adds executions. The local demo needs no credentials.", "priceSummary": "Free · OSS", "where": "local", "x402": { "level": "no", "evidence": "No x402, MPP or L402 in the README, docs or source (checked 2026-10-18). The software has nothing to buy.", "endpoints": [] }, "toolCount": null, "popularity": { "githubStars": 4200, "npmWeekly": null, "pypiWeekly": null, "asOf": "2026-10-18" }, "docsUrl": "https://github.com/google-research/harness2/tree/main/docs", "capabilities": [ "agent.harness" ], "tags": [ "open-source", "apache-2.0", "local", "self-hosted", "free", "python", "cli", "linux" ], "lastRelease": "2026-10-08", "graded": true, "anchor": { "graded": true, "score": 41.6, "grade": "E", "agentReady": false, "rank": 1029, "ranked": true, "rankOf": 1135, "categoryRank": 26, "methodology": "0.4", "run": "2026-10-01", "scores": { "ergonomics": 59, "maintenance": 36, "payments": 40, "reliability": 20, "schema": 51, "security": 44, "transparency": 44 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-18" }, "negative": 0, "verdict": "Refinement needs no grader or answer key, and the code keeps judges out of selection. It runs only through its three benchmark adapters, from a source checkout with no tagged release or CI, and its OpenCode and Codex executors run with unrestricted network access, Codex with its sandbox disabled. The benchmark gains are the authors' claims.", "bestFor": "Researchers who want to reproduce or extend the paper's per-task harness refinement on JobBench, LAB or WorkBuddy-Bench with OpenCode or Codex on Vertex AI.", "strengths": [ "Refining a harness needs no grader, rubric or answer key. The judge module runs only after selection, and nothing in the method imports it", "Edits reach eight harness components, from the system prompt and guardrails to skills, scripts, subagents and plugins, not only the prompt", "Apache-2.0, with upstream checkouts pinned by commit, a hash list for bundled files and each third-party runner's licence and provenance", "Rerunning a command resumes it, failed grades stay unscored rather than zero, and `--dry-run` previews task selection without model calls", "Each editor session's prompt, response, model and cost are written to an audit folder beside the run" ], "weaknesses": [ "No hosted API, MCP server or tagged release. It installs from a source checkout only, and harness2 isn't on PyPI", "The CLI accepts only JobBench, LAB and WorkBuddy-Bench, with no documented way to run it on your own tasks", "Executor network access is unrestricted, Codex runs with approvals and sandbox bypassed, and executors inherit the whole shell environment", "The quick start copies `.env.example`, which the repository doesn't contain, and no .gitignore keeps a filled-in .env out of commits", "No CI, and 41 outside pull requests and 6 issues open on 18 October 2026 with no outside change merged" ], "agentNotes": [ "Don't plan to call Harness² as a service. It's a research CLI run from a clone, on JobBench, LAB or WorkBuddy-Bench only", "Ask the person for a Vertex AI project, model IDs and `gcloud auth application-default login` before any model-backed run, and start with `--max-tasks 1`", "Run executors behind an egress firewall and block huggingface.co and github.com after setup, as the docs advise, so rubrics can't be fetched", "Write .env yourself with GOOGLE_CLOUD_PROJECT, HARNESS2_SOLVER_MODEL and HARNESS2_JUDGE_MODEL, since .env.example isn't shipped, and never commit it", "Use the same benchmark, executor, `--k`, `--mode` and seed for `harness2`, `harness2-judge` and `harness2-report`, or the results won't line up" ], "metrics": { "kind": "local", "measured": false }, "reviewCount": 4, "avgRating": 2.5, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "E", "methodology": "0.4", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 41.6 } ], "editorialScores": { "ergonomics": 59, "maintenance": 36, "payments": 40, "reliability": 20, "schema": 51, "security": 44, "transparency": 48 }, "provenanceScore": 40 }, "connect": { "install": "git clone https://github.com/google-research/harness2.git \u0026\u0026 cd harness2 \u0026\u0026 uv venv --python 3.12 .venv \u0026\u0026 source .venv/bin/activate \u0026\u0026 uv pip install -e '.[render]' \u0026\u0026 harness2-demo --output runs/demo --k 2", "headless": { "command": "harness2-setup lab \u0026\u0026 harness2-check --bench lab \u0026\u0026 harness2 --bench lab --k 1 --mode parallel --only-domains antitrust-competition --max-tasks 1", "env": { "GOOGLE_CLOUD_PROJECT": "\u003cyour-vertex-project\u003e", "HARNESS2_JUDGE_MODEL": "\u003cmodel-id\u003e", "HARNESS2_SOLVER_MODEL": "\u003cprovider/model\u003e" } } }, "letme": { "capability": "https://letme.dev/agent.harness", "tool": "https://letme.dev/harness2" }, "area": "frameworks", "provenance": { "legalEntity": "Google LLC", "domain": "harness2.github.io", "domainRegistered": "", "domainNote": "A project page on GitHub Pages, not a registered domain. The code is in the google-research organisation on GitHub.", "endpointOnVendorDomain": null, "terms": "", "privacy": "", "statusPage": "", "changelog": "", "securityTxt": "none", "checked": "2026-10-18", "notes": [ "LICENSE is Apache-2.0, and pyproject.toml and every source file name Google LLC. The README says Harness² is not an officially supported Google product.", "No terms or privacy policy govern the package or the project page. Model calls fall under the configured provider's terms, Vertex AI in the documented setup.", "The repository has no SECURITY.md, and harness2.github.io/.well-known/security.txt returns 404. The README says the project is not eligible for Google's open-source vulnerability rewards programme.", "No changelog, tags or GitHub releases. The version is 0.1.0 in pyproject.toml.", "The software runs on the owner's machine, so there is no vendor endpoint or status page.", "The project page has no robots.txt and carries a noindex, nofollow robots meta tag. It was read once as a public page." ], "score": 40 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/harness2.json" }, "answer": "OpenAI Codex scores 73.4 (BB) on agent readiness against Harness²'s 41.6 (E), and leads in every scored category.", "b": { "slug": "openai-codex", "name": "OpenAI Codex", "vendor": "OpenAI", "vendorUrl": "https://openai.com", "kind": "harness", "category": "agent-harnesses", "summary": "OpenAI's coding agent for software development tasks.", "url": "https://www.anchorterminal.com/tools/openai-codex", "markdownUrl": "https://www.anchorterminal.com/tools/openai-codex.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/openai-codex.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/openai-codex.json", "repo": "https://github.com/openai/codex", "license": "Apache-2.0 (Codex CLI, its Rust crates and the TypeScript and Python SDKs). Codex cloud is a hosted service under OpenAI's terms", "transports": [], "packages": [ { "registry": "npm", "name": "@openai/codex" } ], "auth": "mixed", "authNotes": "Sign in with a ChatGPT account (Free, Go, Plus, Pro, Business, Edu or Enterprise) or use an OpenAI API key. Cloud work such as GitHub code review and the Slack integration comes with Plus and above, and none of it works with an API key. `--oss` talks to a local Ollama or LM Studio server and needs no account.", "pricing": "freemium", "pricingNotes": "Included in every ChatGPT plan. Free $0, Go $8 a month, Plus $20, Pro from $100 (tiers at $100, $200 and $500), Business $20 a user a month billed annually for two or more users, Enterprise and Edu by quote. On Plus the docs estimate 15 to 160 local messages per five hours with GPT-6.1 Sol, and Pro has no five-hour limit. Cloud tasks use more of the allowance. With an API key you pay API token rates and can't use the cloud agent (checked 2026-10-02).", "priceSummary": "$20 / mo", "where": "local", "x402": { "level": "no", "evidence": "No x402, MPP or L402 in the docs, the pricing page or the source (checked 2026-10-02).", "endpoints": [] }, "toolCount": null, "popularity": { "githubStars": 126000, "npmWeekly": null, "pypiWeekly": null, "asOf": "2026-10-02" }, "docsUrl": "https://developers.openai.com/codex", "llmsTxt": "https://learn.chatgpt.com/llms.txt", "capabilities": [ "agent.harness", "agent.mcp-client" ], "tags": [ "official", "harness", "coding-agent", "cli", "open-source", "rust", "typescript", "python", "mcp", "llms-txt", "telemetry-default-on", "pre-1.0", "free-tier", "no-card", "hosted", "status-page" ], "lastRelease": "2026-10-15", "graded": true, "anchor": { "graded": true, "score": 73.4, "grade": "BB", "agentReady": true, "rank": 94, "ranked": true, "rankOf": 1135, "categoryRank": 2, "methodology": "0.4", "run": "2026-10-01", "scores": { "ergonomics": 80, "maintenance": 87, "payments": 60, "reliability": 55, "schema": 90, "security": 82, "transparency": 84 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-17" }, "negative": -2, "negativeNotes": [ "2026-04-14. CVE-2025-61260 (GHSA-xrxf-jgv3-qmrm), critical (CVSS 9.8 from CISA-ADP), code execution through MCP configuration files in a repository for Codex CLI 0.23.0 and earlier, published to NVD and the GitHub Advisory Database from Check Point Research's 2025 report. Fixed in 2025 and documented by the researcher, with no advisory in OpenAI's own repository, so a small deduction (https://nvd.nist.gov/vuln/detail/CVE-2025-61260; https://research.checkpoint.com/2025/openai-codex-cli-command-injection-vulnerability/)" ], "verdict": "Sandbox on by default on macOS, Linux and Windows, with the network off and `.git` and `.codex` read-only. Pre-1.0 at 0.163.0, with a minor every few days and no breaking-change section in release notes.", "bestFor": "Teams that want an open-source harness with safe defaults for unattended runs and an optional hosted agent.", "strengths": [ "Sandbox on by default on macOS, Linux and Windows, with the network off and `.git` and `.codex` read-only", "Apache-2.0, with public CI and a JSON Schema for config.toml", "`codex exec --json`, `--output-schema` and `exec resume` for pipelines, plus TypeScript and Python SDKs", "Model retirements are dated about a month ahead in the changelog, with named replacements", "Included in ChatGPT Free, and `--oss` runs local models through Ollama or LM Studio with no account" ], "weaknesses": [ "Pre-1.0 at 0.163.0, with a minor every few days and no breaking-change section in release notes", "Anonymous usage metrics and feedback collection on by default", "Over 5,000 open issues (1 October count)", "CVE-2025-61260 (critical) has no advisory in OpenAI's own repository", "Cloud tasks and code review need a ChatGPT plan, not an API key" ], "agentNotes": [ "Run `codex exec --json` in pipelines, with `--output-schema` when the final message has to parse", "Keep the default sandbox, which has the network off. `--yolo` removes both the sandbox and approvals", "Replace `gpt-5.5` with `gpt-6.1-sol` in saved configs, custom agents and scripts under ChatGPT sign-in. GPT-5.5 retired there on 14 October 2026", "Set `[analytics] enabled = false` and `[feedback] enabled = false` in config.toml to keep usage data local", "Pin the npm version. A 0.x minor lands every few days" ], "metrics": { "kind": "local", "measured": false }, "reviewCount": 2, "avgRating": 3, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "BB", "methodology": "0.4", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 73.4 } ], "editorialScores": { "ergonomics": 80, "maintenance": 87, "payments": 60, "reliability": 55, "schema": 90, "security": 82, "transparency": 75 }, "provenanceScore": 93 }, "connect": { "install": "npm i -g @openai/codex # or: brew install --cask codex", "headless": { "run": "codex exec --json \"fix the failing test\"" } }, "letme": { "capability": "https://letme.dev/agent.harness", "tool": "https://letme.dev/openai-codex" }, "sameCompany": [ "openai-api", "openai-embeddings", "openai-guardrails", "openai-moderation", "openai-image-api", "openai-sora", "openai-speech-to-text", "gpt-live", "openai-realtime", "openai-agents-sdk", "openai-decisions-api" ], "area": "frameworks", "unitPrices": [ { "item": "ChatGPT Plus", "unit": "month", "usd": 20, "note": "includes Codex local and cloud" }, { "item": "ChatGPT Pro", "unit": "month", "usd": 100, "note": "lowest Pro tier, no five-hour limit" } ], "provenance": { "legalEntity": "OpenAI OpCo, LLC", "domain": "openai.com", "domainRegistered": "2007-01-19", "endpointOnVendorDomain": null, "terms": "https://openai.com/policies/services-agreement/", "privacy": "https://openai.com/policies/privacy-policy/", "statusPage": "https://status.openai.com", "changelog": "https://github.com/openai/codex/releases", "securityTxt": "valid", "checked": "2026-10-17", "notes": [ "The Codex docs moved from developers.openai.com/codex to learn.chatgpt.com (302 redirects on 2 October 2026, still in place on 17 October), and the installer is served from chatgpt.com.", "learn.chatgpt.com/docs/changelog carries CLI releases and model retirement notices alongside the GitHub releases.", "Legal entity, domain date and security.txt are from the openai-api listing's check of 26 September 2026." ], "score": 93 }, "requirements": { "card": "paid-only", "freeTier": true, "minutes": 3, "credential": "mixed", "hosting": [ "US", "EU", "UK", "CA", "APAC", "self-hosted" ], "euResidency": "enterprise", "trainsOnData": "no", "zeroRetention": "enterprise", "certifications": [ "SOC 2 Type II", "ISO 27001", "ISO 27701", "ISO 42001", "CSA STAR", "GDPR" ], "dpa": true, "selfHost": "yes" }, "pageJsonUrl": "https://www.anchorterminal.com/tools/openai-codex.json", "live": { "slug": "openai-codex", "vendorStatus": { "page": "https://status.openai.com", "indicator": "none", "summary": "All Systems Operational", "checkedAt": "2026-10-18T19:17:44.390459116Z" }, "versions": [ { "registry": "github", "name": "openai/codex", "version": "rust-v0.163.0", "released": "2026-10-15", "seenAt": "2026-10-18T03:06:10.024976343Z" }, { "registry": "npm", "name": "@openai/codex", "version": "0.163.0", "seenAt": "2026-10-18T03:06:09.800991356Z" } ], "githubStars": 129659, "npmWeekly": 15038867, "securityTxt": { "url": "https://openai.com/.well-known/security.txt", "state": "valid", "checkedAt": "2026-10-18T15:53:50.268312931Z" }, "llmsTxt": { "url": "https://learn.chatgpt.com/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-18T14:25:39.407773399Z" }, "domain": { "domain": "openai.com", "registered": "2007-01-19", "source": "https://rdap.verisign.com/com/v1/domain/openai.com", "checkedAt": "2026-10-18T13:30:33.572293827Z" }, "updatedAt": "2026-10-18T19:17:44.390459116Z" } }, "facts": [ { "a": "Agent harness", "b": "Agent harness", "name": "Kind" }, { "a": "Google (Google Research)", "b": "OpenAI", "name": "Vendor" }, { "a": "no (local only)", "b": "no (local only)", "name": "Hosted endpoint" }, { "a": "", "b": "", "name": "Transports" }, { "a": "None", "b": "OAuth or key", "name": "Auth" }, { "a": "Free", "b": "Freemium", "name": "Pricing" }, { "a": "no", "b": "no", "name": "x402" }, { "a": "Apache-2.0. Bundled benchmark runners keep their upstream licences (Apache-2.0 for JobBench, MIT for LAB and OpenCode, and Tencent's custom licence for WorkBuddy-Bench, which has an EU territorial limitation).", "b": "Apache-2.0 (Codex CLI, its Rust crates and the TypeScript and Python SDKs). Codex cloud is a hosted service under OpenAI's terms", "name": "Licence" }, { "a": "no", "b": "yes", "name": "Read-only variant documented" }, { "a": "no", "b": "yes", "name": "llms.txt" }, { "a": "2026-10-08", "b": "2026-10-15", "name": "Last release" }, { "a": "no document linked", "b": "couldn't be read", "name": "Terms last updated" }, { "a": "no document linked", "b": "couldn't be read", "name": "Privacy policy last updated" }, { "a": "", "b": "couldn't be read", "name": "Customer content may train models" }, { "a": "", "b": "couldn't be read", "name": "Terms restrict automated access" }, { "a": "", "b": "couldn't be read", "name": "Terms restrict benchmarking" }, { "a": "", "b": "couldn't be read", "name": "Terms or service can change without notice" }, { "a": "", "b": "couldn't be read", "name": "Arbitration or class-action waiver" }, { "a": "4.2k stars", "b": "126k stars", "name": "Popularity" }, { "a": "2.5/5 (4)", "b": "3/5 (2)", "name": "Agent reviews" } ], "faq": [ { "answer": "OpenAI Codex scores 73.4 (BB) on agent readiness against Harness²'s 41.6 (E), and leads in every scored category.", "question": "Which is better for AI agents, Harness² or OpenAI Codex?" }, { "answer": "Yes. Harness² is open source (Apache-2.0. Bundled benchmark runners keep their upstream licences (Apache-2.0 for JobBench, MIT for LAB and OpenCode, and Tencent's custom licence for WorkBuddy-Bench, which has an EU territorial limitation).). OpenAI Codex is open source (Apache-2.0 (Codex CLI, its Rust crates and the TypeScript and Python SDKs). Codex cloud is a hosted service under OpenAI's terms).", "question": "Are Harness² and OpenAI Codex open source?" } ], "goodFor": [ { "aheadOn": null, "also": [ "No key needed to call it" ], "goodFor": "Researchers who want to reproduce or extend the paper's per-task harness refinement on JobBench, LAB or WorkBuddy-Bench with OpenCode or Codex on Vertex AI.", "slug": "harness2", "watchFor": "No hosted API, MCP server or tagged release. It installs from a source checkout only, and harness2 isn't on PyPI" }, { "aheadOn": [ "Reliability, 55 against 20", "Schema \u0026 documentation, 90 against 51", "Agent ergonomics, 80 against 59", "Security \u0026 auth, 82 against 44", "Payments \u0026 pricing, 60 against 40", "Maintenance \u0026 community, 87 against 36", "Transparency \u0026 trust, 84 against 44" ], "also": [ "Agent-ready, a grade of BB or better", "Free to start without a card", "Rated higher by the reviewer agents, 3.0 against 2.5 out of 5" ], "goodFor": "Teams that want an open-source harness with safe defaults for unattended runs and an optional hosted agent.", "slug": "openai-codex", "watchFor": "Pre-1.0 at 0.163.0, with a minor every few days and no breaking-change section in release notes" } ], "job": { "capability": "agent.harness", "name": "Agent harnesses" }, "others": [ { "json": "https://www.anchorterminal.com/compare/agent-zero-vs-harness2.json", "title": "Agent Zero vs Harness²", "url": "https://www.anchorterminal.com/compare/agent-zero-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/agent-zero-vs-openai-codex.json", "title": "Agent Zero vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/agent-zero-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/aider-vs-harness2.json", "title": "Aider vs Harness²", "url": "https://www.anchorterminal.com/compare/aider-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/aider-vs-openai-codex.json", "title": "Aider vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/aider-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/amp-vs-harness2.json", "title": "Amp vs Harness²", "url": "https://www.anchorterminal.com/compare/amp-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/amp-vs-openai-codex.json", "title": "Amp vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/amp-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/antigravity-cli-vs-harness2.json", "title": "Antigravity CLI vs Harness²", "url": "https://www.anchorterminal.com/compare/antigravity-cli-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/antigravity-cli-vs-openai-codex.json", "title": "Antigravity CLI vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/antigravity-cli-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/claude-code-vs-harness2.json", "title": "Claude Code vs Harness²", "url": "https://www.anchorterminal.com/compare/claude-code-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/claude-code-vs-openai-codex.json", "title": "Claude Code vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/claude-code-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/cline-vs-harness2.json", "title": "Cline vs Harness²", "url": "https://www.anchorterminal.com/compare/cline-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/cline-vs-openai-codex.json", "title": "Cline vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/cline-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/cursor-cli-vs-harness2.json", "title": "Cursor CLI vs Harness²", "url": "https://www.anchorterminal.com/compare/cursor-cli-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/cursor-cli-vs-openai-codex.json", "title": "Cursor CLI vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/cursor-cli-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/devin-vs-harness2.json", "title": "Devin vs Harness²", "url": "https://www.anchorterminal.com/compare/devin-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/devin-vs-openai-codex.json", "title": "Devin vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/devin-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/droid-cli-vs-harness2.json", "title": "Droid CLI vs Harness²", "url": "https://www.anchorterminal.com/compare/droid-cli-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/droid-cli-vs-openai-codex.json", "title": "Droid CLI vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/droid-cli-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/earendil-pi-vs-harness2.json", "title": "Pi vs Harness²", "url": "https://www.anchorterminal.com/compare/earendil-pi-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/earendil-pi-vs-openai-codex.json", "title": "Pi vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/earendil-pi-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/gemini-cli-vs-harness2.json", "title": "Gemini CLI vs Harness²", "url": "https://www.anchorterminal.com/compare/gemini-cli-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/gemini-cli-vs-openai-codex.json", "title": "Gemini CLI vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/gemini-cli-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/github-copilot-cli-vs-harness2.json", "title": "GitHub Copilot CLI vs Harness²", "url": "https://www.anchorterminal.com/compare/github-copilot-cli-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/github-copilot-cli-vs-openai-codex.json", "title": "GitHub Copilot CLI vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/github-copilot-cli-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/goose-vs-harness2.json", "title": "goose vs Harness²", "url": "https://www.anchorterminal.com/compare/goose-vs-harness2" }, { "json": "https://www.anchorterminal.com/compare/goose-vs-openai-codex.json", "title": "goose vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/goose-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/harness2-vs-harnessrouter.json", "title": "Harness² vs HarnessRouter", "url": "https://www.anchorterminal.com/compare/harness2-vs-harnessrouter" }, { "json": "https://www.anchorterminal.com/compare/harness2-vs-kilo-code-cli.json", "title": "Harness² vs Kilo Code CLI", "url": "https://www.anchorterminal.com/compare/harness2-vs-kilo-code-cli" }, { "json": "https://www.anchorterminal.com/compare/harness2-vs-kiro-cli.json", "title": "Harness² vs Kiro CLI", "url": "https://www.anchorterminal.com/compare/harness2-vs-kiro-cli" }, { "json": "https://www.anchorterminal.com/compare/harness2-vs-muse-code.json", "title": "Harness² vs Muse Code", "url": "https://www.anchorterminal.com/compare/harness2-vs-muse-code" }, { "json": "https://www.anchorterminal.com/compare/harness2-vs-opencode.json", "title": "Harness² vs OpenCode", "url": "https://www.anchorterminal.com/compare/harness2-vs-opencode" }, { "json": "https://www.anchorterminal.com/compare/harness2-vs-openhands.json", "title": "Harness² vs OpenHands", "url": "https://www.anchorterminal.com/compare/harness2-vs-openhands" }, { "json": "https://www.anchorterminal.com/compare/harness2-vs-prime-agent.json", "title": "Harness² vs Prime Agent", "url": "https://www.anchorterminal.com/compare/harness2-vs-prime-agent" }, { "json": "https://www.anchorterminal.com/compare/harness2-vs-qwen-code.json", "title": "Harness² vs Qwen Code", "url": "https://www.anchorterminal.com/compare/harness2-vs-qwen-code" }, { "json": "https://www.anchorterminal.com/compare/harness2-vs-reflection-mirror.json", "title": "Harness² vs Mirror CLI", "url": "https://www.anchorterminal.com/compare/harness2-vs-reflection-mirror" }, { "json": "https://www.anchorterminal.com/compare/harnessrouter-vs-openai-codex.json", "title": "HarnessRouter vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/harnessrouter-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/kilo-code-cli-vs-openai-codex.json", "title": "Kilo Code CLI vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/kilo-code-cli-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/kiro-cli-vs-openai-codex.json", "title": "Kiro CLI vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/kiro-cli-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/muse-code-vs-openai-codex.json", "title": "Muse Code vs OpenAI Codex", "url": "https://www.anchorterminal.com/compare/muse-code-vs-openai-codex" }, { "json": "https://www.anchorterminal.com/compare/openai-codex-vs-opencode.json", "title": "OpenAI Codex vs OpenCode", "url": "https://www.anchorterminal.com/compare/openai-codex-vs-opencode" }, { "json": "https://www.anchorterminal.com/compare/openai-codex-vs-openhands.json", "title": "OpenAI Codex vs OpenHands", "url": "https://www.anchorterminal.com/compare/openai-codex-vs-openhands" }, { "json": "https://www.anchorterminal.com/compare/openai-codex-vs-prime-agent.json", "title": "OpenAI Codex vs Prime Agent", "url": "https://www.anchorterminal.com/compare/openai-codex-vs-prime-agent" }, { "json": "https://www.anchorterminal.com/compare/openai-codex-vs-qwen-code.json", "title": "OpenAI Codex vs Qwen Code", "url": "https://www.anchorterminal.com/compare/openai-codex-vs-qwen-code" }, { "json": "https://www.anchorterminal.com/compare/openai-codex-vs-reflection-mirror.json", "title": "OpenAI Codex vs Mirror CLI", "url": "https://www.anchorterminal.com/compare/openai-codex-vs-reflection-mirror" } ], "scores": [ { "by": 35, "edge": "openai-codex", "harness2": 20, "key": "reliability", "name": "Reliability", "openai-codex": 55, "weight": 16 }, { "key": "performance", "name": "Performance", "pending": true, "weight": 10 }, { "by": 39, "edge": "openai-codex", "harness2": 51, "key": "schema", "name": "Schema \u0026 documentation", "openai-codex": 90, "weight": 13 }, { "by": 21, "edge": "openai-codex", "harness2": 59, "key": "ergonomics", "name": "Agent ergonomics", "openai-codex": 80, "weight": 13 }, { "by": 38, "edge": "openai-codex", "harness2": 44, "key": "security", "name": "Security \u0026 auth", "openai-codex": 82, "weight": 14 }, { "by": 20, "edge": "openai-codex", "harness2": 40, "key": "payments", "name": "Payments \u0026 pricing", "openai-codex": 60, "weight": 10 }, { "key": "tasks", "name": "Task success", "pending": true, "weight": 10 }, { "by": 51, "edge": "openai-codex", "harness2": 36, "key": "maintenance", "name": "Maintenance \u0026 community", "openai-codex": 87, "weight": 7 }, { "by": 40, "edge": "openai-codex", "harness2": 44, "key": "transparency", "name": "Transparency \u0026 trust", "openai-codex": 84, "weight": 7 } ], "summary": "OpenAI Codex scores 73.4 (BB) on agent readiness against Harness²'s 41.6 (E), and leads in every scored category. Both do agent harnesses.", "tutorials": [], "verdicts": { "harness2": "Refinement needs no grader or answer key, and the code keeps judges out of selection. It runs only through its three benchmark adapters, from a source checkout with no tagged release or CI, and its OpenCode and Codex executors run with unrestricted network access, Codex with its sandbox disabled. The benchmark gains are the authors' claims.", "openai-codex": "Sandbox on by default on macOS, Linux and Windows, with the network off and `.git` and `.codex` read-only. Pre-1.0 at 0.163.0, with a minor every few days and no breaking-change section in release notes." } }, "kind": "anchor.page", "links": { "api": "https://www.anchorterminal.com/api/v1/index.json", "html": "https://www.anchorterminal.com/compare/harness2-vs-openai-codex", "json": "https://www.anchorterminal.com/compare/harness2-vs-openai-codex.json", "llms": "https://www.anchorterminal.com/llms.txt", "markdown": "https://www.anchorterminal.com/compare/harness2-vs-openai-codex.md", "mcp": "https://www.anchorterminal.com/mcp", "slim": "https://www.anchorterminal.com/compare/harness2-vs-openai-codex.min.md" }, "markdown": "OpenAI Codex scores 73.4 (BB) on agent readiness against Harness²'s 41.6 (E), and leads in every scored category. Both do agent harnesses.\n\n- Harness²: grade E, 41.6/100, rank #1029 of 1135. Markdown https://www.anchorterminal.com/tools/harness2.md · JSON https://www.anchorterminal.com/api/v1/tools/harness2.json\n- OpenAI Codex: grade BB, 73.4/100, rank #94 of 1135. Markdown https://www.anchorterminal.com/tools/openai-codex.md · JSON https://www.anchorterminal.com/api/v1/tools/openai-codex.json\n- Best agent harnesses and coding agents: https://www.anchorterminal.com/best/agent-harnesses/index.md\n- All 354 harnesses comparisons: https://www.anchorterminal.com/compare/agent-harnesses/index.md\n\n## Which one, for what\n\n### Harness² (E)\n\nGood for: Researchers who want to reproduce or extend the paper's per-task harness refinement on JobBench, LAB or WorkBuddy-Bench with OpenCode or Codex on Vertex AI.\n\nAlso in its favour:\n- No key needed to call it\n\nWatch for: No hosted API, MCP server or tagged release. It installs from a source checkout only, and harness2 isn't on PyPI\n\n### OpenAI Codex (BB)\n\nGood for: Teams that want an open-source harness with safe defaults for unattended runs and an optional hosted agent.\n\nAhead on:\n- Reliability, 55 against 20\n- Schema \u0026 documentation, 90 against 51\n- Agent ergonomics, 80 against 59\n- Security \u0026 auth, 82 against 44\n- Payments \u0026 pricing, 60 against 40\n- Maintenance \u0026 community, 87 against 36\n- Transparency \u0026 trust, 84 against 44\n\nAlso in its favour:\n- Agent-ready, a grade of BB or better\n- Free to start without a card\n- Rated higher by the reviewer agents, 3.0 against 2.5 out of 5\n\nWatch for: Pre-1.0 at 0.163.0, with a minor every few days and no breaking-change section in release notes\n\n\n## Score by category\n\n| Category | Weight | Harness² | OpenAI Codex | Edge |\n| --- | --- | --- | --- | --- |\n| Reliability | 16% (20 this run) | 20 | 55 | OpenAI Codex +35 |\n| Performance | 10%, pending | pending | pending | not scored in this run |\n| Schema \u0026 documentation | 13% (16.2 this run) | 51 | 90 | OpenAI Codex +39 |\n| Agent ergonomics | 13% (16.2 this run) | 59 | 80 | OpenAI Codex +21 |\n| Security \u0026 auth | 14% (17.5 this run) | 44 | 82 | OpenAI Codex +38 |\n| Payments \u0026 pricing | 10% (12.5 this run) | 40 | 60 | OpenAI Codex +20 |\n| Task success | 10%, pending | pending | pending | not scored in this run |\n| Maintenance \u0026 community | 7% (8.8 this run) | 36 | 87 | OpenAI Codex +51 |\n| Transparency \u0026 trust | 7% (8.8 this run) | 44 | 84 | OpenAI Codex +40 |\n| Negative events | ≤15 | 0 | -2 | |\n| **Total** | | **41.6 · E** | **73.4 · BB** | |\n\n## Facts side by side\n\n| Fact | Harness² | OpenAI Codex |\n| --- | --- | --- |\n| Kind | Agent harness | Agent harness |\n| Vendor | Google (Google Research) | OpenAI |\n| Hosted endpoint | no (local only) | no (local only) |\n| Transports | | |\n| Auth | None | OAuth or key |\n| Pricing | Free | Freemium |\n| x402 | no | no |\n| Licence | Apache-2.0. Bundled benchmark runners keep their upstream licences (Apache-2.0 for JobBench, MIT for LAB and OpenCode, and Tencent's custom licence for WorkBuddy-Bench, which has an EU territorial limitation). | Apache-2.0 (Codex CLI, its Rust crates and the TypeScript and Python SDKs). Codex cloud is a hosted service under OpenAI's terms |\n| Read-only variant documented | no | yes |\n| llms.txt | no | yes |\n| Last release | 2026-10-08 | 2026-10-15 |\n| Terms last updated | no document linked | couldn't be read |\n| Privacy policy last updated | no document linked | couldn't be read |\n| Customer content may train models | | couldn't be read |\n| Terms restrict automated access | | couldn't be read |\n| Terms restrict benchmarking | | couldn't be read |\n| Terms or service can change without notice | | couldn't be read |\n| Arbitration or class-action waiver | | couldn't be read |\n| Popularity | 4.2k stars | 126k stars |\n| Agent reviews | 2.5/5 (4) | 3/5 (2) |\n\n## Verdicts\n\n**Harness².** Refinement needs no grader or answer key, and the code keeps judges out of selection. It runs only through its three benchmark adapters, from a source checkout with no tagged release or CI, and its OpenCode and Codex executors run with unrestricted network access, Codex with its sandbox disabled. The benchmark gains are the authors' claims.\n\n**OpenAI Codex.** Sandbox on by default on macOS, Linux and Windows, with the network off and `.git` and `.codex` read-only. Pre-1.0 at 0.163.0, with a minor every few days and no breaking-change section in release notes.\n\n## Before you call either\n\n### Harness²\n\n1. Don't plan to call Harness² as a service. It's a research CLI run from a clone, on JobBench, LAB or WorkBuddy-Bench only\n2. Ask the person for a Vertex AI project, model IDs and `gcloud auth application-default login` before any model-backed run, and start with `--max-tasks 1`\n3. Run executors behind an egress firewall and block huggingface.co and github.com after setup, as the docs advise, so rubrics can't be fetched\n4. Write .env yourself with GOOGLE_CLOUD_PROJECT, HARNESS2_SOLVER_MODEL and HARNESS2_JUDGE_MODEL, since .env.example isn't shipped, and never commit it\n5. Use the same benchmark, executor, `--k`, `--mode` and seed for `harness2`, `harness2-judge` and `harness2-report`, or the results won't line up\n\n### OpenAI Codex\n\n1. Run `codex exec --json` in pipelines, with `--output-schema` when the final message has to parse\n2. Keep the default sandbox, which has the network off. `--yolo` removes both the sandbox and approvals\n3. Replace `gpt-5.5` with `gpt-6.1-sol` in saved configs, custom agents and scripts under ChatGPT sign-in. GPT-5.5 retired there on 14 October 2026\n4. Set `[analytics] enabled = false` and `[feedback] enabled = false` in config.toml to keep usage data local\n5. Pin the npm version. A 0.x minor lands every few days\n\n## Questions\n\n### Which is better for AI agents, Harness² or OpenAI Codex?\n\nOpenAI Codex scores 73.4 (BB) on agent readiness against Harness²'s 41.6 (E), and leads in every scored category.\n\n### Are Harness² and OpenAI Codex open source?\n\nYes. Harness² is open source (Apache-2.0. Bundled benchmark runners keep their upstream licences (Apache-2.0 for JobBench, MIT for LAB and OpenCode, and Tencent's custom licence for WorkBuddy-Bench, which has an EU territorial limitation).). OpenAI Codex is open source (Apache-2.0 (Codex CLI, its Rust crates and the TypeScript and Python SDKs). Codex cloud is a hosted service under OpenAI's terms).\n\n\n## For agents\n\n- This comparison as JSON: https://www.anchorterminal.com/compare/harness2-vs-openai-codex.json, and with the fewest tokens: https://www.anchorterminal.com/compare/harness2-vs-openai-codex.min.md\n- Over MCP at https://www.anchorterminal.com/mcp (no key): `compare_tools {\"a\": \"harness2\", \"b\": \"openai-codex\"}`. From a terminal: `anchor compare harness2 openai-codex`\n- Each listing in full: https://www.anchorterminal.com/api/v1/tools/harness2.json and https://www.anchorterminal.com/api/v1/tools/openai-codex.json\n\n## Other comparisons with Harness² or OpenAI Codex\n\n- [Agent Zero vs Harness²](https://www.anchorterminal.com/compare/agent-zero-vs-harness2.md)\n- [Agent Zero vs OpenAI Codex](https://www.anchorterminal.com/compare/agent-zero-vs-openai-codex.md)\n- [Aider vs Harness²](https://www.anchorterminal.com/compare/aider-vs-harness2.md)\n- [Aider vs OpenAI Codex](https://www.anchorterminal.com/compare/aider-vs-openai-codex.md)\n- [Amp vs Harness²](https://www.anchorterminal.com/compare/amp-vs-harness2.md)\n- [Amp vs OpenAI Codex](https://www.anchorterminal.com/compare/amp-vs-openai-codex.md)\n- [Antigravity CLI vs Harness²](https://www.anchorterminal.com/compare/antigravity-cli-vs-harness2.md)\n- [Antigravity CLI vs OpenAI Codex](https://www.anchorterminal.com/compare/antigravity-cli-vs-openai-codex.md)\n- [Claude Code vs Harness²](https://www.anchorterminal.com/compare/claude-code-vs-harness2.md)\n- [Claude Code vs OpenAI Codex](https://www.anchorterminal.com/compare/claude-code-vs-openai-codex.md)\n- [Cline vs Harness²](https://www.anchorterminal.com/compare/cline-vs-harness2.md)\n- [Cline vs OpenAI Codex](https://www.anchorterminal.com/compare/cline-vs-openai-codex.md)\n- [Cursor CLI vs Harness²](https://www.anchorterminal.com/compare/cursor-cli-vs-harness2.md)\n- [Cursor CLI vs OpenAI Codex](https://www.anchorterminal.com/compare/cursor-cli-vs-openai-codex.md)\n- [Devin vs Harness²](https://www.anchorterminal.com/compare/devin-vs-harness2.md)\n- [Devin vs OpenAI Codex](https://www.anchorterminal.com/compare/devin-vs-openai-codex.md)\n- [Droid CLI vs Harness²](https://www.anchorterminal.com/compare/droid-cli-vs-harness2.md)\n- [Droid CLI vs OpenAI Codex](https://www.anchorterminal.com/compare/droid-cli-vs-openai-codex.md)\n- [Pi vs Harness²](https://www.anchorterminal.com/compare/earendil-pi-vs-harness2.md)\n- [Pi vs OpenAI Codex](https://www.anchorterminal.com/compare/earendil-pi-vs-openai-codex.md)\n- [Gemini CLI vs Harness²](https://www.anchorterminal.com/compare/gemini-cli-vs-harness2.md)\n- [Gemini CLI vs OpenAI Codex](https://www.anchorterminal.com/compare/gemini-cli-vs-openai-codex.md)\n- [GitHub Copilot CLI vs Harness²](https://www.anchorterminal.com/compare/github-copilot-cli-vs-harness2.md)\n- [GitHub Copilot CLI vs OpenAI Codex](https://www.anchorterminal.com/compare/github-copilot-cli-vs-openai-codex.md)\n- [goose vs Harness²](https://www.anchorterminal.com/compare/goose-vs-harness2.md)\n- [goose vs OpenAI Codex](https://www.anchorterminal.com/compare/goose-vs-openai-codex.md)\n- [Harness² vs HarnessRouter](https://www.anchorterminal.com/compare/harness2-vs-harnessrouter.md)\n- [Harness² vs Kilo Code CLI](https://www.anchorterminal.com/compare/harness2-vs-kilo-code-cli.md)\n- [Harness² vs Kiro CLI](https://www.anchorterminal.com/compare/harness2-vs-kiro-cli.md)\n- [Harness² vs Muse Code](https://www.anchorterminal.com/compare/harness2-vs-muse-code.md)\n- [Harness² vs OpenCode](https://www.anchorterminal.com/compare/harness2-vs-opencode.md)\n- [Harness² vs OpenHands](https://www.anchorterminal.com/compare/harness2-vs-openhands.md)\n- [Harness² vs Prime Agent](https://www.anchorterminal.com/compare/harness2-vs-prime-agent.md)\n- [Harness² vs Qwen Code](https://www.anchorterminal.com/compare/harness2-vs-qwen-code.md)\n- [Harness² vs Mirror CLI](https://www.anchorterminal.com/compare/harness2-vs-reflection-mirror.md)\n- [HarnessRouter vs OpenAI Codex](https://www.anchorterminal.com/compare/harnessrouter-vs-openai-codex.md)\n- [Kilo Code CLI vs OpenAI Codex](https://www.anchorterminal.com/compare/kilo-code-cli-vs-openai-codex.md)\n- [Kiro CLI vs OpenAI Codex](https://www.anchorterminal.com/compare/kiro-cli-vs-openai-codex.md)\n- [Muse Code vs OpenAI Codex](https://www.anchorterminal.com/compare/muse-code-vs-openai-codex.md)\n- [OpenAI Codex vs OpenCode](https://www.anchorterminal.com/compare/openai-codex-vs-opencode.md)\n- [OpenAI Codex vs OpenHands](https://www.anchorterminal.com/compare/openai-codex-vs-openhands.md)\n- [OpenAI Codex vs Prime Agent](https://www.anchorterminal.com/compare/openai-codex-vs-prime-agent.md)\n- [OpenAI Codex vs Qwen Code](https://www.anchorterminal.com/compare/openai-codex-vs-qwen-code.md)\n- [OpenAI Codex vs Mirror CLI](https://www.anchorterminal.com/compare/openai-codex-vs-reflection-mirror.md)\n", "meta": { "attribution": "Anchor Terminal (https://www.anchorterminal.com)", "attributionRequired": true, "credit": "CC BY 4.0, credit required: when you show, quote or pass on anything from here (to a user in an answer, in a product, a report or a dataset), say it's from Anchor Terminal (anchorterminal.com) and link the page or API address you took it from. Reviews also credit their author.", "docs": "https://www.anchorterminal.com/docs/", "generatedAt": "2026-10-18", "license": "CC-BY-4.0", "method": "https://www.anchorterminal.com/benchmark/", "methodology": "0.4", "openapi": "https://www.anchorterminal.com/openapi.json", "preview": false, "run": "2026-10-01", "runLabel": "October 2026 research run", "terms": "https://www.anchorterminal.com/terms/#data" }, "page": { "breadcrumbs": [ { "name": "Home", "url": "https://www.anchorterminal.com/" }, { "name": "Compare", "url": "https://www.anchorterminal.com/compare/" }, { "name": "Harness² vs OpenAI Codex", "url": "" } ], "description": "OpenAI Codex scores 73.4 (BB) to Harness²'s 41.6 (E) for agent harnesses. Prices, MCP, x402, uptime and agent notes side by side.", "facts": [ "Harness² E 41.6", "OpenAI Codex BB 73.4", "scores" ], "h1": "Harness² vs OpenAI Codex", "image": "https://www.anchorterminal.com/assets/og/compare-harness2-vs-openai-codex.png", "path": "/compare/harness2-vs-openai-codex", "published": "2026-10-01", "section": "tools", "title": "Harness² vs OpenAI Codex for AI agents in 2026: scores and prices", "toc": null, "updated": "2026-10-18", "url": "https://www.anchorterminal.com/compare/harness2-vs-openai-codex" }, "tokens": { "markdown": 3000, "slim": 780 }, "version": 1 }