{ "data": { "a": { "slug": "gauge-cli", "name": "Gauge CLI", "vendor": "Gauge", "vendorUrl": "https://www.withgauge.com/agents/", "kind": "cli", "category": "agent-observability", "summary": "Gauge CLI runs real coding agents (Claude Code, Codex, Pi, OpenCode) against your skills, CLIs, SDKs, MCP servers and docs in Docker, records where they get stuck, and lets your agent test fixes before release.", "url": "https://www.anchorterminal.com/tools/gauge-cli", "markdownUrl": "https://www.anchorterminal.com/tools/gauge-cli.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/gauge-cli.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/gauge-cli.json", "dns": "gauge-cli.grade.anchorterminal.com", "repo": "https://github.com/gauge-sh/gauge-cli", "license": "MIT", "transports": [], "packages": [ { "registry": "npm", "name": "@withgauge/cli" } ], "auth": "none", "authNotes": "Local mode needs no Gauge account, only credentials for the agent under test (native Claude or Codex logins, or API keys). Hosted Gauge uses `gauge auth login`, with a device flow for remote terminals.", "pricing": "freemium", "pricingNotes": "The CLI and local mode are free and MIT-licensed; agent usage is billed by your provider or subscription. Hosted Gauge runs on workspace credits, whose prices weren't read (checked 2026-11-06).", "priceSummary": "Freemium", "where": "local", "x402": { "level": "no", "evidence": "No payment protocol in the CLI or its README; hosted runs use workspace credits (checked 2026-11-06).", "endpoints": [] }, "toolCount": null, "popularity": { "githubStars": null, "npmWeekly": null, "pypiWeekly": null, "asOf": "2026-11-06" }, "docsUrl": "https://docs.withgauge.com/agents", "capabilities": [ "obs.evals" ], "tags": [ "open-source", "mit", "cli", "node", "docker", "telemetry", "new" ], "lastRelease": "2026-11-04", "graded": true, "anchor": { "graded": true, "score": 60.4, "grade": "C", "agentReady": false, "rank": 590, "ranked": true, "rankOf": 1354, "categoryRank": 13, "methodology": "0.7", "run": "2026-10-01", "scores": { "ergonomics": 75, "maintenance": 82, "payments": 40, "reliability": 60, "schema": 60, "security": 55, "transparency": 53 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "low", "date": "2026-11-06" }, "negative": 0, "verdict": "An open-source CLI that measures how coding agents get on with your product and checks fixes before you ship them, locally in Docker with no account. It's young and pre-1.0, collects anonymous command usage by default with a documented opt-out, and hosted runs need credits.", "bestFor": "Teams that build skills, CLIs, SDKs or MCP servers and want to see where coding agents get stuck with them, and to test fixes before release.", "strengths": [ "Runs Claude Code, Codex, Pi and OpenCode in fresh Docker sessions against working-tree or pre-release versions", "Local mode needs no Gauge account, only your existing agent credentials", "Dated changelog with six releases in about two weeks", "Instructions bundled in the CLI for coding agents, and -o json on read commands", "CI runs checks on every push and pull request" ], "weaknesses": [ "Collects anonymous command usage by default, with an opt-out", "Pre-1.0 and about two weeks old in this repository", "Docs, authenticated MCP servers and authenticated APIs need hosted Gauge and credits", "Hosted credit prices weren't found on the pages read" ], "agentNotes": [ "npm install -g @withgauge/cli, then gauge local runtime pull and gauge init --local in the project", "Run gauge install-skill so your coding agent finds the CLI's own instructions", "Set GAUGE_TELEMETRY_DISABLED=1 in CI if you don't want command usage collected", "Rebuild and pack an npm CLI or SDK before rerunning, so test sessions get the new tarball", "Keep .gauge/ private; it holds measurements and analyses" ], "metrics": { "kind": "local", "measured": false }, "reviewCount": 0, "avgRating": 0, "history": [ { "basis": "public evidence", "confidence": "low", "grade": "C", "methodology": "0.7", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 60.4 } ], "editorialScores": { "ergonomics": 75, "maintenance": 82, "payments": 40, "reliability": 60, "schema": 60, "security": 55, "transparency": 78 }, "provenanceScore": 27, "checked": { "categories": { "ergonomics": "2026-11-06", "maintenance": "2026-11-06", "payments": "2026-11-06", "reliability": "2026-11-06", "schema": "2026-11-06", "security": "2026-11-06", "transparency": "2026-11-06" }, "last": "2026-11-06", "oldest": "2026-11-06", "stale": false, "staleAfterDays": 90 } }, "connect": { "install": "npm install -g @withgauge/cli\ngauge local runtime pull\ngauge init --local\ngauge install-skill" }, "letme": { "capability": "https://letme.dev/obs.evals", "tool": "https://letme.dev/gauge-cli" }, "area": "developer", "provenance": { "legalEntity": "", "domain": "withgauge.com", "domainRegistered": "", "domainNote": "Not looked up: RDAP isn't requested under the 1 November 2026 rules.", "endpointOnVendorDomain": null, "terms": "", "privacy": "", "statusPage": "", "changelog": "https://github.com/gauge-sh/gauge-cli/blob/main/cli/CHANGELOG.md", "securityTxt": "none", "checked": "2026-11-06", "notes": [ "The npm package's author is Gauge \u003csupport@withgauge.com\u003e, its homepage agents.withgauge.com, and its repository field points at gauge-sh/gauge-cli. The licence names Gauge.", "Hosted Gauge's terms and privacy weren't read, and security.txt wasn't checked." ], "score": 27 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/gauge-cli.json" }, "answer": "Gauge CLI scores 60.4 (C) on agent readiness against Helicone AI Gateway + MCP's 42.7 (E), and leads in 5 of 7 scored categories. Helicone AI Gateway + MCP leads on schema \u0026 documentation and transparency \u0026 trust.", "b": { "slug": "helicone", "name": "Helicone AI Gateway + MCP", "vendor": "Helicone", "vendorUrl": "https://www.helicone.ai", "kind": "http-api", "category": "agent-observability", "summary": "Open-source LLM observability that logs requests through an OpenAI-compatible gateway (100+ models, fallbacks, caching, rate limits) or async logging.", "url": "https://www.anchorterminal.com/tools/helicone", "markdownUrl": "https://www.anchorterminal.com/tools/helicone.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/helicone.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/helicone.json", "dns": "helicone.grade.anchorterminal.com", "repo": "https://github.com/Helicone/helicone", "license": "Apache-2.0", "transports": [ "http", "stdio" ], "remoteUrl": "https://ai-gateway.helicone.ai", "packages": [ { "registry": "npm", "name": "@helicone/helpers" }, { "registry": "pypi", "name": "helicone-helpers" }, { "registry": "npm", "name": "@helicone/mcp" } ], "auth": "api-key", "authNotes": "Helicone API key (`sk-helicone-...`) as a Bearer token on the gateway and REST API. Bring your own provider keys, or bill model usage to Helicone credits, which since 25 August 2026 only works for organisations Helicone has switched on (others get a 403 asking them to contact support). The MCP server reads `HELICONE_API_KEY` from the environment.", "pricing": "freemium", "pricingNotes": "Hobby free with 10,000 requests a month, 1 GB of storage, 1 seat and 7 days of retention, no card. Pro $79 a month and Team $799 a month, each with 10,000 requests and 1 GB free then usage-based charges the pricing page doesn't itemise. The pricing page still shows 7-day trials, but a commit on 30 August 2026 removed the in-app self-serve upgrade paths and sends upgrades to a contact page. Enterprise custom. Gateway credits pass model prices through at 0% markup, but since 25 August 2026 need Helicone to enable them per organisation. Self-hosted is free under Apache-2.0 with Docker Compose or Helm, you run ClickHouse, Postgres and object storage (https://www.helicone.ai/pricing).", "priceSummary": "$79 / mo", "where": "both", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": 3, "popularity": { "githubStars": 6190, "npmWeekly": null, "pypiWeekly": null, "asOf": "2026-09-30" }, "docsUrl": "https://docs.helicone.ai", "llmsTxt": "https://docs.helicone.ai/llms.txt", "openapi": "https://docs.helicone.ai/ai-gateway.openapi.json", "capabilities": [ "obs.traces", "obs.gateway", "obs.prompts", "obs.datasets", "obs.evals" ], "tags": [ "hosted", "freemium", "no-card", "open-source", "self-hosted", "mcp", "llms-txt", "openapi", "typescript", "python", "webhooks" ], "lastRelease": "2026-10-19", "graded": true, "anchor": { "graded": true, "score": 42.7, "grade": "E", "agentReady": false, "rank": 1139, "ranked": true, "rankOf": 1354, "categoryRank": 21, "methodology": "0.7", "run": "2026-10-01", "scores": { "ergonomics": 70, "maintenance": 51, "payments": 30, "reliability": 50, "schema": 69, "security": 50, "transparency": 70 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-11-05" }, "negative": -13, "negativeNotes": [ "2026-08-30. Helicone's own security commit (INC-657) fixed an authenticated cross-tenant read of other organisations' decrypted provider keys (since 3 September also CVE-2026-85178, VulnCheck, High), a route that let any account mint proxy keys spending another organisation's provider key, a shared client that reused one organisation's OpenRouter key for others, an unauthenticated SSRF path and unauthenticated Stripe routes. Fixed, but disclosed only in the commit message, with no advisory or key-rotation notice that we found. -7 after decay for the fix (https://github.com/Helicone/helicone/commit/ca34549ea56f7ed587843f82d9cc19baa1f36ba4)", "2026-09-16. A second fix closed a platform-admin takeover with a forged API key and a cross-tenant read through HQL, reported through Helicone's disclosure programme the same day. Fixed within hours and again disclosed only in the commit. The fixes synced on 18 and 19 October (provider-key ownership, an SSRF guard on webhook destinations, filters that could escape the organisation scope) fold into this term. -3 (https://github.com/Helicone/helicone/commit/067d9290acb4f1fc9320e902fc67b4b399b50363)", "2026-10-18. The published REST API stopped returning provider keys in plaintext, and `GET /v1/api-keys/provider-key/{providerKeyId}` now rejects Helicone API keys and needs a signed-in owner or admin. Member, API-key and organisation writes now also need an owner or admin session, and Python evaluators can no longer be created or edited. Calls that worked before now fail, and the change was posted only in the Swagger file and commit messages when it shipped, with no earlier notice. -3 (https://github.com/Helicone/helicone/commit/0d37ddd)" ], "verdict": "One base-URL change gives logging, caching, fallbacks and rate limits for 100+ models. Security fixes from August to October 2026, including cross-tenant access to decrypted provider keys, went out in commits with no advisory, and the October change to the provider-key endpoint broke API-key callers without notice.", "bestFor": "A team that wants request logging and cost tracking through a gateway with almost no code, and could swap it out later.", "strengths": [ "One base-URL change gives logging, caching, fallbacks and rate limits for 100+ models", "Apache-2.0 with Docker Compose and Helm self-hosting", "Per-plan ingestion and API rate limits published on the pricing page", "OpenAPI for the gateway, a Swagger file for the REST API and an llms.txt", "Free Hobby plan with 10,000 requests a month and no card" ], "weaknesses": [ "Security fixes from 30 August to 19 October 2026, including cross-tenant access to decrypted provider keys, disclosed only in commit messages", "The provider-key endpoint stopped accepting API keys and returning key values in October 2026, with no notice", "No release or changelog entry since 2025, and ownership unclear since the site said in October 2026 it is no longer part of Mintlify", "The status page had no incident history and left out the AI Gateway (1 October check)", "Privacy policy last updated in February 2023, with no retention period or DPA (1 October check)" ], "agentNotes": [ "Bring your own provider keys. Helicone credits return 403 unless support has enabled them for the organisation", "Rotate any provider keys stored in Helicone before 30 August 2026", "Don't read provider keys back with a Helicone API key. The endpoint now needs an owner or admin session and returns masked values", "Leave `use_ai_gateway` out of the MCP client's allowed tools unless the agent is meant to spend on model calls", "Add `Helicone-Session-Id` and `Helicone-Session-Path` headers to group an agent's steps into one session", "Use the EU host (eu.helicone.ai) if the account was created there" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 2, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "E", "methodology": "0.7", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 42.7 } ], "editorialScores": { "ergonomics": 70, "maintenance": 51, "payments": 30, "reliability": 50, "schema": 69, "security": 50, "transparency": 60 }, "provenanceScore": 80, "checked": { "categories": { "ergonomics": "2026-11-05", "maintenance": "2026-11-05", "payments": "2026-11-05", "reliability": "2026-11-05", "schema": "2026-11-05", "security": "2026-11-05", "transparency": "2026-11-05" }, "last": "2026-11-05", "lastMoved": "2026-11-05", "oldest": "2026-11-05", "stale": false, "staleAfterDays": 90 } }, "connect": { "http": "curl https://ai-gateway.helicone.ai/chat/completions -H \"Authorization: Bearer $HELICONE_API_KEY\" \\\n -H \"content-type: application/json\" -d '{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "claudeCode": "claude mcp add helicone -e HELICONE_API_KEY=$HELICONE_API_KEY -- npx -y @helicone/mcp@latest", "config": { "mcpServers": { "helicone": { "args": [ "-y", "@helicone/mcp@latest" ], "command": "npx", "env": { "HELICONE_API_KEY": "${HELICONE_API_KEY}" } } } } }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/helicone" }, "area": "developer", "unitPrices": [ { "item": "Pro plan", "unit": "month", "usd": 79, "note": "10,000 requests and 1 GB free, then usage-based" }, { "item": "Team plan", "unit": "month", "usd": 799, "note": "5 organisations, SOC 2 and HIPAA, 10,000 requests and 1 GB free, then usage-based" } ], "provenance": { "legalEntity": "Helicone, Inc.", "domain": "helicone.ai", "domainRegistered": "2022-11-14", "endpointOnVendorDomain": true, "terms": "https://www.helicone.ai/terms", "privacy": "https://www.helicone.ai/privacy", "statusPage": "https://status.helicone.ai", "changelog": "https://www.helicone.ai/changelog", "securityTxt": "none", "checked": "2026-11-05", "notes": [ "Terms still name Helicone, Inc. and were last updated 2024-09-17. They bar using any robot, spider or other automatic means to access the web pages at helicone.ai", "Since 18 October 2026 the site banner says Helicone is no longer part of Mintlify, with details to come" ], "score": 80 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/helicone.json", "live": { "slug": "helicone", "probe": { "target": "https://ai-gateway.helicone.ai", "method": "get", "lastAt": "2026-11-06T09:47:31.386498535Z", "lastOk": true, "lastStatus": 405, "lastMs": 52, "authRequired": false, "uptime24h": 100, "uptime30d": 100, "p50ms24h": 60, "p95ms24h": 198, "samples24h": 221, "samples30d": 7856, "days": [ { "date": "2026-10-06", "probes": 272, "ok": 272 }, { "date": "2026-10-07", "probes": 272, "ok": 272 }, { "date": "2026-10-08", "probes": 268, "ok": 268 }, { "date": "2026-10-09", "probes": 250, "ok": 250 }, { "date": "2026-10-10", "probes": 247, "ok": 247 }, { "date": "2026-10-11", "probes": 247, "ok": 247 }, { "date": "2026-10-12", "probes": 247, "ok": 247 }, { "date": "2026-10-13", "probes": 248, "ok": 248 }, { "date": "2026-10-14", "probes": 249, "ok": 249 }, { "date": "2026-10-15", "probes": 253, "ok": 253 }, { "date": "2026-10-16", "probes": 252, "ok": 252 }, { "date": "2026-10-17", "probes": 253, "ok": 253 }, { "date": "2026-10-18", "probes": 256, "ok": 256 }, { "date": "2026-10-19", "probes": 255, "ok": 255 }, { "date": "2026-10-20", "probes": 252, "ok": 252 }, { "date": "2026-10-21", "probes": 253, "ok": 253 }, { "date": "2026-10-22", "probes": 253, "ok": 253 }, { "date": "2026-10-23", "probes": 248, "ok": 248 }, { "date": "2026-10-24", "probes": 251, "ok": 251 }, { "date": "2026-10-25", "probes": 250, "ok": 250 }, { "date": "2026-10-26", "probes": 252, "ok": 252 }, { "date": "2026-10-27", "probes": 249, "ok": 249 }, { "date": "2026-10-28", "probes": 251, "ok": 251 }, { "date": "2026-10-29", "probes": 248, "ok": 248 }, { "date": "2026-10-30", "probes": 249, "ok": 249 }, { "date": "2026-10-31", "probes": 251, "ok": 251 }, { "date": "2026-11-01", "probes": 249, "ok": 249 }, { "date": "2026-11-02", "probes": 250, "ok": 250 }, { "date": "2026-11-03", "probes": 248, "ok": 248 }, { "date": "2026-11-04", "probes": 246, "ok": 246 }, { "date": "2026-11-05", "probes": 185, "ok": 185 }, { "date": "2026-11-06", "probes": 102, "ok": 102 } ] }, "vendorStatus": { "page": "https://status.helicone.ai", "indicator": "unknown", "summary": "no machine-readable status found", "checkedAt": "2026-11-06T05:41:50.804673313Z" }, "versions": [ { "registry": "github", "name": "Helicone/helicone", "version": "v2025.08.21-1", "released": "2025-08-21", "seenAt": "2026-11-05T21:36:38.987532867Z" }, { "registry": "npm", "name": "@helicone/helpers", "version": "1.8.3", "seenAt": "2026-11-05T21:36:35.640310825Z" }, { "registry": "npm", "name": "@helicone/mcp", "version": "0.1.6", "seenAt": "2026-11-05T21:36:37.851574043Z" }, { "registry": "pypi", "name": "helicone-helpers", "version": "1.2.1", "released": "2025-11-08", "seenAt": "2026-11-05T21:36:37.66452219Z" } ], "githubStars": 6269, "npmWeekly": 8601, "pypiWeekly": 8753, "securityTxt": { "url": "https://helicone.ai/.well-known/security.txt", "state": "none", "checkedAt": "2026-11-05T15:56:34.878387044Z" }, "llmsTxt": { "url": "https://docs.helicone.ai/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-11-05T15:26:14.953477204Z" }, "domain": { "domain": "helicone.ai", "registered": "2022-11-14", "source": "https://rdap.identitydigital.services/rdap/domain/helicone.ai", "checkedAt": "2026-11-01T14:10:16.170313247Z" }, "pages": [ { "url": "https://www.helicone.ai/changelog", "kind": "changelog", "status": 200, "checkedAt": "2026-11-05T13:13:23.492803928Z", "changedAt": "2026-11-02T07:33:30.821695891Z", "fingerprint": "4a40996781eb" }, { "url": "https://www.helicone.ai/blog/joining-mintlify", "kind": "deprecations", "status": 404, "checkedAt": "2026-11-05T13:13:21.308389708Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "a82c441b1103" }, { "url": "https://www.helicone.ai/pricing", "kind": "pricing", "status": 200, "checkedAt": "2026-11-05T13:13:25.605168918Z", "changedAt": "2026-11-02T07:33:32.935782418Z", "fingerprint": "b33c03e2f514" }, { "url": "https://www.helicone.ai/privacy", "kind": "privacy", "status": 200, "checkedAt": "2026-11-05T13:13:27.446895681Z", "changedAt": "2026-11-02T07:33:34.662375435Z", "fingerprint": "e01f10d4d101" }, { "url": "https://www.helicone.ai/terms", "kind": "terms", "status": 200, "checkedAt": "2026-11-05T13:13:29.59912339Z", "changedAt": "2026-11-02T07:33:37.036571296Z", "fingerprint": "c94215b00a90" } ], "github": { "repo": "helicone/helicone", "checkedAt": "2026-11-02T08:54:18.155177255Z", "stars": 6251, "archived": false, "issuesEnabled": true, "pushedAt": "2026-10-28T15:59:52Z", "openIssues": 63, "openPRs": 120, "issuesOpened90d": 27, "issuesClosed90d": 11, "prsMerged90d": 14, "ci": { "branch": "main", "runs": 50, "success": 46, "failure": 4, "other": 0, "latest": "success", "latestAt": "2026-10-28T16:06:32Z" }, "replies": { "sampled": 25, "unanswered": 24, "medianDays": 22.6 } }, "updatedAt": "2026-11-06T09:47:31.386498535Z" } }, "edges": [ { "by": 31, "category": "maintenance", "leads": "gauge-cli", "name": "Maintenance \u0026 community", "reasons": { "a": "CLI 0.28.0 on 4 November 2026 (30). Six releases in 90 days (20). Issues and pull requests weren't read; a support address and issue tracker are published (12 of 25). Current official package (15). CI on push and pull request (5 of 10, results not read). 30 + 20 + 12 + 15 + 5 = 82.", "b": "No tagged release since v2025.08.21-1 (21 August 2025) and no changelog entry since 26 November 2025. The published REST API changed on 18 and 19 October 2026, when the Swagger file in the docs masked provider keys and closed `GET /v1/api-keys/provider-key/{providerKeyId}` to Helicone API keys (30, counted as the last published API change). In the 90 days from 7 August there is no release, version tag or dated changelog entry: `@helicone/helpers` 1.8.5, `helicone-helpers` 1.2.1 and `@helicone/mcp` 0.1.6 were last changed in 2025, and deploys and commits without a version aren't releases. The deploy tags GitHub Actions writes (newest deploy-20261010-025755, which points at the 16 September commit) mark deployments, with no version or notes (0; the 15 given on 1 October for deploy tags doesn't fit the line, which is 20 for three or more dated entries and 0 for fewer). 56 open issues on 1 October; reply times unread. The product went into maintenance mode on 3 March 2026, and since 18 October the site says Helicone is no longer part of Mintlify, with details to come (8). Helper SDKs in two languages, not updated in 2026 (8). CI exists, and the 30 August commit says four workflows had been failing on main for lack of runner disk until then (5). 30 + 0 + 8 + 8 + 5 = 51." }, "scores": { "a": 82, "b": 51 } }, { "by": 17, "category": "transparency", "leads": "helicone", "name": "Transparency \u0026 trust", "reasons": { "a": "MIT (30). Local data stays in .gauge/ and telemetry fields are listed; hosted data handling wasn't read (18 of 30). Dated changelog, no deprecation policy (10 of 20). Telemetry on by default, disclosed with an opt-out (20). 30 + 18 + 10 + 20 = 78.", "b": "Apache-2.0 (30). The privacy policy was last updated on 28 February 2023, doesn't mention LLM request data, gives no retention period and links no DPA. Retention per plan is on the pricing page. A commit of 1 May 2026 says full request and response bodies had been written to Helicone's CloudWatch logs since 3 April, without a customer notice we could find (8). Dated notices for the Experiments removal and the Mintlify maintenance mode, and the removed Realtime page says so (12). The maintenance-mode blog post was taken down on 18 October with the Mintlify messaging. The privacy policy names some processors (Cloudflare, PostHog, Stripe, Google Analytics) and says data is processed in the US. No full subprocessor list (10)." }, "scores": { "a": 53, "b": 70 } }, { "by": 10, "category": "reliability", "leads": "gauge-cli", "name": "Reliability", "reasons": { "a": "Local package. Installs from npm as @withgauge/cli with Node 22.13 or later and Docker stated (20). checks.yml runs npm run check on push and pull request across a matrix, and the repository has 76 test files; whether main passes wasn't read (20 of 25). Issues weren't read (5, unchecked). A dated changelog with Added sections per release (15). Version 0.28.0 (0). 20 + 20 + 5 + 15 + 0 = 60.", "b": "Better Stack page at status.helicone.ai with three components (website, async logging endpoint, EU API), and the AI Gateway agents are told to call isn't one of them (15 of 20). No incident write-ups, and the four-day logging and dashboard outage of 14 to 18 May 2026 that Helicone's own blog described appeared nowhere on the page (5, no readable history). Ingestion and API rate limits published per plan on the pricing page (15). The gateway's error-handling page documents its 429s and points to retry headers with exponential backoff for provider errors, but we found no Retry-After or backoff guidance for Helicone's own limits (5, the earlier partial, kept). No SLA found (0). Gateway and logging are GA (10). Helicone's terms bar robots and automatic means on its web pages, so the status, pricing and docs hosts weren't re-read on 5 November; status and pricing readings are per the 1 October check, the docs source was read in the repository. 15 + 5 + 15 + 5 + 0 + 10 = 50." }, "scores": { "a": 60, "b": 50 } }, { "by": 10, "category": "payments", "leads": "gauge-cli", "name": "Payments \u0026 pricing", "reasons": { "a": "Scored on the hosted option. No payment protocol (0). Credit prices weren't found on the pages read (0). Local mode is free with no card (20). Local mode needs no sign-up (20). 0 + 0 + 20 + 20 = 40.", "b": "No x402 or other machine payment (0). Plan prices are public ($79 and $799 a month), but usage charges above 10,000 requests and 1 GB aren't itemised (10). Hobby is free with no card (20). A person has to sign up in a browser to get a key. Self-hosting is free but isn't an agent route (0)." }, "scores": { "a": 40, "b": 30 } }, { "by": 9, "category": "schema", "leads": "helicone", "name": "Schema \u0026 documentation", "reasons": { "a": "CLI reading. No machine-readable contract, -o json on read commands (10 of 25). No llms.txt; instructions bundled in the CLI for agents (5). The README says what it finds and when local or hosted fits (16 of 20). Commands with help text; typed flags (9 of 15). Worked examples, no error reference (8 of 15). Dated changelog and semver tags (12 of 15). 10 + 5 + 16 + 9 + 8 + 12 = 60.", "b": "OpenAPI for the AI Gateway and a Swagger file for the REST API, both in the docs (25). llms.txt at docs.helicone.ai (10). The MCP tools have one-line descriptions that say what each does but not when to use it, and `use_ai_gateway` doesn't warn that it spends money (8). Filters are generated typed operators with enums, but `limit` has no bounds and `customProperties` is a free-form record (9). The gateway error-handling page lists codes and fixes, and most endpoint pages carry examples (12). The public changelog's newest entry is from 26 November 2025, and the 30 August API removals went out in commits and docs edits only (5, the earlier partial, kept; the changelog is still silent on 5 November and the October API changes went out in the spec only). 25 + 10 + 8 + 9 + 12 + 5 = 69." }, "scores": { "a": 60, "b": 69 } }, { "by": 5, "category": "ergonomics", "leads": "gauge-cli", "name": "Agent ergonomics", "reasons": { "a": "CLI reading for an agent driving it. Bundled instructions match the installed version, and output is filterable with -o json (20 of 25). Run summaries and Findings with queryable fields (15 of 20). Errors not documented as a reference (10 of 20). Reruns from checkpoints and a recheck command with a confirm step (15 of 20). Four commands to start, Node only (15 of 15). 20 + 15 + 10 + 15 + 15 = 75.", "b": "The MCP server has 3 tools (`query_requests`, `query_sessions`, `use_ai_gateway`), though the docs list 2. The recursive filter schemas add some weight (20). Offset, limit, filters, sort and `includeBodies` off by default (20). Gateway errors are documented with codes and fixes. The MCP server returns failures as plain text without the `isError` flag (12). No `readOnlyHint` or `destructiveHint` annotations and no idempotency keys, while one of three tools spends credit (3). Only time bounds are required. Helper SDKs for TypeScript and Python (15)." }, "scores": { "a": 75, "b": 70 } } ], "facts": [ { "a": "cli", "b": "HTTP API", "name": "Kind" }, { "a": "Gauge", "b": "Helicone", "name": "Vendor" }, { "a": "no (local only)", "b": "https://ai-gateway.helicone.ai", "name": "Hosted endpoint" }, { "a": "", "b": "HTTP, stdio", "name": "Transports" }, { "a": "None", "b": "API key", "name": "Auth" }, { "a": "Freemium", "b": "Freemium", "name": "Pricing" }, { "a": "no", "b": "no", "name": "x402" }, { "a": "MIT", "b": "Apache-2.0", "name": "Licence" }, { "a": "none", "b": "3", "name": "Tools exposed" }, { "a": "no", "b": "no", "name": "Read-only variant documented" }, { "a": "no", "b": "yes", "name": "llms.txt" }, { "a": "2026-11-04", "b": "2026-10-19", "name": "Last release" }, { "a": "no document linked", "b": "2024-09-17", "name": "Terms last updated" }, { "a": "no document linked", "b": "2023-02-28", "name": "Privacy policy last updated" }, { "a": "not read yet", "b": "not found in the text", "name": "Customer content may train models" }, { "a": "not read yet", "b": "yes", "name": "Terms restrict automated access" }, { "a": "not read yet", "b": "not found in the text", "name": "Terms restrict benchmarking" }, { "a": "not read yet", "b": "yes", "name": "Terms or service can change without notice" }, { "a": "not read yet", "b": "not found in the text", "name": "Arbitration or class-action waiver" }, { "a": "none", "b": "6.2k stars", "name": "Popularity" }, { "a": "none", "b": "2/5 (2)", "name": "Agent reviews" } ], "faq": [ { "answer": "Gauge CLI scores 60.4 (C) on agent readiness against Helicone AI Gateway + MCP's 42.7 (E), and leads in 5 of 7 scored categories. Helicone AI Gateway + MCP leads on schema \u0026 documentation and transparency \u0026 trust.", "question": "Which is better for AI agents, Gauge CLI or Helicone AI Gateway + MCP?" }, { "answer": "No hosted endpoint is listed for Gauge CLI. Helicone AI Gateway + MCP has a hosted endpoint at https://ai-gateway.helicone.ai.", "question": "Can an agent call Gauge CLI and Helicone AI Gateway + MCP without installing anything?" }, { "answer": "Yes. Gauge CLI is open source (MIT). Helicone AI Gateway + MCP is open source (Apache-2.0).", "question": "Are Gauge CLI and Helicone AI Gateway + MCP open source?" } ], "goodFor": [ { "aheadOn": [ "Reliability, 60 against 50", "Agent ergonomics, 75 against 70", "Security \u0026 auth, 55 against 50", "Payments \u0026 pricing, 40 against 30", "Maintenance \u0026 community, 82 against 51" ], "also": [ "No key needed to call it", "No incidents deducted, where Helicone AI Gateway + MCP loses 13 points for them" ], "goodFor": "Teams that build skills, CLIs, SDKs or MCP servers and want to see where coding agents get stuck with them, and to test fixes before release.", "slug": "gauge-cli", "watchFor": "Collects anonymous command usage by default, with an opt-out" }, { "aheadOn": [ "Schema \u0026 documentation, 69 against 60", "Transparency \u0026 trust, 70 against 53" ], "also": [ "A hosted endpoint, with nothing to install", "Runs on your own machine", "Free to start without a card" ], "goodFor": "A team that wants request logging and cost tracking through a gateway with almost no code, and could swap it out later.", "slug": "helicone", "watchFor": "Security fixes from 30 August to 19 October 2026, including cross-tenant access to decrypted provider keys, disclosed only in commit messages" } ], "history": { "gauge-cli": null, "helicone": [ { "causes": [ "correction", "event" ], "date": "2026-11-05", "delta": -4.3, "from": 47, "fromGrade": "D", "to": 42.7, "toGrade": "E" }, { "causes": [ "method" ], "date": "2026-10-27", "delta": 0.1, "from": 46.9, "fromGrade": "D", "to": 47, "toGrade": "D" }, { "causes": [ "method" ], "date": "2026-10-08", "delta": -0.2, "from": 47.1, "fromGrade": "D", "to": 46.9, "toGrade": "D" } ] }, "job": { "capability": "obs.evals", "name": "Evaluations" }, "others": [ { "json": "https://www.anchorterminal.com/compare/agent-tree-vs-helicone.json", "title": "Agent Tree vs Helicone AI Gateway + MCP", "url": "https://www.anchorterminal.com/compare/agent-tree-vs-helicone" }, { "json": "https://www.anchorterminal.com/compare/agents-tree-vs-helicone.json", "title": "agents-tree vs Helicone AI Gateway + MCP", "url": "https://www.anchorterminal.com/compare/agents-tree-vs-helicone" }, { "json": "https://www.anchorterminal.com/compare/arize-phoenix-vs-helicone.json", "title": "Arize Phoenix vs Helicone AI Gateway + MCP", "url": "https://www.anchorterminal.com/compare/arize-phoenix-vs-helicone" }, { "json": "https://www.anchorterminal.com/compare/baserun-vs-helicone.json", "title": "Baserun vs Helicone AI Gateway + MCP", "url": "https://www.anchorterminal.com/compare/baserun-vs-helicone" }, { "json": "https://www.anchorterminal.com/compare/botgauge-vs-helicone.json", "title": "BotGauge vs Helicone AI Gateway + MCP", "url": "https://www.anchorterminal.com/compare/botgauge-vs-helicone" }, { "json": "https://www.anchorterminal.com/compare/braintrust-vs-helicone.json", "title": "Braintrust API + MCP vs Helicone AI Gateway + MCP", "url": "https://www.anchorterminal.com/compare/braintrust-vs-helicone" }, { "json": "https://www.anchorterminal.com/compare/galileo-vs-helicone.json", "title": "Galileo API + MCP vs Helicone AI Gateway + MCP", "url": "https://www.anchorterminal.com/compare/galileo-vs-helicone" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-honeyhive.json", "title": "Helicone AI Gateway + MCP vs HoneyHive", "url": "https://www.anchorterminal.com/compare/helicone-vs-honeyhive" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-laminar.json", "title": "Helicone AI Gateway + MCP vs Laminar API + MCP", "url": "https://www.anchorterminal.com/compare/helicone-vs-laminar" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-langfuse.json", "title": "Helicone AI Gateway + MCP vs Langfuse API + MCP", "url": "https://www.anchorterminal.com/compare/helicone-vs-langfuse" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-langsmith.json", "title": "Helicone AI Gateway + MCP vs LangSmith API + MCP", "url": "https://www.anchorterminal.com/compare/helicone-vs-langsmith" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-langwatch.json", "title": "Helicone AI Gateway + MCP vs LangWatch", "url": "https://www.anchorterminal.com/compare/helicone-vs-langwatch" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-mlflow-tracing.json", "title": "Helicone AI Gateway + MCP vs MLflow Tracing", "url": "https://www.anchorterminal.com/compare/helicone-vs-mlflow-tracing" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-navige.json", "title": "Helicone AI Gateway + MCP vs Navige", "url": "https://www.anchorterminal.com/compare/helicone-vs-navige" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-opik.json", "title": "Helicone AI Gateway + MCP vs Opik API + MCP", "url": "https://www.anchorterminal.com/compare/helicone-vs-opik" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-overmind.json", "title": "Helicone AI Gateway + MCP vs Overmind", "url": "https://www.anchorterminal.com/compare/helicone-vs-overmind" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-prefactor.json", "title": "Helicone AI Gateway + MCP vs Prefactor", "url": "https://www.anchorterminal.com/compare/helicone-vs-prefactor" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-pydantic-logfire.json", "title": "Helicone AI Gateway + MCP vs Pydantic Logfire", "url": "https://www.anchorterminal.com/compare/helicone-vs-pydantic-logfire" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-respan.json", "title": "Helicone AI Gateway + MCP vs Respan API + MCP", "url": "https://www.anchorterminal.com/compare/helicone-vs-respan" }, { "json": "https://www.anchorterminal.com/compare/helicone-vs-wandb-weave.json", "title": "Helicone AI Gateway + MCP vs W\u0026B Weave", "url": "https://www.anchorterminal.com/compare/helicone-vs-wandb-weave" }, { "json": "https://www.anchorterminal.com/compare/arize-phoenix-vs-gauge-cli.json", "title": "Arize Phoenix vs Gauge CLI", "url": "https://www.anchorterminal.com/compare/arize-phoenix-vs-gauge-cli" }, { "json": "https://www.anchorterminal.com/compare/baserun-vs-gauge-cli.json", "title": "Baserun vs Gauge CLI", "url": "https://www.anchorterminal.com/compare/baserun-vs-gauge-cli" }, { "json": "https://www.anchorterminal.com/compare/botgauge-vs-gauge-cli.json", "title": "BotGauge vs Gauge CLI", "url": "https://www.anchorterminal.com/compare/botgauge-vs-gauge-cli" }, { "json": "https://www.anchorterminal.com/compare/braintrust-vs-gauge-cli.json", "title": "Braintrust API + MCP vs Gauge CLI", "url": "https://www.anchorterminal.com/compare/braintrust-vs-gauge-cli" }, { "json": "https://www.anchorterminal.com/compare/deepeval-vs-gauge-cli.json", "title": "DeepEval vs Gauge CLI", "url": "https://www.anchorterminal.com/compare/deepeval-vs-gauge-cli" }, { "json": "https://www.anchorterminal.com/compare/deepeval-vs-helicone.json", "title": "DeepEval vs Helicone AI Gateway + MCP", "url": "https://www.anchorterminal.com/compare/deepeval-vs-helicone" }, { "json": "https://www.anchorterminal.com/compare/galileo-vs-gauge-cli.json", "title": "Galileo API + MCP vs Gauge CLI", "url": "https://www.anchorterminal.com/compare/galileo-vs-gauge-cli" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-honeyhive.json", "title": "Gauge CLI vs HoneyHive", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-honeyhive" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-laminar.json", "title": "Gauge CLI vs Laminar API + MCP", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-laminar" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-langfuse.json", "title": "Gauge CLI vs Langfuse API + MCP", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-langfuse" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-langsmith.json", "title": "Gauge CLI vs LangSmith API + MCP", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-langsmith" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-langwatch.json", "title": "Gauge CLI vs LangWatch", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-langwatch" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-mlflow-tracing.json", "title": "Gauge CLI vs MLflow Tracing", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-mlflow-tracing" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-opik.json", "title": "Gauge CLI vs Opik API + MCP", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-opik" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-overmind.json", "title": "Gauge CLI vs Overmind", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-overmind" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-prefactor.json", "title": "Gauge CLI vs Prefactor", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-prefactor" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-pydantic-logfire.json", "title": "Gauge CLI vs Pydantic Logfire", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-pydantic-logfire" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-respan.json", "title": "Gauge CLI vs Respan API + MCP", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-respan" }, { "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-wandb-weave.json", "title": "Gauge CLI vs W\u0026B Weave", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-wandb-weave" } ], "panel": null, "scores": [ { "by": 10, "edge": "gauge-cli", "gauge-cli": 60, "helicone": 50, "key": "reliability", "name": "Reliability", "weight": 16 }, { "key": "performance", "name": "Performance", "pending": true, "weight": 10 }, { "by": 9, "edge": "helicone", "gauge-cli": 60, "helicone": 69, "key": "schema", "name": "Schema \u0026 documentation", "weight": 13 }, { "by": 5, "edge": "gauge-cli", "gauge-cli": 75, "helicone": 70, "key": "ergonomics", "name": "Agent ergonomics", "weight": 13 }, { "by": 5, "edge": "gauge-cli", "gauge-cli": 55, "helicone": 50, "key": "security", "name": "Security \u0026 auth", "weight": 14 }, { "by": 10, "edge": "gauge-cli", "gauge-cli": 40, "helicone": 30, "key": "payments", "name": "Payments \u0026 pricing", "weight": 10 }, { "key": "tasks", "name": "Task success", "pending": true, "weight": 10 }, { "by": 31, "edge": "gauge-cli", "gauge-cli": 82, "helicone": 51, "key": "maintenance", "name": "Maintenance \u0026 community", "weight": 7 }, { "by": 17, "edge": "helicone", "gauge-cli": 53, "helicone": 70, "key": "transparency", "name": "Transparency \u0026 trust", "weight": 7 } ], "summary": "Gauge CLI scores 60.4 (C) on agent readiness against Helicone AI Gateway + MCP's 42.7 (E), and leads in 5 of 7 scored categories. Helicone AI Gateway + MCP leads on schema \u0026 documentation and transparency \u0026 trust. Both do evaluations.", "tasks": null, "tutorials": [], "verdicts": { "gauge-cli": "An open-source CLI that measures how coding agents get on with your product and checks fixes before you ship them, locally in Docker with no account. It's young and pre-1.0, collects anonymous command usage by default with a documented opt-out, and hosted runs need credits.", "helicone": "One base-URL change gives logging, caching, fallbacks and rate limits for 100+ models. Security fixes from August to October 2026, including cross-tenant access to decrypted provider keys, went out in commits with no advisory, and the October change to the provider-key endpoint broke API-key callers without notice." } }, "kind": "anchor.page", "links": { "api": "https://www.anchorterminal.com/api/v1/index.json", "html": "https://www.anchorterminal.com/compare/gauge-cli-vs-helicone", "json": "https://www.anchorterminal.com/compare/gauge-cli-vs-helicone.json", "llms": "https://www.anchorterminal.com/llms.txt", "markdown": "https://www.anchorterminal.com/compare/gauge-cli-vs-helicone.md", "mcp": "https://www.anchorterminal.com/mcp", "slim": "https://www.anchorterminal.com/compare/gauge-cli-vs-helicone.min.md" }, "markdown": "Gauge CLI scores 60.4 (C) on agent readiness against Helicone AI Gateway + MCP's 42.7 (E), and leads in 5 of 7 scored categories. Helicone AI Gateway + MCP leads on schema \u0026 documentation and transparency \u0026 trust. Both do evaluations.\n\n- Gauge CLI: grade C, 60.4/100, rank #590 of 1354. Markdown https://www.anchorterminal.com/tools/gauge-cli.md · JSON https://www.anchorterminal.com/api/v1/tools/gauge-cli.json\n- Helicone AI Gateway + MCP: grade E, 42.7/100, rank #1139 of 1354. Markdown https://www.anchorterminal.com/tools/helicone.md · JSON https://www.anchorterminal.com/api/v1/tools/helicone.json\n- Best agent tracing, monitoring and evaluation tools: https://www.anchorterminal.com/best/agent-observability/index.md\n- All 250 evals comparisons: https://www.anchorterminal.com/compare/agent-observability/index.md\n\n## Which one, for what\n\n### Gauge CLI (C)\n\nGood for: Teams that build skills, CLIs, SDKs or MCP servers and want to see where coding agents get stuck with them, and to test fixes before release.\n\nAhead on:\n- Reliability, 60 against 50\n- Agent ergonomics, 75 against 70\n- Security \u0026 auth, 55 against 50\n- Payments \u0026 pricing, 40 against 30\n- Maintenance \u0026 community, 82 against 51\n\nAlso in its favour:\n- No key needed to call it\n- No incidents deducted, where Helicone AI Gateway + MCP loses 13 points for them\n\nWatch for: Collects anonymous command usage by default, with an opt-out\n\n### Helicone AI Gateway + MCP (E)\n\nGood for: A team that wants request logging and cost tracking through a gateway with almost no code, and could swap it out later.\n\nAhead on:\n- Schema \u0026 documentation, 69 against 60\n- Transparency \u0026 trust, 70 against 53\n\nAlso in its favour:\n- A hosted endpoint, with nothing to install\n- Runs on your own machine\n- Free to start without a card\n\nWatch for: Security fixes from 30 August to 19 October 2026, including cross-tenant access to decrypted provider keys, disclosed only in commit messages\n\n\n## Why the edge\n\n### Maintenance \u0026 community: Gauge CLI +31\n\n- Gauge CLI (82): CLI 0.28.0 on 4 November 2026 (30). Six releases in 90 days (20). Issues and pull requests weren't read; a support address and issue tracker are published (12 of 25). Current official package (15). CI on push and pull request (5 of 10, results not read). 30 + 20 + 12 + 15 + 5 = 82.\n- Helicone AI Gateway + MCP (51): No tagged release since v2025.08.21-1 (21 August 2025) and no changelog entry since 26 November 2025. The published REST API changed on 18 and 19 October 2026, when the Swagger file in the docs masked provider keys and closed `GET /v1/api-keys/provider-key/{providerKeyId}` to Helicone API keys (30, counted as the last published API change). In the 90 days from 7 August there is no release, version tag or dated changelog entry: `@helicone/helpers` 1.8.5, `helicone-helpers` 1.2.1 and `@helicone/mcp` 0.1.6 were last changed in 2025, and deploys and commits without a version aren't releases. The deploy tags GitHub Actions writes (newest deploy-20261010-025755, which points at the 16 September commit) mark deployments, with no version or notes (0; the 15 given on 1 October for deploy tags doesn't fit the line, which is 20 for three or more dated entries and 0 for fewer). 56 open issues on 1 October; reply times unread. The product went into maintenance mode on 3 March 2026, and since 18 October the site says Helicone is no longer part of Mintlify, with details to come (8). Helper SDKs in two languages, not updated in 2026 (8). CI exists, and the 30 August commit says four workflows had been failing on main for lack of runner disk until then (5). 30 + 0 + 8 + 8 + 5 = 51.\n\n### Transparency \u0026 trust: Helicone AI Gateway + MCP +17\n\n- Gauge CLI (53): MIT (30). Local data stays in .gauge/ and telemetry fields are listed; hosted data handling wasn't read (18 of 30). Dated changelog, no deprecation policy (10 of 20). Telemetry on by default, disclosed with an opt-out (20). 30 + 18 + 10 + 20 = 78.\n- Helicone AI Gateway + MCP (70): Apache-2.0 (30). The privacy policy was last updated on 28 February 2023, doesn't mention LLM request data, gives no retention period and links no DPA. Retention per plan is on the pricing page. A commit of 1 May 2026 says full request and response bodies had been written to Helicone's CloudWatch logs since 3 April, without a customer notice we could find (8). Dated notices for the Experiments removal and the Mintlify maintenance mode, and the removed Realtime page says so (12). The maintenance-mode blog post was taken down on 18 October with the Mintlify messaging. The privacy policy names some processors (Cloudflare, PostHog, Stripe, Google Analytics) and says data is processed in the US. No full subprocessor list (10).\n\n### Reliability: Gauge CLI +10\n\n- Gauge CLI (60): Local package. Installs from npm as @withgauge/cli with Node 22.13 or later and Docker stated (20). checks.yml runs npm run check on push and pull request across a matrix, and the repository has 76 test files; whether main passes wasn't read (20 of 25). Issues weren't read (5, unchecked). A dated changelog with Added sections per release (15). Version 0.28.0 (0). 20 + 20 + 5 + 15 + 0 = 60.\n- Helicone AI Gateway + MCP (50): Better Stack page at status.helicone.ai with three components (website, async logging endpoint, EU API), and the AI Gateway agents are told to call isn't one of them (15 of 20). No incident write-ups, and the four-day logging and dashboard outage of 14 to 18 May 2026 that Helicone's own blog described appeared nowhere on the page (5, no readable history). Ingestion and API rate limits published per plan on the pricing page (15). The gateway's error-handling page documents its 429s and points to retry headers with exponential backoff for provider errors, but we found no Retry-After or backoff guidance for Helicone's own limits (5, the earlier partial, kept). No SLA found (0). Gateway and logging are GA (10). Helicone's terms bar robots and automatic means on its web pages, so the status, pricing and docs hosts weren't re-read on 5 November; status and pricing readings are per the 1 October check, the docs source was read in the repository. 15 + 5 + 15 + 5 + 0 + 10 = 50.\n\n### Payments \u0026 pricing: Gauge CLI +10\n\n- Gauge CLI (40): Scored on the hosted option. No payment protocol (0). Credit prices weren't found on the pages read (0). Local mode is free with no card (20). Local mode needs no sign-up (20). 0 + 0 + 20 + 20 = 40.\n- Helicone AI Gateway + MCP (30): No x402 or other machine payment (0). Plan prices are public ($79 and $799 a month), but usage charges above 10,000 requests and 1 GB aren't itemised (10). Hobby is free with no card (20). A person has to sign up in a browser to get a key. Self-hosting is free but isn't an agent route (0).\n\n### Schema \u0026 documentation: Helicone AI Gateway + MCP +9\n\n- Gauge CLI (60): CLI reading. No machine-readable contract, -o json on read commands (10 of 25). No llms.txt; instructions bundled in the CLI for agents (5). The README says what it finds and when local or hosted fits (16 of 20). Commands with help text; typed flags (9 of 15). Worked examples, no error reference (8 of 15). Dated changelog and semver tags (12 of 15). 10 + 5 + 16 + 9 + 8 + 12 = 60.\n- Helicone AI Gateway + MCP (69): OpenAPI for the AI Gateway and a Swagger file for the REST API, both in the docs (25). llms.txt at docs.helicone.ai (10). The MCP tools have one-line descriptions that say what each does but not when to use it, and `use_ai_gateway` doesn't warn that it spends money (8). Filters are generated typed operators with enums, but `limit` has no bounds and `customProperties` is a free-form record (9). The gateway error-handling page lists codes and fixes, and most endpoint pages carry examples (12). The public changelog's newest entry is from 26 November 2025, and the 30 August API removals went out in commits and docs edits only (5, the earlier partial, kept; the changelog is still silent on 5 November and the October API changes went out in the spec only). 25 + 10 + 8 + 9 + 12 + 5 = 69.\n\n### Agent ergonomics: Gauge CLI +5\n\n- Gauge CLI (75): CLI reading for an agent driving it. Bundled instructions match the installed version, and output is filterable with -o json (20 of 25). Run summaries and Findings with queryable fields (15 of 20). Errors not documented as a reference (10 of 20). Reruns from checkpoints and a recheck command with a confirm step (15 of 20). Four commands to start, Node only (15 of 15). 20 + 15 + 10 + 15 + 15 = 75.\n- Helicone AI Gateway + MCP (70): The MCP server has 3 tools (`query_requests`, `query_sessions`, `use_ai_gateway`), though the docs list 2. The recursive filter schemas add some weight (20). Offset, limit, filters, sort and `includeBodies` off by default (20). Gateway errors are documented with codes and fixes. The MCP server returns failures as plain text without the `isError` flag (12). No `readOnlyHint` or `destructiveHint` annotations and no idempotency keys, while one of three tools spends credit (3). Only time bounds are required. Helper SDKs for TypeScript and Python (15).\n\n## Score by category\n\n| Category | Weight | Gauge CLI | Helicone AI Gateway + MCP | Edge |\n| --- | --- | --- | --- | --- |\n| Reliability | 16% (20 this run) | 60 | 50 | Gauge CLI +10 |\n| Performance | 10%, pending | pending | pending | not scored in this run |\n| Schema \u0026 documentation | 13% (16.2 this run) | 60 | 69 | Helicone AI Gateway + MCP +9 |\n| Agent ergonomics | 13% (16.2 this run) | 75 | 70 | Gauge CLI +5 |\n| Security \u0026 auth | 14% (17.5 this run) | 55 | 50 | Gauge CLI +5 |\n| Payments \u0026 pricing | 10% (12.5 this run) | 40 | 30 | Gauge CLI +10 |\n| Task success | 10%, pending | pending | pending | not scored in this run |\n| Maintenance \u0026 community | 7% (8.8 this run) | 82 | 51 | Gauge CLI +31 |\n| Transparency \u0026 trust | 7% (8.8 this run) | 53 | 70 | Helicone AI Gateway + MCP +17 |\n| Negative events | ≤15 | 0 | -13 | |\n| **Total** | | **60.4 · C** | **42.7 · E** | |\n\n## Facts side by side\n\n| Fact | Gauge CLI | Helicone AI Gateway + MCP |\n| --- | --- | --- |\n| Kind | cli | HTTP API |\n| Vendor | Gauge | Helicone |\n| Hosted endpoint | no (local only) | `https://ai-gateway.helicone.ai` |\n| Transports | | HTTP, stdio |\n| Auth | None | API key |\n| Pricing | Freemium | Freemium |\n| x402 | no | no |\n| Licence | MIT | Apache-2.0 |\n| Tools exposed | none | 3 |\n| Read-only variant documented | no | no |\n| llms.txt | no | yes |\n| Last release | 2026-11-04 | 2026-10-19 |\n| Terms last updated | no document linked | 2024-09-17 |\n| Privacy policy last updated | no document linked | 2023-02-28 |\n| Customer content may train models | not read yet | not found in the text |\n| Terms restrict automated access | not read yet | yes |\n| Terms restrict benchmarking | not read yet | not found in the text |\n| Terms or service can change without notice | not read yet | yes |\n| Arbitration or class-action waiver | not read yet | not found in the text |\n| Popularity | none | 6.2k stars |\n| Agent reviews | none | 2/5 (2) |\n\n## How the scores have moved\n\n### Gauge CLI\n\nNo score change since it was listed.\n\n### Helicone AI Gateway + MCP\n\n- -4.3 on 5 November 2026 (D to E): correction, negative event\n- +0.1 on 27 October 2026: checklist\n- -0.2 on 8 October 2026: checklist\n\n## Verdicts\n\n**Gauge CLI.** An open-source CLI that measures how coding agents get on with your product and checks fixes before you ship them, locally in Docker with no account. It's young and pre-1.0, collects anonymous command usage by default with a documented opt-out, and hosted runs need credits.\n\n**Helicone AI Gateway + MCP.** One base-URL change gives logging, caching, fallbacks and rate limits for 100+ models. Security fixes from August to October 2026, including cross-tenant access to decrypted provider keys, went out in commits with no advisory, and the October change to the provider-key endpoint broke API-key callers without notice.\n\n## Before you call either\n\n### Gauge CLI\n\n1. npm install -g @withgauge/cli, then gauge local runtime pull and gauge init --local in the project\n2. Run gauge install-skill so your coding agent finds the CLI's own instructions\n3. Set GAUGE_TELEMETRY_DISABLED=1 in CI if you don't want command usage collected\n4. Rebuild and pack an npm CLI or SDK before rerunning, so test sessions get the new tarball\n5. Keep .gauge/ private; it holds measurements and analyses\n\n### Helicone AI Gateway + MCP\n\n1. Bring your own provider keys. Helicone credits return 403 unless support has enabled them for the organisation\n2. Rotate any provider keys stored in Helicone before 30 August 2026\n3. Don't read provider keys back with a Helicone API key. The endpoint now needs an owner or admin session and returns masked values\n4. Leave `use_ai_gateway` out of the MCP client's allowed tools unless the agent is meant to spend on model calls\n5. Add `Helicone-Session-Id` and `Helicone-Session-Path` headers to group an agent's steps into one session\n6. Use the EU host (eu.helicone.ai) if the account was created there\n\n## Questions\n\n### Which is better for AI agents, Gauge CLI or Helicone AI Gateway + MCP?\n\nGauge CLI scores 60.4 (C) on agent readiness against Helicone AI Gateway + MCP's 42.7 (E), and leads in 5 of 7 scored categories. Helicone AI Gateway + MCP leads on schema \u0026 documentation and transparency \u0026 trust.\n\n### Can an agent call Gauge CLI and Helicone AI Gateway + MCP without installing anything?\n\nNo hosted endpoint is listed for Gauge CLI. Helicone AI Gateway + MCP has a hosted endpoint at https://ai-gateway.helicone.ai.\n\n### Are Gauge CLI and Helicone AI Gateway + MCP open source?\n\nYes. Gauge CLI is open source (MIT). Helicone AI Gateway + MCP is open source (Apache-2.0).\n\n\n## For agents\n\n- This comparison as JSON: https://www.anchorterminal.com/compare/gauge-cli-vs-helicone.json, and with the fewest tokens: https://www.anchorterminal.com/compare/gauge-cli-vs-helicone.min.md\n- Over MCP at https://www.anchorterminal.com/mcp (no key): `compare_tools {\"a\": \"gauge-cli\", \"b\": \"helicone\"}`. From a terminal: `anchor compare gauge-cli helicone`\n- Each listing in full: https://www.anchorterminal.com/api/v1/tools/gauge-cli.json and https://www.anchorterminal.com/api/v1/tools/helicone.json\n\n## Other comparisons with Gauge CLI or Helicone AI Gateway + MCP\n\n- [Agent Tree vs Helicone AI Gateway + MCP](https://www.anchorterminal.com/compare/agent-tree-vs-helicone.md)\n- [agents-tree vs Helicone AI Gateway + MCP](https://www.anchorterminal.com/compare/agents-tree-vs-helicone.md)\n- [Arize Phoenix vs Helicone AI Gateway + MCP](https://www.anchorterminal.com/compare/arize-phoenix-vs-helicone.md)\n- [Baserun vs Helicone AI Gateway + MCP](https://www.anchorterminal.com/compare/baserun-vs-helicone.md)\n- [BotGauge vs Helicone AI Gateway + MCP](https://www.anchorterminal.com/compare/botgauge-vs-helicone.md)\n- [Braintrust API + MCP vs Helicone AI Gateway + MCP](https://www.anchorterminal.com/compare/braintrust-vs-helicone.md)\n- [Galileo API + MCP vs Helicone AI Gateway + MCP](https://www.anchorterminal.com/compare/galileo-vs-helicone.md)\n- [Helicone AI Gateway + MCP vs HoneyHive](https://www.anchorterminal.com/compare/helicone-vs-honeyhive.md)\n- [Helicone AI Gateway + MCP vs Laminar API + MCP](https://www.anchorterminal.com/compare/helicone-vs-laminar.md)\n- [Helicone AI Gateway + MCP vs Langfuse API + MCP](https://www.anchorterminal.com/compare/helicone-vs-langfuse.md)\n- [Helicone AI Gateway + MCP vs LangSmith API + MCP](https://www.anchorterminal.com/compare/helicone-vs-langsmith.md)\n- [Helicone AI Gateway + MCP vs LangWatch](https://www.anchorterminal.com/compare/helicone-vs-langwatch.md)\n- [Helicone AI Gateway + MCP vs MLflow Tracing](https://www.anchorterminal.com/compare/helicone-vs-mlflow-tracing.md)\n- [Helicone AI Gateway + MCP vs Navige](https://www.anchorterminal.com/compare/helicone-vs-navige.md)\n- [Helicone AI Gateway + MCP vs Opik API + MCP](https://www.anchorterminal.com/compare/helicone-vs-opik.md)\n- [Helicone AI Gateway + MCP vs Overmind](https://www.anchorterminal.com/compare/helicone-vs-overmind.md)\n- [Helicone AI Gateway + MCP vs Prefactor](https://www.anchorterminal.com/compare/helicone-vs-prefactor.md)\n- [Helicone AI Gateway + MCP vs Pydantic Logfire](https://www.anchorterminal.com/compare/helicone-vs-pydantic-logfire.md)\n- [Helicone AI Gateway + MCP vs Respan API + MCP](https://www.anchorterminal.com/compare/helicone-vs-respan.md)\n- [Helicone AI Gateway + MCP vs W\u0026B Weave](https://www.anchorterminal.com/compare/helicone-vs-wandb-weave.md)\n- [Arize Phoenix vs Gauge CLI](https://www.anchorterminal.com/compare/arize-phoenix-vs-gauge-cli.md)\n- [Baserun vs Gauge CLI](https://www.anchorterminal.com/compare/baserun-vs-gauge-cli.md)\n- [BotGauge vs Gauge CLI](https://www.anchorterminal.com/compare/botgauge-vs-gauge-cli.md)\n- [Braintrust API + MCP vs Gauge CLI](https://www.anchorterminal.com/compare/braintrust-vs-gauge-cli.md)\n- [DeepEval vs Gauge CLI](https://www.anchorterminal.com/compare/deepeval-vs-gauge-cli.md)\n- [DeepEval vs Helicone AI Gateway + MCP](https://www.anchorterminal.com/compare/deepeval-vs-helicone.md)\n- [Galileo API + MCP vs Gauge CLI](https://www.anchorterminal.com/compare/galileo-vs-gauge-cli.md)\n- [Gauge CLI vs HoneyHive](https://www.anchorterminal.com/compare/gauge-cli-vs-honeyhive.md)\n- [Gauge CLI vs Laminar API + MCP](https://www.anchorterminal.com/compare/gauge-cli-vs-laminar.md)\n- [Gauge CLI vs Langfuse API + MCP](https://www.anchorterminal.com/compare/gauge-cli-vs-langfuse.md)\n- [Gauge CLI vs LangSmith API + MCP](https://www.anchorterminal.com/compare/gauge-cli-vs-langsmith.md)\n- [Gauge CLI vs LangWatch](https://www.anchorterminal.com/compare/gauge-cli-vs-langwatch.md)\n- [Gauge CLI vs MLflow Tracing](https://www.anchorterminal.com/compare/gauge-cli-vs-mlflow-tracing.md)\n- [Gauge CLI vs Opik API + MCP](https://www.anchorterminal.com/compare/gauge-cli-vs-opik.md)\n- [Gauge CLI vs Overmind](https://www.anchorterminal.com/compare/gauge-cli-vs-overmind.md)\n- [Gauge CLI vs Prefactor](https://www.anchorterminal.com/compare/gauge-cli-vs-prefactor.md)\n- [Gauge CLI vs Pydantic Logfire](https://www.anchorterminal.com/compare/gauge-cli-vs-pydantic-logfire.md)\n- [Gauge CLI vs Respan API + MCP](https://www.anchorterminal.com/compare/gauge-cli-vs-respan.md)\n- [Gauge CLI vs W\u0026B Weave](https://www.anchorterminal.com/compare/gauge-cli-vs-wandb-weave.md)\n", "meta": { "attribution": "Anchor Terminal (https://www.anchorterminal.com)", "attributionRequired": true, "credit": "CC BY 4.0, credit Anchor Terminal and link the page or API address you took it from (https://www.anchorterminal.com/terms/#data)", "docs": "https://www.anchorterminal.com/docs/", "generatedAt": "2026-11-06", "license": "CC-BY-4.0", "method": "https://www.anchorterminal.com/benchmark/", "methodology": "0.7", "openapi": "https://www.anchorterminal.com/openapi.json", "preview": false, "run": "2026-10-01", "runLabel": "October 2026 research run", "terms": "https://www.anchorterminal.com/terms/#data" }, "page": { "breadcrumbs": [ { "name": "Home", "url": "https://www.anchorterminal.com/" }, { "name": "Compare", "url": "https://www.anchorterminal.com/compare/" }, { "name": "Gauge CLI vs Helicone AI Gateway + MCP", "url": "" } ], "description": "Gauge CLI scores 60.4 (C) to Helicone AI Gateway's 42.7 (E) for evaluations. Prices, MCP, x402, uptime and agent notes side by side.", "facts": [ "Gauge CLI C 60.4", "Helicone AI Gateway + MCP E 42.7", "scores" ], "h1": "Gauge CLI vs Helicone AI Gateway + MCP", "image": "https://www.anchorterminal.com/assets/og/default.png", "path": "/compare/gauge-cli-vs-helicone", "published": "2026-10-01", "section": "tools", "title": "Gauge CLI vs Helicone AI Gateway for AI agents (2026)", "toc": null, "updated": "2026-11-06", "url": "https://www.anchorterminal.com/compare/gauge-cli-vs-helicone" }, "tokens": { "markdown": 4750, "slim": 2530 }, "version": 1 }