{ "data": { "category": { "area": "developer", "capabilities": [ "obs.traces", "obs.evals", "obs.prompts", "obs.gateway", "obs.datasets" ], "description": "Tools that trace agent runs, tool calls and model calls, and run evaluations on them. Compared on what appears in a trace, how fast a failure can be found, whether an evaluation can be reproduced, setup time and cost.", "indexed": [ { "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/undercurrentholdings-aegis.json", "kind": "mcp", "name": "aegis", "slug": "undercurrentholdings-aegis", "url": "https://www.anchorterminal.com/tools/undercurrentholdings-aegis" }, { "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/argosvix-server.json", "kind": "mcp", "name": "argosvix.com MCP server", "slug": "argosvix-server", "url": "https://www.anchorterminal.com/tools/argosvix-server" }, { "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/completionkit-evals.json", "kind": "mcp", "name": "CompletionKit", "slug": "completionkit-evals", "url": "https://www.anchorterminal.com/tools/completionkit-evals" }, { "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/openfeature-mcp.json", "kind": "mcp", "name": "openfeature.dev MCP server", "slug": "openfeature-mcp", "url": "https://www.anchorterminal.com/tools/openfeature-mcp" }, { "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/promptot-mcp.json", "kind": "mcp", "name": "PromptOT", "slug": "promptot-mcp", "url": "https://www.anchorterminal.com/tools/promptot-mcp" } ], "indexedCount": 5, "json": "https://www.anchorterminal.com/categories/agent-observability.json", "name": "Agent observability \u0026 evals", "slug": "agent-observability", "test": "The same agent run instrumented in every tool, including a failed tool call and a retry. We check which steps appear in the trace, whether the failure can be found quickly, whether an evaluation can be reproduced, how long setup takes and what it costs.", "title": "Agent tracing, monitoring and evaluation tools", "toolCount": 10, "tools": [ "arize-phoenix", "langfuse", "langsmith", "respan", "braintrust", "laminar", "honeyhive", "galileo", "helicone", "baserun" ], "url": "https://www.anchorterminal.com/categories/agent-observability" }, "tools": [ { "slug": "arize-phoenix", "name": "Arize Phoenix", "vendor": "Arize AI", "vendorUrl": "https://phoenix.arize.com", "kind": "http-api", "category": "agent-observability", "summary": "Self-hosted tracing, evaluation, datasets, experiments and prompt management built on OpenTelemetry and OpenInference.", "url": "https://www.anchorterminal.com/tools/arize-phoenix", "markdownUrl": "https://www.anchorterminal.com/tools/arize-phoenix.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/arize-phoenix.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/arize-phoenix.json", "repo": "https://github.com/Arize-ai/phoenix", "license": "Elastic-2.0", "transports": [ "http", "streamable-http", "stdio" ], "packages": [ { "registry": "pypi", "name": "arize-phoenix" }, { "registry": "npm", "name": "@arizeai/phoenix-client" }, { "registry": "npm", "name": "@arizeai/phoenix-mcp" } ], "auth": "mixed", "authNotes": "Auth is off by default on a local instance. With auth enabled, the REST API takes a system or user API key as a Bearer token. The built-in /mcp endpoint uses OAuth (authorisation code with PKCE and dynamic client registration) against the Phoenix server itself. The older @arizeai/phoenix-mcp stdio package takes PHOENIX_API_KEY.", "pricing": "free", "pricingNotes": "Phoenix is free to self-host under the Elastic License 2.0; you pay for your own compute and Postgres or SQLite storage. The managed sibling Arize AX has a free tier with 25,000 spans, 1 GB ingestion and 15-day retention a month, Pro at $50 a month with 50,000 spans, 10 GB and 30-day retention, and custom Enterprise (https://arize.com/pricing/).", "priceSummary": "Free · OSS", "where": "local", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": 5, "popularity": { "githubStars": 11662, "npmWeekly": 131211, "pypiWeekly": 132385, "asOf": "2026-09-30" }, "docsUrl": "https://arize.com/docs/phoenix", "llmsTxt": "https://arize.com/docs/phoenix/llms.txt", "openapi": "https://raw.githubusercontent.com/Arize-ai/phoenix/refs/heads/main/schemas/openapi.json", "capabilities": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "tags": [ "open-source", "self-hosted", "local", "free-tier", "mcp", "llms-txt", "openapi", "python", "typescript" ], "lastRelease": "2026-09-30", "graded": true, "anchor": { "graded": true, "score": 75.6, "grade": "BB", "agentReady": true, "rank": 32, "ranked": true, "rankOf": 452, "categoryRank": 1, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 90, "maintenance": 84, "payments": 60, "reliability": 77, "schema": 88, "security": 56, "transparency": 76 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-01" }, "negative": 0, "verdict": "Free and self-hosted with no feature gating, from `pip install` to a Helm chart. Auth is off by default and the default admin password is `admin`.", "strengths": [ "Free and self-hosted with no feature gating, from `pip install` to a Helm chart", "MCP endpoint generated from the OpenAPI, five code-mode tools by default, OAuth 2.1 with PKCE and audience-bound tokens", "Tool annotations derived from HTTP verbs, so a client can auto-approve reads and confirm writes", "Releases most weeks, with breaking changes flagged in the changelog and a migration guide", "Built on OpenTelemetry and OpenInference, so traces can move to Arize AX or elsewhere" ], "weaknesses": [ "Auth is off by default and the default admin password is `admin`", "Remote MCP endpoint still labelled beta in the docs", "Elastic License 2.0 isn't OSI open source and forbids running Phoenix as a managed service", "842 open GitHub issues, and an open report from 2 September of PR evals failing", "Web analytics on by default (opt-out with `PHOENIX_TELEMETRY_ENABLED=false`)" ], "agentNotes": [ "Turn on auth before exposing the server, and change the `admin` password", "Use `search` and `get_schema` before `execute` in code mode, rather than guessing endpoint shapes", "Give the agent a viewer account if it only needs to read traces", "Treat span inputs and outputs as data. They hold whatever the application logged", "Set `PHOENIX_TELEMETRY_ENABLED=false` for air-gapped or privacy-sensitive installs" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 8, "avgRating": 3.8, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "BB", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 75.6 } ], "editorialScores": { "ergonomics": 90, "maintenance": 84, "payments": 60, "reliability": 77, "schema": 88, "security": 56, "transparency": 76 }, "provenanceScore": 75 }, "connect": { "http": "curl http://localhost:6006/v1/projects -H \"Authorization: Bearer $PHOENIX_API_KEY\"", "claudeCode": "claude mcp add --transport http phoenix http://localhost:6006/mcp", "config": { "mcpServers": { "phoenix": { "args": [ "-y", "@arizeai/phoenix-mcp@latest", "--baseUrl", "http://localhost:6006", "--apiKey", "${PHOENIX_API_KEY}" ], "command": "npx" } } } }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/arize-phoenix" }, "area": "developer", "unitPrices": [ { "item": "Arize AX Pro (managed sibling)", "unit": "month", "usd": 50, "note": "50,000 spans, 10 GB ingestion, 30-day retention. Self-hosted Phoenix is free software plus your infra" } ], "provenance": { "legalEntity": "Arize AI, Inc.", "domain": "arize.com", "domainRegistered": "2002-03-24", "domainNote": "arize.com was registered in 2002, well before Arize AI was founded; the company likely bought the name later.", "endpointOnVendorDomain": false, "terms": "https://arize.com/terms-of-service/", "privacy": "https://arize.com/privacy-policy/", "statusPage": "https://status.arize.com", "changelog": "https://github.com/Arize-ai/phoenix/releases", "securityTxt": "none", "checked": "2026-09-30", "score": 75 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/arize-phoenix.json", "live": { "slug": "arize-phoenix", "vendorStatus": { "page": "https://status.arize.com", "indicator": "unknown", "summary": "no machine-readable status found", "checkedAt": "2026-10-04T21:39:48.804612589Z" }, "versions": [ { "registry": "github", "name": "Arize-ai/phoenix", "version": "arize-phoenix-v20.19.0", "released": "2026-10-01", "seenAt": "2026-10-04T16:20:53.33744975Z" }, { "registry": "npm", "name": "@arizeai/phoenix-client", "version": "7.16.0", "seenAt": "2026-10-04T16:20:51.131454134Z" }, { "registry": "npm", "name": "@arizeai/phoenix-mcp", "version": "4.3.15", "seenAt": "2026-10-04T16:20:51.575630222Z" }, { "registry": "pypi", "name": "arize-phoenix", "version": "20.19.0", "released": "2026-10-01", "seenAt": "2026-10-04T16:20:50.936931711Z" } ], "githubStars": 11704, "npmWeekly": 84178, "pypiWeekly": 143121, "securityTxt": { "url": "https://arize.com/.well-known/security.txt", "state": "none", "checkedAt": "2026-10-04T15:16:01.290278815Z" }, "llmsTxt": { "url": "https://arize.com/docs/phoenix/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-04T15:17:15.318838737Z" }, "domain": { "domain": "arize.com", "registered": "2002-03-24", "source": "https://rdap.verisign.com/com/v1/domain/arize.com", "checkedAt": "2026-10-04T13:04:54.314321516Z" }, "pages": [ { "url": "https://arize.com/pricing/", "kind": "pricing", "status": 200, "checkedAt": "2026-10-04T15:41:15.874853632Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "4450553fe0f8" }, { "url": "https://arize.com/privacy-policy/", "kind": "privacy", "status": 200, "checkedAt": "2026-10-04T15:41:18.402560606Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "8ced75f284e2" }, { "url": "https://arize.com/terms-of-service/", "kind": "terms", "status": 200, "checkedAt": "2026-10-04T15:41:20.923451914Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "d9ef61d742b4" } ], "updatedAt": "2026-10-04T21:39:48.804612589Z" } }, { "slug": "langfuse", "name": "Langfuse API + MCP", "vendor": "Langfuse (ClickHouse)", "vendorUrl": "https://langfuse.com", "kind": "http-api", "category": "agent-observability", "summary": "Open-source tracing, evaluation, prompt management and datasets for LLM apps and agents, built on OpenTelemetry.", "url": "https://www.anchorterminal.com/tools/langfuse", "markdownUrl": "https://www.anchorterminal.com/tools/langfuse.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/langfuse.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/langfuse.json", "repo": "https://github.com/langfuse/langfuse", "license": "MIT (core), commercial licence for the ee directories", "transports": [ "http", "streamable-http" ], "remoteUrl": "https://cloud.langfuse.com/api/public", "packages": [ { "registry": "pypi", "name": "langfuse" }, { "registry": "npm", "name": "@langfuse/tracing" }, { "registry": "npm", "name": "@langfuse/client" } ], "auth": "api-key", "authNotes": "Project-scoped key pair (public `pk-lf-...`, secret `sk-lf-...`) sent as HTTP Basic auth, for both the REST API and the MCP server at `/api/public/mcp`. Organisation-level keys exist for SCIM and admin APIs on Enterprise.", "pricing": "freemium", "pricingNotes": "Cloud Hobby is free with 50,000 units a month, 30 days of data and 2 users, no card. Core $29 a month with 100,000 units and 90 days, Pro $199 a month with 3 years of data, Enterprise $2,499 a month. Extra units cost $8 per 100,000 on paid plans, lower with volume. A unit is one trace, observation or score. Teams add-on $300 a month for SSO and fine-grained RBAC. Self-hosted open source is free with unlimited usage, you pay for your own Postgres, ClickHouse, Redis and blob storage. Self-hosted Enterprise is custom and sold on top of a ClickHouse plan (https://langfuse.com/pricing).", "priceSummary": "$29 / mo", "where": "hosted", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": 89, "popularity": { "githubStars": 35235, "npmWeekly": 3039593, "pypiWeekly": 5865664, "asOf": "2026-09-30" }, "docsUrl": "https://langfuse.com/docs", "llmsTxt": "https://langfuse.com/llms.txt", "openapi": "https://cloud.langfuse.com/generated/api/openapi.yml", "capabilities": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "tags": [ "hosted", "freemium", "no-card", "open-source", "self-hosted", "mcp", "llms-txt", "openapi", "python", "typescript", "webhooks" ], "lastRelease": "2026-10-01", "graded": true, "anchor": { "graded": true, "score": 72.8, "grade": "BB", "agentReady": true, "rank": 66, "ranked": true, "rankOf": 452, "categoryRank": 2, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 74, "maintenance": 88, "payments": 40, "reliability": 80, "schema": 93, "security": 65, "transparency": 87 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "high", "date": "2026-10-01" }, "negative": -2, "negativeNotes": [ "2025-11-09 to 2026-01-21. Three moderate advisories, cross-organisation enumeration of member and invitation lists (GHSA-94hf-6gqq-pj69), SSO account takeover through CSRF or phishing (GHSA-w9pw-c549-5m6w) and an unauthenticated Slack OAuth install that could link Slack to any project (GHSA-pvq7-vvfj-p98x). All fixed and published, eight to eleven months old, so -2 (https://github.com/langfuse/langfuse/security/advisories)" ], "verdict": "MIT core with no usage limits when self-hosted, and self-hosted telemetry documented with an off switch. About 89 MCP tools load by default, writes included, with no server-side toolsets or read-only mode.", "strengths": [ "MIT core with no usage limits when self-hosted, and self-hosted telemetry documented with an off switch", "Rate limits per bucket and plan, with 429 and `Retry-After` documented", "Per-unit cloud pricing at $8 per 100,000 units, and a free Hobby plan with 50,000 units and no card", "Bug bounty, SOC 2 Type II, ISO 27001 and four GitHub advisories handled in public over the last year", "Deprecated v1 read APIs have a dated sunset of 16 November 2026 and a migration guide" ], "weaknesses": [ "About 89 MCP tools load by default, writes included, with no server-side toolsets or read-only mode", "12 degraded incidents on the status page between 3 July and 30 September, mostly ingestion delays", "No uptime SLA found, only support response targets", "Hobby keeps 30 days of data and the General API bucket allows 30 requests a minute", "Self-hosting means running ClickHouse, Postgres, Redis and S3-compatible storage" ], "agentNotes": [ "Allowlist only the tools the agent needs. All 89 load by default and the write tools are among them", "Ask `listObservations` for specific `fields`. Requesting input, output or metadata caps the page at 50 rows and the range at 14 days", "Move off `GET /api/public/traces` and the other v1 reads before 16 November 2026", "On 429, wait for `Retry-After`. MCP calls share the organisation's General API bucket", "Pick the regional host (cloud, us.cloud, jp.cloud, hipaa.cloud) that matches the project's keys" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 4, "history": [ { "basis": "public evidence", "confidence": "high", "grade": "BB", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 72.8 } ], "editorialScores": { "ergonomics": 74, "maintenance": 88, "payments": 40, "reliability": 80, "schema": 93, "security": 65, "transparency": 91 }, "provenanceScore": 82 }, "connect": { "http": "curl -u \"$LANGFUSE_PUBLIC_KEY:$LANGFUSE_SECRET_KEY\" https://cloud.langfuse.com/api/public/projects", "claudeCode": "claude mcp add --transport http langfuse https://cloud.langfuse.com/api/public/mcp \\\n --header \"Authorization: Basic $(printf '%s:%s' \"$LANGFUSE_PUBLIC_KEY\" \"$LANGFUSE_SECRET_KEY\" | base64)\"" }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/langfuse" }, "area": "developer", "unitPrices": [ { "item": "Core plan", "unit": "month", "usd": 29, "note": "100,000 units included, 90 days of data" }, { "item": "Pro plan", "unit": "month", "usd": 199, "note": "100,000 units included, 3 years of data" }, { "item": "Enterprise plan", "unit": "month", "usd": 2499, "note": "100,000 units included, audit logs, SCIM, SLAs" }, { "item": "Teams add-on", "unit": "month", "usd": 300, "note": "SSO, SSO enforcement and fine-grained RBAC on Pro" } ], "provenance": { "legalEntity": "Langfuse GmbH (a ClickHouse, Inc. company)", "domain": "langfuse.com", "domainRegistered": "2023-04-20", "endpointOnVendorDomain": true, "terms": "https://clickhouse.com/legal/clickhouse-general-terms-and-conditions", "privacy": "https://langfuse.com/privacy", "statusPage": "https://status.langfuse.com", "changelog": "https://langfuse.com/changelog", "securityTxt": "none", "checked": "2026-09-30", "notes": [ "The imprint names Langfuse GmbH in Berlin. The privacy notice is issued by ClickHouse, Inc. and langfuse.com/terms redirects to the ClickHouse general terms" ], "score": 82 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/langfuse.json", "live": { "slug": "langfuse", "probe": { "target": "https://cloud.langfuse.com/api/public", "method": "get", "lastAt": "2026-10-04T22:35:25.621515874Z", "lastOk": true, "lastStatus": 404, "lastMs": 52, "authRequired": false, "uptime24h": 100, "uptime30d": 100, "p50ms24h": 53, "p95ms24h": 95, "samples24h": 272, "samples30d": 1086, "days": [ { "date": "2026-09-30", "probes": 35, "ok": 35 }, { "date": "2026-10-01", "probes": 276, "ok": 276 }, { "date": "2026-10-02", "probes": 248, "ok": 248 }, { "date": "2026-10-03", "probes": 271, "ok": 271 }, { "date": "2026-10-04", "probes": 256, "ok": 256 } ] }, "vendorStatus": { "page": "https://status.langfuse.com", "indicator": "none", "summary": "All Systems Operational", "checkedAt": "2026-10-04T22:33:59.334958987Z" }, "versions": [ { "registry": "github", "name": "langfuse/langfuse", "version": "v4.50.0", "released": "2026-10-02", "seenAt": "2026-10-04T16:31:18.139005808Z" }, { "registry": "npm", "name": "@langfuse/client", "version": "5.11.1", "seenAt": "2026-10-04T16:31:16.823169285Z" }, { "registry": "npm", "name": "@langfuse/tracing", "version": "5.11.1", "seenAt": "2026-10-04T16:31:15.89253951Z" }, { "registry": "pypi", "name": "langfuse", "version": "4.16.0", "released": "2026-09-30", "seenAt": "2026-10-04T16:31:15.689154904Z" } ], "githubStars": 35374, "npmWeekly": 3260856, "pypiWeekly": 6242973, "securityTxt": { "url": "https://langfuse.com/.well-known/security.txt", "state": "none", "checkedAt": "2026-10-04T15:15:59.271988313Z" }, "llmsTxt": { "url": "https://langfuse.com/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-04T15:17:55.905804251Z" }, "domain": { "domain": "langfuse.com", "registered": "2023-04-20", "source": "https://rdap.verisign.com/com/v1/domain/langfuse.com", "checkedAt": "2026-10-04T13:09:17.712369699Z" }, "pages": [ { "url": "https://langfuse.com/changelog", "kind": "changelog", "status": 404, "checkedAt": "2026-10-04T15:45:22.907270008Z", "changedAt": "0001-01-01T00:00:00Z" }, { "url": "https://langfuse.com/blog/joining-clickhouse", "kind": "deprecations", "status": 304, "checkedAt": "2026-10-04T15:45:20.82840104Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "4e8ff98bb557" }, { "url": "https://langfuse.com/pricing", "kind": "pricing", "status": 304, "checkedAt": "2026-10-04T15:45:25.072991864Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "fa6797a4f9e7" }, { "url": "https://langfuse.com/privacy", "kind": "privacy", "status": 304, "checkedAt": "2026-10-04T15:45:26.884030862Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "9e1e582ef9a9" }, { "url": "https://clickhouse.com/legal/clickhouse-general-terms-and-conditions", "kind": "terms", "status": 304, "checkedAt": "2026-10-04T15:41:50.81939355Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "892d76cd652d" } ], "updatedAt": "2026-10-04T22:35:25.621515874Z" } }, { "slug": "langsmith", "name": "LangSmith API + MCP", "vendor": "LangChain", "vendorUrl": "https://www.langchain.com/langsmith", "kind": "http-api", "category": "agent-observability", "summary": "LangChain's hosted tracing, evaluation, prompt hub and dataset platform, framework-agnostic with Python and TypeScript SDKs and OpenTelemetry ingest.", "url": "https://www.anchorterminal.com/tools/langsmith", "markdownUrl": "https://www.anchorterminal.com/tools/langsmith.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/langsmith.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/langsmith.json", "repo": "https://github.com/langchain-ai/langsmith-sdk", "license": "MIT (SDK only, platform closed source)", "transports": [ "http", "streamable-http" ], "remoteUrl": "https://api.smith.langchain.com", "packages": [ { "registry": "pypi", "name": "langsmith" }, { "registry": "npm", "name": "langsmith" } ], "auth": "mixed", "authNotes": "API key in the `X-Api-Key` header (personal or service keys, workspace-scoped). The remote MCP server uses OAuth 2.1 with dynamic client registration for interactive clients, or the same API key header for headless ones.", "pricing": "freemium", "pricingNotes": "Developer is free for 1 seat with 5,000 base traces a month. Plus $39 a seat a month with 10,000 base traces, unlimited seats and one free small deployment. Enterprise custom. Beyond the included traces, a base trace costs 0.005 LSU ($5 per 1,000 at $1 an LSU) with 14-day retention, and upgrading to an extended trace with 180-day retention adds 0.0025 LSU. Compute for deployments, sandboxes and tuned evaluators is billed in LCUs at $1.50 each. Self-hosting and BYOC are Enterprise only (https://www.langchain.com/pricing).", "priceSummary": "$39 / seat-mo", "where": "hosted", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": null, "popularity": { "githubStars": 1064, "npmWeekly": 7588622, "pypiWeekly": 17835288, "asOf": "2026-09-30" }, "docsUrl": "https://docs.langchain.com/langsmith/observability", "llmsTxt": "https://docs.langchain.com/llms.txt", "openapi": "https://api.smith.langchain.com/openapi.json", "capabilities": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets", "obs.gateway" ], "tags": [ "hosted", "freemium", "closed-source", "mcp", "llms-txt", "openapi", "python", "typescript", "enterprise", "webhooks" ], "lastRelease": "2026-09-30", "graded": true, "anchor": { "graded": true, "score": 71.3, "grade": "BB", "agentReady": true, "rank": 85, "ranked": true, "rankOf": 452, "categoryRank": 3, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 76, "maintenance": 85, "payments": 40, "reliability": 81, "schema": 86, "security": 77, "transparency": 78 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-01" }, "negative": -4, "negativeNotes": [ "2026-02-09 to 2026-06-19. Five published advisories in the LangSmith SDK. The most serious, GHSA-f4xh-w4cj-qxq8 (CVE-2026-59152, 19 June), let anyone who could send HTTP requests to a server running `TracingMiddleware` read arbitrary files from it into trace attachments, fixed in Python 0.8.18. GitHub's list rates it critical and the advisory scores it CVSS 5.0. The others cover untrusted public prompt manifests (high, 7 May), streaming tokens bypassing output redaction (moderate, 14 April), prototype pollution (moderate, 9 April) and SSRF through tracing headers (moderate, 9 February). All fixed and disclosed in public, so -4 (https://github.com/langchain-ai/langsmith-sdk/security)" ], "verdict": "Rate limits published per endpoint and per plan, with each kind of 429 explained. Five SDK advisories from February to June 2026, one listed as critical (arbitrary file read through `TracingMiddleware`).", "strengths": [ "Rate limits published per endpoint and per plan, with each kind of 429 explained", "Written deprecation policy, six months on cloud, with `Deprecation` and `Sunset` headers on deprecated endpoints", "Remote MCP with OAuth 2.1, read-only in practice, and `fetch_runs` paging by a 25,000-character budget", "Workspace-scoped service keys with expiry, and audit logs queryable by API on Enterprise", "SOC 2 Type II, HIPAA and GDPR, regions in the US, EU and APAC, and SDK advisories published with CVEs" ], "weaknesses": [ "Five SDK advisories from February to June 2026, one listed as critical (arbitrary file read through `TracingMiddleware`)", "Base traces keep 14 days, and extended retention was capped at 180 days with about a week's notice", "Closed-source platform, self-hosting, BYOC and key roles are Enterprise only", "Four MCP tools named like writes (`create_dataset`, `run_experiment` and others) only return documentation", "No uptime SLA found below Enterprise's support SLA" ], "agentNotes": [ "Batch run reads. `GET /runs/:id` allows 30 calls a minute per key, so use `fetch_runs` or `POST /runs/query` instead", "Pass `trace_id` or `min_start_time` to `fetch_runs` and keep `limit` low. Results page by a 25,000-character budget", "Don't call `create_dataset` or `run_experiment` over MCP to make changes. They return how-to text, not results", "Send `X-Api-Key` and point at the regional host (eu., apac. or aws.api.smith.langchain.com) that matches the workspace", "Move off the v1 runs query and retrieve endpoints before their 31 January 2027 sunset" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 3, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "BB", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 71.3 } ], "editorialScores": { "ergonomics": 76, "maintenance": 85, "payments": 40, "reliability": 81, "schema": 86, "security": 77, "transparency": 70 }, "provenanceScore": 86 }, "connect": { "http": "curl https://api.smith.langchain.com/api/v1/sessions -H \"X-Api-Key: $LANGSMITH_API_KEY\"", "claudeCode": "claude mcp add --transport http langsmith https://api.smith.langchain.com/mcp" }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/langsmith" }, "sameCompany": [ "langgraph" ], "area": "developer", "unitPrices": [ { "item": "Plus plan", "unit": "seat-month", "usd": 39, "note": "10,000 base traces a month included" }, { "item": "Base traces", "unit": "1k-requests", "usd": 5, "note": "0.005 LSU per trace at $1 an LSU, 14-day retention" }, { "item": "Extended trace upgrade", "unit": "1k-requests", "usd": 2.5, "note": "0.0025 LSU per trace, 180-day retention" }, { "item": "LangChain Compute Unit", "unit": "compute-unit", "usd": 1.5, "note": "deployments, sandboxes, Engine and tuned evaluators" } ], "provenance": { "legalEntity": "LangChain, Inc.", "domain": "langchain.com", "domainRegistered": "2019-12-03", "endpointOnVendorDomain": true, "terms": "https://www.langchain.com/terms-of-service", "privacy": "https://www.langchain.com/privacy-policy", "statusPage": "https://status.smith.langchain.com", "changelog": "https://docs.langchain.com/langsmith/changelog", "securityTxt": "none", "checked": "2026-09-30", "score": 86 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/langsmith.json", "live": { "slug": "langsmith", "probe": { "target": "https://api.smith.langchain.com", "method": "get", "lastAt": "2026-10-04T22:35:25.63934858Z", "lastOk": true, "lastStatus": 404, "lastMs": 137, "authRequired": false, "uptime24h": 100, "uptime30d": 100, "p50ms24h": 144, "p95ms24h": 181, "samples24h": 272, "samples30d": 1086, "days": [ { "date": "2026-09-30", "probes": 35, "ok": 35 }, { "date": "2026-10-01", "probes": 276, "ok": 276 }, { "date": "2026-10-02", "probes": 248, "ok": 248 }, { "date": "2026-10-03", "probes": 271, "ok": 271 }, { "date": "2026-10-04", "probes": 256, "ok": 256 } ] }, "vendorStatus": { "page": "https://status.smith.langchain.com", "indicator": "unknown", "summary": "no machine-readable status found", "checkedAt": "2026-10-04T21:40:11.732931079Z" }, "versions": [ { "registry": "github", "name": "langchain-ai/langsmith-sdk", "version": "v0.14.4", "released": "2026-10-02", "seenAt": "2026-10-04T16:31:25.515882778Z" }, { "registry": "npm", "name": "langsmith", "version": "0.10.8", "seenAt": "2026-10-04T16:31:24.686804575Z" }, { "registry": "pypi", "name": "langsmith", "version": "0.14.4", "released": "2026-10-02", "seenAt": "2026-10-04T16:31:24.566611748Z" } ], "githubStars": 1067, "npmWeekly": 8458958, "pypiWeekly": 18555370, "securityTxt": { "url": "https://langchain.com/.well-known/security.txt", "state": "none", "checkedAt": "2026-10-04T15:15:56.149171337Z" }, "llmsTxt": { "url": "https://docs.langchain.com/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-04T15:17:57.813298707Z" }, "domain": { "domain": "langchain.com", "registered": "2019-12-03", "source": "https://rdap.verisign.com/com/v1/domain/langchain.com", "checkedAt": "2026-10-04T13:04:29.722113647Z" }, "pages": [ { "url": "https://docs.langchain.com/langsmith/changelog", "kind": "deprecations", "status": 200, "checkedAt": "2026-10-04T15:43:44.062855903Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "2c81953979b2" }, { "url": "https://www.langchain.com/pricing", "kind": "pricing", "status": 304, "checkedAt": "2026-10-04T15:50:56.911482719Z", "changedAt": "2026-10-02T15:27:03.289812971Z", "fingerprint": "27094c5a18c8" }, { "url": "https://www.langchain.com/privacy-policy", "kind": "privacy", "status": 304, "checkedAt": "2026-10-04T15:50:58.933650784Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "83d57e07b1e6" }, { "url": "https://www.langchain.com/terms-of-service", "kind": "terms", "status": 304, "checkedAt": "2026-10-04T15:51:00.937320286Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "bafbaf0129ca" } ], "updatedAt": "2026-10-04T22:35:25.63934858Z" } }, { "slug": "respan", "name": "Respan API + MCP", "vendor": "Respan (formerly Keywords AI)", "vendorUrl": "https://www.respan.ai", "kind": "http-api", "category": "agent-observability", "summary": "Hosted tracing, evals, prompt management and an LLM gateway to 250+ models, from the company formerly called Keywords AI.", "url": "https://www.anchorterminal.com/tools/respan", "markdownUrl": "https://www.anchorterminal.com/tools/respan.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/respan.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/respan.json", "repo": "https://github.com/respanai/respan", "license": "Apache-2.0 (SDKs only, platform closed source)", "transports": [ "http", "streamable-http" ], "remoteUrl": "https://api.respan.ai/api", "packages": [ { "registry": "pypi", "name": "respan-tracing" }, { "registry": "npm", "name": "@respan/tracing" } ], "auth": "mixed", "authNotes": "API key as a Bearer token for the REST API and MCP. Keys carry Read, Write, Gateway and Admin scopes (new keys default to Read, Write and Gateway), can have an expiry and spend caps, and are bound to one project. The hosted MCP server also supports OAuth sign-in, or a JWT copied from mcp.respan.ai/login. A Respan-Enabled-Tools header limits which MCP tools a client sees.", "pricing": "freemium", "pricingNotes": "Free plan with 100,000 logs, 1,000 scores, 7-day retention, unlimited seats and 412 requests a minute on the gateway. Team $199 a month billed yearly with 5 seats ($15 per extra member), 10,000 scores, 30-day retention and 8,400 gateway requests a minute. Extra logs $8 per 100,000, extra scores $1 per 1,000, HIPAA add-on $249 a month. Enterprise is custom (https://www.respan.ai/pricing).", "priceSummary": "$199 / mo", "where": "hosted", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": 67, "popularity": { "githubStars": null, "npmWeekly": 270, "pypiWeekly": 12515, "asOf": "2026-09-30" }, "docsUrl": "https://www.respan.ai/docs", "llmsTxt": "https://www.respan.ai/docs/llms.txt", "openapi": "https://respan.ai/docs/openapi.json", "capabilities": [ "obs.traces", "obs.evals", "obs.prompts", "obs.gateway", "obs.datasets" ], "tags": [ "hosted", "freemium", "mcp", "llms-txt", "openapi", "python", "typescript", "closed-source", "webhooks", "enterprise" ], "lastRelease": "2026-09-30", "graded": true, "anchor": { "graded": true, "score": 65.9, "grade": "B", "agentReady": false, "rank": 165, "ranked": true, "rankOf": 452, "categoryRank": 4, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 66, "maintenance": 85, "payments": 30, "reliability": 73, "schema": 89, "security": 66, "transparency": 61 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-01" }, "negative": -2, "negativeNotes": [ "2026-07-30 and 2026-08-28. The changelog lists \"Closed an unauthorized live-event stream vulnerability\" and \"Patched some security vulnerabilities\" with no advisory, scope or statement on whether customer data was exposed. Fixed, but disclosed in one line each. -2 (https://www.respan.ai/docs/changelog/changelog)" ], "verdict": "API keys with Read, Write, Gateway and Admin scopes, expiry and spend caps, bound to one project. 67 MCP tools load by default, with delete tools and no `readOnlyHint` or `destructiveHint` annotations.", "strengths": [ "API keys with Read, Write, Gateway and Admin scopes, expiry and spend caps, bound to one project", "OpenAPI 3.1 spec, llms.txt and Markdown for every docs page", "`Respan-Enabled-Tools` header trims the MCP server's tool list server-side", "`list_logs` returns summary fields by default with pages of up to 50 rows", "Published overage prices ($8 per 100,000 logs) and per-plan gateway and ingestion limits" ], "weaknesses": [ "67 MCP tools load by default, with delete tools and no `readOnlyHint` or `destructiveHint` annotations", "A 13-hour 47-minute gateway incident on 26 September 2026, while the status bars show 100 per cent", "Security fixes on 30 July and 28 August 2026 disclosed only as one-line changelog entries", "Free plan keeps logs for 7 days", "No deprecation policy, and the legacy `/api/generate` endpoint was retired on 11 September 2026" ], "agentNotes": [ "Create a key with only the Read scope for agents that analyse logs. New keys get Read, Write and Gateway by default", "Set `Respan-Enabled-Tools` to a short list such as list_traces,get_trace_tree,list_logs. All 67 tools load otherwise", "Pass `filters` to `list_logs` rather than paging through everything. The default window is the last hour", "Call `list_organizations` first when results look empty over OAuth. Every tool acts on the active organisation only", "Append `.md` to any docs URL for clean Markdown" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 2.5, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "B", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 65.9 } ], "editorialScores": { "ergonomics": 66, "maintenance": 85, "payments": 30, "reliability": 73, "schema": 89, "security": 66, "transparency": 47 }, "provenanceScore": 75 }, "connect": { "http": "curl -X POST https://api.respan.ai/api/traces/list/ -H \"Authorization: Bearer $RESPAN_API_KEY\" \\\n -H \"content-type: application/json\" -d '{}'", "claudeCode": "claude mcp add --transport http --header \"Authorization: Bearer $RESPAN_API_KEY\" respan https://mcp.respan.ai/mcp", "config": { "mcpServers": { "respan": { "headers": { "Authorization": "Bearer ${RESPAN_API_KEY}" }, "type": "http", "url": "https://mcp.respan.ai/mcp" } } } }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/respan" }, "area": "developer", "unitPrices": [ { "item": "Team plan", "unit": "month", "usd": 199, "note": "billed yearly, 5 seats, 10,000 scores, 30-day retention" }, { "item": "Extra Team member", "unit": "seat-month", "usd": 15 }, { "item": "HIPAA add-on", "unit": "month", "usd": 249 } ], "provenance": { "legalEntity": "Keywords AI Inc. (d/b/a Respan)", "domain": "respan.ai", "domainRegistered": "2026-01-13", "domainNote": "respan.ai shows a 2026-01-13 registration, matching the rename from Keywords AI (keywordsai.co).", "endpointOnVendorDomain": true, "terms": "https://www.respan.ai/legal/terms-of-use", "privacy": "https://www.respan.ai/legal/privacy-policy", "statusPage": "https://status.respan.ai", "changelog": "https://www.respan.ai/docs/changelog/changelog", "securityTxt": "none", "checked": "2026-09-30", "score": 75 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/respan.json", "live": { "slug": "respan", "probe": { "target": "https://api.respan.ai/api", "method": "get", "lastAt": "2026-10-04T22:35:30.310685129Z", "lastOk": true, "lastStatus": 404, "lastMs": 269, "authRequired": false, "uptime24h": 100, "uptime30d": 100, "p50ms24h": 263, "p95ms24h": 351, "samples24h": 272, "samples30d": 1086, "days": [ { "date": "2026-09-30", "probes": 35, "ok": 35 }, { "date": "2026-10-01", "probes": 276, "ok": 276 }, { "date": "2026-10-02", "probes": 248, "ok": 248 }, { "date": "2026-10-03", "probes": 271, "ok": 271 }, { "date": "2026-10-04", "probes": 256, "ok": 256 } ] }, "vendorStatus": { "page": "https://status.respan.ai", "indicator": "unknown", "summary": "no machine-readable status found", "checkedAt": "2026-10-04T21:40:26.483050488Z" }, "versions": [ { "registry": "npm", "name": "@respan/tracing", "version": "1.6.1", "seenAt": "2026-10-04T16:38:26.934877475Z" }, { "registry": "pypi", "name": "respan-tracing", "version": "2.20.1", "released": "2026-09-30", "seenAt": "2026-10-04T16:38:26.745912023Z" } ], "githubStars": 48, "npmWeekly": 261, "pypiWeekly": 14951, "securityTxt": { "url": "https://respan.ai/.well-known/security.txt", "state": "none", "checkedAt": "2026-10-04T15:15:47.892233894Z" }, "llmsTxt": { "url": "https://www.respan.ai/docs/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-04T15:18:11.693900288Z" }, "domain": { "domain": "respan.ai", "registered": "2026-01-13", "source": "https://rdap.identitydigital.services/rdap/domain/respan.ai", "checkedAt": "2026-10-04T13:04:19.8662822Z" }, "pages": [ { "url": "https://www.respan.ai/docs/changelog/changelog", "kind": "changelog", "status": 304, "checkedAt": "2026-10-04T15:51:52.1620977Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "8b11c1ae998c" }, { "url": "https://www.respan.ai/pricing", "kind": "pricing", "status": 200, "checkedAt": "2026-10-04T15:51:58.388862278Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "4df3d42fb60f" }, { "url": "https://www.respan.ai/legal/privacy-policy", "kind": "privacy", "status": 200, "checkedAt": "2026-10-04T15:51:55.022515419Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "560ff946cfa5" }, { "url": "https://www.respan.ai/legal/terms-of-use", "kind": "terms", "status": 200, "checkedAt": "2026-10-04T15:51:56.646058071Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "868dea928c7d" } ], "updatedAt": "2026-10-04T22:35:30.310685129Z" } }, { "slug": "braintrust", "name": "Braintrust API + MCP", "vendor": "Braintrust", "vendorUrl": "https://www.braintrust.dev", "kind": "http-api", "category": "agent-observability", "summary": "Hosted tracing, logging and evaluation for LLM apps and agents, with experiments, datasets, prompts, online scorers and a model gateway.", "url": "https://www.anchorterminal.com/tools/braintrust", "markdownUrl": "https://www.anchorterminal.com/tools/braintrust.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/braintrust.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/braintrust.json", "repo": "https://github.com/braintrustdata/braintrust-sdk-javascript", "license": "Apache-2.0 (SDKs only, platform closed source)", "transports": [ "http", "streamable-http" ], "remoteUrl": "https://api.braintrust.dev/v1", "packages": [ { "registry": "npm", "name": "braintrust" }, { "registry": "pypi", "name": "braintrust" } ], "auth": "mixed", "authNotes": "REST API and SDKs take an org API key as a Bearer token (BRAINTRUST_API_KEY). The hosted MCP server accepts OAuth 2.0 with dynamic client registration or the same API key, and acts with the permissions of the key's account. EU and self-hosted data planes have their own API and MCP URLs.", "pricing": "freemium", "pricingNotes": "Starter is free with no card, unlimited users, 1 GB processed data and 10,000 scores a month, 14-day retention and $10 of model credit. Overage $4 a GB and $2.50 per 1,000 scores. Pro $249 a month with 5 GB, 50,000 scores, $100 of model credit and 30-day retention, then $3 a GB, $1.50 per 1,000 scores and $0.50 a GB a month for longer retention. Qualifying startups can get Pro free for 6 to 12 months. Enterprise is custom and adds on-prem or hybrid deployment, a BAA and SLAs (https://www.braintrust.dev/pricing).", "priceSummary": "$249 / mo", "where": "hosted", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": 42, "popularity": { "githubStars": null, "npmWeekly": 2018529, "pypiWeekly": 1701906, "asOf": "2026-09-30" }, "docsUrl": "https://www.braintrust.dev/docs", "llmsTxt": "https://www.braintrust.dev/docs/llms.txt", "openapi": "https://www.braintrust.dev/docs/openapi.json", "registryName": "io.github.braintrustdata/braintrust", "capabilities": [ "obs.traces", "obs.evals", "obs.prompts", "obs.gateway", "obs.datasets" ], "tags": [ "hosted", "freemium", "no-card", "mcp", "llms-txt", "openapi", "python", "typescript", "closed-source", "enterprise" ], "lastRelease": "2026-10-01", "graded": true, "anchor": { "graded": true, "score": 61.3, "grade": "C", "agentReady": false, "rank": 229, "ranked": true, "rankOf": 452, "categoryRank": 5, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 66, "maintenance": 85, "payments": 40, "reliability": 61, "schema": 85, "security": 58, "transparency": 68 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-01" }, "negative": -4, "negativeNotes": [ "Fixed 2026-07-16. The Python SDK's LiteLLM instrumentation in versions 0.2.0 to 0.27.x wrote provider credentials into trace metadata, among them API keys, `Authorization` headers, Azure tokens and AWS access and secret keys. Fixed in 0.28.0 with a metadata allowlist, and a knowledge-base page tells users to rotate. It reads as an advisory but isn't filed as one and carries no date. -2 after decay (https://braintrust.dev/docs/kb/litellm-instrumentation-may-record-provider-credentials.md)", "Fixed 2026-07-16. The TypeScript SDK from 0.4.0 to 3.23.0 recorded API keys passed to the Claude Agent SDK through `options.apiKey` into trace metadata. The 3.23.1 changelog line calls the fix \"Clean up span metadata\", and only the knowledge-base page explains the exposure and asks for rotation. -2 after decay (https://braintrust.dev/docs/kb/claude-agent-sdk-api-keys-recorded-in-trace-metadata.md)" ], "verdict": "OpenAPI 3.0.3 spec with 75 paths, 429 and `Retry-After` declared on 154 operations. Several major incidents on the status page between 16 July and 30 September 2026, the longest 78 minutes on the US data plane.", "strengths": [ "OpenAPI 3.0.3 spec with 75 paths, 429 and `Retry-After` declared on 154 operations", "Hosted MCP server with 42 tools, OAuth 2.0 or API key, and `sql_query` that hands results over 1 MB back as a signed URL", "Service tokens can be scoped to single projects through permission groups on every plan", "Starter is free with no card, unlimited users and published per-GB and per-score overage prices", "TypeScript and Python SDKs both released on 2026-10-01, with 34 releases between them since 16 July" ], "weaknesses": [ "Several major incidents on the status page between 16 July and 30 September 2026, the longest 78 minutes on the US data plane", "Python SDK up to 0.27.x and TypeScript SDK up to 3.23.0 recorded provider credentials in trace metadata, fixed on 2026-07-16 without a formal advisory", "All 42 MCP tools load by default and write tools act with the full permissions of the signed-in account", "Only BTQL has a published rate limit, about 20 queries a minute on Starter and Pro", "Privacy notice dates from September 2023 and excludes customer data. Starter keeps data for 14 days" ], "agentNotes": [ "Connect a project-scoped service token rather than a personal key, because MCP write tools act with the key's full permissions", "Set the client to confirm MCP write tools such as `edit_dataset_rows` and `create_threshold_alert`. The server has no read-only mode", "Cache `sql_query` results. Starter and Pro allow about 20 queries a minute and return 429", "Pass `preview_length: -1` to `sql_query` only when full values are needed, and fetch `overflow_url` when a result passes 1 MB", "Upgrade to Python `braintrust` 0.28.0 or TypeScript 3.23.1 or later and rotate any provider keys traced before July 2026" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 3, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "C", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 61.3 } ], "editorialScores": { "ergonomics": 66, "maintenance": 85, "payments": 40, "reliability": 61, "schema": 85, "security": 58, "transparency": 50 }, "provenanceScore": 86 }, "connect": { "http": "curl https://api.braintrust.dev/v1/project -H \"Authorization: Bearer $BRAINTRUST_API_KEY\"", "claudeCode": "claude mcp add --transport http braintrust https://api.braintrust.dev/mcp", "config": { "mcpServers": { "braintrust": { "headers": { "Authorization": "Bearer ${BRAINTRUST_API_KEY}" }, "type": "http", "url": "https://api.braintrust.dev/mcp" } } } }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/braintrust" }, "area": "developer", "unitPrices": [ { "item": "Pro plan", "unit": "month", "usd": 249, "note": "5 GB processed data, 50,000 scores, 30-day retention" }, { "item": "Processed data on Starter", "unit": "gb", "usd": 4 }, { "item": "Processed data on Pro", "unit": "gb", "usd": 3 }, { "item": "Extended retention on Pro", "unit": "gb", "usd": 0.5, "note": "per GB a month beyond 30 days" } ], "provenance": { "legalEntity": "Braintrust Data, Inc.", "domain": "braintrust.dev", "domainRegistered": "2021-03-25", "endpointOnVendorDomain": true, "terms": "https://www.braintrust.dev/legal/terms-of-service", "privacy": "https://www.braintrust.dev/legal/privacy-policy", "statusPage": "https://status.braintrust.dev", "changelog": "https://www.braintrust.dev/docs/changelog", "securityTxt": "none", "checked": "2026-09-30", "score": 86 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/braintrust.json", "live": { "slug": "braintrust", "probe": { "target": "https://api.braintrust.dev/v1", "method": "get", "lastAt": "2026-10-04T22:35:20.229030202Z", "lastOk": true, "lastStatus": 200, "lastMs": 104, "authRequired": false, "uptime24h": 99.63, "uptime30d": 99.82, "p50ms24h": 108, "p95ms24h": 186, "samples24h": 272, "samples30d": 1086, "days": [ { "date": "2026-09-30", "probes": 35, "ok": 34 }, { "date": "2026-10-01", "probes": 276, "ok": 276 }, { "date": "2026-10-02", "probes": 248, "ok": 248 }, { "date": "2026-10-03", "probes": 271, "ok": 271 }, { "date": "2026-10-04", "probes": 256, "ok": 255 } ] }, "vendorStatus": { "page": "https://status.braintrust.dev", "indicator": "none", "summary": "All Systems Operational", "checkedAt": "2026-10-04T22:33:47.681674393Z" }, "versions": [ { "registry": "github", "name": "braintrustdata/braintrust-sdk-javascript", "version": "@braintrust/temporal@1.0.1", "released": "2026-10-01", "seenAt": "2026-10-04T16:22:40.159832334Z" }, { "registry": "mcp-registry", "name": "io.github.braintrustdata/braintrust", "version": "1.0.0", "seenAt": "2026-10-03T23:29:28.630222764Z" }, { "registry": "npm", "name": "braintrust", "version": "3.36.0", "seenAt": "2026-10-04T16:22:39.707776024Z" }, { "registry": "pypi", "name": "braintrust", "version": "0.44.0", "released": "2026-10-01", "seenAt": "2026-10-04T16:22:39.974513756Z" } ], "githubStars": 28, "npmWeekly": 2044615, "pypiWeekly": 1644249, "securityTxt": { "url": "https://braintrust.dev/.well-known/security.txt", "state": "none", "checkedAt": "2026-10-04T15:15:41.228122743Z" }, "llmsTxt": { "url": "https://www.braintrust.dev/docs/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-04T15:17:21.471688058Z" }, "domain": { "domain": "braintrust.dev", "registered": "2021-03-25", "source": "https://pubapi.registry.google/rdap/domain/braintrust.dev", "checkedAt": "2026-10-04T13:08:21.062714804Z" }, "pages": [ { "url": "https://www.braintrust.dev/docs/changelog", "kind": "changelog", "status": 200, "checkedAt": "2026-10-04T15:49:33.413638455Z", "changedAt": "2026-10-03T15:37:29.257129491Z", "fingerprint": "92cfa827a3cc" }, { "url": "https://www.braintrust.dev/pricing", "kind": "pricing", "status": 200, "checkedAt": "2026-10-04T15:49:39.826623842Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "aeaf56390ac0" }, { "url": "https://www.braintrust.dev/legal/privacy-policy", "kind": "privacy", "status": 200, "checkedAt": "2026-10-04T15:49:35.778324091Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "834b4aa10e1f" }, { "url": "https://www.braintrust.dev/legal/terms-of-service", "kind": "terms", "status": 200, "checkedAt": "2026-10-04T15:49:37.707572087Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "2f9eb5dc8e8d" } ], "updatedAt": "2026-10-04T22:35:20.229030202Z" } }, { "slug": "laminar", "name": "Laminar API + MCP", "vendor": "Laminar (LMNR AI)", "vendorUrl": "https://laminar.sh", "kind": "http-api", "category": "agent-observability", "summary": "Open-source tracing, evaluations and an agent debugger built on OpenTelemetry, with browser session recordings synced to traces for browser agents.", "url": "https://www.anchorterminal.com/tools/laminar", "markdownUrl": "https://www.anchorterminal.com/tools/laminar.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/laminar.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/laminar.json", "repo": "https://github.com/lmnr-ai/lmnr", "license": "Apache-2.0", "transports": [ "http", "streamable-http" ], "remoteUrl": "https://api.lmnr.ai/v1", "packages": [ { "registry": "pypi", "name": "lmnr" }, { "registry": "npm", "name": "@lmnr-ai/lmnr" } ], "auth": "api-key", "authNotes": "Project API key as a Bearer token, for ingestion, the REST API and the MCP server. One key maps to one project.", "pricing": "freemium", "pricingNotes": "Free $0 a month with 1 GB of data, 7 days of retention, 1 project and 1 seat, no overage. Starter $30 a month with 3 GB then $2 a GB, 30 days, unlimited projects and seats. Pro $150 a month with 10 GB then $1.50 a GB, 6 months. Enterprise custom with on-premise. Signals (LLM analysis of traces) comes with $2.50, $7.50 or $25 of credit, then $0.05 per million input tokens and $0.35 per million output tokens. Self-hosted open source is free, you run Postgres, ClickHouse and Quickwit yourself. Signals and Slack alerts on self-hosted need an enterprise licence key (https://laminar.sh/pricing).", "priceSummary": "$30 / mo", "where": "hosted", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": 3, "popularity": { "githubStars": 3288, "npmWeekly": 79857, "pypiWeekly": 1698863, "asOf": "2026-09-30" }, "docsUrl": "https://laminar.sh/docs", "llmsTxt": "https://laminar.sh/docs/llms.txt", "openapi": "https://laminar.sh/docs/openapi/openapi.yaml", "capabilities": [ "obs.traces", "obs.evals", "obs.datasets" ], "tags": [ "hosted", "freemium", "open-source", "self-hosted", "mcp", "llms-txt", "openapi", "python", "typescript" ], "lastRelease": "2026-09-23", "graded": true, "anchor": { "graded": true, "score": 57, "grade": "C", "agentReady": false, "rank": 298, "ranked": true, "rankOf": 452, "categoryRank": 6, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 80, "maintenance": 83, "payments": 30, "reliability": 50, "schema": 88, "security": 45, "transparency": 66 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-01" }, "negative": -5, "negativeNotes": [ "2026-08-27. Laminar's own fix says a client-supplied `projectId` in the body of the SQL export-job route overrode the one the server had authorised, allowing cross-tenant SQL execution and bulk data export (CWE-862). The route dates from July 2025. Fixed with a one-line change, disclosed only in the commit message with no advisory or customer notice we could find. -5 after decay for the fix (https://github.com/lmnr-ai/lmnr/commit/5059c7d951fe47d4c91c8016a542a11e82090ade)" ], "verdict": "Browser session recordings can be matched to agent traces. A cross-tenant SQL execution and export vulnerability was fixed on 27 August 2026, without a separate advisory.", "strengths": [ "Browser session recordings synced to traces for Browser Use, Stagehand, Playwright and Puppeteer", "Three MCP tools with descriptions that say when to use each, and SQL that runs SELECT-only on a read-only client", "Apache-2.0 with Docker Compose and Helm self-hosting, and self-hosted telemetry that switches off with one variable", "Priced by GB stored, $2 a GB on Starter and $1.50 on Pro, published without login", "SDK releases most weeks, TypeScript 0.8.49 on 2026-09-23 and Python 0.7.64 on 2026-09-21" ], "weaknesses": [ "A cross-tenant SQL execution and export hole was fixed on 27 August 2026 in a commit, with no advisory", "No SECURITY.md, security.txt or published advisories", "Rate limits exist per project but no numbers are published", "Status page covers only the homepage and an echo probe", "Signals, clusters and Slack or email alerts aren't in the open-source build" ], "agentNotes": [ "Use `query_laminar_sql` to find trace IDs, then `get_trace_context` for a compact summary of one run", "Always add a time range and `LIMIT` to SQL. Server caps on time, memory and result size reject wide scans", "On 429 from the SQL API, back off. The per-project limit isn't published", "Set `flush_by_size` when spans carry whole conversation histories, or large batches get rejected", "Keep the project API key in a header, not in the URL" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 3.5, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "C", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 57 } ], "editorialScores": { "ergonomics": 80, "maintenance": 83, "payments": 30, "reliability": 50, "schema": 88, "security": 45, "transparency": 57 }, "provenanceScore": 75 }, "connect": { "http": "curl https://api.lmnr.ai/v1/project -H \"Authorization: Bearer $LMNR_PROJECT_API_KEY\"", "claudeCode": "claude mcp add --transport http laminar https://api.lmnr.ai/v1/mcp \\\n --header \"Authorization: Bearer $LMNR_PROJECT_API_KEY\"" }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/laminar" }, "area": "developer", "unitPrices": [ { "item": "Starter plan", "unit": "month", "usd": 30, "note": "3 GB of data, 30 days retention, unlimited seats" }, { "item": "Pro plan", "unit": "month", "usd": 150, "note": "10 GB of data, 6 months retention" }, { "item": "Extra data on Starter", "unit": "gb", "usd": 2 }, { "item": "Extra data on Pro", "unit": "gb", "usd": 1.5 } ], "provenance": { "legalEntity": "LMNR AI, Inc.", "domain": "laminar.sh", "domainRegistered": "", "domainNote": "The .sh registry has no RDAP service. The former domain lmnr.ai, registered 2024-02-26, now redirects to laminar.sh and still serves the API.", "endpointOnVendorDomain": true, "terms": "https://laminar.sh/policies/terms", "privacy": "https://laminar.sh/policies/privacy", "statusPage": "https://status.laminar.sh", "changelog": "https://laminar.sh/docs/changelog", "securityTxt": "none", "checked": "2026-09-30", "score": 75 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/laminar.json", "live": { "slug": "laminar", "probe": { "target": "https://api.lmnr.ai/v1", "method": "get", "lastAt": "2026-10-04T22:35:25.588503544Z", "lastOk": true, "lastStatus": 404, "lastMs": 257, "authRequired": false, "uptime24h": 100, "uptime30d": 100, "p50ms24h": 260, "p95ms24h": 332, "samples24h": 272, "samples30d": 1086, "days": [ { "date": "2026-09-30", "probes": 35, "ok": 35 }, { "date": "2026-10-01", "probes": 276, "ok": 276 }, { "date": "2026-10-02", "probes": 248, "ok": 248 }, { "date": "2026-10-03", "probes": 271, "ok": 271 }, { "date": "2026-10-04", "probes": 256, "ok": 256 } ] }, "vendorStatus": { "page": "https://status.laminar.sh", "indicator": "unknown", "summary": "no machine-readable status found", "checkedAt": "2026-10-04T21:40:11.503101655Z" }, "versions": [ { "registry": "github", "name": "lmnr-ai/lmnr", "version": "v0.2.5", "released": "2026-09-13", "seenAt": "2026-10-04T16:31:08.795419199Z" }, { "registry": "npm", "name": "@lmnr-ai/lmnr", "version": "0.8.49", "seenAt": "2026-10-04T16:31:07.940881866Z" }, { "registry": "pypi", "name": "lmnr", "version": "0.7.64", "released": "2026-09-21", "seenAt": "2026-10-04T16:31:07.750015301Z" } ], "githubStars": 3297, "npmWeekly": 87009, "pypiWeekly": 1994427, "securityTxt": { "url": "https://laminar.sh/.well-known/security.txt", "state": "none", "checkedAt": "2026-10-04T15:15:42.001487057Z" }, "llmsTxt": { "url": "https://laminar.sh/docs/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-04T15:17:55.609508329Z" }, "domain": { "domain": "laminar.sh", "checkedAt": "2026-10-04T13:08:33.087077121Z" }, "pages": [ { "url": "https://laminar.sh/docs/changelog", "kind": "changelog", "status": 200, "checkedAt": "2026-10-04T15:45:16.56490294Z", "changedAt": "2026-10-03T15:33:21.857290455Z", "fingerprint": "26ea3af58e20" }, { "url": "https://laminar.sh/pricing", "kind": "pricing", "status": 200, "checkedAt": "2026-10-04T15:45:22.900150315Z", "changedAt": "2026-10-03T15:33:28.075146176Z", "fingerprint": "97b56e4f6b09" }, { "url": "https://laminar.sh/policies/privacy", "kind": "privacy", "status": 200, "checkedAt": "2026-10-04T15:45:19.093421142Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "9cf47952b952" }, { "url": "https://laminar.sh/policies/terms", "kind": "terms", "status": 200, "checkedAt": "2026-10-04T15:45:20.742564071Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "30d7a2d7eb44" } ], "updatedAt": "2026-10-04T22:35:25.588503544Z" } }, { "slug": "honeyhive", "name": "HoneyHive", "vendor": "HoneyHive", "vendorUrl": "https://www.honeyhive.ai", "kind": "http-api", "category": "agent-observability", "summary": "Hosted tracing, evaluation, datasets and prompt management for agents, built on OpenTelemetry.", "url": "https://www.anchorterminal.com/tools/honeyhive", "markdownUrl": "https://www.anchorterminal.com/tools/honeyhive.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/honeyhive.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/honeyhive.json", "repo": "https://github.com/honeyhiveai/python-sdk", "license": "MIT (SDK, CLI and OpenAPI specs only, platform closed source)", "transports": [ "http" ], "remoteUrl": "https://api.dp1.us.honeyhive.ai", "packages": [ { "registry": "pypi", "name": "honeyhive" }, { "registry": "npm", "name": "honeyhive" } ], "auth": "api-key", "authNotes": "Bearer API keys in four types. Project keys (`hh_`) and read-only project keys (`hh_ro_`) for the data plane, ingestion keys (`hh_ingst_`) that can only send traces and events to one project, and fine-grained keys (`hh_fgcp_`) for the control plane, scoped to an organisation or workspace with chosen permissions and a required expiry of at most 365 days. Data plane at api.dp1.us.honeyhive.ai, control plane at api.cp.us.honeyhive.ai. Since 24 September 2026 invalid keys and permission failures return 404. Self-hosted deployments can use an OIDC identity provider.", "pricing": "freemium", "pricingNotes": "Free Developer plan with 10,000 events a month, up to 5 users and 30-day retention. An event is one trace span or one metric label. Enterprise is custom with unlimited users and workspaces, custom retention and self-hosting. No self-serve paid tier is published (https://www.honeyhive.ai/pricing).", "priceSummary": "Freemium", "where": "hosted", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": null, "popularity": { "githubStars": null, "npmWeekly": 60, "pypiWeekly": 2105, "asOf": "2026-09-30" }, "docsUrl": "https://docs.honeyhive.ai", "llmsTxt": "https://docs.honeyhive.ai/llms.txt", "openapi": "https://raw.githubusercontent.com/honeyhiveai/honeyhive-openapi/main/data_plane_openapi.yaml", "capabilities": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "tags": [ "hosted", "freemium", "llms-txt", "openapi", "python", "typescript", "closed-source", "enterprise", "self-hosted" ], "lastRelease": "2026-09-29", "graded": true, "anchor": { "graded": true, "score": 55.9, "grade": "C", "agentReady": false, "rank": 310, "ranked": true, "rankOf": 452, "categoryRank": 7, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 63, "maintenance": 77, "payments": 20, "reliability": 44, "schema": 86, "security": 61, "transparency": 68 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-01" }, "negative": -3, "negativeNotes": [ "2026-09-22 to 2026-09-24. The API stopped returning 401 for invalid, revoked or expired keys and 403 for permission failures, and now answers 404 for all of them, on every client version. It was announced in the CLI 1.7.0 changelog on 22 September and the product changelog on 24 September, with no deprecation window. -3 (https://github.com/honeyhiveai/honeyhive-cli/blob/main/CHANGELOG.md)" ], "verdict": "Read-only (`hh_ro_`), ingestion-only (`hh_ingst_`) and expiring fine-grained (`hh_fgcp_`) API keys. Status page showed 80.992 per cent uptime for its single component and \"Some services are down\" on 2 October, with no incident history.", "strengths": [ "Read-only (`hh_ro_`), ingestion-only (`hh_ingst_`) and expiring fine-grained (`hh_fgcp_`) API keys", "Public OpenAPI 3.1 specs for data plane and control plane under MIT, with an oasdiff-generated changelog", "CLI generated one to one from the specs, plus agent skills and Markdown docs", "Python SDK 1.6.1 and CLI 1.7.0 released in the last two weeks of September 2026", "Free plan with 30-day retention, 1,000 requests a minute and no card" ], "weaknesses": [ "Status page showed 80.992 per cent uptime for its single component and \"Some services are down\" on 2 October, with no incident history", "No MCP server for trace data, only a docs search MCP", "No self-serve paid tier, only Free and a sales-led Enterprise plan", "Since 24 September 2026 invalid keys and permission failures return 404, with no deprecation window", "Event search returns up to 1,000 rows by default with no field selection" ], "agentNotes": [ "Give the agent a read-only `hh_ro_` key for analysis and a separate `hh_ingst_` key for sending traces", "Treat a 404 as possibly a bad, revoked or expired key before assuming the record is missing", "Set `limit` on `POST /v1/events/search`. The default is 1,000 rows", "Use the CLI (`brew tap honeyhiveai/tap \u0026\u0026 brew install honeyhive`) from a coding agent. It maps one command to each API endpoint", "Control plane commands need an `hh_fgcp_` key and the control plane host api.cp.us.honeyhive.ai" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 2.5, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "C", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 55.9 } ], "editorialScores": { "ergonomics": 63, "maintenance": 77, "payments": 20, "reliability": 44, "schema": 86, "security": 61, "transparency": 54 }, "provenanceScore": 82 }, "connect": { "http": "curl https://api.dp1.us.honeyhive.ai/v1/datasets -H \"Authorization: Bearer $HH_API_KEY\"" }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/honeyhive" }, "area": "developer", "provenance": { "legalEntity": "HoneyHive Inc.", "domain": "honeyhive.ai", "domainRegistered": "2022-07-28", "endpointOnVendorDomain": true, "terms": "https://www.honeyhive.ai/legal/terms-of-service", "privacy": "https://www.honeyhive.ai/legal/privacy-policy", "statusPage": "https://status.honeyhive.ai", "changelog": "https://docs.honeyhive.ai/v2/changelog/product", "securityTxt": "none", "checked": "2026-09-30", "score": 82 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/honeyhive.json", "live": { "slug": "honeyhive", "probe": { "target": "https://api.dp1.us.honeyhive.ai", "method": "get", "lastAt": "2026-10-04T22:35:24.621348994Z", "lastOk": true, "lastStatus": 404, "lastMs": 420, "authRequired": false, "uptime24h": 100, "uptime30d": 100, "p50ms24h": 447, "p95ms24h": 481, "samples24h": 272, "samples30d": 1086, "days": [ { "date": "2026-09-30", "probes": 35, "ok": 35 }, { "date": "2026-10-01", "probes": 276, "ok": 276 }, { "date": "2026-10-02", "probes": 248, "ok": 248 }, { "date": "2026-10-03", "probes": 271, "ok": 271 }, { "date": "2026-10-04", "probes": 256, "ok": 256 } ] }, "vendorStatus": { "page": "https://status.honeyhive.ai", "indicator": "unknown", "summary": "no machine-readable status found", "checkedAt": "2026-10-04T21:40:07.679425636Z" }, "versions": [ { "registry": "github", "name": "honeyhiveai/python-sdk", "version": "v1.6.1", "released": "2026-09-29", "seenAt": "2026-10-04T16:29:47.051274003Z" }, { "registry": "npm", "name": "honeyhive", "version": "1.0.45", "seenAt": "2026-10-04T16:29:46.131902271Z" }, { "registry": "pypi", "name": "honeyhive", "version": "1.6.1", "released": "2026-09-29", "seenAt": "2026-10-04T16:29:45.944378114Z" } ], "githubStars": 3, "npmWeekly": 115, "pypiWeekly": 2157, "securityTxt": { "url": "https://honeyhive.ai/.well-known/security.txt", "state": "none", "checkedAt": "2026-10-04T15:15:43.408713013Z" }, "llmsTxt": { "url": "https://docs.honeyhive.ai/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-04T15:17:52.608156452Z" }, "domain": { "domain": "honeyhive.ai", "registered": "2022-07-28", "source": "https://rdap.identitydigital.services/rdap/domain/honeyhive.ai", "checkedAt": "2026-10-04T13:04:07.481546397Z" }, "pages": [ { "url": "https://docs.honeyhive.ai/v2/changelog/product", "kind": "changelog", "status": 200, "checkedAt": "2026-10-04T15:43:41.843601235Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "d1e93acad1c0" }, { "url": "https://www.honeyhive.ai/pricing", "kind": "pricing", "status": 304, "checkedAt": "2026-10-04T15:50:44.12758554Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "5861207b5e2f" }, { "url": "https://www.honeyhive.ai/legal/privacy-policy", "kind": "privacy", "status": 304, "checkedAt": "2026-10-04T15:50:40.106324522Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "728d616153ce" }, { "url": "https://www.honeyhive.ai/legal/terms-of-service", "kind": "terms", "status": 304, "checkedAt": "2026-10-04T15:50:42.132533735Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "68e3e45c1001" } ], "updatedAt": "2026-10-04T22:35:24.621348994Z" } }, { "slug": "galileo", "name": "Galileo API + MCP", "vendor": "Galileo (now Splunk Agent Observability, Cisco)", "vendorUrl": "https://galileo.ai", "kind": "http-api", "category": "agent-observability", "summary": "Hosted tracing, evaluation metrics and guardrails for LLM apps and agents, with a REST API, Python and TypeScript SDKs and an 8-tool MCP server in preview.", "url": "https://www.anchorterminal.com/tools/galileo", "markdownUrl": "https://www.anchorterminal.com/tools/galileo.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/galileo.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/galileo.json", "repo": "https://github.com/rungalileo/galileo-python", "license": "Apache-2.0 (SDKs only, platform closed source)", "transports": [ "http", "streamable-http" ], "remoteUrl": "https://api.galileo.ai/v2", "packages": [ { "registry": "pypi", "name": "galileo" }, { "registry": "npm", "name": "galileo" } ], "auth": "api-key", "authNotes": "API key in a header. The current OpenAPI spec names it `Splunk-AO-API-Key` and older Galileo docs `Galileo-API-Key`. The API also issues a JWT from `/login/api_key` that expires after 24 hours and accepts HTTP Basic with a username and password. The MCP server takes the same API key header. New SaaS accounts on Splunk Observability Cloud use `https://app.{realm}.observability.splunkcloud.com/ao/api/` and Splunk's own authentication. Self-hosted deployments use their own api.* host.", "pricing": "freemium", "pricingNotes": "Free plan with 5,000 traces a month, unlimited users and unlimited custom evals. Pro $100 a month billed yearly with 50,000 traces, then priced by trace volume with no published rate. Enterprise is custom with unlimited traces, custom rate limits and hosted, VPC or on-prem deployment. The pricing page doesn't mention Splunk, but the docs say customers onboarded after 2026-08-07 should use the Splunk docs (https://galileo.ai/pricing).", "priceSummary": "$100 / mo", "where": "hosted", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": 8, "popularity": { "githubStars": null, "npmWeekly": 1431, "pypiWeekly": 6160, "asOf": "2026-09-30" }, "docsUrl": "https://docs.galileo.ai", "llmsTxt": "https://docs.galileo.ai/llms.txt", "openapi": "https://api.galileo.ai/public/v2/openapi.json", "capabilities": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "tags": [ "hosted", "freemium", "mcp", "llms-txt", "openapi", "python", "typescript", "closed-source", "enterprise" ], "lastRelease": "2026-10-01", "graded": true, "anchor": { "graded": true, "score": 48, "grade": "D", "agentReady": false, "rank": 378, "ranked": true, "rankOf": 452, "categoryRank": 8, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 73, "maintenance": 74, "payments": 20, "reliability": 18, "schema": 79, "security": 33, "transparency": 56 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-01" }, "negative": 0, "verdict": "OpenAPI 3.1 spec with 184 paths and 244 operations. No status page, no published rate limits and no SLA.", "strengths": [ "OpenAPI 3.1 spec with 184 paths and 244 operations", "Error catalogue of 111 entries with HTTP status, fix and a retriable flag", "Free plan with 5,000 traces a month and unlimited users", "TypeScript SDK released three times in September 2026, most recently 2.3.2 on 1 October", "Enterprise deployment in your VPC or on-prem" ], "weaknesses": [ "No status page, no published rate limits and no SLA", "Split between Galileo and Splunk since 7 August 2026, with two doc sites, two API hosts and no timeline for the old ones", "MCP server in preview with 8 tools, only `Get Signals` reads production data", "No security page, trust portal, security.txt or certification claim found, and the privacy policy dates from November 2024", "Python SDK quiet since 30 July and its CHANGELOG.md stuck at v0.10.0" ], "agentNotes": [ "Check which side the account lives on first. Pre-August Galileo accounts use api.galileo.ai, Splunk SaaS accounts use app.{realm}.observability.splunkcloud.com/ao/api/", "Send the key as `Splunk-AO-API-Key`. The current spec no longer lists `Galileo-API-Key`", "Page list endpoints with `limit` and `starting_token`", "Use the error catalogue's retriable flag to decide whether to retry. 429s carry no documented `Retry-After`", "Use the REST API to read traces. The MCP server mostly creates datasets and prompts" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 2.5, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "D", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 48 } ], "editorialScores": { "ergonomics": 73, "maintenance": 74, "payments": 20, "reliability": 18, "schema": 79, "security": 33, "transparency": 36 }, "provenanceScore": 76 }, "connect": { "http": "curl https://api.galileo.ai/v2/current_user -H \"Galileo-API-Key: $GALILEO_API_KEY\"", "claudeCode": "claude mcp add --transport http --header \"Galileo-API-Key: $GALILEO_API_KEY\" galileo https://api.galileo.ai/mcp/http/mcp", "config": { "mcpServers": { "galileo": { "headers": { "Accept": "text/event-stream", "Galileo-API-Key": "${GALILEO_API_KEY}" }, "type": "http", "url": "https://api.galileo.ai/mcp/http/mcp" } } } }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/galileo" }, "alsoIn": [ "guardrails" ], "area": "developer", "unitPrices": [ { "item": "Pro plan", "unit": "month", "usd": 100, "note": "billed yearly, 50,000 traces a month" } ], "provenance": { "legalEntity": "Galileo Technologies, Inc.", "domain": "galileo.ai", "domainRegistered": "2020-05-05", "endpointOnVendorDomain": true, "terms": "https://galileo.ai/terms-of-service", "privacy": "https://galileo.ai/privacy-policy", "statusPage": "", "changelog": "https://docs.galileo.ai/release-notes", "securityTxt": "none", "checked": "2026-09-30", "notes": [ "Now owned by Cisco and sold as Splunk Agent Observability; the terms still name Galileo Technologies, Inc." ], "score": 76 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/galileo.json", "live": { "slug": "galileo", "probe": { "target": "https://api.galileo.ai/v2", "method": "get", "lastAt": "2026-10-04T22:35:23.779975691Z", "lastOk": true, "lastStatus": 404, "lastMs": 1273, "authRequired": false, "uptime24h": 99.26, "uptime30d": 99.72, "p50ms24h": 262, "p95ms24h": 3880, "samples24h": 272, "samples30d": 1086, "days": [ { "date": "2026-09-30", "probes": 35, "ok": 35 }, { "date": "2026-10-01", "probes": 276, "ok": 276 }, { "date": "2026-10-02", "probes": 248, "ok": 247 }, { "date": "2026-10-03", "probes": 271, "ok": 271 }, { "date": "2026-10-04", "probes": 256, "ok": 254 } ] }, "versions": [ { "registry": "github", "name": "rungalileo/galileo-python", "version": "v2.6.0", "released": "2026-07-30", "seenAt": "2026-10-04T16:27:40.611176298Z" }, { "registry": "npm", "name": "galileo", "version": "2.3.2", "seenAt": "2026-10-04T16:27:40.188712083Z" }, { "registry": "pypi", "name": "galileo", "version": "2.6.0", "released": "2026-07-30", "seenAt": "2026-10-04T16:27:40.002157835Z" } ], "githubStars": 22, "npmWeekly": 1892, "securityTxt": { "url": "https://galileo.ai/.well-known/security.txt", "state": "none", "checkedAt": "2026-10-04T15:15:43.807289994Z" }, "llmsTxt": { "url": "https://docs.galileo.ai/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-04T15:17:48.251773467Z" }, "domain": { "domain": "galileo.ai", "registered": "2020-05-05", "source": "https://rdap.identitydigital.services/rdap/domain/galileo.ai", "checkedAt": "2026-10-04T13:06:13.135112782Z" }, "pages": [ { "url": "https://docs.galileo.ai/release-notes", "kind": "deprecations", "status": 200, "checkedAt": "2026-10-04T15:43:38.210171478Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "0771d4784c19" }, { "url": "https://galileo.ai/pricing", "kind": "pricing", "status": 304, "checkedAt": "2026-10-04T15:44:51.381606205Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "0145c8cf98fd" }, { "url": "https://galileo.ai/privacy-policy", "kind": "privacy", "status": 304, "checkedAt": "2026-10-04T15:44:53.544097308Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "0a4347a8061e" }, { "url": "https://galileo.ai/terms-of-service", "kind": "terms", "status": 304, "checkedAt": "2026-10-04T15:44:55.585915991Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "fe70a53f84a7" } ], "updatedAt": "2026-10-04T22:35:23.779975691Z" } }, { "slug": "helicone", "name": "Helicone AI Gateway + MCP", "vendor": "Helicone (Mintlify)", "vendorUrl": "https://www.helicone.ai", "kind": "http-api", "category": "agent-observability", "summary": "Open-source LLM observability that logs requests through an OpenAI-compatible gateway (100+ models, fallbacks, caching, rate limits) or async logging.", "url": "https://www.anchorterminal.com/tools/helicone", "markdownUrl": "https://www.anchorterminal.com/tools/helicone.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/helicone.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/helicone.json", "repo": "https://github.com/Helicone/helicone", "license": "Apache-2.0", "transports": [ "http", "stdio" ], "remoteUrl": "https://ai-gateway.helicone.ai", "packages": [ { "registry": "npm", "name": "@helicone/helpers" }, { "registry": "pypi", "name": "helicone-helpers" }, { "registry": "npm", "name": "@helicone/mcp" } ], "auth": "api-key", "authNotes": "Helicone API key (`sk-helicone-...`) as a Bearer token on the gateway and REST API. Bring your own provider keys, or bill model usage to Helicone credits, which since 25 August 2026 only works for organisations Helicone has switched on (others get a 403 asking them to contact support). The MCP server reads `HELICONE_API_KEY` from the environment.", "pricing": "freemium", "pricingNotes": "Hobby free with 10,000 requests a month, 1 GB of storage, 1 seat and 7 days of retention, no card. Pro $79 a month and Team $799 a month, each with 10,000 requests and 1 GB free then usage-based charges the pricing page doesn't itemise. The pricing page still shows 7-day trials, but a commit on 30 August 2026 removed the in-app self-serve upgrade paths and sends upgrades to a contact page. Enterprise custom. Gateway credits pass model prices through at 0% markup, but since 25 August 2026 need Helicone to enable them per organisation. Self-hosted is free under Apache-2.0 with Docker Compose or Helm, you run ClickHouse, Postgres and object storage (https://www.helicone.ai/pricing).", "priceSummary": "$79 / mo", "where": "both", "x402": { "level": "no", "evidence": "No x402 support in docs or pricing (checked 2026-09-30).", "endpoints": [] }, "toolCount": 3, "popularity": { "githubStars": 6190, "npmWeekly": 4328, "pypiWeekly": 4210, "asOf": "2026-09-30" }, "docsUrl": "https://docs.helicone.ai", "llmsTxt": "https://docs.helicone.ai/llms.txt", "openapi": "https://docs.helicone.ai/ai-gateway.openapi.json", "capabilities": [ "obs.traces", "obs.gateway", "obs.prompts", "obs.datasets", "obs.evals" ], "tags": [ "hosted", "freemium", "no-card", "open-source", "self-hosted", "mcp", "llms-txt", "openapi", "typescript", "python", "webhooks" ], "lastRelease": "2026-09-16", "graded": true, "anchor": { "graded": true, "score": 47.1, "grade": "D", "agentReady": false, "rank": 387, "ranked": true, "rankOf": 452, "categoryRank": 9, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 70, "maintenance": 66, "payments": 30, "reliability": 50, "schema": 69, "security": 50, "transparency": 71 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "medium", "date": "2026-10-01" }, "negative": -10, "negativeNotes": [ "2026-08-30. Helicone's own security commit (INC-657) fixed an authenticated cross-tenant read of other organisations' decrypted provider keys, a route that let any account mint proxy keys spending another organisation's provider key, a shared client that reused one organisation's OpenRouter key for others, an unauthenticated SSRF path and unauthenticated Stripe routes. Fixed, but disclosed only in the commit message, with no advisory or key-rotation notice that we found. -7 after decay for the fix (https://github.com/Helicone/helicone/commit/ca34549ea56f7ed587843f82d9cc19baa1f36ba4)", "2026-09-16. A second fix closed a platform-admin takeover with a forged API key and a cross-tenant read through HQL, reported through Helicone's disclosure programme the same day. Fixed within hours and again disclosed only in the commit. -3 (https://github.com/Helicone/helicone/commit/067d9290acb4f1fc9320e902fc67b4b399b50363)" ], "verdict": "One base-URL change gives logging, caching, fallbacks and rate limits for 100+ models. Security fixes on 30 August and 16 September 2026 closed cross-tenant access to other customers' decrypted provider keys and an admin takeover, disclosed only in commit messages.", "strengths": [ "One base-URL change gives logging, caching, fallbacks and rate limits for 100+ models", "Apache-2.0 with Docker Compose and Helm self-hosting", "Per-plan ingestion and API rate limits published on the pricing page", "OpenAPI for the gateway, a Swagger file for the REST API and an llms.txt", "Free Hobby plan with 10,000 requests a month and no card" ], "weaknesses": [ "Security fixes on 30 August and 16 September 2026 closed cross-tenant access to other customers' decrypted provider keys and an admin takeover, disclosed only in commit messages", "Maintenance mode since March 2026, changelog silent since 26 November 2025, and Experiments, Realtime and self-serve upgrades removed on 30 August 2026", "The status page has no incident history, leaves out the AI Gateway and doesn't mention the four-day outage of 14 to 18 May 2026", "Helicone credits need a per-organisation switch since 25 August 2026, so new accounts must bring their own provider keys", "Privacy policy last updated in February 2023, with no retention period or DPA" ], "agentNotes": [ "Bring your own provider keys. Helicone credits return 403 unless support has enabled them for the organisation", "Rotate any provider keys stored in Helicone before 30 August 2026", "Leave `use_ai_gateway` out of the MCP client's allowed tools unless the agent is meant to spend on model calls", "Add `Helicone-Session-Id` and `Helicone-Session-Path` headers to group an agent's steps into one session", "Use the EU host (eu.helicone.ai) if the account was created there" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 2, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "D", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 47.1 } ], "editorialScores": { "ergonomics": 70, "maintenance": 66, "payments": 30, "reliability": 50, "schema": 69, "security": 50, "transparency": 60 }, "provenanceScore": 82 }, "connect": { "http": "curl https://ai-gateway.helicone.ai/chat/completions -H \"Authorization: Bearer $HELICONE_API_KEY\" \\\n -H \"content-type: application/json\" -d '{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}'", "claudeCode": "claude mcp add helicone -e HELICONE_API_KEY=$HELICONE_API_KEY -- npx -y @helicone/mcp@latest", "config": { "mcpServers": { "helicone": { "args": [ "-y", "@helicone/mcp@latest" ], "command": "npx", "env": { "HELICONE_API_KEY": "${HELICONE_API_KEY}" } } } } }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/helicone" }, "area": "developer", "unitPrices": [ { "item": "Pro plan", "unit": "month", "usd": 79, "note": "10,000 requests and 1 GB free, then usage-based" }, { "item": "Team plan", "unit": "month", "usd": 799, "note": "5 organisations, SOC 2 and HIPAA, 10,000 requests and 1 GB free, then usage-based" } ], "provenance": { "legalEntity": "Helicone, Inc.", "domain": "helicone.ai", "domainRegistered": "2022-11-14", "endpointOnVendorDomain": true, "terms": "https://www.helicone.ai/terms", "privacy": "https://www.helicone.ai/privacy", "statusPage": "https://status.helicone.ai", "changelog": "https://www.helicone.ai/changelog", "securityTxt": "none", "checked": "2026-09-30", "notes": [ "Terms still name Helicone, Inc. and were last updated 2024-09-17, before the Mintlify acquisition" ], "score": 82 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/helicone.json", "live": { "slug": "helicone", "probe": { "target": "https://ai-gateway.helicone.ai", "method": "get", "lastAt": "2026-10-04T22:35:24.372188232Z", "lastOk": true, "lastStatus": 405, "lastMs": 44, "authRequired": false, "uptime24h": 100, "uptime30d": 100, "p50ms24h": 44, "p95ms24h": 222, "samples24h": 272, "samples30d": 1086, "days": [ { "date": "2026-09-30", "probes": 35, "ok": 35 }, { "date": "2026-10-01", "probes": 276, "ok": 276 }, { "date": "2026-10-02", "probes": 248, "ok": 248 }, { "date": "2026-10-03", "probes": 271, "ok": 271 }, { "date": "2026-10-04", "probes": 256, "ok": 256 } ] }, "vendorStatus": { "page": "https://status.helicone.ai", "indicator": "unknown", "summary": "no machine-readable status found", "checkedAt": "2026-10-04T21:40:07.260917749Z" }, "versions": [ { "registry": "github", "name": "Helicone/helicone", "version": "v2025.08.21-1", "released": "2025-08-21", "seenAt": "2026-10-04T16:29:35.379994311Z" }, { "registry": "npm", "name": "@helicone/helpers", "version": "1.8.3", "seenAt": "2026-10-04T16:29:33.162290788Z" }, { "registry": "npm", "name": "@helicone/mcp", "version": "0.1.6", "seenAt": "2026-10-04T16:29:34.165843645Z" }, { "registry": "pypi", "name": "helicone-helpers", "version": "1.2.1", "released": "2025-11-08", "seenAt": "2026-10-04T16:29:33.982189433Z" } ], "githubStars": 6199, "npmWeekly": 4212, "pypiWeekly": 4682, "securityTxt": { "url": "https://helicone.ai/.well-known/security.txt", "state": "none", "checkedAt": "2026-10-04T15:15:42.63667923Z" }, "llmsTxt": { "url": "https://docs.helicone.ai/llms.txt", "ok": true, "status": 200, "checkedAt": "2026-10-04T15:17:52.198561612Z" }, "domain": { "domain": "helicone.ai", "registered": "2022-11-14", "source": "https://rdap.identitydigital.services/rdap/domain/helicone.ai", "checkedAt": "2026-10-04T13:08:06.915944951Z" }, "pages": [ { "url": "https://www.helicone.ai/changelog", "kind": "changelog", "status": 200, "checkedAt": "2026-10-04T15:50:40.612062964Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "1af05c56483a" }, { "url": "https://www.helicone.ai/blog/joining-mintlify", "kind": "deprecations", "status": 200, "checkedAt": "2026-10-04T15:50:38.173666232Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "a82c441b1103" }, { "url": "https://www.helicone.ai/pricing", "kind": "pricing", "status": 200, "checkedAt": "2026-10-04T15:50:42.468836302Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "4b1e79419d28" }, { "url": "https://www.helicone.ai/privacy", "kind": "privacy", "status": 200, "checkedAt": "2026-10-04T15:50:44.402436735Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "47e9ad5d395a" }, { "url": "https://www.helicone.ai/terms", "kind": "terms", "status": 200, "checkedAt": "2026-10-04T15:50:46.370943239Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "adbbffec4ecd" } ], "updatedAt": "2026-10-04T22:35:24.372188232Z" } }, { "slug": "baserun", "name": "Baserun", "vendor": "Baserun", "vendorUrl": "https://baserun.ai", "kind": "http-api", "category": "agent-observability", "summary": "Discontinued LLM testing and monitoring platform with Python and JavaScript SDKs for tracing model calls.", "url": "https://www.anchorterminal.com/tools/baserun", "markdownUrl": "https://www.anchorterminal.com/tools/baserun.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/baserun.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/baserun.json", "transports": [ "http" ], "remoteUrl": "https://app.baserun.ai", "packages": [ { "registry": "pypi", "name": "baserun" }, { "registry": "npm", "name": "baserun" } ], "auth": "api-key", "authNotes": "`BASERUN_API_KEY` per project, read by the SDKs. No service is left to accept it.", "pricing": "freemium", "pricingNotes": "No longer sold. The last archived pricing page listed Starter free with 10,000 traces a month, 7 days of retention and 2 users, Pro $49 a seat with 100,000 traces then $10 per 10,000 and 30 days of retention, and Enterprise custom (https://web.archive.org/web/20240718042849/https://www.baserun.ai/pricing).", "priceSummary": "$49 / seat-mo", "where": "hosted", "x402": { "level": "no", "endpoints": [] }, "toolCount": null, "popularity": { "githubStars": null, "npmWeekly": 115, "pypiWeekly": 209, "asOf": "2026-09-30" }, "docsUrl": "https://docs.baserun.ai", "capabilities": [ "obs.traces", "obs.evals", "obs.prompts" ], "tags": [ "closed-source", "retired", "python", "typescript" ], "lastRelease": "2024-06-26", "graded": true, "anchor": { "graded": true, "score": 7.3, "grade": "F", "agentReady": false, "rank": 0, "ranked": false, "notRankedWhy": "Shut down 2024-09-30, so out of the ranking", "rankOf": 452, "categoryRank": 10, "methodology": "0.3", "run": "2026-10-01", "scores": { "ergonomics": 5, "maintenance": 0, "payments": 0, "reliability": 0, "schema": 15, "security": 5, "transparency": 36 }, "pending": [ "performance", "tasks" ], "assessment": { "confidence": "high", "date": "2026-10-01" }, "negative": 0, "verdict": "SDKs were MIT licensed and the Python source is still public at github.com/baserun-ai/baserun-py. Service offline since late 2024. api.baserun.ai doesn't resolve and app.baserun.ai serves an expired certificate.", "strengths": [ "SDKs were MIT licensed and the Python source is still public at github.com/baserun-ai/baserun-py", "Docs remain readable at docs.baserun.ai, with an llms.txt index of 37 pages, for anyone migrating old code" ], "weaknesses": [ "Service offline since late 2024. api.baserun.ai doesn't resolve and app.baserun.ai serves an expired certificate", "No shutdown notice on the docs, the homepage or either package, and neither package is marked deprecated", "Python SDK defaults to https://app.baserun.ai, so old installs keep trying to send trace data there", "No announced data export, deletion statement or migration path", "Last SDK releases were PyPI 2.0.9 on 2024-06-26 and npm 2.1.3" ], "agentNotes": [ "Don't install `baserun` from PyPI or npm. Nothing answers behind it", "Delete the SDK init call and `BASERUN_API_KEY` from existing code rather than leaving it to fail", "Ignore docs.baserun.ai. It describes a service that no longer runs and doesn't say so" ], "metrics": { "kind": "remote", "measured": false }, "reviewCount": 2, "avgRating": 1, "history": [ { "basis": "public evidence", "confidence": "high", "grade": "F", "methodology": "0.3", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 7.3 } ], "editorialScores": { "ergonomics": 5, "maintenance": 0, "payments": 0, "reliability": 0, "schema": 15, "security": 5, "transparency": 10 }, "provenanceScore": 62 }, "connect": { "http": "# Retired. baserun.ai and api.baserun.ai no longer answer.\npip install baserun # or: npm i baserun" }, "letme": { "capability": "https://letme.dev/obs.traces", "tool": "https://letme.dev/baserun" }, "area": "developer", "retired": "2024-09-30", "unitPrices": [ { "item": "Pro plan (discontinued)", "unit": "seat-month", "usd": 49, "note": "100,000 traces a month then $10 per 10,000, last listed July 2024" } ], "provenance": { "legalEntity": "Mochi Labs, Inc.", "domain": "baserun.ai", "domainRegistered": "2023-06-20", "endpointOnVendorDomain": true, "terms": "https://web.archive.org/web/20240908185548/https://www.baserun.ai/terms", "privacy": "https://web.archive.org/web/20240908185719/https://www.baserun.ai/privacy", "statusPage": "", "changelog": "", "securityTxt": "none", "checked": "2026-10-01", "notes": [ "Legal entity from the SDK licence (MIT, copyright 2023 Mochi Labs, Inc.) on PyPI and in github.com/baserun-ai/baserun-py", "app.baserun.ai serves an expired certificate, api.baserun.ai doesn't resolve, and baserun.ai failed the TLS handshake for /.well-known/security.txt on 2026-10-01", "One fetch of www.baserun.ai on 2026-10-01 returned the old marketing copy with no dates, prices or working sign-in links" ], "score": 62 }, "pageJsonUrl": "https://www.anchorterminal.com/tools/baserun.json" } ] }, "kind": "anchor.page", "links": { "api": "https://www.anchorterminal.com/api/v1/index.json", "html": "https://www.anchorterminal.com/categories/agent-observability", "json": "https://www.anchorterminal.com/categories/agent-observability.json", "llms": "https://www.anchorterminal.com/llms.txt", "markdown": "https://www.anchorterminal.com/categories/agent-observability.md", "slim": "https://www.anchorterminal.com/categories/agent-observability.min.md" }, "markdown": "Tools that trace agent runs, tool calls and model calls, and run evaluations on them. Compared on what appears in a trace, how fast a failure can be found, whether an evaluation can be reproduced, setup time and cost.\n\n- Tools ranked: 10 · agent-ready (BB or better): 3 · accept x402: 0 · hosted endpoints: 9 · desk reviews by the panel: 26\n- JSON: https://www.anchorterminal.com/api/v1/tools.json (list) · https://www.anchorterminal.com/api/v1/rankings.json (ranked) · https://www.anchorterminal.com/api/v1/x402.json (payable) · https://www.anchorterminal.com/api/v1/capabilities.json (by capability)\n- Grades run AA, A, BB, B, C, D, E, F · methodology: https://www.anchorterminal.com/benchmark/\n\n- Capabilities in this category: obs.traces, obs.evals, obs.prompts, obs.gateway, obs.datasets\n- https://letme.dev/obs.traces picks the top-graded tool in this list and says how to call it direct; calling through letme comes later (https://www.anchorterminal.com/letme/index.md)\n\n## Ranking\n\n| # | Tool | Vendor | Kind | Category | Grade | Score | Confidence | x402 | Auth | Where | Reviews | Page |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| 32 | Arize Phoenix | Arize AI | HTTP API | Evals | BB | 75.6 | medium | no | OAuth or key | local | 3.8/5 (8) | https://www.anchorterminal.com/tools/arize-phoenix.md |\n| 66 | Langfuse API + MCP | Langfuse (ClickHouse) | HTTP API | Evals | BB | 72.8 | high | no | API key | hosted | 4/5 (2) | https://www.anchorterminal.com/tools/langfuse.md |\n| 85 | LangSmith API + MCP | LangChain | HTTP API | Evals | BB | 71.3 | medium | no | OAuth or key | hosted | 3/5 (2) | https://www.anchorterminal.com/tools/langsmith.md |\n| 165 | Respan API + MCP | Respan (formerly Keywords AI) | HTTP API | Evals | B | 65.9 | medium | no | OAuth or key | hosted | 2.5/5 (2) | https://www.anchorterminal.com/tools/respan.md |\n| 229 | Braintrust API + MCP | Braintrust | HTTP API | Evals | C | 61.3 | medium | no | OAuth or key | hosted | 3/5 (2) | https://www.anchorterminal.com/tools/braintrust.md |\n| 298 | Laminar API + MCP | Laminar (LMNR AI) | HTTP API | Evals | C | 57 | medium | no | API key | hosted | 3.5/5 (2) | https://www.anchorterminal.com/tools/laminar.md |\n| 310 | HoneyHive | HoneyHive | HTTP API | Evals | C | 55.9 | medium | no | API key | hosted | 2.5/5 (2) | https://www.anchorterminal.com/tools/honeyhive.md |\n| 378 | Galileo API + MCP | Galileo (now Splunk Agent Observability, Cisco) | HTTP API | Evals | D | 48 | medium | no | API key | hosted | 2.5/5 (2) | https://www.anchorterminal.com/tools/galileo.md |\n| 387 | Helicone AI Gateway + MCP | Helicone (Mintlify) | HTTP API | Evals | D | 47.1 | medium | no | API key | hosted + local | 2/5 (2) | https://www.anchorterminal.com/tools/helicone.md |\n| not ranked, shut down | Baserun | Baserun | HTTP API | Evals | F | 7.3 | high | no | API key | hosted | 1/5 (2) | https://www.anchorterminal.com/tools/baserun.md |\n\nScores are from public evidence against the published checklist (https://www.anchorterminal.com/benchmark/), with Performance and Task success pending. p95 latency and context cost come from our probes, which haven't run yet.\n\n## Summaries\n\n### 32. Arize Phoenix, BB (75.6)\n\nSelf-hosted tracing, evaluation, datasets, experiments and prompt management built on OpenTelemetry and OpenInference. Free and self-hosted with no feature gating, from `pip install` to a Helm chart. Auth is off by default and the default admin password is `admin`.\n\n- Page: https://www.anchorterminal.com/tools/arize-phoenix · Markdown: https://www.anchorterminal.com/tools/arize-phoenix.md · JSON: https://www.anchorterminal.com/api/v1/tools/arize-phoenix.json\n- Capabilities: obs.traces, obs.evals, obs.prompts, obs.datasets\n\n### 66. Langfuse API + MCP, BB (72.8)\n\nOpen-source tracing, evaluation, prompt management and datasets for LLM apps and agents, built on OpenTelemetry. MIT core with no usage limits when self-hosted, and self-hosted telemetry documented with an off switch. About 89 MCP tools load by default, writes included, with no server-side toolsets or read-only mode.\n\n- Page: https://www.anchorterminal.com/tools/langfuse · Markdown: https://www.anchorterminal.com/tools/langfuse.md · JSON: https://www.anchorterminal.com/api/v1/tools/langfuse.json\n- Capabilities: obs.traces, obs.evals, obs.prompts, obs.datasets · endpoint: `https://cloud.langfuse.com/api/public`\n\n### 85. LangSmith API + MCP, BB (71.3)\n\nLangChain's hosted tracing, evaluation, prompt hub and dataset platform, framework-agnostic with Python and TypeScript SDKs and OpenTelemetry ingest. Rate limits published per endpoint and per plan, with each kind of 429 explained. Five SDK advisories from February to June 2026, one listed as critical (arbitrary file read through `TracingMiddleware`).\n\n- Page: https://www.anchorterminal.com/tools/langsmith · Markdown: https://www.anchorterminal.com/tools/langsmith.md · JSON: https://www.anchorterminal.com/api/v1/tools/langsmith.json\n- Capabilities: obs.traces, obs.evals, obs.prompts, obs.datasets, obs.gateway · endpoint: `https://api.smith.langchain.com`\n\n### 165. Respan API + MCP, B (65.9)\n\nHosted tracing, evals, prompt management and an LLM gateway to 250+ models, from the company formerly called Keywords AI. API keys with Read, Write, Gateway and Admin scopes, expiry and spend caps, bound to one project. 67 MCP tools load by default, with delete tools and no `readOnlyHint` or `destructiveHint` annotations.\n\n- Page: https://www.anchorterminal.com/tools/respan · Markdown: https://www.anchorterminal.com/tools/respan.md · JSON: https://www.anchorterminal.com/api/v1/tools/respan.json\n- Capabilities: obs.traces, obs.evals, obs.prompts, obs.gateway, obs.datasets · endpoint: `https://api.respan.ai/api`\n\n### 229. Braintrust API + MCP, C (61.3)\n\nHosted tracing, logging and evaluation for LLM apps and agents, with experiments, datasets, prompts, online scorers and a model gateway. OpenAPI 3.0.3 spec with 75 paths, 429 and `Retry-After` declared on 154 operations. Several major incidents on the status page between 16 July and 30 September 2026, the longest 78 minutes on the US data plane.\n\n- Page: https://www.anchorterminal.com/tools/braintrust · Markdown: https://www.anchorterminal.com/tools/braintrust.md · JSON: https://www.anchorterminal.com/api/v1/tools/braintrust.json\n- Capabilities: obs.traces, obs.evals, obs.prompts, obs.gateway, obs.datasets · endpoint: `https://api.braintrust.dev/v1`\n\n### 298. Laminar API + MCP, C (57)\n\nOpen-source tracing, evaluations and an agent debugger built on OpenTelemetry, with browser session recordings synced to traces for browser agents. Browser session recordings can be matched to agent traces. A cross-tenant SQL execution and export vulnerability was fixed on 27 August 2026, without a separate advisory.\n\n- Page: https://www.anchorterminal.com/tools/laminar · Markdown: https://www.anchorterminal.com/tools/laminar.md · JSON: https://www.anchorterminal.com/api/v1/tools/laminar.json\n- Capabilities: obs.traces, obs.evals, obs.datasets · endpoint: `https://api.lmnr.ai/v1`\n\n### 310. HoneyHive, C (55.9)\n\nHosted tracing, evaluation, datasets and prompt management for agents, built on OpenTelemetry. Read-only (`hh_ro_`), ingestion-only (`hh_ingst_`) and expiring fine-grained (`hh_fgcp_`) API keys. Status page showed 80.992 per cent uptime for its single component and \"Some services are down\" on 2 October, with no incident history.\n\n- Page: https://www.anchorterminal.com/tools/honeyhive · Markdown: https://www.anchorterminal.com/tools/honeyhive.md · JSON: https://www.anchorterminal.com/api/v1/tools/honeyhive.json\n- Capabilities: obs.traces, obs.evals, obs.prompts, obs.datasets · endpoint: `https://api.dp1.us.honeyhive.ai`\n\n### 378. Galileo API + MCP, D (48)\n\nHosted tracing, evaluation metrics and guardrails for LLM apps and agents, with a REST API, Python and TypeScript SDKs and an 8-tool MCP server in preview. OpenAPI 3.1 spec with 184 paths and 244 operations. No status page, no published rate limits and no SLA.\n\n- Page: https://www.anchorterminal.com/tools/galileo · Markdown: https://www.anchorterminal.com/tools/galileo.md · JSON: https://www.anchorterminal.com/api/v1/tools/galileo.json\n- Capabilities: obs.traces, obs.evals, obs.prompts, obs.datasets · endpoint: `https://api.galileo.ai/v2`\n\n### 387. Helicone AI Gateway + MCP, D (47.1)\n\nOpen-source LLM observability that logs requests through an OpenAI-compatible gateway (100+ models, fallbacks, caching, rate limits) or async logging. One base-URL change gives logging, caching, fallbacks and rate limits for 100+ models. Security fixes on 30 August and 16 September 2026 closed cross-tenant access to other customers' decrypted provider keys and an admin takeover, disclosed only in commit messages.\n\n- Page: https://www.anchorterminal.com/tools/helicone · Markdown: https://www.anchorterminal.com/tools/helicone.md · JSON: https://www.anchorterminal.com/api/v1/tools/helicone.json\n- Capabilities: obs.traces, obs.gateway, obs.prompts, obs.datasets, obs.evals · endpoint: `https://ai-gateway.helicone.ai`\n\n### Baserun, F (7.3), retired, not ranked\n\nDiscontinued LLM testing and monitoring platform with Python and JavaScript SDKs for tracing model calls. SDKs were MIT licensed and the Python source is still public at github.com/baserun-ai/baserun-py. Service offline since late 2024. api.baserun.ai doesn't resolve and app.baserun.ai serves an expired certificate.\n\n- Page: https://www.anchorterminal.com/tools/baserun · Markdown: https://www.anchorterminal.com/tools/baserun.md · JSON: https://www.anchorterminal.com/api/v1/tools/baserun.json\n- Capabilities: obs.traces, obs.evals, obs.prompts · endpoint: `https://app.baserun.ai`\n\n## How we test this category\n\nThe same agent run instrumented in every tool, including a failed tool call and a retry. We check which steps appear in the trace, whether the failure can be found quickly, whether an evaluation can be reproduced, how long setup takes and what it costs. This test hasn't run yet, so Task success is pending and the grades here come from the categories assessed from public evidence.\n\n## Indexed, not reviewed (5)\n\nSorted into this category from public catalogues, with facts and our own checks but no score, grade or rank (https://www.anchorterminal.com/indexed/index.md).\n\n| Listing | Kind | What it does | Why it's here |\n| --- | --- | --- | --- |\n| [aegis](https://www.anchorterminal.com/tools/undercurrentholdings-aegis.md) | MCP server | Local stdio MCP server for the AEGIS governance client. The hosted evaluation service is offline. | vendor's own |\n| [argosvix.com MCP server](https://www.anchorterminal.com/tools/argosvix-server.md) | MCP server | Observability MCP server: query LLM cost/errors/latency \u0026 operate alerts/evals from Claude/Cursor | vendor's own |\n| [CompletionKit](https://www.anchorterminal.com/tools/completionkit-evals.md) | MCP server | Prompt evals over MCP: run a prompt on your dataset, score each output 1-5 with an LLM judge. | vendor's own |\n| [openfeature.dev MCP server](https://www.anchorterminal.com/tools/openfeature-mcp.md) | MCP server | MCP server providing OpenFeature SDK installation guides and OFREP flag evaluation | vendor's own |\n| [PromptOT](https://www.anchorterminal.com/tools/promptot-mcp.md) | MCP server | Manage, version, and publish LLM prompts with blocks, variables, and evaluations. | vendor's own |\n\n", "meta": { "attribution": "Anchor Terminal (https://www.anchorterminal.com)", "docs": "https://www.anchorterminal.com/docs/", "generatedAt": "2026-10-04", "license": "CC-BY-4.0", "method": "https://www.anchorterminal.com/benchmark/", "methodology": "0.3", "openapi": "https://www.anchorterminal.com/openapi.json", "preview": false, "run": "2026-10-01", "runLabel": "October 2026 research run" }, "page": { "breadcrumbs": [ { "name": "Home", "url": "https://www.anchorterminal.com/" }, { "name": "Terminal", "url": "https://www.anchorterminal.com/tools/" }, { "name": "Agent observability \u0026 evals", "url": "" } ], "description": "10 agent observability \u0026 evals ranked by the Anchor benchmark. Leader Arize Phoenix (BB). Tools that trace agent runs, tool calls and model calls, and run evaluations on them. Compared on what appears in a trace, how fast a failure can be found, whether an evaluation can be reproduced, setup time and cost.", "facts": [ "Arize Phoenix BB", "Langfuse API + MCP BB", "LangSmith API + MCP BB" ], "h1": "Agent tracing, monitoring and evaluation tools", "image": "https://www.anchorterminal.com/assets/og/categories-agent-observability.png", "path": "/categories/agent-observability", "published": "", "section": "tools", "title": "Agent tracing, monitoring and evaluation tools, ranked", "toc": null, "updated": "2026-10-04", "url": "https://www.anchorterminal.com/categories/agent-observability" }, "tokens": { "markdown": 3050, "slim": 530 }, "version": 1 }