{ "data": { "similar": [ { "grade": "BB", "json": "https://www.anchorterminal.com/tools/arize-phoenix.json", "name": "Arize Phoenix", "score": 75.4, "shared": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "slug": "arize-phoenix" }, { "grade": "BB", "json": "https://www.anchorterminal.com/tools/langfuse.json", "name": "Langfuse API + MCP", "score": 72.7, "shared": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "slug": "langfuse" }, { "grade": "BB", "json": "https://www.anchorterminal.com/tools/langsmith.json", "name": "LangSmith API + MCP", "score": 71.1, "shared": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "slug": "langsmith" }, { "grade": "B", "json": "https://www.anchorterminal.com/tools/wandb-weave.json", "name": "W\u0026B Weave", "score": 66.7, "shared": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "slug": "wandb-weave" }, { "grade": "B", "json": "https://www.anchorterminal.com/tools/respan.json", "name": "Respan API + MCP", "score": 65.5, "shared": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "slug": "respan" }, { "grade": "B", "json": "https://www.anchorterminal.com/tools/langwatch.json", "name": "LangWatch", "score": 65.5, "shared": [ "obs.traces", "obs.evals", "obs.prompts", "obs.datasets" ], "slug": "langwatch" } ], "tool": { "slug": "deepeval", "name": "DeepEval", "vendor": "Confident AI, Inc.", "vendorUrl": "https://www.confident-ai.com", "kind": "sdk", "category": "agent-observability", "summary": "DeepEval is an open-source Python and TypeScript framework from Confident AI for evaluating LLM applications and agents. Evaluations run locally through a pytest plugin and the `deepeval` CLI, with optional reporting to the hosted Confident AI platform.", "url": "https://www.anchorterminal.com/tools/deepeval", "markdownUrl": "https://www.anchorterminal.com/tools/deepeval.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/deepeval.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/deepeval.json", "repo": "https://github.com/confident-ai/deepeval", "license": "Apache 2.0 for the Python and TypeScript packages and the agent skills. Confident AI, the hosted platform, is a proprietary service under its own terms", "transports": [], "packages": [ { "registry": "pypi", "name": "deepeval" }, { "registry": "npm", "name": "deepeval" } ], "auth": "mixed", "authNotes": "No credential is needed to run evaluations locally. Most metrics call an LLM judge, so the owner supplies a model provider key such as `OPENAI_API_KEY` in the environment or a dotenv file. Reporting to Confident AI needs a project API key in `CONFIDENT_API_KEY`. `deepeval login` pairs with a browser through a device code and writes the key to `.env.local`, and `deepeval login --api-key` takes an existing key for CI. Confident AI project keys are named, take an expiry, and can be deactivated, rotated with a grace period or deleted. No per-permission scopes for project keys were found in the page read.", "pricing": "freemium", "pricingNotes": "DeepEval is free under Apache 2.0 and runs without an account. LLM judge calls are billed by the owner's model provider. The hosted Confident AI platform has a Free plan with no card (5 test runs a week, 2 seats, 1 project, 1 GB-month of trace spans), Starter at $200 a month, Team at $2,000 a month, both with $1 per GB-month beyond the included trace spans, and Enterprise by quote (https://www.confident-ai.com/pricing).", "priceSummary": "$200 / mo", "where": "local", "x402": { "level": "no", "evidence": "No x402, MPP or L402 in the repository docs, the Confident AI docs index or the pricing page (checked 2026-10-09).", "endpoints": [] }, "toolCount": null, "popularity": { "githubStars": 18700, "npmWeekly": 35092, "pypiWeekly": 735637, "asOf": "2026-10-09" }, "docsUrl": "https://deepeval.com/docs/getting-started", "llmsTxt": "https://deepeval.com/llms.txt", "capabilities": [ "obs.evals", "obs.traces", "obs.datasets", "obs.prompts" ], "tags": [ "open-source", "apache-2.0", "local", "cli", "python", "typescript", "pytest", "llms-txt", "agent-skills", "opentelemetry", "freemium", "no-card" ], "lastRelease": "2026-10-02", "graded": true, "anchor": { "graded": true, "score": 64.7, "grade": "B", "agentReady": false, "rank": 343, "ranked": true, "rankOf": 950, "categoryRank": 9, "methodology": "0.4", "run": "2026-10-01", "scores": { "ergonomics": 72, "maintenance": 83, "payments": 60, "reliability": 59, "schema": 78, "security": 47, "transparency": 63 }, "pending": [ "performance", "tasks" ], "breakdown": [ { "key": "reliability", "name": "Reliability", "weight": 16, "effectiveWeight": 20, "score": 59, "points": 11.8, "reason": "Scored on the local-software lines, since DeepEval runs where the owner installs it. Official `deepeval` packages on PyPI and npm, with Python 3.9 or later stated (20). Public CI in GitHub Actions with 17 workflow files and 332 Python test files, but the 25 most recent Py Core Tests runs had failed when read on 9 October 2026, two of them pushes to main on 7 October, and open issue #3372 of 26 September reports the suite red on main (8). 322 open issues and 387 open pull requests on a repository with 18.7k stars. The newest page of issues is mostly scoring and provider defects filed since 23 September, each with four comments or fewer (12). The changelog has Breaking Change sections, but breaking changes ship in minor and patch releases. 4.2.0 reversed the score direction of four metrics and 3.9.8 and 3.9.9 removed options (7). Python is at 4.2.8. The TypeScript package is at 0.9.21 (12)." }, { "key": "performance", "name": "Performance", "weight": 10, "effectiveWeight": 0, "pending": true, "points": 0, "reason": "Pending. Latency is measured per call by our probes, which haven't run yet, so this run doesn't score it. Its weight is shared across the assessed categories until the first probe window closes." }, { "key": "schema", "name": "Schema \u0026 documentation", "weight": 13, "effectiveWeight": 16.25, "score": 78, "points": 12.68, "reason": "Read on the framework lines. The Python package ships `py.typed` and pydantic models, the TypeScript package ships type declarations, and the CLI is built on typer. A library has no OpenAPI file to publish (18). llms.txt on deepeval.com indexes the docs, and pages carry a Copy Markdown control (10). Each metric has its own page with required parameters and usage, and three agent skills in the repository tell a coding agent how to add evaluations and tracing (16). Typed test case and metric classes, with enums for evaluation parameters and CLI choices (12). Many runnable examples, a troubleshooting page and an environment variable reference. Error types are not listed in one place (11). Tagged releases for both packages and a public changelog by year, but the 2026 changelog stops at 4.2.0 of 25 August and has no entry for 4.2.1 to 4.2.8 (11)." }, { "key": "ergonomics", "name": "Agent ergonomics", "weight": 13, "effectiveWeight": 16.25, "score": 72, "points": 11.7, "reason": "Read on the framework lines. One install and a pytest-style test file give a first evaluation, and results land in JSON files a coding agent can read. `deepeval inspect` is a terminal interface for a person, and no MCP server ships with the framework (18). `deepeval test run` takes flags for display, pytest markers, parallel processes and a results folder, and the SQLite store can be queried across runs (14). `--ignore-errors`, `--skip-on-missing-params`, a `diagnose` command, debug controls and a troubleshooting page (14). `--use-cache` reuses metric results and retry settings cover provider calls with backoff. Scores from LLM judges can differ between runs (12). Python and TypeScript packages with few required parameters. The default judge needs `OPENAI_API_KEY` (14)." }, { "key": "security", "name": "Security \u0026 auth", "weight": 14, "effectiveWeight": 17.5, "score": 47, "points": 8.23, "reason": "Local runs need no credential. Provider keys and `CONFIDENT_API_KEY` are read from the environment or a dotenv file, and storing keys in the legacy `.deepeval/.deepeval` file is deprecated in the source. Confident AI project keys take a name and an expiry and can be deactivated, rotated with a grace period or deleted. No per-permission scopes were found (20). The framework has no read-only mode. Once logged in, test runs and traces upload to Confident AI (8). Metrics pass application output to an LLM judge, and no guidance on untrusted content reaching the judge was found in the pages read (5). Each run is recorded in local JSON or SQLite. Confident AI documents audit log exports, which we did not read (7). GitHub shows no security policy and no published advisories, and security.txt answered 404 on both domains. No bug bounty was found. The Confident AI site states SOC 2 Type II and HIPAA, and its trust centre is drawn by script and was not read (7)." }, { "key": "payments", "name": "Payments \u0026 pricing", "weight": 10, "effectiveWeight": 12.5, "score": 60, "points": 7.5, "reason": "No x402, MPP or L402 (0). DeepEval is free, and the paid option is the hosted Confident AI platform, whose prices are public without a login. Starter is $200 a month, Team $2,000 a month, and trace spans beyond the plan are $1 per GB-month (20). The framework is free and Confident AI's Free plan needs no card (20). `pip install -U deepeval` and a local run need no account, so an agent can start without a person signing up. Confident AI itself needs a browser sign-up, and its auth.md says there is no self-serve registration endpoint for API keys. Scoring this line on the local framework is a judgement call (20)." }, { "key": "tasks", "name": "Task success", "weight": 10, "effectiveWeight": 0, "pending": true, "points": 0, "reason": "Pending. Task success needs the category task suites run through each tool, which haven't run yet, so this run doesn't score it. Its weight is shared across the assessed categories until then. A data provider's data-quality score is published on its listing now and becomes half of this category when it's scored." }, { "key": "maintenance", "name": "Maintenance \u0026 community", "weight": 7, "effectiveWeight": 8.75, "score": 83, "points": 7.26, "reason": "Python 4.2.8 was tagged on 2 October 2026, seven days before the check (30). 14 Python releases tagged since 9 August, from 4.1.6 to 4.2.8 (20). 124 commits in the 30 days to 9 October, most by three maintainers, and a Discord community. 322 issues and 387 pull requests are open and reply times were not read (14). Current official packages in Python and TypeScript, the latter at 0.9.21 on 30 September (15). A `poetry.lock` file and CI on pull requests. No Dependabot configuration was found and the core test workflow is failing on main (4)." }, { "key": "transparency", "name": "Transparency \u0026 trust", "weight": 7, "effectiveWeight": 8.75, "score": 63, "points": 5.51, "note": "editorial 73, provenance 52", "reason": "Apache-2.0 for the whole repository (30). The docs state what telemetry collects, and Confident AI publishes terms, a privacy policy, a DPA and a sub-processor list with regions. They disagree in places. The privacy policy says the DPA is available on request while it is published at /dpa, and the DeepEval docs say data sits in a private cloud on AWS while the sub-processor list names Supabase, ClickHouse and Railway for storage and hosting (20). The changelog has Breaking Change sections and the source warns at run time about the reversed metrics and the legacy key file. No deprecation policy or notice period was found (9). Telemetry through PostHog is on by default and `DEEPEVAL_TELEMETRY_OPT_OUT=1` turns it off. The source also sends the judge provider and model, the runtime kind, the CLI command and test case counts, which the docs list omits (14)." } ], "assessment": { "date": "2026-10-09", "basis": "public evidence", "confidence": "medium", "notes": { "ergonomics": "Read on the framework lines. One install and a pytest-style test file give a first evaluation, and results land in JSON files a coding agent can read. `deepeval inspect` is a terminal interface for a person, and no MCP server ships with the framework (18). `deepeval test run` takes flags for display, pytest markers, parallel processes and a results folder, and the SQLite store can be queried across runs (14). `--ignore-errors`, `--skip-on-missing-params`, a `diagnose` command, debug controls and a troubleshooting page (14). `--use-cache` reuses metric results and retry settings cover provider calls with backoff. Scores from LLM judges can differ between runs (12). Python and TypeScript packages with few required parameters. The default judge needs `OPENAI_API_KEY` (14).", "maintenance": "Python 4.2.8 was tagged on 2 October 2026, seven days before the check (30). 14 Python releases tagged since 9 August, from 4.1.6 to 4.2.8 (20). 124 commits in the 30 days to 9 October, most by three maintainers, and a Discord community. 322 issues and 387 pull requests are open and reply times were not read (14). Current official packages in Python and TypeScript, the latter at 0.9.21 on 30 September (15). A `poetry.lock` file and CI on pull requests. No Dependabot configuration was found and the core test workflow is failing on main (4).", "payments": "No x402, MPP or L402 (0). DeepEval is free, and the paid option is the hosted Confident AI platform, whose prices are public without a login. Starter is $200 a month, Team $2,000 a month, and trace spans beyond the plan are $1 per GB-month (20). The framework is free and Confident AI's Free plan needs no card (20). `pip install -U deepeval` and a local run need no account, so an agent can start without a person signing up. Confident AI itself needs a browser sign-up, and its auth.md says there is no self-serve registration endpoint for API keys. Scoring this line on the local framework is a judgement call (20).", "reliability": "Scored on the local-software lines, since DeepEval runs where the owner installs it. Official `deepeval` packages on PyPI and npm, with Python 3.9 or later stated (20). Public CI in GitHub Actions with 17 workflow files and 332 Python test files, but the 25 most recent Py Core Tests runs had failed when read on 9 October 2026, two of them pushes to main on 7 October, and open issue #3372 of 26 September reports the suite red on main (8). 322 open issues and 387 open pull requests on a repository with 18.7k stars. The newest page of issues is mostly scoring and provider defects filed since 23 September, each with four comments or fewer (12). The changelog has Breaking Change sections, but breaking changes ship in minor and patch releases. 4.2.0 reversed the score direction of four metrics and 3.9.8 and 3.9.9 removed options (7). Python is at 4.2.8. The TypeScript package is at 0.9.21 (12).", "schema": "Read on the framework lines. The Python package ships `py.typed` and pydantic models, the TypeScript package ships type declarations, and the CLI is built on typer. A library has no OpenAPI file to publish (18). llms.txt on deepeval.com indexes the docs, and pages carry a Copy Markdown control (10). Each metric has its own page with required parameters and usage, and three agent skills in the repository tell a coding agent how to add evaluations and tracing (16). Typed test case and metric classes, with enums for evaluation parameters and CLI choices (12). Many runnable examples, a troubleshooting page and an environment variable reference. Error types are not listed in one place (11). Tagged releases for both packages and a public changelog by year, but the 2026 changelog stops at 4.2.0 of 25 August and has no entry for 4.2.1 to 4.2.8 (11).", "security": "Local runs need no credential. Provider keys and `CONFIDENT_API_KEY` are read from the environment or a dotenv file, and storing keys in the legacy `.deepeval/.deepeval` file is deprecated in the source. Confident AI project keys take a name and an expiry and can be deactivated, rotated with a grace period or deleted. No per-permission scopes were found (20). The framework has no read-only mode. Once logged in, test runs and traces upload to Confident AI (8). Metrics pass application output to an LLM judge, and no guidance on untrusted content reaching the judge was found in the pages read (5). Each run is recorded in local JSON or SQLite. Confident AI documents audit log exports, which we did not read (7). GitHub shows no security policy and no published advisories, and security.txt answered 404 on both domains. No bug bounty was found. The Confident AI site states SOC 2 Type II and HIPAA, and its trust centre is drawn by script and was not read (7).", "transparency": "Apache-2.0 for the whole repository (30). The docs state what telemetry collects, and Confident AI publishes terms, a privacy policy, a DPA and a sub-processor list with regions. They disagree in places. The privacy policy says the DPA is available on request while it is published at /dpa, and the DeepEval docs say data sits in a private cloud on AWS while the sub-processor list names Supabase, ClickHouse and Railway for storage and hosting (20). The changelog has Breaking Change sections and the source warns at run time about the reversed metrics and the legacy key file. No deprecation policy or notice period was found (9). Telemetry through PostHog is on by default and `DEEPEVAL_TELEMETRY_OPT_OUT=1` turns it off. The source also sends the judge provider and model, the runtime kind, the CLI command and test case counts, which the docs list omits (14)." }, "sources": [ { "what": "robots.txt for deepeval.com, HTTP 200, allows every path, Content-Signal ai-input=yes", "url": "https://deepeval.com/robots.txt", "seen": "2026-10-09" }, { "what": "robots.txt for www.confident-ai.com, HTTP 200, ai-input=yes, disallows /app/, /api/ and /docs-openapi", "url": "https://www.confident-ai.com/robots.txt", "seen": "2026-10-09" }, { "what": "robots.txt for github.com, HTTP 200", "url": "https://github.com/robots.txt", "seen": "2026-10-09" }, { "what": "DeepEval docs index for agents", "url": "https://deepeval.com/llms.txt", "seen": "2026-10-09" }, { "what": "data privacy page, telemetry fields and opt-out", "url": "https://deepeval.com/docs/data-privacy", "seen": "2026-10-09" }, { "what": "repository home, 18.7k stars, 322 open issues, 387 open pull requests", "url": "https://github.com/confident-ai/deepeval", "seen": "2026-10-09" }, { "what": "repository clone at commit 0fb05d0 of 7 October 2026, read for tags, package files, CLI, telemetry and docs source", "url": "https://github.com/confident-ai/deepeval", "seen": "2026-10-09" }, { "what": "Python package metadata, version 4.2.8, Python 3.9 or later, Apache-2.0", "url": "https://github.com/confident-ai/deepeval/blob/main/pyproject.toml", "seen": "2026-10-09" }, { "what": "TypeScript package metadata, version 0.9.21", "url": "https://github.com/confident-ai/deepeval/blob/main/typescript/package.json", "seen": "2026-10-09" }, { "what": "telemetry source, PostHog host and event properties", "url": "https://github.com/confident-ai/deepeval/blob/main/deepeval/telemetry/properties.py", "seen": "2026-10-09" }, { "what": "2026 changelog source, Breaking Change entries for 4.2.0, 3.9.9 and 3.9.8, last entry 4.2.0", "url": "https://github.com/confident-ai/deepeval/blob/main/docs/content/changelog/changelog-2026.mdx", "seen": "2026-10-09" }, { "what": "CLI reference source, commands, test run flags and secrets handling", "url": "https://github.com/confident-ai/deepeval/blob/main/docs/content/docs/command-line-interface.mdx", "seen": "2026-10-09" }, { "what": "local backend storage docs source", "url": "https://github.com/confident-ai/deepeval/blob/main/docs/content/docs/evaluation-local-backend-storage.mdx", "seen": "2026-10-09" }, { "what": "environment variables docs source, retry settings", "url": "https://github.com/confident-ai/deepeval/blob/main/docs/content/docs/environment-variables.mdx", "seen": "2026-10-09" }, { "what": "device-code login flow source", "url": "https://github.com/confident-ai/deepeval/blob/main/deepeval/cli/auth/flow.py", "seen": "2026-10-09" }, { "what": "agent skills and installation", "url": "https://github.com/confident-ai/deepeval/blob/main/skills/README.md", "seen": "2026-10-09" }, { "what": "Py Core Tests workflow runs, 25 most recent all failed", "url": "https://github.com/confident-ai/deepeval/actions/workflows/test_core.yml", "seen": "2026-10-09" }, { "what": "open issues, first page", "url": "https://github.com/confident-ai/deepeval/issues", "seen": "2026-10-09" }, { "what": "security tab, no policy and no advisories", "url": "https://github.com/confident-ai/deepeval/security", "seen": "2026-10-09" }, { "what": "Confident AI index for agents", "url": "https://www.confident-ai.com/llms.txt", "seen": "2026-10-09" }, { "what": "Confident AI agent authentication file", "url": "https://www.confident-ai.com/auth.md", "seen": "2026-10-09" }, { "what": "Confident AI pricing page", "url": "https://www.confident-ai.com/pricing", "seen": "2026-10-09" }, { "what": "Terms of Service, last modified 2 March 2026", "url": "https://www.confident-ai.com/terms", "seen": "2026-10-09" }, { "what": "privacy policy, last modified 26 May 2026", "url": "https://www.confident-ai.com/privacy-policy", "seen": "2026-10-09" }, { "what": "sub-processor list", "url": "https://www.confident-ai.com/subprocessors-list", "seen": "2026-10-09" }, { "what": "data processing agreement", "url": "https://www.confident-ai.com/dpa", "seen": "2026-10-09" }, { "what": "Confident AI docs index", "url": "https://www.confident-ai.com/docs/llms.txt", "seen": "2026-10-09" }, { "what": "project API keys, expiry, rotation and deactivation", "url": "https://www.confident-ai.com/docs/settings/project/api-keys.md", "seen": "2026-10-09" }, { "what": "data residency, US and EU hosts, stated compliance", "url": "https://www.confident-ai.com/docs/settings/data-residency.md", "seen": "2026-10-09" }, { "what": "Confident AI MCP server introduction", "url": "https://www.confident-ai.com/docs/reference/mcp.md", "seen": "2026-10-09" }, { "what": "security.txt, HTTP 404", "url": "https://www.confident-ai.com/.well-known/security.txt", "seen": "2026-10-09" }, { "what": "security.txt, HTTP 404", "url": "https://deepeval.com/.well-known/security.txt", "seen": "2026-10-09" }, { "what": "npm weekly downloads, 35,092 for 1 to 7 October 2026", "url": "https://api.npmjs.org/downloads/point/last-week/deepeval", "seen": "2026-10-09" }, { "what": "PyPI weekly downloads, 735,637", "url": "https://pypistats.org/api/packages/deepeval/recent", "seen": "2026-10-09" }, { "what": "domain registration for confident-ai.com, 2023-08-15", "url": "https://rdap.verisign.com/com/v1/domain/confident-ai.com", "seen": "2026-10-09" }, { "what": "domain registration for deepeval.com, 2023-07-15", "url": "https://rdap.verisign.com/com/v1/domain/deepeval.com", "seen": "2026-10-09" } ], "openQuestions": [ "unchecked: the trust centre at trust.oneleet.com/confident-ai, which is drawn by script. Its robots.txt request returned the application's HTML shell. SOC 2 Type II and HIPAA are the vendor's statements on its own site", "unchecked: whether Confident AI has a status page. No link to one was found on the pages read", "unchecked: Confident AI's REST API reference, OpenAPI file, MCP tool reference, audit log and data retention pages. The hosted platform is not the surface graded, and robots.txt disallows /docs-openapi", "unchecked: GitHub release notes for 4.2.1 to 4.2.8. The tags page is closed by robots.txt, and the dates come from the cloned tags", "unchecked: reply times on issues and pull requests. Only the first page of open issues was read", "unchecked: the Node.js versions the TypeScript package supports. No `engines` field was found in `typescript/package.json`", "Confident AI's REST API and hosted MCP server at https://mcp.confident-ai.com/mcp are a separately documented surface. They may merit a listing of their own", "The Payments onboarding line is scored on the local framework, which needs no account. Scored on Confident AI it would be 0, and Payments would be 40", "Confident AI's terms forbid using its services for competitive analysis. No clause on automated access was found. This matters before any probe of the hosted platform is run", "The robots.txt request answered 400 on rdap.verisign.com and 404 on api.npmjs.org and pypistats.org, so those hosts publish no rules and were read on that basis", "The terms and privacy links in `provenance` are Confident AI's and cover the optional hosted platform. The framework itself is governed by the Apache-2.0 licence" ] }, "negative": 0, "verdict": "DeepEval runs evaluations and tracing locally under Apache 2.0 with no account, and writes each test run to JSON or SQLite. The 25 most recent core test runs on GitHub had failed on 9 October 2026, two of them on the main branch, and the repository has no security policy.", "bestFor": "Teams that want evaluations in pytest or a CLI on their own machines, with agent, RAG, multi-turn and MCP metrics.", "strengths": [ "Apache-2.0 framework that runs evaluations and tracing locally with no account. Test runs are written to JSON files or a SQLite database on the owner's disk", "Python 4.2.8 was tagged on 2 October 2026, with 14 Python releases tagged since 9 August and TypeScript 0.9.21 on 30 September", "`deepeval test run` takes flags for parallel processes, repeats, a result cache, ignoring errors and skipping cases with missing parameters", "llms.txt on deepeval.com, three agent skills in the repository and plugin manifests for Claude Code and Cursor", "Telemetry goes only to PostHog per the docs and the source, and `DEEPEVAL_TELEMETRY_OPT_OUT=1` turns it off" ], "weaknesses": [ "The 25 most recent Py Core Tests runs had failed when read on 9 October 2026, two of them pushes to main. Issue #3372 of 26 September reports the same", "No SECURITY.md, no published advisories and no security.txt on deepeval.com or confident-ai.com. The trust centre is drawn by script and was not read", "Release 4.2.0 reversed the score direction of four safety metrics in a minor version. The changelog marks it as breaking and the code warns at run time", "The 2026 changelog stops at 4.2.0 of 25 August. Releases 4.2.1 to 4.2.8 have no entries in it", "Usage telemetry is on by default and sends the public IP address. The source also sends the judge model, runtime kind and CLI command, which the docs list omits", "Confident AI's terms forbid using its services for competitive analysis. This matters before any probe of the hosted platform is run" ], "agentNotes": [ "Set `DEEPEVAL_TELEMETRY_OPT_OUT=1` before the first run if usage events and the public IP address should not go to PostHog", "Set a judge model key such as `OPENAI_API_KEY`, or use the non-LLM metrics. Most metrics call an LLM judge and bill the owner's provider account", "Review thresholds for `BiasMetric`, `HallucinationMetric`, `MisuseMetric` and `ToxicityMetric` when upgrading past 4.2.0. Higher scores now mean better", "Read results from `.deepeval/.latest_run_full.json` or a `results_folder`. `deepeval inspect` opens a terminal interface meant for a person", "Pass an existing key with `deepeval login --api-key` in CI. Plain `deepeval login` opens a browser, and results then upload to Confident AI" ], "metrics": { "kind": "local", "measured": false }, "reviewCount": 0, "avgRating": 0, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "B", "methodology": "0.4", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 64.7 } ], "editorialScores": { "ergonomics": 72, "maintenance": 83, "payments": 60, "reliability": 59, "schema": 78, "security": 47, "transparency": 73 }, "provenanceScore": 52 }, "connect": { "install": "pip install -U deepeval" }, "letme": { "capability": "https://letme.dev/obs.evals", "tool": "https://letme.dev/deepeval" }, "notable": [ "The Python package is `deepeval` 4.2.8, tagged on 2 October 2026, for Python 3.9 and later. It installs a pytest plugin and the `deepeval` CLI (https://github.com/confident-ai/deepeval/blob/main/pyproject.toml)", "The TypeScript package is `deepeval` 0.9.21 on npm, tagged on 30 September 2026, with its own `deepeval` CLI run through `npx` (https://github.com/confident-ai/deepeval/tree/main/typescript)", "Test runs are stored locally as JSON by default or in SQLite after `deepeval set-local-store sqlite`, under `.deepeval/` or a chosen results folder (https://deepeval.com/docs/evaluation-local-backend-storage)", "The tracing docs say tracing and component-level evaluation run locally with no account, and that Confident AI draws the same trace data in a dashboard (https://deepeval.com/docs/evaluation-llm-tracing)", "Telemetry uses PostHog and is on by default. The docs list event names, metric names, notebook use, the local store type, an anonymous UUID and the public IP address, and give `DEEPEVAL_TELEMETRY_OPT_OUT=1` as the opt-out (https://deepeval.com/docs/data-privacy)", "Release 4.2.0 of 25 August 2026 changed `BiasMetric`, `HallucinationMetric`, `MisuseMetric` and `ToxicityMetric` so that higher scores are better. The changelog lists it under Breaking Change (https://deepeval.com/changelog)", "The 25 most recent runs of the Py Core Tests workflow had failed when read on 9 October 2026, among them two pushes to main (https://github.com/confident-ai/deepeval/actions/workflows/test_core.yml)", "GitHub shows no security policy and no published advisories for the repository (https://github.com/confident-ai/deepeval/security)", "Confident AI, the hosted platform, has its own REST API with project API keys and a hosted MCP server with OAuth at https://mcp.confident-ai.com/mcp. This listing grades the DeepEval framework, not those surfaces (https://www.confident-ai.com/auth.md)", "Confident AI's Free plan allows 5 test runs a week, 2 seats and 1 project. Starter is $200 a month and Team $2,000 a month (https://www.confident-ai.com/pricing)" ], "area": "developer", "details": [ { "label": "Surface graded", "value": "The DeepEval framework and CLI, run by the owner. Confident AI's hosted REST API and MCP server are separate surfaces and are not graded here" }, { "label": "Packages", "value": "`deepeval` on PyPI, 4.2.8 (2 October 2026), Python 3.9 or later. `deepeval` on npm, 0.9.21 (30 September 2026). Both Apache-2.0" }, { "label": "CLI", "value": "`deepeval test run`, `generate`, `inspect`, `diagnose`, `login`, `logout`, `view`, `gate`, `settings`, `set-local-store`, `set-confident-region`, `set-mode`, `set-eval-mode`, `set-debug`, plus model provider `set-*` commands" }, { "label": "Test run flags", "value": "`-n` processes, `-r` repeat, `-c` cache, `-i` ignore errors, `-s` skip on missing parameters, `-x` stop at first failure, `-id` identifier, `-m` pytest marker, `-d` display" }, { "label": "Local storage", "value": "JSON by default (`.deepeval/.latest_run_full.json` and `test_run_\u003ctimestamp\u003e.json` in a results folder) or SQLite (`deepeval.db`). Location set with `DEEPEVAL_CACHE_FOLDER`" }, { "label": "Credentials", "value": "None for local runs. LLM judge keys come from the environment or a dotenv file. `CONFIDENT_API_KEY` is a project key from Confident AI, written to `.env.local` by `deepeval login`" }, { "label": "Retries", "value": "`DEEPEVAL_RETRY_MAX_ATTEMPTS` defaults to 2 with exponential backoff capped at 5 seconds, for LLM provider calls. Retries can be handed to the provider SDK" }, { "label": "Telemetry", "value": "PostHog at us.i.posthog.com, on by default. Opt out with `DEEPEVAL_TELEMETRY_OPT_OUT=1`" }, { "label": "Agent files", "value": "llms.txt on deepeval.com. Skills `deepeval`, `deepeval-tracing` and `deepeval-otel` in the repository, with `.claude-plugin` and `.cursor-plugin` manifests" }, { "label": "Hosted option", "value": "Confident AI. Free with 5 test runs a week, 2 seats, 1 project and 1 GB-month of trace spans. Starter $200 a month, Team $2,000 a month, then $1 per GB-month. Enterprise by quote" }, { "label": "Hosted regions", "value": "US by default at api.confident-ai.com, EU at eu.api.confident-ai.com with `deepeval set-confident-region EU`. Other regions and on-premises hosting through sales" }, { "label": "Repository", "value": "18.7k stars, 322 open issues and 387 open pull requests on 9 October 2026. 17 workflow files and 332 Python test files" } ], "unitPrices": [ { "item": "Confident AI Starter plan", "unit": "month", "usd": 200, "note": "Hosted platform, optional. Unlimited seats, 5 projects, 5 GB-months of trace spans" }, { "item": "Confident AI Team plan", "unit": "month", "usd": 2000, "note": "Hosted platform, optional. Unlimited projects, 75 GB-months of trace spans" }, { "item": "Confident AI trace spans beyond the plan", "unit": "gb-month", "usd": 1, "note": "Per GB-month ingested or retained on Starter and Team" } ], "provenance": { "legalEntity": "Confident AI, Inc.", "domain": "confident-ai.com", "domainRegistered": "2023-08-15", "endpointOnVendorDomain": true, "terms": "", "privacy": "", "statusPage": "", "changelog": "https://deepeval.com/changelog", "securityTxt": "none", "checked": "2026-10-09", "notes": [ "DeepEval is software the owner runs under the Apache-2.0 licence, which stands in for terms here. Confident AI's Terms of Service (https://www.confident-ai.com/terms) and privacy policy (https://www.confident-ai.com/privacy-policy) govern only the optional hosted platform that `deepeval login`, `view` and `gate` use, so they are not linked as this listing's terms.", "The Terms of Service (last modified 2 March 2026) name Confident AI, Inc., a Delaware corporation, with offices at 33rd 8th St, San Francisco, CA 94103, as printed on the page.", "The privacy policy (last modified 26 May 2026) says it does not apply to Customer Data, which Confident AI processes under its data processing agreement. A DPA and a sub-processor list are published at /dpa and /subprocessors-list.", "When logged in, DeepEval sends results to api.confident-ai.com or eu.api.confident-ai.com and traces to otel.confident-ai.com, all on the vendor's domain. Telemetry goes to us.i.posthog.com.", "https://www.confident-ai.com/.well-known/security.txt and https://deepeval.com/.well-known/security.txt both answered 404.", "No status page link was found on the pages read. The trust centre at trust.oneleet.com/confident-ai is drawn by script and was not read.", "RDAP gives 2023-08-15 as the registration date of confident-ai.com and 2023-07-15 for deepeval.com, both through NameCheap, Inc." ], "score": 52, "checks": [ { "check": "Legal entity named", "value": "Confident AI, Inc.", "points": 20, "max": 20, "state": "ok" }, { "check": "Domain age", "value": "confident-ai.com, registered 2023-08-15 (3 years)", "points": 7, "max": 15, "state": "part" }, { "check": "Endpoint on the vendor's domain", "value": "confident-ai.com", "points": 15, "max": 15, "state": "ok" }, { "check": "Terms of service", "value": "not found", "points": 0, "max": 10, "state": "no" }, { "check": "Privacy policy", "value": "not found", "points": 0, "max": 10, "state": "no" }, { "check": "Status page", "value": "not found", "points": 0, "max": 10, "state": "no" }, { "check": "Changelog", "value": "published", "points": 10, "max": 10, "state": "ok" }, { "check": "security.txt", "value": "not found", "points": 0, "max": 10, "state": "no" } ] }, "pageJsonUrl": "https://www.anchorterminal.com/tools/deepeval.json", "live": { "slug": "deepeval", "versions": [ { "registry": "github", "name": "confident-ai/deepeval", "version": "python-v4.2.4", "released": "2026-09-22", "seenAt": "2026-10-09T16:48:57.205584437Z" }, { "registry": "npm", "name": "deepeval", "version": "0.9.22", "seenAt": "2026-10-09T16:48:56.771405961Z" }, { "registry": "pypi", "name": "deepeval", "version": "4.2.8", "released": "2026-10-02", "seenAt": "2026-10-09T16:48:56.555388028Z" } ], "githubStars": 18722, "npmWeekly": 35092, "pypiWeekly": 735637, "pages": [ { "url": "https://deepeval.com/changelog", "kind": "changelog", "status": 200, "checkedAt": "2026-10-09T18:34:45.297656608Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "5c04fa001f0e" }, { "url": "https://www.confident-ai.com/pricing", "kind": "pricing", "status": 200, "checkedAt": "2026-10-09T18:49:15.052218782Z", "changedAt": "0001-01-01T00:00:00Z", "fingerprint": "f37a25bd3065" } ], "updatedAt": "2026-10-09T18:49:15.052218782Z" } }, "verify": { "accepts": "a page on confident-ai.com or one of its subdomains, or the README of github.com/confident-ai/deepeval", "badgeUrl": "https://www.anchorterminal.com/badges/deepeval.svg", "body": { "slug": "deepeval", "url": "the page with the badge or the link" }, "docs": "https://www.anchorterminal.com/builders/#verify", "effect": "none, it never changes a grade, rank or review", "endpoint": "https://www.anchorterminal.com/api/v1/verify", "listingUrl": "https://www.anchorterminal.com/tools/deepeval", "mcpTool": "verify_listing", "recheck": "weekly; two failed checks in a row and it lapses, a later pass restores it", "snippets": { "html": "\u003ca href=\"https://www.anchorterminal.com/tools/deepeval\"\u003e\u003cimg src=\"https://www.anchorterminal.com/badges/deepeval.svg\" alt=\"DeepEval on Anchor Terminal\" height=\"20\"\u003e\u003c/a\u003e", "markdown": "[![DeepEval on Anchor Terminal](https://www.anchorterminal.com/badges/deepeval.svg)](https://www.anchorterminal.com/tools/deepeval)", "link": "\u003ca href=\"https://www.anchorterminal.com/tools/deepeval\"\u003eDeepEval on Anchor Terminal\u003c/a\u003e" } } }, "kind": "anchor.page", "links": { "api": "https://www.anchorterminal.com/api/v1/index.json", "html": "https://www.anchorterminal.com/tools/deepeval", "json": "https://www.anchorterminal.com/tools/deepeval.json", "llms": "https://www.anchorterminal.com/llms.txt", "markdown": "https://www.anchorterminal.com/tools/deepeval.md", "slim": "https://www.anchorterminal.com/tools/deepeval.min.md" }, "markdown": "## Overview\n\n**Grade B · 64.7/100 · rank #343 of 950 · #9 in Agent observability \u0026 evals · not agent-ready · confidence medium**\n\n\n## Assessment\n\nDeepEval runs evaluations and tracing locally under Apache 2.0 with no account, and writes each test run to JSON or SQLite. The 25 most recent core test runs on GitHub had failed on 9 October 2026, two of them on the main branch, and the repository has no security policy.\n\n## Facts\n\n| Field | Value |\n| --- | --- |\n| Vendor | Confident AI, Inc. (https://www.confident-ai.com) |\n| Kind | SDK + MCP |\n| Category | Agent observability \u0026 evals (https://www.anchorterminal.com/categories/agent-observability) |\n| Auth | OAuth or key · No credential is needed to run evaluations locally. Most metrics call an LLM judge, so the owner supplies a model provider key such as `OPENAI_API_KEY` in the environment or a dotenv file. Reporting to Confident AI needs a project API key in `CONFIDENT_API_KEY`. `deepeval login` pairs with a browser through a device code and writes the key to `.env.local`, and `deepeval login --api-key` takes an existing key for CI. Confident AI project keys are named, take an expiry, and can be deactivated, rotated with a grace period or deleted. No per-permission scopes for project keys were found in the page read. |\n| Pricing | Freemium ($200 / mo) · DeepEval is free under Apache 2.0 and runs without an account. LLM judge calls are billed by the owner's model provider. The hosted Confident AI platform has a Free plan with no card (5 test runs a week, 2 seats, 1 project, 1 GB-month of trace spans), Starter at $200 a month, Team at $2,000 a month, both with $1 per GB-month beyond the included trace spans, and Enterprise by quote (https://www.confident-ai.com/pricing). |\n| x402 | No · No x402, MPP or L402 in the repository docs, the Confident AI docs index or the pricing page (checked 2026-10-09). |\n| Licence | Apache 2.0 for the Python and TypeScript packages and the agent skills. Confident AI, the hosted platform, is a proprietary service under its own terms |\n| Packages | pypi: `deepeval`; npm: `deepeval` |\n| Source | https://github.com/confident-ai/deepeval |\n| Docs | https://deepeval.com/docs/getting-started |\n| llms.txt | https://deepeval.com/llms.txt |\n| Last release | 2026-10-02 |\n| GitHub stars | 18,700 (as of 2026-10-09) |\n| npm downloads / week | 35,092 |\n| PyPI downloads / week | 735,637 |\n| Surface graded | The DeepEval framework and CLI, run by the owner. Confident AI's hosted REST API and MCP server are separate surfaces and are not graded here |\n| Packages | `deepeval` on PyPI, 4.2.8 (2 October 2026), Python 3.9 or later. `deepeval` on npm, 0.9.21 (30 September 2026). Both Apache-2.0 |\n| CLI | `deepeval test run`, `generate`, `inspect`, `diagnose`, `login`, `logout`, `view`, `gate`, `settings`, `set-local-store`, `set-confident-region`, `set-mode`, `set-eval-mode`, `set-debug`, plus model provider `set-*` commands |\n| Test run flags | `-n` processes, `-r` repeat, `-c` cache, `-i` ignore errors, `-s` skip on missing parameters, `-x` stop at first failure, `-id` identifier, `-m` pytest marker, `-d` display |\n| Local storage | JSON by default (`.deepeval/.latest_run_full.json` and `test_run_\u003ctimestamp\u003e.json` in a results folder) or SQLite (`deepeval.db`). Location set with `DEEPEVAL_CACHE_FOLDER` |\n| Credentials | None for local runs. LLM judge keys come from the environment or a dotenv file. `CONFIDENT_API_KEY` is a project key from Confident AI, written to `.env.local` by `deepeval login` |\n| Retries | `DEEPEVAL_RETRY_MAX_ATTEMPTS` defaults to 2 with exponential backoff capped at 5 seconds, for LLM provider calls. Retries can be handed to the provider SDK |\n| Telemetry | PostHog at us.i.posthog.com, on by default. Opt out with `DEEPEVAL_TELEMETRY_OPT_OUT=1` |\n| Agent files | llms.txt on deepeval.com. Skills `deepeval`, `deepeval-tracing` and `deepeval-otel` in the repository, with `.claude-plugin` and `.cursor-plugin` manifests |\n| Hosted option | Confident AI. Free with 5 test runs a week, 2 seats, 1 project and 1 GB-month of trace spans. Starter $200 a month, Team $2,000 a month, then $1 per GB-month. Enterprise by quote |\n| Hosted regions | US by default at api.confident-ai.com, EU at eu.api.confident-ai.com with `deepeval set-confident-region EU`. Other regions and on-premises hosting through sales |\n| Repository | 18.7k stars, 322 open issues and 387 open pull requests on 9 October 2026. 17 workflow files and 332 Python test files |\n| Capabilities | obs.evals, obs.traces, obs.datasets, obs.prompts |\n| Tags | open-source, apache-2.0, local, cli, python, typescript, pytest, llms-txt, agent-skills, opentelemetry, freemium, no-card |\n| JSON | https://www.anchorterminal.com/api/v1/tools/deepeval.json |\n\n## Score breakdown (methodology v0.4, October 2026 research run)\n\nAssessed 2026-10-09 from public evidence against the published checklist (https://www.anchorterminal.com/benchmark/#checklist). Confidence: medium. Performance and Task success pending (no score, not in the total); the total is Σ(score × weight) ÷ 80 over the 7 assessed categories. \"This run\" is each category's share of the 100 points.\n\n| Category | Weight | This run | Score (0–100) | Points |\n| --- | --- | --- | --- | --- |\n| Reliability | 16% | 20 | 59 | 11.8 |\n| Performance | 10% | pending | pending | n/a |\n| Schema \u0026 documentation | 13% | 16.2 | 78 | 12.7 |\n| Agent ergonomics | 13% | 16.2 | 72 | 11.7 |\n| Security \u0026 auth | 14% | 17.5 | 47 | 8.2 |\n| Payments \u0026 pricing | 10% | 12.5 | 60 | 7.5 |\n| Task success | 10% | pending | pending | n/a |\n| Maintenance \u0026 community | 7% | 8.8 | 83 | 7.3 |\n| Transparency \u0026 trust (editorial 73, provenance 52) | 7% | 8.8 | 63 | 5.5 |\n| Negative events | up to −15 | up to −15 | none recorded | 0 |\n| **Total** | | | | **64.7 → B** |\n\n### Why each score\n\n- Reliability 59: Scored on the local-software lines, since DeepEval runs where the owner installs it. Official `deepeval` packages on PyPI and npm, with Python 3.9 or later stated (20). Public CI in GitHub Actions with 17 workflow files and 332 Python test files, but the 25 most recent Py Core Tests runs had failed when read on 9 October 2026, two of them pushes to main on 7 October, and open issue #3372 of 26 September reports the suite red on main (8). 322 open issues and 387 open pull requests on a repository with 18.7k stars. The newest page of issues is mostly scoring and provider defects filed since 23 September, each with four comments or fewer (12). The changelog has Breaking Change sections, but breaking changes ship in minor and patch releases. 4.2.0 reversed the score direction of four metrics and 3.9.8 and 3.9.9 removed options (7). Python is at 4.2.8. The TypeScript package is at 0.9.21 (12).\n- Performance: Pending. Latency is measured per call by our probes, which haven't run yet, so this run doesn't score it. Its weight is shared across the assessed categories until the first probe window closes.\n- Schema \u0026 documentation 78: Read on the framework lines. The Python package ships `py.typed` and pydantic models, the TypeScript package ships type declarations, and the CLI is built on typer. A library has no OpenAPI file to publish (18). llms.txt on deepeval.com indexes the docs, and pages carry a Copy Markdown control (10). Each metric has its own page with required parameters and usage, and three agent skills in the repository tell a coding agent how to add evaluations and tracing (16). Typed test case and metric classes, with enums for evaluation parameters and CLI choices (12). Many runnable examples, a troubleshooting page and an environment variable reference. Error types are not listed in one place (11). Tagged releases for both packages and a public changelog by year, but the 2026 changelog stops at 4.2.0 of 25 August and has no entry for 4.2.1 to 4.2.8 (11).\n- Agent ergonomics 72: Read on the framework lines. One install and a pytest-style test file give a first evaluation, and results land in JSON files a coding agent can read. `deepeval inspect` is a terminal interface for a person, and no MCP server ships with the framework (18). `deepeval test run` takes flags for display, pytest markers, parallel processes and a results folder, and the SQLite store can be queried across runs (14). `--ignore-errors`, `--skip-on-missing-params`, a `diagnose` command, debug controls and a troubleshooting page (14). `--use-cache` reuses metric results and retry settings cover provider calls with backoff. Scores from LLM judges can differ between runs (12). Python and TypeScript packages with few required parameters. The default judge needs `OPENAI_API_KEY` (14).\n- Security \u0026 auth 47: Local runs need no credential. Provider keys and `CONFIDENT_API_KEY` are read from the environment or a dotenv file, and storing keys in the legacy `.deepeval/.deepeval` file is deprecated in the source. Confident AI project keys take a name and an expiry and can be deactivated, rotated with a grace period or deleted. No per-permission scopes were found (20). The framework has no read-only mode. Once logged in, test runs and traces upload to Confident AI (8). Metrics pass application output to an LLM judge, and no guidance on untrusted content reaching the judge was found in the pages read (5). Each run is recorded in local JSON or SQLite. Confident AI documents audit log exports, which we did not read (7). GitHub shows no security policy and no published advisories, and security.txt answered 404 on both domains. No bug bounty was found. The Confident AI site states SOC 2 Type II and HIPAA, and its trust centre is drawn by script and was not read (7).\n- Payments \u0026 pricing 60: No x402, MPP or L402 (0). DeepEval is free, and the paid option is the hosted Confident AI platform, whose prices are public without a login. Starter is $200 a month, Team $2,000 a month, and trace spans beyond the plan are $1 per GB-month (20). The framework is free and Confident AI's Free plan needs no card (20). `pip install -U deepeval` and a local run need no account, so an agent can start without a person signing up. Confident AI itself needs a browser sign-up, and its auth.md says there is no self-serve registration endpoint for API keys. Scoring this line on the local framework is a judgement call (20).\n- Task success: Pending. Task success needs the category task suites run through each tool, which haven't run yet, so this run doesn't score it. Its weight is shared across the assessed categories until then. A data provider's data-quality score is published on its listing now and becomes half of this category when it's scored.\n- Maintenance \u0026 community 83: Python 4.2.8 was tagged on 2 October 2026, seven days before the check (30). 14 Python releases tagged since 9 August, from 4.1.6 to 4.2.8 (20). 124 commits in the 30 days to 9 October, most by three maintainers, and a Discord community. 322 issues and 387 pull requests are open and reply times were not read (14). Current official packages in Python and TypeScript, the latter at 0.9.21 on 30 September (15). A `poetry.lock` file and CI on pull requests. No Dependabot configuration was found and the core test workflow is failing on main (4).\n- Transparency \u0026 trust 63: Apache-2.0 for the whole repository (30). The docs state what telemetry collects, and Confident AI publishes terms, a privacy policy, a DPA and a sub-processor list with regions. They disagree in places. The privacy policy says the DPA is available on request while it is published at /dpa, and the DeepEval docs say data sits in a private cloud on AWS while the sub-processor list names Supabase, ClickHouse and Railway for storage and hosting (20). The changelog has Breaking Change sections and the source warns at run time about the reversed metrics and the legacy key file. No deprecation policy or notice period was found (9). Telemetry through PostHog is on by default and `DEEPEVAL_TELEMETRY_OPT_OUT=1` turns it off. The source also sends the judge provider and model, the runtime kind, the CLI command and test case counts, which the docs list omits (14).\n\nFix list for a coding agent, everything this grade says the listing lacks, the biggest gain first (23 items): https://www.anchorterminal.com/fixes/deepeval.md (JSON https://www.anchorterminal.com/fixes/deepeval.json)\n\n### What we couldn't check\n\n- unchecked: the trust centre at trust.oneleet.com/confident-ai, which is drawn by script. Its robots.txt request returned the application's HTML shell. SOC 2 Type II and HIPAA are the vendor's statements on its own site\n- unchecked: whether Confident AI has a status page. No link to one was found on the pages read\n- unchecked: Confident AI's REST API reference, OpenAPI file, MCP tool reference, audit log and data retention pages. The hosted platform is not the surface graded, and robots.txt disallows /docs-openapi\n- unchecked: GitHub release notes for 4.2.1 to 4.2.8. The tags page is closed by robots.txt, and the dates come from the cloned tags\n- unchecked: reply times on issues and pull requests. Only the first page of open issues was read\n- unchecked: the Node.js versions the TypeScript package supports. No `engines` field was found in `typescript/package.json`\n- Confident AI's REST API and hosted MCP server at https://mcp.confident-ai.com/mcp are a separately documented surface. They may merit a listing of their own\n- The Payments onboarding line is scored on the local framework, which needs no account. Scored on Confident AI it would be 0, and Payments would be 40\n- Confident AI's terms forbid using its services for competitive analysis. No clause on automated access was found. This matters before any probe of the hosted platform is run\n- The robots.txt request answered 400 on rdap.verisign.com and 404 on api.npmjs.org and pypistats.org, so those hosts publish no rules and were read on that basis\n- The terms and privacy links in `provenance` are Confident AI's and cover the optional hosted platform. The framework itself is governed by the Apache-2.0 licence\n\n### Sources\n\n- robots.txt for deepeval.com, HTTP 200, allows every path, Content-Signal ai-input=yes: \u003chttps://deepeval.com/robots.txt\u003e (seen 2026-10-09)\n- robots.txt for www.confident-ai.com, HTTP 200, ai-input=yes, disallows /app/, /api/ and /docs-openapi: \u003chttps://www.confident-ai.com/robots.txt\u003e (seen 2026-10-09)\n- robots.txt for github.com, HTTP 200: \u003chttps://github.com/robots.txt\u003e (seen 2026-10-09)\n- DeepEval docs index for agents: \u003chttps://deepeval.com/llms.txt\u003e (seen 2026-10-09)\n- data privacy page, telemetry fields and opt-out: \u003chttps://deepeval.com/docs/data-privacy\u003e (seen 2026-10-09)\n- repository home, 18.7k stars, 322 open issues, 387 open pull requests: \u003chttps://github.com/confident-ai/deepeval\u003e (seen 2026-10-09)\n- repository clone at commit 0fb05d0 of 7 October 2026, read for tags, package files, CLI, telemetry and docs source: \u003chttps://github.com/confident-ai/deepeval\u003e (seen 2026-10-09)\n- Python package metadata, version 4.2.8, Python 3.9 or later, Apache-2.0: \u003chttps://github.com/confident-ai/deepeval/blob/main/pyproject.toml\u003e (seen 2026-10-09)\n- TypeScript package metadata, version 0.9.21: \u003chttps://github.com/confident-ai/deepeval/blob/main/typescript/package.json\u003e (seen 2026-10-09)\n- telemetry source, PostHog host and event properties: \u003chttps://github.com/confident-ai/deepeval/blob/main/deepeval/telemetry/properties.py\u003e (seen 2026-10-09)\n- 2026 changelog source, Breaking Change entries for 4.2.0, 3.9.9 and 3.9.8, last entry 4.2.0: \u003chttps://github.com/confident-ai/deepeval/blob/main/docs/content/changelog/changelog-2026.mdx\u003e (seen 2026-10-09)\n- CLI reference source, commands, test run flags and secrets handling: \u003chttps://github.com/confident-ai/deepeval/blob/main/docs/content/docs/command-line-interface.mdx\u003e (seen 2026-10-09)\n- local backend storage docs source: \u003chttps://github.com/confident-ai/deepeval/blob/main/docs/content/docs/evaluation-local-backend-storage.mdx\u003e (seen 2026-10-09)\n- environment variables docs source, retry settings: \u003chttps://github.com/confident-ai/deepeval/blob/main/docs/content/docs/environment-variables.mdx\u003e (seen 2026-10-09)\n- device-code login flow source: \u003chttps://github.com/confident-ai/deepeval/blob/main/deepeval/cli/auth/flow.py\u003e (seen 2026-10-09)\n- agent skills and installation: \u003chttps://github.com/confident-ai/deepeval/blob/main/skills/README.md\u003e (seen 2026-10-09)\n- Py Core Tests workflow runs, 25 most recent all failed: \u003chttps://github.com/confident-ai/deepeval/actions/workflows/test_core.yml\u003e (seen 2026-10-09)\n- open issues, first page: \u003chttps://github.com/confident-ai/deepeval/issues\u003e (seen 2026-10-09)\n- security tab, no policy and no advisories: \u003chttps://github.com/confident-ai/deepeval/security\u003e (seen 2026-10-09)\n- Confident AI index for agents: \u003chttps://www.confident-ai.com/llms.txt\u003e (seen 2026-10-09)\n- Confident AI agent authentication file: \u003chttps://www.confident-ai.com/auth.md\u003e (seen 2026-10-09)\n- Confident AI pricing page: \u003chttps://www.confident-ai.com/pricing\u003e (seen 2026-10-09)\n- Terms of Service, last modified 2 March 2026: \u003chttps://www.confident-ai.com/terms\u003e (seen 2026-10-09)\n- privacy policy, last modified 26 May 2026: \u003chttps://www.confident-ai.com/privacy-policy\u003e (seen 2026-10-09)\n- sub-processor list: \u003chttps://www.confident-ai.com/subprocessors-list\u003e (seen 2026-10-09)\n- data processing agreement: \u003chttps://www.confident-ai.com/dpa\u003e (seen 2026-10-09)\n- Confident AI docs index: \u003chttps://www.confident-ai.com/docs/llms.txt\u003e (seen 2026-10-09)\n- project API keys, expiry, rotation and deactivation: \u003chttps://www.confident-ai.com/docs/settings/project/api-keys.md\u003e (seen 2026-10-09)\n- data residency, US and EU hosts, stated compliance: \u003chttps://www.confident-ai.com/docs/settings/data-residency.md\u003e (seen 2026-10-09)\n- Confident AI MCP server introduction: \u003chttps://www.confident-ai.com/docs/reference/mcp.md\u003e (seen 2026-10-09)\n- security.txt, HTTP 404: \u003chttps://www.confident-ai.com/.well-known/security.txt\u003e (seen 2026-10-09)\n- security.txt, HTTP 404: \u003chttps://deepeval.com/.well-known/security.txt\u003e (seen 2026-10-09)\n- npm weekly downloads, 35,092 for 1 to 7 October 2026: \u003chttps://api.npmjs.org/downloads/point/last-week/deepeval\u003e (seen 2026-10-09)\n- PyPI weekly downloads, 735,637: \u003chttps://pypistats.org/api/packages/deepeval/recent\u003e (seen 2026-10-09)\n- domain registration for confident-ai.com, 2023-08-15: \u003chttps://rdap.verisign.com/com/v1/domain/confident-ai.com\u003e (seen 2026-10-09)\n- domain registration for deepeval.com, 2023-07-15: \u003chttps://rdap.verisign.com/com/v1/domain/deepeval.com\u003e (seen 2026-10-09)\n\n## Who's behind it (provenance 52/100, checked 2026-10-09)\n\n| Check | Finding | Points |\n| --- | --- | --- |\n| Legal entity named | Confident AI, Inc. | 20/20 |\n| Domain age | confident-ai.com, registered 2023-08-15 (3 years) | 7/15 |\n| Endpoint on the vendor's domain | confident-ai.com | 15/15 |\n| Terms of service | not found | 0/10 |\n| Privacy policy | not found | 0/10 |\n| Status page | not found | 0/10 |\n| Changelog | published | 10/10 |\n| security.txt | not found | 0/10 |\n\nDeepEval is software the owner runs under the Apache-2.0 licence, which stands in for terms here. Confident AI's Terms of Service (https://www.confident-ai.com/terms) and privacy policy (https://www.confident-ai.com/privacy-policy) govern only the optional hosted platform that `deepeval login`, `view` and `gate` use, so they are not linked as this listing's terms.\n\nThe Terms of Service (last modified 2 March 2026) name Confident AI, Inc., a Delaware corporation, with offices at 33rd 8th St, San Francisco, CA 94103, as printed on the page.\n\nThe privacy policy (last modified 26 May 2026) says it does not apply to Customer Data, which Confident AI processes under its data processing agreement. A DPA and a sub-processor list are published at /dpa and /subprocessors-list.\n\nWhen logged in, DeepEval sends results to api.confident-ai.com or eu.api.confident-ai.com and traces to otel.confident-ai.com, all on the vendor's domain. Telemetry goes to us.i.posthog.com.\n\nhttps://www.confident-ai.com/.well-known/security.txt and https://deepeval.com/.well-known/security.txt both answered 404.\n\nNo status page link was found on the pages read. The trust centre at trust.oneleet.com/confident-ai is drawn by script and was not read.\n\nRDAP gives 2023-08-15 as the registration date of confident-ai.com and 2023-07-15 for deepeval.com, both through NameCheap, Inc.\n\n### Terms and privacy, as read\n\nA reading by a fixed set of rules, each answered with the vendor's own sentence. Not legal advice.\n\n**Terms of service**. We found no terms of service published for this product, so there is nothing to read and the check scores 0.\n\n\n**Privacy policy**. We found no privacy policy published for this product, so there is nothing to read and the check scores 0.\n\n\n## Live (updated 2026-10-09 18:49 UTC)\n\n- github `confident-ai/deepeval` python-v4.2.4, released 2026-09-22\n- npm `deepeval` 0.9.22\n- pypi `deepeval` 4.2.8, released 2026-10-02\n- Watching changelog \u003chttps://deepeval.com/changelog\u003e\n- Watching pricing \u003chttps://www.confident-ai.com/pricing\u003e\n- Always current: https://www.anchorterminal.com/api/v1/live/deepeval.json\n\n## Probe metrics\n\nNot measured yet. Our benchmark probes haven't run, so there's no availability, latency or error rate from a run and Performance is pending. Live uptime, where we poll the endpoint, is under Live and doesn't change the score.\n\n## Prices\n\n| Item | Price | Unit | Note |\n| --- | --- | --- | --- |\n| Confident AI Starter plan | $200 | per month (plan) | Hosted platform, optional. Unlimited seats, 5 projects, 5 GB-months of trace spans |\n| Confident AI Team plan | $2000 | per month (plan) | Hosted platform, optional. Unlimited projects, 75 GB-months of trace spans |\n| Confident AI trace spans beyond the plan | $1 | per GB per month | Per GB-month ingested or retained on Starter and Team |\n\nAcross all listings: https://www.anchorterminal.com/prices/index.md\n\n## Strengths\n\n- Apache-2.0 framework that runs evaluations and tracing locally with no account. Test runs are written to JSON files or a SQLite database on the owner's disk\n- Python 4.2.8 was tagged on 2 October 2026, with 14 Python releases tagged since 9 August and TypeScript 0.9.21 on 30 September\n- `deepeval test run` takes flags for parallel processes, repeats, a result cache, ignoring errors and skipping cases with missing parameters\n- llms.txt on deepeval.com, three agent skills in the repository and plugin manifests for Claude Code and Cursor\n- Telemetry goes only to PostHog per the docs and the source, and `DEEPEVAL_TELEMETRY_OPT_OUT=1` turns it off\n\n## Weaknesses\n\n- The 25 most recent Py Core Tests runs had failed when read on 9 October 2026, two of them pushes to main. Issue #3372 of 26 September reports the same\n- No SECURITY.md, no published advisories and no security.txt on deepeval.com or confident-ai.com. The trust centre is drawn by script and was not read\n- Release 4.2.0 reversed the score direction of four safety metrics in a minor version. The changelog marks it as breaking and the code warns at run time\n- The 2026 changelog stops at 4.2.0 of 25 August. Releases 4.2.1 to 4.2.8 have no entries in it\n- Usage telemetry is on by default and sends the public IP address. The source also sends the judge model, runtime kind and CLI command, which the docs list omits\n- Confident AI's terms forbid using its services for competitive analysis. This matters before any probe of the hosted platform is run\n\n## Before you call it (notes for agents)\n\n1. Set `DEEPEVAL_TELEMETRY_OPT_OUT=1` before the first run if usage events and the public IP address should not go to PostHog\n2. Set a judge model key such as `OPENAI_API_KEY`, or use the non-LLM metrics. Most metrics call an LLM judge and bill the owner's provider account\n3. Review thresholds for `BiasMetric`, `HallucinationMetric`, `MisuseMetric` and `ToxicityMetric` when upgrading past 4.2.0. Higher scores now mean better\n4. Read results from `.deepeval/.latest_run_full.json` or a `results_folder`. `deepeval inspect` opens a terminal interface meant for a person\n5. Pass an existing key with `deepeval login --api-key` in CI. Plain `deepeval login` opens a browser, and results then upload to Confident AI\n\n## Connect\n\nInstall:\n\n```bash\npip install -U deepeval\n```\n\nThrough letme (picks today, calling later): https://letme.dev/deepeval. letme answers with the pick and how to call it direct; calling through letme (one key, the vendor's own price) comes later. How it works: https://www.anchorterminal.com/letme/index.md\n\n## Similar tools\n\nRanked by shared capabilities, then score. Same-category tools with no shared capability key are listed last.\n\n| Tool | Grade | Score | Rank | Shared capabilities | x402 | Markdown |\n| --- | --- | --- | --- | --- | --- | --- |\n| Arize Phoenix | BB | 75.4 | 47 | obs.traces, obs.evals, obs.prompts, obs.datasets | no | https://www.anchorterminal.com/tools/arize-phoenix.md |\n| Langfuse API + MCP | BB | 72.7 | 101 | obs.traces, obs.evals, obs.prompts, obs.datasets | no | https://www.anchorterminal.com/tools/langfuse.md |\n| LangSmith API + MCP | BB | 71.1 | 136 | obs.traces, obs.evals, obs.prompts, obs.datasets | no | https://www.anchorterminal.com/tools/langsmith.md |\n| W\u0026B Weave | B | 66.7 | 280 | obs.traces, obs.evals, obs.prompts, obs.datasets | no | https://www.anchorterminal.com/tools/wandb-weave.md |\n| Respan API + MCP | B | 65.5 | 317 | obs.traces, obs.evals, obs.prompts, obs.datasets | no | https://www.anchorterminal.com/tools/respan.md |\n| LangWatch | B | 65.5 | 318 | obs.traces, obs.evals, obs.prompts, obs.datasets | no | https://www.anchorterminal.com/tools/langwatch.md |\n\n## Panel reviews (0)\n\nReviewed by the Anchor panel (https://www.anchorterminal.com/reviewers/index.md): .\n\nDesk reviews, written from public documentation, pricing, terms, source and status history on 1 October 2026. No calls made. For a desk review, the outcome says whether the reviewer's questions could be answered from public material: success, partial or failure. How reviews work: https://www.anchorterminal.com/reviews/how-it-works.md\n\n## Notable\n\n- The Python package is `deepeval` 4.2.8, tagged on 2 October 2026, for Python 3.9 and later. It installs a pytest plugin and the `deepeval` CLI (source: \u003chttps://github.com/confident-ai/deepeval/blob/main/pyproject.toml\u003e)\n- The TypeScript package is `deepeval` 0.9.21 on npm, tagged on 30 September 2026, with its own `deepeval` CLI run through `npx` (source: \u003chttps://github.com/confident-ai/deepeval/tree/main/typescript\u003e)\n- Test runs are stored locally as JSON by default or in SQLite after `deepeval set-local-store sqlite`, under `.deepeval/` or a chosen results folder (source: \u003chttps://deepeval.com/docs/evaluation-local-backend-storage\u003e)\n- The tracing docs say tracing and component-level evaluation run locally with no account, and that Confident AI draws the same trace data in a dashboard (source: \u003chttps://deepeval.com/docs/evaluation-llm-tracing\u003e)\n- Telemetry uses PostHog and is on by default. The docs list event names, metric names, notebook use, the local store type, an anonymous UUID and the public IP address, and give `DEEPEVAL_TELEMETRY_OPT_OUT=1` as the opt-out (source: \u003chttps://deepeval.com/docs/data-privacy\u003e)\n- Release 4.2.0 of 25 August 2026 changed `BiasMetric`, `HallucinationMetric`, `MisuseMetric` and `ToxicityMetric` so that higher scores are better. The changelog lists it under Breaking Change (source: \u003chttps://deepeval.com/changelog\u003e)\n- The 25 most recent runs of the Py Core Tests workflow had failed when read on 9 October 2026, among them two pushes to main (source: \u003chttps://github.com/confident-ai/deepeval/actions/workflows/test_core.yml\u003e)\n- GitHub shows no security policy and no published advisories for the repository (source: \u003chttps://github.com/confident-ai/deepeval/security\u003e)\n- Confident AI, the hosted platform, has its own REST API with project API keys and a hosted MCP server with OAuth at https://mcp.confident-ai.com/mcp. This listing grades the DeepEval framework, not those surfaces (source: \u003chttps://www.confident-ai.com/auth.md\u003e)\n- Confident AI's Free plan allows 5 test runs a week, 2 seats and 1 project. Starter is $200 a month and Team $2,000 a month (source: \u003chttps://www.confident-ai.com/pricing\u003e)\n\n- #9 of 15 in Best agent tracing, monitoring and evaluation tools: https://www.anchorterminal.com/best/agent-observability/index.md\n- All 120 evals comparisons: https://www.anchorterminal.com/compare/agent-observability/index.md\n\n## Compare\n\n- [Arize Phoenix vs DeepEval](https://www.anchorterminal.com/compare/arize-phoenix-vs-deepeval.md): BB 75.4 vs B 64.7\n- [Baserun vs DeepEval](https://www.anchorterminal.com/compare/baserun-vs-deepeval.md): F 7 vs B 64.7\n- [Braintrust API + MCP vs DeepEval](https://www.anchorterminal.com/compare/braintrust-vs-deepeval.md): C 61.1 vs B 64.7\n- [DeepEval vs Galileo API + MCP](https://www.anchorterminal.com/compare/deepeval-vs-galileo.md): B 64.7 vs D 47.7\n- [DeepEval vs Helicone AI Gateway + MCP](https://www.anchorterminal.com/compare/deepeval-vs-helicone.md): B 64.7 vs D 46.9\n- [DeepEval vs HoneyHive](https://www.anchorterminal.com/compare/deepeval-vs-honeyhive.md): B 64.7 vs C 55.7\n- [DeepEval vs Laminar API + MCP](https://www.anchorterminal.com/compare/deepeval-vs-laminar.md): B 64.7 vs C 56.6\n- [DeepEval vs Langfuse API + MCP](https://www.anchorterminal.com/compare/deepeval-vs-langfuse.md): B 64.7 vs BB 72.7\n- [DeepEval vs LangSmith API + MCP](https://www.anchorterminal.com/compare/deepeval-vs-langsmith.md): B 64.7 vs BB 71.1\n- [DeepEval vs LangWatch](https://www.anchorterminal.com/compare/deepeval-vs-langwatch.md): B 64.7 vs B 65.5\n- [DeepEval vs MLflow Tracing](https://www.anchorterminal.com/compare/deepeval-vs-mlflow-tracing.md): B 64.7 vs C 61.2\n- [DeepEval vs Prefactor](https://www.anchorterminal.com/compare/deepeval-vs-prefactor.md): B 64.7 vs B 65.6\n- [DeepEval vs Pydantic Logfire](https://www.anchorterminal.com/compare/deepeval-vs-pydantic-logfire.md): B 64.7 vs B 64.9\n- [DeepEval vs Respan API + MCP](https://www.anchorterminal.com/compare/deepeval-vs-respan.md): B 64.7 vs B 65.5\n- [DeepEval vs W\u0026B Weave](https://www.anchorterminal.com/compare/deepeval-vs-wandb-weave.md): B 64.7 vs B 66.7\n\n## Verify this listing\n\nFor the vendor. The badge or a plain link to this page verifies the listing, from a page on confident-ai.com or one of its subdomains, or the README of github.com/confident-ai/deepeval. It shows the listing is the vendor's and that the vendor knows it's here, and it never changes a grade, rank or review. The vendor sends the page's address to `POST https://www.anchorterminal.com/api/v1/verify` as `{\"slug\": \"deepeval\", \"url\": \"…\"}`, or calls the `verify_listing` tool at https://www.anchorterminal.com/mcp. We fetch the page once, then again every week; two failed checks in a row and the verification lapses, and a later pass restores it. What we check: https://www.anchorterminal.com/builders/index.md#verify\n\nHTML badge:\n\n```html\n\u003ca href=\"https://www.anchorterminal.com/tools/deepeval\"\u003e\u003cimg src=\"https://www.anchorterminal.com/badges/deepeval.svg\" alt=\"DeepEval on Anchor Terminal\" height=\"20\"\u003e\u003c/a\u003e\n```\n\nMarkdown badge, for a README:\n\n```markdown\n[![DeepEval on Anchor Terminal](https://www.anchorterminal.com/badges/deepeval.svg)](https://www.anchorterminal.com/tools/deepeval)\n```\n\nPlain link:\n\n```html\n\u003ca href=\"https://www.anchorterminal.com/tools/deepeval\"\u003eDeepEval on Anchor Terminal\u003c/a\u003e\n```\n\n## Share this listing\n\nFor the vendor. Sharing assets for social media, two PNGs of 1200 × 630 that say DeepEval is listed on Anchor Terminal, with the vendor's logo and this page's address and no grade or score.\n\n- Dark: https://www.anchorterminal.com/assets/share/deepeval-dark.png\n- Light: https://www.anchorterminal.com/assets/share/deepeval-light.png\n", "meta": { "attribution": "Anchor Terminal (https://www.anchorterminal.com)", "docs": "https://www.anchorterminal.com/docs/", "generatedAt": "2026-10-10", "license": "CC-BY-4.0", "method": "https://www.anchorterminal.com/benchmark/", "methodology": "0.4", "openapi": "https://www.anchorterminal.com/openapi.json", "preview": false, "run": "2026-10-01", "runLabel": "October 2026 research run" }, "page": { "breadcrumbs": [ { "name": "Home", "url": "https://www.anchorterminal.com/" }, { "name": "Terminal", "url": "https://www.anchorterminal.com/tools/" }, { "name": "Agent observability \u0026 evals", "url": "https://www.anchorterminal.com/categories/agent-observability" }, { "name": "DeepEval", "url": "" } ], "description": "DeepEval is an open-source Python and TypeScript framework from Confident AI for evaluating LLM applications and agents. Evaluations run locally through a pytest plugin and the deepeval CLI, with optional reporting to the hosted Confident AI platform.", "facts": [ "rank #343 of 950", "OAuth or key auth", "0 desk reviews" ], "h1": "DeepEval", "image": "https://www.anchorterminal.com/assets/og/tools-deepeval.png", "path": "/tools/deepeval", "published": "2026-10-01", "section": "tools", "title": "DeepEval review (2026): pricing, alternatives and grade B", "toc": null, "updated": "2026-10-09", "url": "https://www.anchorterminal.com/tools/deepeval" }, "tokens": { "markdown": 8150, "slim": 1680 }, "version": 1 }