{
  "listings": [
    {
      "slug": "langwatch",
      "name": "LangWatch",
      "company": "Reasoning Engine B.V. (doing business as LangWatch)",
      "grade": "B",
      "fit": "yes",
      "why": "Monitors score each trace or thread as it arrives with a chosen judge model, preconditions and sampling; the judge sees formatted_trace with tool calls and results, and Instant Evals publish their token price.",
      "source": "https://langwatch.ai/docs/evaluations/online-evaluation/setup-monitors.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/langwatch"
    },
    {
      "slug": "arize-phoenix",
      "name": "Arize Phoenix",
      "company": "Arize AI, Inc.",
      "grade": "BB",
      "fit": "caveat",
      "why": "Evals are provider agnostic (OpenAI, Azure, Anthropic, Bedrock and Vertex via LiteLLM) and span annotations are logged by span_id, but server-side evaluators run only on dataset experiments; continuous scoring of incoming traces is your own loop.",
      "source": "https://arize.com/docs/phoenix/evaluation/how-to-evals/configuring-the-llm.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/arize-phoenix"
    },
    {
      "slug": "botgauge",
      "name": "BotGauge",
      "company": "BotGauge AI Inc. (Delaware, United States)",
      "grade": "E",
      "fit": "caveat",
      "why": "Judges run at span, trace or session scope with sampling and API filters; traces wait 2 to 30 minutes (default 10) after their last span, spans are judged at once, retries and quota pauses are shown. The judge model is BotGauge's, not the owner's choice.",
      "source": "https://www.botgauge.com/docs/evals/rules-and-settings.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/botgauge"
    },
    {
      "slug": "braintrust",
      "name": "Braintrust API + MCP",
      "company": "Braintrust Data, Inc.",
      "grade": "B",
      "fit": "caveat",
      "why": "Online scoring rules at trace or span scope with filters, sampling and a 30 second idle timeout; rule activity shows scorer LLM cost; judges run on any configured provider including Bedrock and custom endpoints. Duplicate AI SDK spans doubled scoring runs and cost (992).",
      "source": "https://docs.braintrust.dev/evaluate/score-online.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/braintrust"
    },
    {
      "slug": "galileo",
      "name": "Galileo API + MCP",
      "company": "Galileo Technologies, Inc.",
      "grade": "D",
      "fit": "caveat",
      "why": "Log stream metrics run automatically on incoming traces with sampling, and custom LLM-as-judge metrics target session, trace, LLM or tool spans on any configured integration, custom ones included. Judge cost visibility was not read.",
      "source": "https://docs.galileo.ai/concepts/logging/configure-metrics/configure-metrics.md",
      "confidence": "low",
      "url": "https://www.anchorterminal.com/tools/galileo"
    },
    {
      "slug": "honeyhive",
      "name": "HoneyHive",
      "company": "HoneyHive Inc.",
      "grade": "D",
      "fit": "caveat",
      "why": "Online evaluations run Python or LLM evaluators asynchronously on ingested traces matching event filters, with a sampling percentage, on workspace-configured providers. Judge cost is not shown and the provider list page was not read.",
      "source": "https://docs.honeyhive.ai/v2/monitoring/onlineevals.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/honeyhive"
    },
    {
      "slug": "laminar",
      "name": "Laminar API + MCP",
      "company": "LMNR AI, Inc.",
      "grade": "C",
      "fit": "caveat",
      "why": "Signals run a trace-analysis agent on each finished trace under triggers, filters and a sample rate, write events to SQL and alert Slack or email. The judge model is not the owner's choice in the docs read, and Signals need an enterprise licence when self-hosted.",
      "source": "https://laminar.sh/docs/signals/introduction.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/laminar"
    },
    {
      "slug": "langfuse",
      "name": "Langfuse API + MCP",
      "company": "Langfuse GmbH (a ClickHouse, Inc. company)",
      "grade": "BB",
      "fit": "caveat",
      "why": "Observation-level evaluators run on a worker with filters and the project's LLM connection (Azure, Bedrock, Vertex, OpenAI-compatible). Trace-level evaluators stop on 2026-11-16, a GLM judge failed on non-JSON output (13502) and Bedrock Claude on missing tool calls (9537).",
      "source": "https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/langfuse"
    },
    {
      "slug": "langsmith",
      "name": "LangSmith API + MCP",
      "company": "LangChain, Inc.",
      "grade": "BB",
      "fit": "caveat",
      "why": "Run-level and thread-level LLM-as-judge evaluators with filters, sampling, a weekly spend limit that pauses the evaluator, and a decision-model option. The provider list for the judge was not on the page read, and scored traces are auto-upgraded to paid extended retention.",
      "source": "https://docs.langchain.com/langsmith/online-evaluations-llm-as-judge.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/langsmith"
    },
    {
      "slug": "mlflow-tracing",
      "name": "MLflow Tracing",
      "company": "MLflow Project, a Series of LF Projects, LLC",
      "grade": "C",
      "fit": "caveat",
      "why": "Automatic evaluation runs registered judges on traces or sessions as they are logged, with a sample rate and filter string, but the judge must be an AI Gateway endpoint to run from the UI, only traces under an hour old are picked up, and judge cost is not shown.",
      "source": "https://mlflow.org/docs/latest/genai/eval-monitor/automatic-evaluations.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/mlflow-tracing"
    },
    {
      "slug": "opik",
      "name": "Opik API + MCP",
      "company": "Comet ML, Inc.",
      "grade": "BB",
      "fit": "caveat",
      "why": "Online rules score traces, threads or spans with filters, sampling, a USD cost limit and an agent-as-a-judge that reads the trace; providers include Bedrock, Azure, Vertex, Ollama and any OpenAI-compatible endpoint. A user could not find judge token usage (issue 4017).",
      "source": "https://www.comet.com/docs/opik/production/online-evaluation/rules.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/opik"
    },
    {
      "slug": "overmind",
      "name": "Overmind",
      "company": "Overmind Technology Ltd (England and Wales, company 16585996)",
      "grade": "C",
      "fit": "caveat",
      "why": "Live span scoring writes Span.eval_attributes with bounded attempts, judged by Jev, a decision model, with a generative judge_model only for explanations; the judge is not the owner's choice and no arrival-to-score time is given.",
      "source": "https://docs.overmindlab.ai/reference/evaluation.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/overmind"
    },
    {
      "slug": "prefactor",
      "name": "Prefactor",
      "company": "Prefactor Pty Ltd",
      "grade": "B",
      "fit": "caveat",
      "why": "Hosted judges score other agents' runs on models from LLM providers you connect, with a Preparing status and a pass or fail verdict on the Quality tab. Scope is the run, not a span, and no arrival-to-verdict time is given.",
      "source": "https://docs.prefactor.ai/llms-full.txt",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/prefactor"
    },
    {
      "slug": "pydantic-logfire",
      "name": "Pydantic Logfire",
      "company": "Pydantic Services Inc.",
      "grade": "B",
      "fit": "caveat",
      "why": "The FAQ describes live evaluations grading production traffic in the background from pydantic-evals, but that page was not read, so target level, judge model and cost visibility are unverified.",
      "source": "https://pydantic.dev/docs/logfire/get-started/faq/index.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/pydantic-logfire"
    },
    {
      "slug": "respan",
      "name": "Respan API + MCP",
      "company": "Keywords AI Inc. (d/b/a Respan)",
      "grade": "BB",
      "fit": "caveat",
      "why": "Behaviours score every new span with span-02, Respan's decision model, firing above 65 per cent confidence, with custom plain-language definitions on the Team plan. The judge is not a model of the owner's choosing and past spans are not rescored.",
      "source": "https://www.respan.ai/docs/documentation/features/pulse/behaviors.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/respan"
    },
    {
      "slug": "helicone",
      "name": "Helicone AI Gateway + MCP",
      "company": "Helicone, Inc.",
      "grade": "E",
      "fit": "unchecked",
      "why": "The docs index lists an Eval Scores page and score submission endpoints but no judge runner; those pages were not opened.",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/helicone"
    },
    {
      "slug": "wandb-weave",
      "name": "W\u0026B Weave",
      "company": "Weights and Biases, LLC",
      "grade": "B",
      "fit": "unchecked",
      "why": "The guardrails and monitors page describes inline guardrails that need code changes; a background monitor over incoming traces was not found in the pages read.",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/wandb-weave"
    },
    {
      "slug": "agent-tree",
      "name": "Agent Tree",
      "company": "No legal entity found. The LICENSE, plugin.json and marketplace.json name \"Mike Pollow, Digitalroots\"",
      "grade": "C",
      "fit": "no",
      "why": "No judge.",
      "source": "https://raw.githubusercontent.com/MikeGT4/agent-tree/main/docs/GUIDE.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/agent-tree"
    },
    {
      "slug": "agents-tree",
      "name": "agents-tree",
      "company": "No legal entity found. The LICENSE and pyproject.toml name Karel Blavka",
      "grade": "B",
      "fit": "no",
      "why": "No judge.",
      "source": "https://raw.githubusercontent.com/blavka/agents-tree/main/README.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/agents-tree"
    },
    {
      "slug": "deepeval",
      "name": "DeepEval",
      "company": "Confident AI, Inc.",
      "grade": "B",
      "fit": "no",
      "why": "Production scoring is Confident AI's job; the framework runs offline.",
      "source": "https://deepeval.com/docs/evaluation-llm-tracing",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/deepeval"
    },
    {
      "slug": "gauge-cli",
      "name": "Gauge CLI",
      "company": "Gauge",
      "grade": "B",
      "fit": "no",
      "why": "No production trace ingestion.",
      "source": "https://docs.withgauge.com/agents/index.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/gauge-cli"
    },
    {
      "slug": "navige",
      "name": "Navige",
      "company": "Navige Ltd",
      "grade": "E",
      "fit": "no",
      "why": "Outcomes are reported by you or judged by hand; no LLM judge.",
      "source": "https://app.navige.ai/docs/concepts/transactions",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/navige"
    }
  ],
  "marks": "yes: the docs show it does the job as the success test says. caveat: it does the job with a limit an agent should know. no: it doesn't do this job. unchecked: we couldn't read it.",
  "meta": {
    "attribution": "Anchor Terminal (https://www.anchorterminal.com)",
    "attributionRequired": true,
    "credit": "CC BY 4.0, credit Anchor Terminal and link the page or API address you took it from (https://www.anchorterminal.com/terms/#data)",
    "docs": "https://www.anchorterminal.com/docs/",
    "generatedAt": "2026-11-16",
    "license": "CC-BY-4.0",
    "method": "https://www.anchorterminal.com/benchmark/",
    "methodology": "0.8",
    "openapi": "https://www.anchorterminal.com/openapi.json",
    "preview": false,
    "run": "2026-10-01",
    "runLabel": "October 2026 research run",
    "terms": "https://www.anchorterminal.com/terms/#data"
  },
  "useCase": {
    "accounts": 9,
    "after": [
      "agent-observability.offline-eval-regression",
      "notifications.alert"
    ],
    "area": "developer",
    "asks": [
      "flag bad answers in production",
      "run a judge on every trace",
      "score just the retrieval span",
      "alert me when quality drops"
    ],
    "before": [
      "agent-observability.instrument-app"
    ],
    "blockers": [
      "judge requires a tool-calling or strict-JSON model",
      "span-level evaluation not offered or broken on self-host",
      "judge cost invisible",
      "evaluator silently never fires"
    ],
    "capabilities": [
      "obs.evals",
      "obs.traces"
    ],
    "category": "agent-observability",
    "categoryName": "Agent observability \u0026 evals",
    "checked": "2026-11-12",
    "companies": {
      "fit": 1,
      "fitWithLimit": 14,
      "tagged": 24
    },
    "competition": "one",
    "fits": [
      "langwatch"
    ],
    "fitsWithLimit": [
      "arize-phoenix",
      "botgauge",
      "braintrust",
      "galileo",
      "honeyhive",
      "laminar",
      "langfuse",
      "langsmith",
      "mlflow-tracing",
      "opik",
      "overmind",
      "prefactor",
      "pydantic-logfire",
      "respan"
    ],
    "id": "agent-observability.online-llm-judge",
    "listings": {
      "caveat": 14,
      "no": 5,
      "notRead": 0,
      "tagged": 24,
      "unchecked": 2,
      "yes": 1
    },
    "name": "Score production traces automatically with an LLM judge",
    "status": "draft",
    "success": "Each new trace or chosen span gets a score, the judge runs on the configured model, the judge's own token cost is visible, and scores can be filtered and alerted on. Measured: each score arrives within 10 minutes of the trace or span.",
    "url": "https://www.anchorterminal.com/api/v1/use-cases/agent-observability.online-llm-judge.json",
    "weight": 9,
    "weightBasis": "9 of 86 accounts: Langfuse issues on judge model configuration (GLM, Bedrock), span-level evaluators failing or not triggering, Phoenix multi-hop span evals, Opik judge usage and nested online rules, Braintrust duplicate scoring"
  }
}
