{
  "listings": [
    {
      "slug": "langwatch",
      "name": "LangWatch",
      "company": "Reasoning Engine B.V. (doing business as LangWatch)",
      "grade": "B",
      "fit": "yes",
      "why": "Scenario runs a simulated user against the agent in pytest or vitest and a judge passes or fails each criterion with reasoning; suite evaluators read tool_calls.\u003ctool\u003e.input and output or every span, and a required evaluator fails the run.",
      "source": "https://langwatch.ai/docs/agent-testing/simulated-users.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/langwatch"
    },
    {
      "slug": "opik",
      "name": "Opik API + MCP",
      "company": "Comet ML, Inc.",
      "grade": "BB",
      "fit": "yes",
      "why": "TrajectoryAccuracy scores a ReAct trajectory with a reason, the agent trajectory guide builds tool-call test datasets, and multi-turn evaluation uses an LLM user simulator; issue 2529 asked for precisely this metric.",
      "source": "https://www.comet.com/docs/opik/evaluation/metrics/trajectory_accuracy.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/opik"
    },
    {
      "slug": "arize-phoenix",
      "name": "Arize Phoenix",
      "company": "Arize AI, Inc.",
      "grade": "BB",
      "fit": "caveat",
      "why": "Tool Selection and Tool Invocation evaluators score LLM spans and flag hallucinated or unnecessary tools; ATIF trajectories can be imported. No user simulator and no whole-run divergence score.",
      "source": "https://arize.com/docs/phoenix/evaluation/pre-built-metrics/tool-selection.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/arize-phoenix"
    },
    {
      "slug": "braintrust",
      "name": "Braintrust API + MCP",
      "company": "Braintrust Data, Inc.",
      "grade": "B",
      "fit": "caveat",
      "why": "Trace-scoped scorers get trace.getSpans() and {{thread}} so a judge can read tool calls, and the agents guide covers tool choice and arguments. No built-in trajectory metric and no user simulator.",
      "source": "https://docs.braintrust.dev/evaluate/score-online.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/braintrust"
    },
    {
      "slug": "deepeval",
      "name": "DeepEval",
      "company": "Confident AI, Inc.",
      "grade": "B",
      "fit": "caveat",
      "why": "Trajectory-based evals score the ordered chain of tool calls and handoffs, ToolCorrectness can require ordering and arguments, component metrics attach to spans and ConversationSimulator plays a user. ToolCorrectness scored 20 per cent when no tool was called (dev.to) and no divergence metric exists (issue 3019).",
      "source": "https://deepeval.com/docs/evaluation-trajectory-based-llm-evals",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/deepeval"
    },
    {
      "slug": "galileo",
      "name": "Galileo API + MCP",
      "company": "Galileo Technologies, Inc.",
      "grade": "D",
      "fit": "caveat",
      "why": "Agent Flow validates a session's trajectory against natural-language tests including expected tool or agent calls, and Tool Selection Quality judges tool and argument choice per span. No user simulator.",
      "source": "https://docs.galileo.ai/concepts/metrics/agentic/agent-flow.md",
      "confidence": "low",
      "url": "https://www.anchorterminal.com/tools/galileo"
    },
    {
      "slug": "gauge-cli",
      "name": "Gauge CLI",
      "company": "Gauge",
      "grade": "B",
      "fit": "caveat",
      "why": "Agent Experience judges whether a real coding agent completed a task with your product, with pass criteria, session evidence and recurring-failure grouping. The agent under test is the harness using your product, not your own agent.",
      "source": "https://docs.withgauge.com/agents/concepts/agent-experience.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/gauge-cli"
    },
    {
      "slug": "honeyhive",
      "name": "HoneyHive",
      "company": "HoneyHive Inc.",
      "grade": "D",
      "fit": "caveat",
      "why": "The Trajectory view plots spans by step to spot loops and stuck steps, and conversation evaluator templates score multi-turn sessions. No tool-choice metric or user simulator in the pages read.",
      "source": "https://docs.honeyhive.ai/v2/tracing/trajectory-view.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/honeyhive"
    },
    {
      "slug": "laminar",
      "name": "Laminar API + MCP",
      "company": "LMNR AI, Inc.",
      "grade": "C",
      "fit": "caveat",
      "why": "Agent evaluations run the real app per trial and a verifier scores the state, trajectory and artefacts, with the trajectory attached as LLM and TOOL spans or imported in ATIF; a tool-call evaluation guide exists. No user simulator.",
      "source": "https://laminar.sh/docs/agent-evaluations/introduction.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/laminar"
    },
    {
      "slug": "langsmith",
      "name": "LangSmith API + MCP",
      "company": "LangChain, Inc.",
      "grade": "BB",
      "fit": "caveat",
      "why": "Multi-turn online evaluators measure the trajectory of tool calls across a thread, and datasets can hold expected trajectories. No user simulator and no named divergence metric in the pages read.",
      "source": "https://docs.langchain.com/langsmith/online-evaluations-multi-turn.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/langsmith"
    },
    {
      "slug": "mlflow-tracing",
      "name": "MLflow Tracing",
      "company": "MLflow Project, a Series of LF Projects, LLC",
      "grade": "C",
      "fit": "caveat",
      "why": "ToolCallCorrectness and ToolCallEfficiency judge TOOL spans in a trace, scorers can parse the whole trace for trajectory and routing, and ConversationSimulator (experimental, 3.10) plays a user. No divergence-between-runs metric.",
      "source": "https://mlflow.org/docs/latest/genai/eval-monitor/scorers/llm-judge/tool-call.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/mlflow-tracing"
    },
    {
      "slug": "overmind",
      "name": "Overmind",
      "company": "Overmind Technology Ltd (England and Wales, company 16585996)",
      "grade": "C",
      "fit": "caveat",
      "why": "Stock checks cover tool selection, tool arguments and goal state, and live trajectory and plan-drift scoring is mentioned with bounded attempts. No user simulator.",
      "source": "https://docs.overmindlab.ai/reference/evaluation.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/overmind"
    },
    {
      "slug": "pydantic-logfire",
      "name": "Pydantic Logfire",
      "company": "Pydantic Services Inc.",
      "grade": "B",
      "fit": "unchecked",
      "why": "pydantic-evals may score trajectories, but its docs were not opened and the Logfire pages read show no trajectory metric or user simulator.",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/pydantic-logfire"
    },
    {
      "slug": "agent-tree",
      "name": "Agent Tree",
      "company": "No legal entity found. The LICENSE, plugin.json and marketplace.json name \"Mike Pollow, Digitalroots\"",
      "grade": "C",
      "fit": "no",
      "why": "No scoring.",
      "source": "https://raw.githubusercontent.com/MikeGT4/agent-tree/main/docs/GUIDE.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/agent-tree"
    },
    {
      "slug": "agents-tree",
      "name": "agents-tree",
      "company": "No legal entity found. The LICENSE and pyproject.toml name Karel Blavka",
      "grade": "B",
      "fit": "no",
      "why": "No scoring.",
      "source": "https://raw.githubusercontent.com/blavka/agents-tree/main/README.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/agents-tree"
    },
    {
      "slug": "botgauge",
      "name": "BotGauge",
      "company": "BotGauge AI Inc. (Delaware, United States)",
      "grade": "E",
      "fit": "no",
      "why": "Built-in judges score spans, traces or sessions against criteria; no trajectory metric or simulator in the pages read.",
      "source": "https://www.botgauge.com/docs/evals/built-in-judges.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/botgauge"
    },
    {
      "slug": "helicone",
      "name": "Helicone AI Gateway + MCP",
      "company": "Helicone, Inc.",
      "grade": "E",
      "fit": "no",
      "why": "No trajectory metric, tool-call judge or simulator in the docs index.",
      "source": "https://docs.helicone.ai/llms.txt",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/helicone"
    },
    {
      "slug": "langfuse",
      "name": "Langfuse API + MCP",
      "company": "Langfuse GmbH (a ClickHouse, Inc. company)",
      "grade": "BB",
      "fit": "no",
      "why": "Evaluators see one observation and do not load sibling or child observations; multi-span evaluation is promised for the new data model. No trajectory metric or user simulator in the docs.",
      "source": "https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge.md",
      "confidence": "high",
      "url": "https://www.anchorterminal.com/tools/langfuse"
    },
    {
      "slug": "navige",
      "name": "Navige",
      "company": "Navige Ltd",
      "grade": "E",
      "fit": "no",
      "why": "No trajectory scoring.",
      "source": "https://app.navige.ai/docs/",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/navige"
    },
    {
      "slug": "prefactor",
      "name": "Prefactor",
      "company": "Prefactor Pty Ltd",
      "grade": "B",
      "fit": "no",
      "why": "No trajectory scoring or simulator.",
      "source": "https://docs.prefactor.ai/llms-full.txt",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/prefactor"
    },
    {
      "slug": "respan",
      "name": "Respan API + MCP",
      "company": "Keywords AI Inc. (d/b/a Respan)",
      "grade": "BB",
      "fit": "no",
      "why": "No trajectory metric, tool-choice judge or simulator in the docs index.",
      "source": "https://www.respan.ai/docs/llms.txt",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/respan"
    },
    {
      "slug": "wandb-weave",
      "name": "W\u0026B Weave",
      "company": "Weights and Biases, LLC",
      "grade": "B",
      "fit": "no",
      "why": "Scorers are custom functions; no trajectory metric, tool-choice judge or user simulator appears in the pages read.",
      "source": "https://docs.coreweave.com/products/wandb/weave/guides/evaluation/scorers.md",
      "confidence": "medium",
      "url": "https://www.anchorterminal.com/tools/wandb-weave"
    }
  ],
  "marks": "yes: the docs show it does the job as the success test says. caveat: it does the job with a limit an agent should know. no: it doesn't do this job. unchecked: we couldn't read it.",
  "meta": {
    "attribution": "Anchor Terminal (https://www.anchorterminal.com)",
    "attributionRequired": true,
    "credit": "CC BY 4.0, credit Anchor Terminal and link the page or API address you took it from (https://www.anchorterminal.com/terms/#data)",
    "docs": "https://www.anchorterminal.com/docs/",
    "generatedAt": "2026-11-16",
    "license": "CC-BY-4.0",
    "method": "https://www.anchorterminal.com/benchmark/",
    "methodology": "0.8",
    "openapi": "https://www.anchorterminal.com/openapi.json",
    "preview": false,
    "run": "2026-10-01",
    "runLabel": "October 2026 research run",
    "terms": "https://www.anchorterminal.com/terms/#data"
  },
  "useCase": {
    "accounts": 6,
    "after": [
      "agent-observability.offline-eval-regression"
    ],
    "area": "developer",
    "asks": [
      "did it call the right tool",
      "score the whole session not each turn",
      "simulate a user and see if it books the flight",
      "where did these two runs diverge"
    ],
    "before": [
      "agent-observability.debug-failed-run"
    ],
    "blockers": [
      "metrics score one run against a static expectation only",
      "no simulation of a user",
      "tool calls not in the trace so nothing to judge"
    ],
    "capabilities": [
      "obs.evals",
      "obs.simulation",
      "obs.datasets"
    ],
    "category": "agent-observability",
    "categoryName": "Agent observability \u0026 evals",
    "checked": "2026-11-12",
    "companies": {
      "fit": 2,
      "fitWithLimit": 10,
      "tagged": 20
    },
    "competition": "few",
    "fits": [
      "langwatch",
      "opik"
    ],
    "fitsWithLimit": [
      "arize-phoenix",
      "braintrust",
      "deepeval",
      "galileo",
      "gauge-cli",
      "honeyhive",
      "laminar",
      "langsmith",
      "mlflow-tracing",
      "overmind"
    ],
    "id": "agent-observability.agent-trajectory-eval",
    "listings": {
      "caveat": 10,
      "no": 9,
      "notRead": 0,
      "tagged": 20,
      "unchecked": 1,
      "yes": 2
    },
    "name": "Judge whether the agent took the right steps, not only gave the right answer",
    "status": "draft",
    "success": "A score per trajectory that names the wrong or missing tool call or the turn where the run went off, reproducible on the same trace, with a clear result when the agent answered without calling any tool.",
    "url": "https://www.anchorterminal.com/api/v1/use-cases/agent-observability.agent-trajectory-eval.json",
    "weight": 5,
    "weightBasis": "6 of 86 accounts: the DeepEval dev.to post (ToolCorrectness at 20 per cent), the Opik trajectory metric request, the DeepEval divergence proposal, the Cekura thread's wrong-tool support agent, the Lucidic thread on session-level evals, Phoenix multi-hop evals; LangWatch Scenario appears only in its staff's posts"
  }
}
