{
  "data": {
    "latest": [
      {
        "byLevel": [
          18,
          27,
          12
        ],
        "correct": 57,
        "ended": "2026-11-20T12:27:43Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-8-flash-next.json",
        "mode": "either",
        "model": "qwen3.8-flash-next",
        "modelMinutes": 63.5,
        "name": "Qwen3.8-Flash-Next",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 18.6,
        "passes": 3,
        "paths": {
          "both": 77,
          "native": 100
        },
        "pct": 96.6,
        "points": 98,
        "rank": 1,
        "scored": 59,
        "secPerTask": 70.988,
        "slug": "qwen3-8-flash-next",
        "steady": 55,
        "tokens": 3923165
      },
      {
        "byLevel": [
          19,
          26,
          10
        ],
        "correct": 55,
        "ended": "2026-11-19T22:28:59Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/gemma-4-26b-a4b-it.json",
        "mode": "either",
        "model": "gemma-4-26b-a4b",
        "modelMinutes": 84.23,
        "name": "Gemma 4 26B A4B IT",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 46.5,
        "passes": 3,
        "paths": {
          "both": 74,
          "native": 99,
          "none": 3,
          "program": 1
        },
        "pct": 93.2,
        "points": 92,
        "rank": 2,
        "scored": 59,
        "secPerTask": 91.354,
        "slug": "gemma-4-26b-a4b-it",
        "steady": 47,
        "tokens": 4658656
      },
      {
        "byLevel": [
          18,
          24,
          10
        ],
        "correct": 52,
        "ended": "2026-11-19T17:59:28Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/gemma-4-31b-it.json",
        "mode": "either",
        "model": "gemma-4-31b",
        "modelMinutes": 307.71,
        "name": "Gemma 4 31B IT",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 9.1,
        "passes": 3,
        "paths": {
          "both": 72,
          "native": 91,
          "none": 14
        },
        "pct": 88.1,
        "points": 87,
        "rank": 3,
        "scored": 59,
        "secPerTask": 317.387,
        "slug": "gemma-4-31b-it",
        "steady": 39,
        "tokens": 3818650
      },
      {
        "byLevel": [
          20,
          25,
          5
        ],
        "correct": 50,
        "ended": "2026-11-20T06:15:13Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-6-35b-a3b.json",
        "mode": "either",
        "model": "qwen3.6-35b-a3b",
        "modelMinutes": 51.32,
        "name": "Qwen3.6-35B-A3B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 33.8,
        "passes": 3,
        "paths": {
          "both": 44,
          "native": 131,
          "none": 2
        },
        "pct": 84.7,
        "points": 77,
        "rank": 4,
        "scored": 59,
        "secPerTask": 56.402,
        "slug": "qwen3-6-35b-a3b",
        "steady": 42,
        "tokens": 5522692
      },
      {
        "byLevel": [
          19,
          20,
          5
        ],
        "correct": 44,
        "ended": "2026-11-21T21:03:55Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/glm-4-5-air.json",
        "mode": "either",
        "model": "glm-4.5-air",
        "modelMinutes": 132.12,
        "name": "GLM-4.5-Air",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 9.4,
        "passes": 3,
        "paths": {
          "both": 32,
          "native": 143,
          "none": 2
        },
        "pct": 74.6,
        "points": 67,
        "rank": 5,
        "scored": 59,
        "secPerTask": 137.653,
        "slug": "glm-4-5-air",
        "steady": 36,
        "tokens": 6893322
      },
      {
        "byLevel": [
          18,
          20,
          5
        ],
        "correct": 43,
        "ended": "2026-11-20T08:58:17Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/gpt-oss-120b.json",
        "mode": "either",
        "model": "gpt-oss-120b",
        "modelMinutes": 51.07,
        "name": "gpt-oss-120b",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 29.9,
        "passes": 3,
        "paths": {
          "both": 52,
          "native": 125
        },
        "pct": 72.9,
        "points": 66,
        "rank": 6,
        "scored": 59,
        "secPerTask": 55.268,
        "slug": "gpt-oss-120b",
        "steady": 33,
        "tokens": 5314464
      },
      {
        "byLevel": [
          18,
          23,
          3
        ],
        "correct": 44,
        "ended": "2026-11-21T08:34:05Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-5-9b.json",
        "mode": "either",
        "model": "qwen3.5-9b",
        "modelMinutes": 114.97,
        "name": "Qwen3.5-9B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 17.2,
        "passes": 3,
        "paths": {
          "both": 50,
          "native": 127
        },
        "pct": 74.6,
        "points": 66,
        "rank": 7,
        "scored": 59,
        "secPerTask": 121.887,
        "slug": "qwen3-5-9b",
        "steady": 34,
        "tokens": 8377285
      },
      {
        "byLevel": [
          18,
          18,
          5
        ],
        "correct": 41,
        "ended": "2026-11-21T02:34:30Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/nemotron-3-super-120b-a12b.json",
        "mode": "either",
        "model": "nemotron-3-super-120b-a12b",
        "modelMinutes": 275.02,
        "name": "Nemotron 3 Super 120B A12B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 15,
        "passes": 3,
        "paths": {
          "both": 71,
          "native": 105,
          "none": 1
        },
        "pct": 69.5,
        "points": 63,
        "rank": 8,
        "scored": 59,
        "secPerTask": 287.034,
        "slug": "nemotron-3-super-120b-a12b",
        "steady": 34,
        "tokens": 8259135
      },
      {
        "byLevel": [
          18,
          19,
          4
        ],
        "correct": 41,
        "ended": "2026-11-21T11:57:36Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-5-4b.json",
        "mode": "either",
        "model": "qwen3.5-4b",
        "modelMinutes": 64.69,
        "name": "Qwen3.5-4B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 26.3,
        "passes": 3,
        "paths": {
          "both": 30,
          "native": 147
        },
        "pct": 69.5,
        "points": 62,
        "rank": 9,
        "scored": 59,
        "secPerTask": 68.981,
        "slug": "qwen3-5-4b",
        "steady": 31,
        "tokens": 8250719
      },
      {
        "byLevel": [
          17,
          19,
          4
        ],
        "correct": 40,
        "ended": "2026-11-21T14:17:48Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/gpt-oss-20b.json",
        "mode": "either",
        "model": "gpt-oss-20b",
        "modelMinutes": 42.95,
        "name": "gpt-oss-20b",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 45,
        "passes": 3,
        "paths": {
          "both": 54,
          "native": 123
        },
        "pct": 67.8,
        "points": 61,
        "rank": 10,
        "scored": 59,
        "secPerTask": 47.518,
        "slug": "gpt-oss-20b",
        "steady": 33,
        "tokens": 5709798
      },
      {
        "byLevel": [
          18,
          20,
          3
        ],
        "correct": 41,
        "ended": "2026-11-22T09:07:23Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-coder-next.json",
        "mode": "either",
        "model": "qwen3-coder-next",
        "modelMinutes": 46.07,
        "name": "Qwen3-Coder-Next",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 32.7,
        "passes": 3,
        "paths": {
          "both": 80,
          "native": 95,
          "none": 2
        },
        "pct": 69.5,
        "points": 61,
        "rank": 11,
        "scored": 59,
        "secPerTask": 53.379,
        "slug": "qwen3-coder-next",
        "steady": 23,
        "tokens": 7444410
      },
      {
        "byLevel": [
          14,
          21,
          3
        ],
        "correct": 38,
        "ended": "2026-11-22T15:36:41Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-coder-30b-a3b-instruct.json",
        "mode": "either",
        "model": "qwen3-coder-30b-a3b",
        "modelMinutes": 60.04,
        "name": "Qwen3-Coder-30B-A3B-Instruct",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 34.9,
        "passes": 3,
        "paths": {
          "both": 61,
          "native": 116
        },
        "pct": 64.4,
        "points": 59,
        "rank": 12,
        "scored": 59,
        "secPerTask": 64.063,
        "slug": "qwen3-coder-30b-a3b-instruct",
        "steady": 32,
        "tokens": 5188972
      },
      {
        "byLevel": [
          18,
          19,
          3
        ],
        "correct": 40,
        "ended": "2026-11-22T06:29:54Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/nemotron-3-nano-30b-a3b.json",
        "mode": "either",
        "model": "nemotron-3-nano-30b-a3b",
        "modelMinutes": 183.41,
        "name": "Nemotron 3 Nano 30B A3B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 48.5,
        "passes": 3,
        "paths": {
          "both": 53,
          "native": 121,
          "none": 3
        },
        "pct": 67.8,
        "points": 59,
        "rank": 13,
        "scored": 59,
        "secPerTask": 191.853,
        "slug": "nemotron-3-nano-30b-a3b",
        "steady": 34,
        "tokens": 6493700
      },
      {
        "byLevel": [
          19,
          16,
          3
        ],
        "correct": 38,
        "ended": "2026-11-20T03:28:49Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/glm-4-7-flash.json",
        "mode": "either",
        "model": "glm-4.7-flash",
        "modelMinutes": 93.36,
        "name": "GLM-4.7-Flash",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 14.4,
        "passes": 3,
        "paths": {
          "both": 44,
          "native": 132,
          "program": 1
        },
        "pct": 64.4,
        "points": 55,
        "rank": 14,
        "scored": 59,
        "secPerTask": 101.626,
        "slug": "glm-4-7-flash",
        "steady": 24,
        "tokens": 4845277
      },
      {
        "byLevel": [
          14,
          13,
          4
        ],
        "correct": 31,
        "ended": "2026-11-22T12:27:40Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-30b-a3b-instruct-2507.json",
        "mode": "either",
        "model": "qwen3-30b-a3b-instruct-2507",
        "modelMinutes": 59.8,
        "name": "Qwen3-30B-A3B-Instruct-2507",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 31.7,
        "passes": 3,
        "paths": {
          "both": 90,
          "native": 86,
          "none": 1
        },
        "pct": 52.5,
        "points": 47,
        "rank": 15,
        "scored": 59,
        "secPerTask": 67.88,
        "slug": "qwen3-30b-a3b-instruct-2507",
        "steady": 19,
        "tokens": 8496823
      }
    ],
    "method": "Questions with one right answer about Anchor Terminal, answered through its MCP server, pages and API, three passes each, the model calling the tools itself or writing programs that call them, as it chooses (mode either). The Lab score is the share right with a level 2 question counting twice and a level 3 question three times, from either-mode runs only; a code-mode run (programs only) is listed and scores nothing.",
    "page": "https://www.anchorterminal.com/benchmark/lab/evaluation/models/",
    "questions": "https://github.com/Anchor-Terminal/anchorlab/blob/main/eval/tasks.go",
    "runs": [
      {
        "byLevel": [
          14,
          21,
          3
        ],
        "correct": 38,
        "ended": "2026-11-22T15:36:41Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-coder-30b-a3b-instruct.json",
        "mode": "either",
        "model": "qwen3-coder-30b-a3b",
        "modelMinutes": 60.04,
        "name": "Qwen3-Coder-30B-A3B-Instruct",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 34.9,
        "passes": 3,
        "paths": {
          "both": 61,
          "native": 116
        },
        "pct": 64.4,
        "points": 59,
        "scored": 59,
        "secPerTask": 64.063,
        "slug": "qwen3-coder-30b-a3b-instruct",
        "steady": 32,
        "tokens": 5188972
      },
      {
        "byLevel": [
          14,
          13,
          4
        ],
        "correct": 31,
        "ended": "2026-11-22T12:27:40Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-30b-a3b-instruct-2507.json",
        "mode": "either",
        "model": "qwen3-30b-a3b-instruct-2507",
        "modelMinutes": 59.8,
        "name": "Qwen3-30B-A3B-Instruct-2507",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 31.7,
        "passes": 3,
        "paths": {
          "both": 90,
          "native": 86,
          "none": 1
        },
        "pct": 52.5,
        "points": 47,
        "scored": 59,
        "secPerTask": 67.88,
        "slug": "qwen3-30b-a3b-instruct-2507",
        "steady": 19,
        "tokens": 8496823
      },
      {
        "byLevel": [
          18,
          20,
          3
        ],
        "correct": 41,
        "ended": "2026-11-22T09:07:23Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-coder-next.json",
        "mode": "either",
        "model": "qwen3-coder-next",
        "modelMinutes": 46.07,
        "name": "Qwen3-Coder-Next",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 32.7,
        "passes": 3,
        "paths": {
          "both": 80,
          "native": 95,
          "none": 2
        },
        "pct": 69.5,
        "points": 61,
        "scored": 59,
        "secPerTask": 53.379,
        "slug": "qwen3-coder-next",
        "steady": 23,
        "tokens": 7444410
      },
      {
        "byLevel": [
          18,
          19,
          3
        ],
        "correct": 40,
        "ended": "2026-11-22T06:29:54Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/nemotron-3-nano-30b-a3b.json",
        "mode": "either",
        "model": "nemotron-3-nano-30b-a3b",
        "modelMinutes": 183.41,
        "name": "Nemotron 3 Nano 30B A3B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 48.5,
        "passes": 3,
        "paths": {
          "both": 53,
          "native": 121,
          "none": 3
        },
        "pct": 67.8,
        "points": 59,
        "scored": 59,
        "secPerTask": 191.853,
        "slug": "nemotron-3-nano-30b-a3b",
        "steady": 34,
        "tokens": 6493700
      },
      {
        "byLevel": [
          19,
          20,
          5
        ],
        "correct": 44,
        "ended": "2026-11-21T21:03:55Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/glm-4-5-air.json",
        "mode": "either",
        "model": "glm-4.5-air",
        "modelMinutes": 132.12,
        "name": "GLM-4.5-Air",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 9.4,
        "passes": 3,
        "paths": {
          "both": 32,
          "native": 143,
          "none": 2
        },
        "pct": 74.6,
        "points": 67,
        "scored": 59,
        "secPerTask": 137.653,
        "slug": "glm-4-5-air",
        "steady": 36,
        "tokens": 6893322
      },
      {
        "byLevel": [
          17,
          19,
          4
        ],
        "correct": 40,
        "ended": "2026-11-21T14:17:48Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/gpt-oss-20b.json",
        "mode": "either",
        "model": "gpt-oss-20b",
        "modelMinutes": 42.95,
        "name": "gpt-oss-20b",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 45,
        "passes": 3,
        "paths": {
          "both": 54,
          "native": 123
        },
        "pct": 67.8,
        "points": 61,
        "scored": 59,
        "secPerTask": 47.518,
        "slug": "gpt-oss-20b",
        "steady": 33,
        "tokens": 5709798
      },
      {
        "byLevel": [
          18,
          19,
          4
        ],
        "correct": 41,
        "ended": "2026-11-21T11:57:36Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-5-4b.json",
        "mode": "either",
        "model": "qwen3.5-4b",
        "modelMinutes": 64.69,
        "name": "Qwen3.5-4B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 26.3,
        "passes": 3,
        "paths": {
          "both": 30,
          "native": 147
        },
        "pct": 69.5,
        "points": 62,
        "scored": 59,
        "secPerTask": 68.981,
        "slug": "qwen3-5-4b",
        "steady": 31,
        "tokens": 8250719
      },
      {
        "byLevel": [
          18,
          23,
          3
        ],
        "correct": 44,
        "ended": "2026-11-21T08:34:05Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-5-9b.json",
        "mode": "either",
        "model": "qwen3.5-9b",
        "modelMinutes": 114.97,
        "name": "Qwen3.5-9B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 17.2,
        "passes": 3,
        "paths": {
          "both": 50,
          "native": 127
        },
        "pct": 74.6,
        "points": 66,
        "scored": 59,
        "secPerTask": 121.887,
        "slug": "qwen3-5-9b",
        "steady": 34,
        "tokens": 8377285
      },
      {
        "byLevel": [
          18,
          18,
          5
        ],
        "correct": 41,
        "ended": "2026-11-21T02:34:30Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/nemotron-3-super-120b-a12b.json",
        "mode": "either",
        "model": "nemotron-3-super-120b-a12b",
        "modelMinutes": 275.02,
        "name": "Nemotron 3 Super 120B A12B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 15,
        "passes": 3,
        "paths": {
          "both": 71,
          "native": 105,
          "none": 1
        },
        "pct": 69.5,
        "points": 63,
        "scored": 59,
        "secPerTask": 287.034,
        "slug": "nemotron-3-super-120b-a12b",
        "steady": 34,
        "tokens": 8259135
      },
      {
        "byLevel": [
          18,
          27,
          12
        ],
        "correct": 57,
        "ended": "2026-11-20T12:27:43Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-8-flash-next.json",
        "mode": "either",
        "model": "qwen3.8-flash-next",
        "modelMinutes": 63.5,
        "name": "Qwen3.8-Flash-Next",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 18.6,
        "passes": 3,
        "paths": {
          "both": 77,
          "native": 100
        },
        "pct": 96.6,
        "points": 98,
        "scored": 59,
        "secPerTask": 70.988,
        "slug": "qwen3-8-flash-next",
        "steady": 55,
        "tokens": 3923165
      },
      {
        "byLevel": [
          18,
          20,
          5
        ],
        "correct": 43,
        "ended": "2026-11-20T08:58:17Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/gpt-oss-120b.json",
        "mode": "either",
        "model": "gpt-oss-120b",
        "modelMinutes": 51.07,
        "name": "gpt-oss-120b",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 29.9,
        "passes": 3,
        "paths": {
          "both": 52,
          "native": 125
        },
        "pct": 72.9,
        "points": 66,
        "scored": 59,
        "secPerTask": 55.268,
        "slug": "gpt-oss-120b",
        "steady": 33,
        "tokens": 5314464
      },
      {
        "byLevel": [
          20,
          25,
          5
        ],
        "correct": 50,
        "ended": "2026-11-20T06:15:13Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-6-35b-a3b.json",
        "mode": "either",
        "model": "qwen3.6-35b-a3b",
        "modelMinutes": 51.32,
        "name": "Qwen3.6-35B-A3B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 33.8,
        "passes": 3,
        "paths": {
          "both": 44,
          "native": 131,
          "none": 2
        },
        "pct": 84.7,
        "points": 77,
        "scored": 59,
        "secPerTask": 56.402,
        "slug": "qwen3-6-35b-a3b",
        "steady": 42,
        "tokens": 5522692
      },
      {
        "byLevel": [
          19,
          16,
          3
        ],
        "correct": 38,
        "ended": "2026-11-20T03:28:49Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/glm-4-7-flash.json",
        "mode": "either",
        "model": "glm-4.7-flash",
        "modelMinutes": 93.36,
        "name": "GLM-4.7-Flash",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 14.4,
        "passes": 3,
        "paths": {
          "both": 44,
          "native": 132,
          "program": 1
        },
        "pct": 64.4,
        "points": 55,
        "scored": 59,
        "secPerTask": 101.626,
        "slug": "glm-4-7-flash",
        "steady": 24,
        "tokens": 4845277
      },
      {
        "byLevel": [
          19,
          26,
          10
        ],
        "correct": 55,
        "ended": "2026-11-19T22:28:59Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/gemma-4-26b-a4b-it.json",
        "mode": "either",
        "model": "gemma-4-26b-a4b",
        "modelMinutes": 84.23,
        "name": "Gemma 4 26B A4B IT",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 46.5,
        "passes": 3,
        "paths": {
          "both": 74,
          "native": 99,
          "none": 3,
          "program": 1
        },
        "pct": 93.2,
        "points": 92,
        "scored": 59,
        "secPerTask": 91.354,
        "slug": "gemma-4-26b-a4b-it",
        "steady": 47,
        "tokens": 4658656
      },
      {
        "byLevel": [
          18,
          24,
          10
        ],
        "correct": 52,
        "ended": "2026-11-19T17:59:28Z",
        "forbidden": 0,
        "harness": "2026.11.18-2308",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/gemma-4-31b-it.json",
        "mode": "either",
        "model": "gemma-4-31b",
        "modelMinutes": 307.71,
        "name": "Gemma 4 31B IT",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 9.1,
        "passes": 3,
        "paths": {
          "both": 72,
          "native": 91,
          "none": 14
        },
        "pct": 88.1,
        "points": 87,
        "scored": 59,
        "secPerTask": 317.387,
        "slug": "gemma-4-31b-it",
        "steady": 39,
        "tokens": 3818650
      },
      {
        "byLevel": [
          20,
          24,
          10
        ],
        "correct": 54,
        "ended": "2026-11-18T15:10:45Z",
        "forbidden": 0,
        "harness": "2026.11.18-0557",
        "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/qwen3-8-27b.json",
        "mode": "code",
        "model": "qwen3.8-27b",
        "modelMinutes": 175.5,
        "name": "Qwen3.8-27B",
        "ofLevel": [
          20,
          27,
          12
        ],
        "outputTps": 7.1,
        "passes": 3,
        "pct": 91.5,
        "points": 89,
        "scored": 59,
        "secPerTask": 187.523,
        "slug": "qwen3-8-27b",
        "steady": 45,
        "tokens": 1924154
      }
    ],
    "set": "anchor-lab-eval/1",
    "tasks": 59
  },
  "kind": "anchor.page",
  "links": {
    "api": "https://www.anchorterminal.com/api/v1/index.json",
    "html": "https://www.anchorterminal.com/benchmark/lab/evaluation/models/",
    "json": "https://www.anchorterminal.com/benchmark/lab/evaluation/models/index.json",
    "llms": "https://www.anchorterminal.com/llms.txt",
    "markdown": "https://www.anchorterminal.com/benchmark/lab/evaluation/models/index.md",
    "mcp": "https://www.anchorterminal.com/mcp",
    "slim": "https://www.anchorterminal.com/benchmark/lab/evaluation/models/index.min.md"
  },
  "markdown": "Each open-weight model the lab runs answers 59 questions about Anchor Terminal through our MCP server, pages and API, three times over, calling the tools itself or writing programs that call them, as it chooses. The right answers come from the site when each question is asked. The Lab score, 0 to 100, is the questions it got right with harder ones counting more.\n\n## Each model's latest\n\n| # | Model | Lab score | Right | One lookup | A few | Hard | How it got there | A question takes | Tokens a second | Date |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| 1 | [Qwen3.8-Flash-Next](https://www.anchorterminal.com/tools/qwen3-8-flash-next.md) (qwen3.8-flash-next) | 98 | 57 of 59 | 18 of 20 | 27 of 27 | 12 of 12 | own calls 56%, both 44% | 71 s | 18.6 | 20 November 2026 |\n| 2 | [Gemma 4 26B A4B IT](https://www.anchorterminal.com/tools/gemma-4-26b-a4b-it.md) (gemma-4-26b-a4b) | 92 | 55 of 59 | 19 of 20 | 26 of 27 | 10 of 12 | own calls 57%, programs 1%, both 43% | 91 s | 46.5 | 19 November 2026 |\n| 3 | [Gemma 4 31B IT](https://www.anchorterminal.com/tools/gemma-4-31b-it.md) (gemma-4-31b) | 87 | 52 of 59 | 18 of 20 | 24 of 27 | 10 of 12 | own calls 56%, both 44% | 5.3 min | 9.1 | 19 November 2026 |\n| 4 | [Qwen3.6-35B-A3B](https://www.anchorterminal.com/tools/qwen3-6-35b-a3b.md) (qwen3.6-35b-a3b) | 77 | 50 of 59 | 20 of 20 | 25 of 27 | 5 of 12 | own calls 75%, both 25% | 56 s | 33.8 | 20 November 2026 |\n| 5 | [GLM-4.5-Air](https://www.anchorterminal.com/tools/glm-4-5-air.md) (glm-4.5-air) | 67 | 44 of 59 | 19 of 20 | 20 of 27 | 5 of 12 | own calls 82%, both 18% | 2.3 min | 9.4 | 21 November 2026 |\n| 6 | [gpt-oss-120b](https://www.anchorterminal.com/tools/gpt-oss-120b.md) (gpt-oss-120b) | 66 | 43 of 59 | 18 of 20 | 20 of 27 | 5 of 12 | own calls 71%, both 29% | 55 s | 29.9 | 20 November 2026 |\n| 7 | [Qwen3.5-9B](https://www.anchorterminal.com/tools/qwen3-5-9b.md) (qwen3.5-9b) | 66 | 44 of 59 | 18 of 20 | 23 of 27 | 3 of 12 | own calls 72%, both 28% | 2.0 min | 17.2 | 21 November 2026 |\n| 8 | [Nemotron 3 Super 120B A12B](https://www.anchorterminal.com/tools/nemotron-3-super-120b-a12b.md) (nemotron-3-super-120b-a12b) | 63 | 41 of 59 | 18 of 20 | 18 of 27 | 5 of 12 | own calls 60%, both 40% | 4.8 min | 15.0 | 21 November 2026 |\n| 9 | [Qwen3.5-4B](https://www.anchorterminal.com/tools/qwen3-5-4b.md) (qwen3.5-4b) | 62 | 41 of 59 | 18 of 20 | 19 of 27 | 4 of 12 | own calls 83%, both 17% | 69 s | 26.3 | 21 November 2026 |\n| 10 | [gpt-oss-20b](https://www.anchorterminal.com/tools/gpt-oss-20b.md) (gpt-oss-20b) | 61 | 40 of 59 | 17 of 20 | 19 of 27 | 4 of 12 | own calls 69%, both 31% | 48 s | 45.0 | 21 November 2026 |\n| 11 | [Qwen3-Coder-Next](https://www.anchorterminal.com/tools/qwen3-coder-next.md) (qwen3-coder-next) | 61 | 41 of 59 | 18 of 20 | 20 of 27 | 3 of 12 | own calls 54%, both 46% | 53 s | 32.7 | 22 November 2026 |\n| 12 | [Qwen3-Coder-30B-A3B-Instruct](https://www.anchorterminal.com/tools/qwen3-coder-30b-a3b-instruct.md) (qwen3-coder-30b-a3b) | 59 | 38 of 59 | 14 of 20 | 21 of 27 | 3 of 12 | own calls 66%, both 34% | 64 s | 34.9 | 22 November 2026 |\n| 13 | [Nemotron 3 Nano 30B A3B](https://www.anchorterminal.com/tools/nemotron-3-nano-30b-a3b.md) (nemotron-3-nano-30b-a3b) | 59 | 40 of 59 | 18 of 20 | 19 of 27 | 3 of 12 | own calls 70%, both 30% | 3.2 min | 48.5 | 22 November 2026 |\n| 14 | [GLM-4.7-Flash](https://www.anchorterminal.com/tools/glm-4-7-flash.md) (glm-4.7-flash) | 55 | 38 of 59 | 19 of 20 | 16 of 27 | 3 of 12 | own calls 75%, programs 1%, both 25% | 102 s | 14.4 | 20 November 2026 |\n| 15 | [Qwen3-30B-A3B-Instruct-2507](https://www.anchorterminal.com/tools/qwen3-30b-a3b-instruct-2507.md) (qwen3-30b-a3b-instruct-2507) | 47 | 31 of 59 | 14 of 20 | 13 of 27 | 4 of 12 | own calls 49%, both 51% | 68 s | 31.7 | 22 November 2026 |\n\n## Every evaluation\n\nA run in code mode, where the model may only write programs, is kept beside the Lab score's runs and scores nothing.\n\n| Date | Model | Mode | Score | Right |\n| --- | --- | --- | --- | --- |\n| 22 November 2026 | qwen3-coder-30b-a3b | either | 59 | 38 of 59 |\n| 22 November 2026 | qwen3-30b-a3b-instruct-2507 | either | 47 | 31 of 59 |\n| 22 November 2026 | qwen3-coder-next | either | 61 | 41 of 59 |\n| 22 November 2026 | nemotron-3-nano-30b-a3b | either | 59 | 40 of 59 |\n| 21 November 2026 | glm-4.5-air | either | 67 | 44 of 59 |\n| 21 November 2026 | gpt-oss-20b | either | 61 | 40 of 59 |\n| 21 November 2026 | qwen3.5-4b | either | 62 | 41 of 59 |\n| 21 November 2026 | qwen3.5-9b | either | 66 | 44 of 59 |\n| 21 November 2026 | nemotron-3-super-120b-a12b | either | 63 | 41 of 59 |\n| 20 November 2026 | qwen3.8-flash-next | either | 98 | 57 of 59 |\n| 20 November 2026 | gpt-oss-120b | either | 66 | 43 of 59 |\n| 20 November 2026 | qwen3.6-35b-a3b | either | 77 | 50 of 59 |\n| 20 November 2026 | glm-4.7-flash | either | 55 | 38 of 59 |\n| 19 November 2026 | gemma-4-26b-a4b | either | 92 | 55 of 59 |\n| 19 November 2026 | gemma-4-31b | either | 87 | 52 of 59 |\n| 18 November 2026 | qwen3.8-27b | code only | 89 | 54 of 59 |\n\nThe harness is open source (Go, standard library only, MIT): https://github.com/Anchor-Terminal/anchorlab\n\nThe questions: https://github.com/Anchor-Terminal/anchorlab/blob/main/eval/tasks.go\n\nAn answer is checked against the site before and after the model answered, a question whose subject left the site or that has two right answers is skipped, and an answer with no lookup behind it counts as wrong. A question counts as right when it's right in more than half of three passes; a level 2 question counts twice and a level 3 question three times.\n\nJSON: https://www.anchorterminal.com/api/v1/lab-evals.json\n",
  "meta": {
    "attribution": "Anchor Terminal (https://www.anchorterminal.com)",
    "attributionRequired": true,
    "credit": "CC BY 4.0, credit Anchor Terminal and link the page or API address you took it from (https://www.anchorterminal.com/terms/#data)",
    "docs": "https://www.anchorterminal.com/docs/",
    "generatedAt": "2026-11-22",
    "license": "CC-BY-4.0",
    "method": "https://www.anchorterminal.com/benchmark/",
    "methodology": "1.0",
    "openapi": "https://www.anchorterminal.com/openapi.json",
    "preview": false,
    "run": "2026-10-01",
    "runLabel": "October 2026 research run",
    "terms": "https://www.anchorterminal.com/terms/#data"
  },
  "page": {
    "breadcrumbs": [
      {
        "name": "Home",
        "url": "https://www.anchorterminal.com/"
      },
      {
        "name": "Benchmark",
        "url": "https://www.anchorterminal.com/benchmark/"
      },
      {
        "name": "Lab",
        "url": "https://www.anchorterminal.com/benchmark/lab/"
      },
      {
        "name": "Evaluations",
        "url": "https://www.anchorterminal.com/benchmark/lab/evaluation/"
      },
      {
        "name": "Open-weight models",
        "url": ""
      }
    ],
    "description": "Open-weight models answering 59 questions about Anchor Terminal through its MCP server, pages and API, scored 0 to 100, with every run dated.",
    "facts": [
      "59 questions",
      "three passes",
      "15 models"
    ],
    "h1": "Lab score: open-weight models on our own evaluation",
    "image": "https://www.anchorterminal.com/assets/og/benchmark-open-weights-runs.png",
    "path": "/benchmark/lab/evaluation/models/",
    "published": "",
    "section": "benchmark",
    "title": "Lab score: open-weight models on our own evaluation | Anchor Terminal",
    "toc": null,
    "updated": "2026-11-22",
    "url": "https://www.anchorterminal.com/benchmark/lab/evaluation/models/"
  },
  "tokens": {
    "markdown": 1600,
    "slim": 1430
  },
  "version": 1
}
