{
  "model": "deepseek/deepseek-v4.1-flash-1017",
  "date": "2026-10-17",
  "provider": "TensorX",
  "request_reasoning": {
    "reasoning_effort": "xhigh"
  },
  "harness_commit": "526f8506909d30d3d46d420eef8f8f0efafb67e2",
  "harness_note": "Measured working tree based on this commit, including local changes; exact source hashes are recorded below. Older leaderboard rows were not rerun.",
  "source_sha256": {
    "aa": {
      "scripts/eval_dashboard_path.py": "6f88edb08f2d3b4c8f618a076d7284cfe0bdd892acabbd1128a9b0c45fe1a741",
      "evaluate_cases.py": "b6b03776f4063bd2af154e4e61986f10bebc03648c3c828038d7395007983b61",
      "web_server/api.py": "5dc8eb0d6cfcc86102120f76972d0fa30a771669591c28e82c615ed99073b3df",
      "client/agent_cli.py": "20b2983fdada9975a9ed6ee6b60566e1156879b5e66a592498b686754e974586",
      "client/agent_guards.py": "6c7cae002e965bad65c40d40d0bcc66765891e01592b0e35dc3142f962c4fe6f",
      "client/agent_skill.md": "2e6715f3512d02082862fda0bc6f210d34475b99c98e4393e76f1fe09b26e153",
      "mcp_server/tools.py": "d9c7a9d994c0581e1b7a3f25bb89df1a267961b975f7189caa693b48edb7c19c",
      "ludwig/evaluation_results_sectionfix.csv": "12aa051856aa8c3657316f6b40e05fb90c9fab31dd253d834ee68af7687f2a6c"
    },
    "bb": {
      "scripts/benger_eval.py": "304baab4f34fc3fbf13a4c623d6f140bfb9923191039eab3500e085ccd737b71",
      "bench/benger/prompts.py": "35afc286246b2b5853c35c839b626cf699dac19dfb9b22bbe0cbb71937f43315",
      "bench/benger/data/benchathon.json": "e7cd6d4bbdbcf6bcecaf15c18056c879a6cc5511b8fec5ef0311b0563892b0e3",
      "client/agent_cli.py": "20b2983fdada9975a9ed6ee6b60566e1156879b5e66a592498b686754e974586"
    }
  },
  "bb_max_output_tokens": 64000,
  "aa_judge": "openai/gpt-5.5",
  "bb_judge": "openai/gpt-5.4-mini",
  "aa": {
    "count": 10,
    "score": 90.0,
    "valid_answers": 10,
    "tokens_in": 588587,
    "tokens_out": 33017,
    "mean_seconds": 17.65,
    "tasks": [
      {
        "task": "aa-00",
        "score": 90.0,
        "tokens_in": 57702,
        "tokens_out": 4217,
        "seconds": 21.8,
        "attempts": 1,
        "case": "BGH IX ZR 143/20 15.04.2021",
        "steps": 5
      },
      {
        "task": "aa-01",
        "score": 90.0,
        "tokens_in": 48764,
        "tokens_out": 2227,
        "seconds": 13.2,
        "attempts": 1,
        "case": "BAG, 22.07.2021, 2 AZR 193/21",
        "steps": 5
      },
      {
        "task": "aa-02",
        "score": 80.0,
        "tokens_in": 42320,
        "tokens_out": 5151,
        "seconds": 20.8,
        "attempts": 1,
        "case": "BAG, 19.05.2021, 5 AS 2/21",
        "steps": 5
      },
      {
        "task": "aa-03",
        "score": 90.0,
        "tokens_in": 60679,
        "tokens_out": 2609,
        "seconds": 14.2,
        "attempts": 1,
        "case": "BAG, 20.05.2021, 2 AZR 457/20",
        "steps": 5
      },
      {
        "task": "aa-04",
        "score": 90.0,
        "tokens_in": 59071,
        "tokens_out": 3433,
        "seconds": 19.4,
        "attempts": 1,
        "case": "BAG, 20.05.2021, 2 AZR 560/20",
        "steps": 5
      },
      {
        "task": "aa-05",
        "score": 90.0,
        "tokens_in": 75003,
        "tokens_out": 4021,
        "seconds": 21.0,
        "attempts": 1,
        "case": "BAG, 28.04.2021, 10 AZR 404/18",
        "steps": 5
      },
      {
        "task": "aa-06",
        "score": 100.0,
        "tokens_in": 59281,
        "tokens_out": 3666,
        "seconds": 18.9,
        "attempts": 1,
        "case": "BAG, 27.04.2021, 2 AZR 540/20",
        "steps": 5
      },
      {
        "task": "aa-07",
        "score": 90.0,
        "tokens_in": 68812,
        "tokens_out": 2699,
        "seconds": 14.0,
        "attempts": 1,
        "case": "BAG, 27.04.2021, 2 AZR 342/20",
        "steps": 6
      },
      {
        "task": "aa-08",
        "score": 90.0,
        "tokens_in": 60007,
        "tokens_out": 2508,
        "seconds": 16.7,
        "attempts": 1,
        "case": "BAG, 27.04.2021, 9 AZR 662/19",
        "steps": 5
      },
      {
        "task": "aa-09",
        "score": 90.0,
        "tokens_in": 56948,
        "tokens_out": 2486,
        "seconds": 16.5,
        "attempts": 1,
        "case": "BAG, 31.03.2021, 5 AZR 197/20",
        "steps": 5
      }
    ]
  },
  "bb": {
    "count": 15,
    "score": 64.46666666666667,
    "valid_answers": 15,
    "tokens_in": 37828,
    "tokens_out": 349901,
    "mean_seconds": 129.51333333333335,
    "tasks": [
      {
        "task": "bb-00",
        "score": 53.0,
        "tokens_in": 2350,
        "tokens_out": 20324,
        "seconds": 121.9,
        "attempts": 1,
        "area": "Zivilrecht",
        "grade_points": 4.0,
        "passed": true
      },
      {
        "task": "bb-01",
        "score": 54.5,
        "tokens_in": 2123,
        "tokens_out": 15981,
        "seconds": 99.0,
        "attempts": 1,
        "area": "Zivilrecht",
        "grade_points": 5.0,
        "passed": true
      },
      {
        "task": "bb-02",
        "score": 79.0,
        "tokens_in": 2064,
        "tokens_out": 11714,
        "seconds": 94.4,
        "attempts": 1,
        "area": "Zivilrecht",
        "grade_points": 12.0,
        "passed": true
      },
      {
        "task": "bb-03",
        "score": 69.5,
        "tokens_in": 2377,
        "tokens_out": 13617,
        "seconds": 104.5,
        "attempts": 1,
        "area": "Zivilrecht",
        "grade_points": 9.0,
        "passed": true
      },
      {
        "task": "bb-04",
        "score": 58.0,
        "tokens_in": 2420,
        "tokens_out": 12439,
        "seconds": 70.9,
        "attempts": 1,
        "area": "Zivilrecht",
        "grade_points": 6.0,
        "passed": true
      },
      {
        "task": "bb-05",
        "score": 49.0,
        "tokens_in": 3459,
        "tokens_out": 21832,
        "seconds": 127.8,
        "attempts": 1,
        "area": "Öffentliches Recht",
        "grade_points": 3.0,
        "passed": false
      },
      {
        "task": "bb-06",
        "score": 70.5,
        "tokens_in": 2651,
        "tokens_out": 19757,
        "seconds": 111.2,
        "attempts": 1,
        "area": "Öffentliches Recht",
        "grade_points": 10.0,
        "passed": true
      },
      {
        "task": "bb-07",
        "score": 79.0,
        "tokens_in": 2623,
        "tokens_out": 42677,
        "seconds": 208.8,
        "attempts": 1,
        "area": "Öffentliches Recht",
        "grade_points": 12.0,
        "passed": true
      },
      {
        "task": "bb-08",
        "score": 75.0,
        "tokens_in": 3332,
        "tokens_out": 55332,
        "seconds": 217.0,
        "attempts": 1,
        "area": "Öffentliches Recht",
        "grade_points": 11.0,
        "passed": true
      },
      {
        "task": "bb-09",
        "score": 57.0,
        "tokens_in": 1983,
        "tokens_out": 19826,
        "seconds": 86.3,
        "attempts": 1,
        "area": "Öffentliches Recht",
        "grade_points": 6.0,
        "passed": true
      },
      {
        "task": "bb-10",
        "score": 61.0,
        "tokens_in": 2362,
        "tokens_out": 20281,
        "seconds": 119.7,
        "attempts": 1,
        "area": "Strafrecht",
        "grade_points": 7.0,
        "passed": true
      },
      {
        "task": "bb-11",
        "score": 62.0,
        "tokens_in": 2614,
        "tokens_out": 18792,
        "seconds": 119.0,
        "attempts": 1,
        "area": "Strafrecht",
        "grade_points": 7.0,
        "passed": true
      },
      {
        "task": "bb-12",
        "score": 69.5,
        "tokens_in": 2452,
        "tokens_out": 32646,
        "seconds": 237.6,
        "attempts": 1,
        "area": "Strafrecht",
        "grade_points": 9.0,
        "passed": true
      },
      {
        "task": "bb-13",
        "score": 60.5,
        "tokens_in": 2680,
        "tokens_out": 28681,
        "seconds": 153.7,
        "attempts": 1,
        "area": "Strafrecht",
        "grade_points": 7.0,
        "passed": true
      },
      {
        "task": "bb-14",
        "score": 69.5,
        "tokens_in": 2338,
        "tokens_out": 16002,
        "seconds": 70.9,
        "attempts": 1,
        "area": "Strafrecht",
        "grade_points": 9.0,
        "passed": true
      }
    ],
    "grade_points_mean": 7.8,
    "pass_rate_percent": 93.33333333333333,
    "retry_attempts": 0
  },
  "lg_score": 77.23333333333333,
  "total_cost_usd": 0.8875845,
  "average_cost_usd_per_task": 0.03550338,
  "average_cost_eur_per_task": 0.031550146627565986,
  "eur_usd_reference_rate": 1.1253,
  "eur_usd_reference_date": "2026-10-16",
  "eur_usd_reference_url": "https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/eurofxref-graph-usd.en.html",
  "input_usd_per_million": 0.5,
  "output_usd_per_million": 1.5,
  "pricing_url": "https://tensorx.ai/pricing/",
  "pricing_as_of": "2026-10-18",
  "cost_note": "No-cache TensorX list prices for the 25 scored tasks only. Excludes judges, smoke tests, and the abandoned 16k diagnostic run; no retries in the scored 64k Benchathon run.",
  "scoring_note": "LG = (AA mean on 0-100 + BB mean on 0-100) / 2. All ten AA and fifteen BB tasks are included. Judge scores and grade points are preserved unchanged.",
  "protocol_note": "Benchathon uses a 64000-token output limit instead of the published runner’s 32000. This is a disclosed 64k variant, not an identical-budget leaderboard comparison.",
  "validation": {
    "all_25_tasks_scored": true,
    "benchathon_generation_errors": 0,
    "benchathon_token_limit_hits": 0,
    "benchmark_sources_unchanged": true,
    "own10_csv_unchanged": true
  },
  "diagnostic_run": {
    "max_output_tokens": 16000,
    "included_in_score": false,
    "included_in_cost": false,
    "reason": "Stopped after two tasks exhausted the reasoning budget without a visible answer; all 15 tasks were then rerun uniformly at 64k."
  },
  "downloads": {
    "aa": "deepseek-v41-flash-1017-aa-2026-10-17.csv",
    "bb": "deepseek-v41-flash-1017-bb-2026-10-17.csv"
  },
  "original_result_sha256": {
    "deepseek-v41-flash-1017-aa-2026-10-17.csv": "e9155916536d01a8c82a2176607c7e4697fa4c5f30b562f2b225ef868a4ea935",
    "deepseek-v41-flash-1017-bb-2026-10-17.csv": "3335492a454a3bbd701c2cf3c7177e65784036677cba778971eeb768228ecb57"
  },
  "download_note": "CSV downloads contain per-task scores and usage metrics. Hashes of the original detailed research-output CSVs are recorded separately.",
  "result_sha256": {
    "deepseek-v41-flash-1017-aa-2026-10-17.csv": "53746eebab74ba0d792afe2b6b5337d101cc1a959b6bc9e8e59d2ae5b337cb82",
    "deepseek-v41-flash-1017-bb-2026-10-17.csv": "574bfbc8bed3910d58ea5cd030bca8a06b4c258b51a0376fb37840457bdc9431"
  }
}