{
  "schema_version": 1,
  "decoder": {
    "num_beams": 1,
    "no_repeat_ngram_size": 4,
    "repetition_penalty": 1.1,
    "length_penalty": 1.0
  },
  "bootstrap": {
    "samples": 10000,
    "seed": "kuku-version-probe-2026-07-14"
  },
  "models": {
    "step2770": {
      "lexicon": {
        "overall": {
          "rows": 297,
          "raw_exact_count": 41,
          "raw_exact_percent": 13.804713804713806,
          "normalized_exact_count": 42,
          "normalized_exact_percent": 14.14141414141414,
          "token_contains_reference_percent": 14.14141414141414,
          "mean_minimum_character_error_rate": 0.806539121185586,
          "mean_maximum_sentence_chrf": 28.063929993425884,
          "empty_outputs": 0,
          "source_copy_outputs": 8
        },
        "strata": {
          "target_seen_in_training": {
            "rows": 192,
            "raw_exact_count": 39,
            "raw_exact_percent": 20.3125,
            "normalized_exact_count": 40,
            "normalized_exact_percent": 20.833333333333336,
            "token_contains_reference_percent": 20.833333333333336,
            "mean_minimum_character_error_rate": 0.7576932615995117,
            "mean_maximum_sentence_chrf": 34.15660006687014,
            "empty_outputs": 0,
            "source_copy_outputs": 6
          },
          "target_unseen_in_training": {
            "rows": 105,
            "raw_exact_count": 2,
            "raw_exact_percent": 1.9047619047619049,
            "normalized_exact_count": 2,
            "normalized_exact_percent": 1.9047619047619049,
            "token_contains_reference_percent": 1.9047619047619049,
            "mean_minimum_character_error_rate": 0.8958572644286928,
            "mean_maximum_sentence_chrf": 16.923047573413545,
            "empty_outputs": 0,
            "source_copy_outputs": 2
          },
          "identity_translation": {
            "rows": 14,
            "raw_exact_count": 5,
            "raw_exact_percent": 35.714285714285715,
            "normalized_exact_count": 5,
            "normalized_exact_percent": 35.714285714285715,
            "token_contains_reference_percent": 35.714285714285715,
            "mean_minimum_character_error_rate": 0.646825396825397,
            "mean_maximum_sentence_chrf": 41.44030211808232,
            "empty_outputs": 0,
            "source_copy_outputs": 5
          },
          "non_identity_translation": {
            "rows": 283,
            "raw_exact_count": 36,
            "raw_exact_percent": 12.7208480565371,
            "normalized_exact_count": 37,
            "normalized_exact_percent": 13.074204946996467,
            "token_contains_reference_percent": 13.074204946996467,
            "mean_minimum_character_error_rate": 0.81444015348609,
            "mean_maximum_sentence_chrf": 27.402201337082445,
            "empty_outputs": 0,
            "source_copy_outputs": 3
          },
          "isolated_source_unseen_in_training": {
            "rows": 297,
            "raw_exact_count": 41,
            "raw_exact_percent": 13.804713804713806,
            "normalized_exact_count": 42,
            "normalized_exact_percent": 14.14141414141414,
            "token_contains_reference_percent": 14.14141414141414,
            "mean_minimum_character_error_rate": 0.806539121185586,
            "mean_maximum_sentence_chrf": 28.063929993425884,
            "empty_outputs": 0,
            "source_copy_outputs": 8
          }
        }
      },
      "elder": {
        "rows": 43,
        "exact_match_count": 0,
        "exact_match_percent": 0.0,
        "corpus_chrf": 29.18478628369407,
        "mean_sentence_chrf": 29.625329220447906,
        "empty_outputs": 0,
        "source_copy_outputs": 0
      },
      "prediction_files": {
        "lexicon": "/workspace/kuku-eval-kit/runs/kuku-version-benchmark-runpod-a40-r2-20260714T104027Z/predictions/step2770.predictions.json",
        "elder": "/workspace/kuku-eval-kit/runs/kuku-version-benchmark-runpod-a40-r2-20260714T104027Z/predictions/step2770.predictions.json"
      }
    },
    "step3120": {
      "lexicon": {
        "overall": {
          "rows": 297,
          "raw_exact_count": 41,
          "raw_exact_percent": 13.804713804713806,
          "normalized_exact_count": 43,
          "normalized_exact_percent": 14.47811447811448,
          "token_contains_reference_percent": 14.47811447811448,
          "mean_minimum_character_error_rate": 0.8109505814051268,
          "mean_maximum_sentence_chrf": 28.828852293065456,
          "empty_outputs": 0,
          "source_copy_outputs": 8
        },
        "strata": {
          "target_seen_in_training": {
            "rows": 192,
            "raw_exact_count": 39,
            "raw_exact_percent": 20.3125,
            "normalized_exact_count": 41,
            "normalized_exact_percent": 21.354166666666664,
            "token_contains_reference_percent": 21.354166666666664,
            "mean_minimum_character_error_rate": 0.7478781924094423,
            "mean_maximum_sentence_chrf": 35.301790812873925,
            "empty_outputs": 0,
            "source_copy_outputs": 6
          },
          "target_unseen_in_training": {
            "rows": 105,
            "raw_exact_count": 2,
            "raw_exact_percent": 1.9047619047619049,
            "normalized_exact_count": 2,
            "normalized_exact_percent": 1.9047619047619049,
            "token_contains_reference_percent": 1.9047619047619049,
            "mean_minimum_character_error_rate": 0.9262829498543782,
            "mean_maximum_sentence_chrf": 16.9926218568443,
            "empty_outputs": 0,
            "source_copy_outputs": 2
          },
          "identity_translation": {
            "rows": 14,
            "raw_exact_count": 3,
            "raw_exact_percent": 21.428571428571427,
            "normalized_exact_count": 4,
            "normalized_exact_percent": 28.57142857142857,
            "token_contains_reference_percent": 28.57142857142857,
            "mean_minimum_character_error_rate": 0.7413832199546485,
            "mean_maximum_sentence_chrf": 30.1039605801977,
            "empty_outputs": 0,
            "source_copy_outputs": 4
          },
          "non_identity_translation": {
            "rows": 283,
            "raw_exact_count": 38,
            "raw_exact_percent": 13.427561837455832,
            "normalized_exact_count": 39,
            "normalized_exact_percent": 13.780918727915195,
            "token_contains_reference_percent": 13.780918727915195,
            "mean_minimum_character_error_rate": 0.8143920763178712,
            "mean_maximum_sentence_chrf": 28.765772731157856,
            "empty_outputs": 0,
            "source_copy_outputs": 4
          },
          "isolated_source_unseen_in_training": {
            "rows": 297,
            "raw_exact_count": 41,
            "raw_exact_percent": 13.804713804713806,
            "normalized_exact_count": 43,
            "normalized_exact_percent": 14.47811447811448,
            "token_contains_reference_percent": 14.47811447811448,
            "mean_minimum_character_error_rate": 0.8109505814051268,
            "mean_maximum_sentence_chrf": 28.828852293065456,
            "empty_outputs": 0,
            "source_copy_outputs": 8
          }
        }
      },
      "elder": {
        "rows": 43,
        "exact_match_count": 0,
        "exact_match_percent": 0.0,
        "corpus_chrf": 29.259574714072752,
        "mean_sentence_chrf": 29.660768416903206,
        "empty_outputs": 0,
        "source_copy_outputs": 0
      },
      "prediction_files": {
        "lexicon": "/workspace/kuku-eval-kit/runs/kuku-version-benchmark-runpod-a40-r2-20260714T104027Z/predictions/step3120.predictions.json",
        "elder": "/workspace/kuku-eval-kit/runs/kuku-version-benchmark-runpod-a40-r2-20260714T104027Z/predictions/step3120.predictions.json"
      }
    },
    "step4155_guarded": {
      "lexicon": {
        "overall": {
          "rows": 297,
          "raw_exact_count": 47,
          "raw_exact_percent": 15.824915824915825,
          "normalized_exact_count": 48,
          "normalized_exact_percent": 16.161616161616163,
          "token_contains_reference_percent": 16.161616161616163,
          "mean_minimum_character_error_rate": 0.7817935898743978,
          "mean_maximum_sentence_chrf": 30.378303910852274,
          "empty_outputs": 0,
          "source_copy_outputs": 10
        },
        "strata": {
          "target_seen_in_training": {
            "rows": 192,
            "raw_exact_count": 45,
            "raw_exact_percent": 23.4375,
            "normalized_exact_count": 46,
            "normalized_exact_percent": 23.958333333333336,
            "token_contains_reference_percent": 23.958333333333336,
            "mean_minimum_character_error_rate": 0.7053581545769045,
            "mean_maximum_sentence_chrf": 37.563360949489045,
            "empty_outputs": 0,
            "source_copy_outputs": 7
          },
          "target_unseen_in_training": {
            "rows": 105,
            "raw_exact_count": 2,
            "raw_exact_percent": 1.9047619047619049,
            "normalized_exact_count": 2,
            "normalized_exact_percent": 1.9047619047619049,
            "token_contains_reference_percent": 1.9047619047619049,
            "mean_minimum_character_error_rate": 0.9215612429898143,
            "mean_maximum_sentence_chrf": 17.239913897345016,
            "empty_outputs": 0,
            "source_copy_outputs": 3
          },
          "identity_translation": {
            "rows": 14,
            "raw_exact_count": 5,
            "raw_exact_percent": 35.714285714285715,
            "normalized_exact_count": 5,
            "normalized_exact_percent": 35.714285714285715,
            "token_contains_reference_percent": 35.714285714285715,
            "mean_minimum_character_error_rate": 0.6723356009070295,
            "mean_maximum_sentence_chrf": 41.466187241929276,
            "empty_outputs": 0,
            "source_copy_outputs": 5
          },
          "non_identity_translation": {
            "rows": 283,
            "raw_exact_count": 42,
            "raw_exact_percent": 14.840989399293287,
            "normalized_exact_count": 43,
            "normalized_exact_percent": 15.19434628975265,
            "token_contains_reference_percent": 15.19434628975265,
            "mean_minimum_character_error_rate": 0.78720847272084,
            "mean_maximum_sentence_chrf": 29.82978671426189,
            "empty_outputs": 0,
            "source_copy_outputs": 5
          },
          "isolated_source_unseen_in_training": {
            "rows": 297,
            "raw_exact_count": 47,
            "raw_exact_percent": 15.824915824915825,
            "normalized_exact_count": 48,
            "normalized_exact_percent": 16.161616161616163,
            "token_contains_reference_percent": 16.161616161616163,
            "mean_minimum_character_error_rate": 0.7817935898743978,
            "mean_maximum_sentence_chrf": 30.378303910852274,
            "empty_outputs": 0,
            "source_copy_outputs": 10
          }
        }
      },
      "elder": {
        "rows": 43,
        "exact_match_count": 0,
        "exact_match_percent": 0.0,
        "corpus_chrf": 29.43278345989157,
        "mean_sentence_chrf": 29.892685027115533,
        "empty_outputs": 0,
        "source_copy_outputs": 0
      },
      "prediction_files": {
        "lexicon": "/workspace/kuku-eval-kit/runs/kuku-version-benchmark-runpod-a40-r2-20260714T104027Z/predictions/step4155_guarded.predictions.json",
        "elder": "/workspace/kuku-eval-kit/runs/kuku-version-benchmark-runpod-a40-r2-20260714T104027Z/predictions/step4155_guarded.predictions.json"
      }
    }
  },
  "pairwise": [
    {
      "left": "step2770",
      "right": "step3120",
      "lexicon_normalized_exact_percentage_points": {
        "difference": -0.33670033670033794,
        "ci95_low": -1.6835016835016834,
        "ci95_high": 1.0101010101010102
      },
      "lexicon_mean_sentence_chrf": {
        "difference": -0.7649222996395721,
        "ci95_low": -2.2801160454768574,
        "ci95_high": 0.6884420805003528
      },
      "elder_mean_sentence_chrf": {
        "difference": -0.03543919645530025,
        "ci95_low": -0.8408615220510275,
        "ci95_high": 0.8631706044299656
      }
    },
    {
      "left": "step2770",
      "right": "step4155_guarded",
      "lexicon_normalized_exact_percentage_points": {
        "difference": -2.020202020202022,
        "ci95_low": -3.7037037037037033,
        "ci95_high": -0.6734006734006733
      },
      "lexicon_mean_sentence_chrf": {
        "difference": -2.3143739174263906,
        "ci95_low": -3.981203541851546,
        "ci95_high": -0.8869502462660079
      },
      "elder_mean_sentence_chrf": {
        "difference": -0.26735580666762715,
        "ci95_low": -1.221928017486759,
        "ci95_high": 0.5089606077751778
      }
    },
    {
      "left": "step3120",
      "right": "step4155_guarded",
      "lexicon_normalized_exact_percentage_points": {
        "difference": -1.6835016835016843,
        "ci95_low": -3.3670033670033668,
        "ci95_high": -0.33670033670033667
      },
      "lexicon_mean_sentence_chrf": {
        "difference": -1.5494516177868185,
        "ci95_low": -2.9497021557279592,
        "ci95_high": -0.3657296804362016
      },
      "elder_mean_sentence_chrf": {
        "difference": -0.2319166102123269,
        "ci95_low": -0.9928708975529179,
        "ci95_high": 0.4191929719909702
      }
    }
  ]
}