{
  "schema_version": 1,
  "decoder": {
    "num_beams": 1,
    "no_repeat_ngram_size": 4,
    "repetition_penalty": 1.1,
    "length_penalty": 1.0
  },
  "bootstrap": {
    "samples": 10000,
    "seed": "kuku-version-probe-2026-07-14"
  },
  "models": {
    "step2770": {
      "lexicon": {
        "overall": {
          "rows": 297,
          "raw_exact_count": 41,
          "raw_exact_percent": 13.804713804713806,
          "normalized_exact_count": 42,
          "normalized_exact_percent": 14.14141414141414,
          "token_contains_reference_percent": 14.14141414141414,
          "mean_minimum_character_error_rate": 0.8065391211855858,
          "mean_maximum_sentence_chrf": 28.063929993425884,
          "empty_outputs": 0,
          "source_copy_outputs": 8
        },
        "strata": {
          "target_seen_in_training": {
            "rows": 192,
            "raw_exact_count": 39,
            "raw_exact_percent": 20.3125,
            "normalized_exact_count": 40,
            "normalized_exact_percent": 20.833333333333336,
            "token_contains_reference_percent": 20.833333333333336,
            "mean_minimum_character_error_rate": 0.7576932615995116,
            "mean_maximum_sentence_chrf": 34.15660006687013,
            "empty_outputs": 0,
            "source_copy_outputs": 6
          },
          "target_unseen_in_training": {
            "rows": 105,
            "raw_exact_count": 2,
            "raw_exact_percent": 1.9047619047619049,
            "normalized_exact_count": 2,
            "normalized_exact_percent": 1.9047619047619049,
            "token_contains_reference_percent": 1.9047619047619049,
            "mean_minimum_character_error_rate": 0.8958572644286931,
            "mean_maximum_sentence_chrf": 16.923047573413537,
            "empty_outputs": 0,
            "source_copy_outputs": 2
          },
          "identity_translation": {
            "rows": 14,
            "raw_exact_count": 5,
            "raw_exact_percent": 35.714285714285715,
            "normalized_exact_count": 5,
            "normalized_exact_percent": 35.714285714285715,
            "token_contains_reference_percent": 35.714285714285715,
            "mean_minimum_character_error_rate": 0.6468253968253969,
            "mean_maximum_sentence_chrf": 41.44030211808232,
            "empty_outputs": 0,
            "source_copy_outputs": 5
          },
          "non_identity_translation": {
            "rows": 283,
            "raw_exact_count": 36,
            "raw_exact_percent": 12.7208480565371,
            "normalized_exact_count": 37,
            "normalized_exact_percent": 13.074204946996467,
            "token_contains_reference_percent": 13.074204946996467,
            "mean_minimum_character_error_rate": 0.8144401534860899,
            "mean_maximum_sentence_chrf": 27.402201337082452,
            "empty_outputs": 0,
            "source_copy_outputs": 3
          },
          "isolated_source_unseen_in_training": {
            "rows": 297,
            "raw_exact_count": 41,
            "raw_exact_percent": 13.804713804713806,
            "normalized_exact_count": 42,
            "normalized_exact_percent": 14.14141414141414,
            "token_contains_reference_percent": 14.14141414141414,
            "mean_minimum_character_error_rate": 0.8065391211855858,
            "mean_maximum_sentence_chrf": 28.063929993425884,
            "empty_outputs": 0,
            "source_copy_outputs": 8
          }
        }
      },
      "elder": {
        "rows": 43,
        "exact_match_count": 0,
        "exact_match_percent": 0.0,
        "corpus_chrf": 29.18478628369407,
        "mean_sentence_chrf": 29.625329220447906,
        "empty_outputs": 0,
        "source_copy_outputs": 0
      },
      "prediction_files": {
        "lexicon": "/mnt/donto-data/donto-resources/research/translation-training/kuku-yalanji-runpod-2026-06-30/analysis/v22-lexicon-elder-version-probe-2026-07-14/step2770.predictions.json",
        "elder": "/mnt/donto-data/donto-resources/research/translation-training/kuku-yalanji-runpod-2026-06-30/analysis/v22-lexicon-elder-version-probe-2026-07-14/step2770.predictions.json"
      }
    },
    "step3120": {
      "lexicon": {
        "overall": {
          "rows": 297,
          "raw_exact_count": 41,
          "raw_exact_percent": 13.804713804713806,
          "normalized_exact_count": 43,
          "normalized_exact_percent": 14.47811447811448,
          "token_contains_reference_percent": 14.47811447811448,
          "mean_minimum_character_error_rate": 0.8109505814051269,
          "mean_maximum_sentence_chrf": 28.828852293065463,
          "empty_outputs": 0,
          "source_copy_outputs": 8
        },
        "strata": {
          "target_seen_in_training": {
            "rows": 192,
            "raw_exact_count": 39,
            "raw_exact_percent": 20.3125,
            "normalized_exact_count": 41,
            "normalized_exact_percent": 21.354166666666664,
            "token_contains_reference_percent": 21.354166666666664,
            "mean_minimum_character_error_rate": 0.7478781924094423,
            "mean_maximum_sentence_chrf": 35.30179081287391,
            "empty_outputs": 0,
            "source_copy_outputs": 6
          },
          "target_unseen_in_training": {
            "rows": 105,
            "raw_exact_count": 2,
            "raw_exact_percent": 1.9047619047619049,
            "normalized_exact_count": 2,
            "normalized_exact_percent": 1.9047619047619049,
            "token_contains_reference_percent": 1.9047619047619049,
            "mean_minimum_character_error_rate": 0.9262829498543784,
            "mean_maximum_sentence_chrf": 16.99262185684429,
            "empty_outputs": 0,
            "source_copy_outputs": 2
          },
          "identity_translation": {
            "rows": 14,
            "raw_exact_count": 3,
            "raw_exact_percent": 21.428571428571427,
            "normalized_exact_count": 4,
            "normalized_exact_percent": 28.57142857142857,
            "token_contains_reference_percent": 28.57142857142857,
            "mean_minimum_character_error_rate": 0.7413832199546485,
            "mean_maximum_sentence_chrf": 30.103960580197704,
            "empty_outputs": 0,
            "source_copy_outputs": 4
          },
          "non_identity_translation": {
            "rows": 283,
            "raw_exact_count": 38,
            "raw_exact_percent": 13.427561837455832,
            "normalized_exact_count": 39,
            "normalized_exact_percent": 13.780918727915195,
            "token_contains_reference_percent": 13.780918727915195,
            "mean_minimum_character_error_rate": 0.8143920763178714,
            "mean_maximum_sentence_chrf": 28.76577273115786,
            "empty_outputs": 0,
            "source_copy_outputs": 4
          },
          "isolated_source_unseen_in_training": {
            "rows": 297,
            "raw_exact_count": 41,
            "raw_exact_percent": 13.804713804713806,
            "normalized_exact_count": 43,
            "normalized_exact_percent": 14.47811447811448,
            "token_contains_reference_percent": 14.47811447811448,
            "mean_minimum_character_error_rate": 0.8109505814051269,
            "mean_maximum_sentence_chrf": 28.828852293065463,
            "empty_outputs": 0,
            "source_copy_outputs": 8
          }
        }
      },
      "elder": {
        "rows": 43,
        "exact_match_count": 0,
        "exact_match_percent": 0.0,
        "corpus_chrf": 29.259574714072752,
        "mean_sentence_chrf": 29.660768416903213,
        "empty_outputs": 0,
        "source_copy_outputs": 0
      },
      "prediction_files": {
        "lexicon": "/mnt/donto-data/donto-resources/research/translation-training/kuku-yalanji-runpod-2026-06-30/analysis/v22-lexicon-elder-version-probe-2026-07-14/step3120.predictions.json",
        "elder": "/mnt/donto-data/donto-resources/research/translation-training/kuku-yalanji-runpod-2026-06-30/analysis/v22-lexicon-elder-version-probe-2026-07-14/step3120.predictions.json"
      }
    },
    "step4155_guarded": {
      "lexicon": {
        "overall": {
          "rows": 297,
          "raw_exact_count": 47,
          "raw_exact_percent": 15.824915824915825,
          "normalized_exact_count": 48,
          "normalized_exact_percent": 16.161616161616163,
          "token_contains_reference_percent": 16.161616161616163,
          "mean_minimum_character_error_rate": 0.781793589874398,
          "mean_maximum_sentence_chrf": 30.378303910852257,
          "empty_outputs": 0,
          "source_copy_outputs": 10
        },
        "strata": {
          "target_seen_in_training": {
            "rows": 192,
            "raw_exact_count": 45,
            "raw_exact_percent": 23.4375,
            "normalized_exact_count": 46,
            "normalized_exact_percent": 23.958333333333336,
            "token_contains_reference_percent": 23.958333333333336,
            "mean_minimum_character_error_rate": 0.7053581545769045,
            "mean_maximum_sentence_chrf": 37.56336094948903,
            "empty_outputs": 0,
            "source_copy_outputs": 7
          },
          "target_unseen_in_training": {
            "rows": 105,
            "raw_exact_count": 2,
            "raw_exact_percent": 1.9047619047619049,
            "normalized_exact_count": 2,
            "normalized_exact_percent": 1.9047619047619049,
            "token_contains_reference_percent": 1.9047619047619049,
            "mean_minimum_character_error_rate": 0.9215612429898145,
            "mean_maximum_sentence_chrf": 17.23991389734501,
            "empty_outputs": 0,
            "source_copy_outputs": 3
          },
          "identity_translation": {
            "rows": 14,
            "raw_exact_count": 5,
            "raw_exact_percent": 35.714285714285715,
            "normalized_exact_count": 5,
            "normalized_exact_percent": 35.714285714285715,
            "token_contains_reference_percent": 35.714285714285715,
            "mean_minimum_character_error_rate": 0.6723356009070295,
            "mean_maximum_sentence_chrf": 41.466187241929276,
            "empty_outputs": 0,
            "source_copy_outputs": 5
          },
          "non_identity_translation": {
            "rows": 283,
            "raw_exact_count": 42,
            "raw_exact_percent": 14.840989399293287,
            "normalized_exact_count": 43,
            "normalized_exact_percent": 15.19434628975265,
            "token_contains_reference_percent": 15.19434628975265,
            "mean_minimum_character_error_rate": 0.7872084727208403,
            "mean_maximum_sentence_chrf": 29.82978671426188,
            "empty_outputs": 0,
            "source_copy_outputs": 5
          },
          "isolated_source_unseen_in_training": {
            "rows": 297,
            "raw_exact_count": 47,
            "raw_exact_percent": 15.824915824915825,
            "normalized_exact_count": 48,
            "normalized_exact_percent": 16.161616161616163,
            "token_contains_reference_percent": 16.161616161616163,
            "mean_minimum_character_error_rate": 0.781793589874398,
            "mean_maximum_sentence_chrf": 30.378303910852257,
            "empty_outputs": 0,
            "source_copy_outputs": 10
          }
        }
      },
      "elder": {
        "rows": 43,
        "exact_match_count": 0,
        "exact_match_percent": 0.0,
        "corpus_chrf": 29.43278345989157,
        "mean_sentence_chrf": 29.892685027115533,
        "empty_outputs": 0,
        "source_copy_outputs": 0
      },
      "prediction_files": {
        "lexicon": "/mnt/donto-data/donto-resources/research/translation-training/kuku-yalanji-runpod-2026-06-30/analysis/v22-lexicon-elder-version-probe-2026-07-14/step4155_guarded.predictions.json",
        "elder": "/mnt/donto-data/donto-resources/research/translation-training/kuku-yalanji-runpod-2026-06-30/analysis/v22-lexicon-elder-version-probe-2026-07-14/step4155_guarded.predictions.json"
      }
    }
  },
  "pairwise": [
    {
      "left": "step2770",
      "right": "step3120",
      "lexicon_normalized_exact_percentage_points": {
        "difference": -0.33670033670033794,
        "ci95_low": -1.6835016835016834,
        "ci95_high": 1.0101010101010102
      },
      "lexicon_mean_sentence_chrf": {
        "difference": -0.7649222996395793,
        "ci95_low": -2.2801160454768588,
        "ci95_high": 0.688442080500353
      },
      "elder_mean_sentence_chrf": {
        "difference": -0.035439196455307354,
        "ci95_low": -0.8408615220510277,
        "ci95_high": 0.8631706044299653
      }
    },
    {
      "left": "step2770",
      "right": "step4155_guarded",
      "lexicon_normalized_exact_percentage_points": {
        "difference": -2.020202020202022,
        "ci95_low": -3.7037037037037033,
        "ci95_high": -0.6734006734006733
      },
      "lexicon_mean_sentence_chrf": {
        "difference": -2.314373917426373,
        "ci95_low": -3.981203541851546,
        "ci95_high": -0.8869502462660083
      },
      "elder_mean_sentence_chrf": {
        "difference": -0.26735580666762715,
        "ci95_low": -1.221928017486759,
        "ci95_high": 0.5089606077751778
      }
    },
    {
      "left": "step3120",
      "right": "step4155_guarded",
      "lexicon_normalized_exact_percentage_points": {
        "difference": -1.6835016835016843,
        "ci95_low": -3.3670033670033668,
        "ci95_high": -0.33670033670033667
      },
      "lexicon_mean_sentence_chrf": {
        "difference": -1.5494516177867936,
        "ci95_low": -2.9497021557279592,
        "ci95_high": -0.36572968043620163
      },
      "elder_mean_sentence_chrf": {
        "difference": -0.2319166102123198,
        "ci95_low": -0.9928708975529177,
        "ci95_high": 0.4191929719909702
      }
    }
  ]
}