{
  "schema_version": 1,
  "benchmark_id": "kuku-yalanji-lexicon-elder-version-probe-v1",
  "frozen_at": "2026-07-14T07:12:00Z",
  "task": "English to Kuku Yalanji machine translation diagnostics",
  "direction": "eng-gvn",
  "source_lang": "eng_Latn",
  "target_lang": "gvn_Latn",
  "data": {
    "relative_path": "data/combined_lexicon_elder_probe.eng-gvn.jsonl",
    "sha256": "6e525f16eb7475c6bc5450b978d70db0fbf1a0d71f2af9f143fb80c4bf60c01f",
    "rows": 340,
    "directions": {
      "eng-gvn": 340
    },
    "pair_kinds": {
      "dictionary_single_gloss_probe": 297,
      "elder_sentence_pair_parallel": 43
    }
  },
  "models": [
    {
      "label": "step2770",
      "directory": "step2770",
      "optimizer_step": 2770,
      "role": "reconstructed earlier point on the v21.2/v22 trajectory",
      "model_safetensors_sha256": "1f2a31593a350f373f2c4345f1bd4f16a29ece18b53fa552ba1828acff7c84f7",
      "files": {
        "added_tokens.json": "26d6bc46ee6972f04c2cdd8025ef3fb6b84b98b1ee0fb84a4e53233de26a6d7e",
        "config.json": "7c512de4e3796c41ebeb761d7abfc16444be773ab4ab32e2224f39a0455d88eb",
        "generation_config.json": "56869e2f435f78b97bb8d57b47da2ebd922de3770707cc79ec9aa66e8dfdf060",
        "model.safetensors": "1f2a31593a350f373f2c4345f1bd4f16a29ece18b53fa552ba1828acff7c84f7",
        "sentencepiece.bpe.model": "14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a",
        "special_tokens_map.json": "1f75de3da1463f716bf3a3d9f6c30da35ffb5131dcb3003b6f82908dedf24177",
        "tokenizer.json": "9041375d4d92d6b87628b57b64103d0ce1974559b7ccf146e871656b754fc8ed",
        "tokenizer_config.json": "f6929bc5e3321c4bcbdd9f5251362e4d8c7451f29a1e890cce192cf348b7f314"
      }
    },
    {
      "label": "step3120",
      "directory": "step3120",
      "optimizer_step": 3120,
      "role": "v22.0 step-matched failed-promotion checkpoint",
      "model_safetensors_sha256": "8ce31236b051195fbbd01df8d7149ce729249a30959592e9762b2dabf404c2aa",
      "files": {
        "added_tokens.json": "26d6bc46ee6972f04c2cdd8025ef3fb6b84b98b1ee0fb84a4e53233de26a6d7e",
        "config.json": "7c512de4e3796c41ebeb761d7abfc16444be773ab4ab32e2224f39a0455d88eb",
        "generation_config.json": "56869e2f435f78b97bb8d57b47da2ebd922de3770707cc79ec9aa66e8dfdf060",
        "model.safetensors": "8ce31236b051195fbbd01df8d7149ce729249a30959592e9762b2dabf404c2aa",
        "sentencepiece.bpe.model": "14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a",
        "special_tokens_map.json": "1f75de3da1463f716bf3a3d9f6c30da35ffb5131dcb3003b6f82908dedf24177",
        "tokenizer.json": "9041375d4d92d6b87628b57b64103d0ce1974559b7ccf146e871656b754fc8ed",
        "tokenizer_config.json": "f6929bc5e3321c4bcbdd9f5251362e4d8c7451f29a1e890cce192cf348b7f314"
      }
    },
    {
      "label": "step4155_guarded",
      "directory": "step4155_guarded",
      "optimizer_step": 4155,
      "role": "current v21.2 weights under the guarded serving decoder",
      "model_safetensors_sha256": "7f9d0fe325e9e4568e45f13179adb336b93bbd53e83ddab2826e999eba3c76f7",
      "files": {
        "added_tokens.json": "26d6bc46ee6972f04c2cdd8025ef3fb6b84b98b1ee0fb84a4e53233de26a6d7e",
        "config.json": "8b1ff7577221948cb9b9ffd9e2e64dda4e19be42c1465fe1d4b23f3a7e3e5f00",
        "generation_config.json": "56869e2f435f78b97bb8d57b47da2ebd922de3770707cc79ec9aa66e8dfdf060",
        "model.safetensors": "7f9d0fe325e9e4568e45f13179adb336b93bbd53e83ddab2826e999eba3c76f7",
        "sentencepiece.bpe.model": "14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a",
        "special_tokens_map.json": "1f75de3da1463f716bf3a3d9f6c30da35ffb5131dcb3003b6f82908dedf24177",
        "tokenizer.json": "9041375d4d92d6b87628b57b64103d0ce1974559b7ccf146e871656b754fc8ed",
        "tokenizer_config.json": "f6929bc5e3321c4bcbdd9f5251362e4d8c7451f29a1e890cce192cf348b7f314"
      }
    }
  ],
  "decoder": {
    "batch_size": 8,
    "num_beams": 1,
    "no_repeat_ngram_size": 4,
    "repetition_penalty": 1.1,
    "length_penalty": 1.0,
    "max_source_length": 192,
    "max_new_tokens": 208,
    "dtype": "float32",
    "do_sample": false,
    "deterministic": true,
    "seed": 0,
    "allow_tf32": false
  },
  "runtime": {
    "image": "runpod/pytorch:2.4.0-py3.11-cuda12.4.1-devel-ubuntu22.04",
    "python": "3.11.10",
    "torch": "2.4.1+cu124",
    "cuda": "12.4",
    "gpu": "NVIDIA A40",
    "packages": {
      "transformers": "4.48.3",
      "tokenizers": "0.21.4",
      "sacrebleu": "2.6.0",
      "sentencepiece": "0.2.0",
      "safetensors": "0.8.0"
    }
  },
  "statistics": {
    "paired_bootstrap_samples": 10000,
    "seed": "kuku-version-probe-2026-07-14"
  },
  "interpretation_limits": [
    "Dictionary exact match measures canonical-form retrieval, not free-sentence adequacy.",
    "The dictionary informed earlier corpus development and is not an untouched test sample.",
    "The elder rows were excluded from training but observed in earlier evaluations.",
    "Automatic overlap metrics do not establish grammaticality, adequacy, dialect fit, cultural acceptability, or speaker certification."
  ]
}