Fill-Mask
Transformers
TensorBoard
Safetensors
Latin
xlm-roberta
medieval-latin
domain-adaptation
legal-history
digital-humanities
canon-law
Instructions to use mschonhardt/MediBerta with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use mschonhardt/MediBerta with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("fill-mask", model="mschonhardt/MediBerta")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForMaskedLM tokenizer = AutoTokenizer.from_pretrained("mschonhardt/MediBerta") model = AutoModelForMaskedLM.from_pretrained("mschonhardt/MediBerta", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Add files using upload-large-folder tool
Browse files- .gitattributes +1 -0
- all_results.json +15 -0
- config.json +30 -0
- dataset_stats.json +12 -0
- eval_representativeness.json +15 -0
- final_eval_results.json +10 -0
- model.safetensors +3 -0
- run_metadata.json +81 -0
- runs/Jun21_18-29-32_linglit-gpu/events.out.tfevents.1782059372.linglit-gpu +3 -0
- runs/Jun21_18-29-32_linglit-gpu/events.out.tfevents.1782093628.linglit-gpu +3 -0
- tokenizer.json +3 -0
- tokenizer_config.json +15 -0
- tokenizer_diagnostics.json +16 -0
- train.log +87 -0
- train.py +1178 -0
- train_results.json +9 -0
- trainer_state.json +1043 -0
- training_args.bin +3 -0
- training_log.jsonl +102 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
all_results.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"epoch": 10.0,
|
| 3 |
+
"final_val_loss": 1.0483721494674683,
|
| 4 |
+
"final_val_masked_accuracy": 0.7699716443872566,
|
| 5 |
+
"final_val_perplexity": 2.853003073356264,
|
| 6 |
+
"final_val_runtime": 25.2687,
|
| 7 |
+
"final_val_samples_per_second": 220.351,
|
| 8 |
+
"final_val_steps_per_second": 3.443,
|
| 9 |
+
"num_input_tokens_seen": 1464637440,
|
| 10 |
+
"total_flos": 4.917799558028206e+18,
|
| 11 |
+
"train_loss": 21.80173199032777,
|
| 12 |
+
"train_runtime": 34212.9191,
|
| 13 |
+
"train_samples_per_second": 83.612,
|
| 14 |
+
"train_steps_per_second": 0.131
|
| 15 |
+
}
|
config.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_cross_attention": false,
|
| 3 |
+
"architectures": [
|
| 4 |
+
"XLMRobertaForMaskedLM"
|
| 5 |
+
],
|
| 6 |
+
"attention_probs_dropout_prob": 0.1,
|
| 7 |
+
"bos_token_id": 0,
|
| 8 |
+
"classifier_dropout": null,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"eos_token_id": 2,
|
| 11 |
+
"hidden_act": "gelu",
|
| 12 |
+
"hidden_dropout_prob": 0.1,
|
| 13 |
+
"hidden_size": 1024,
|
| 14 |
+
"initializer_range": 0.02,
|
| 15 |
+
"intermediate_size": 4096,
|
| 16 |
+
"is_decoder": false,
|
| 17 |
+
"layer_norm_eps": 1e-05,
|
| 18 |
+
"max_position_embeddings": 514,
|
| 19 |
+
"model_type": "xlm-roberta",
|
| 20 |
+
"num_attention_heads": 16,
|
| 21 |
+
"num_hidden_layers": 24,
|
| 22 |
+
"output_past": true,
|
| 23 |
+
"pad_token_id": 1,
|
| 24 |
+
"position_embedding_type": "absolute",
|
| 25 |
+
"tie_word_embeddings": true,
|
| 26 |
+
"transformers_version": "5.12.1",
|
| 27 |
+
"type_vocab_size": 1,
|
| 28 |
+
"use_cache": false,
|
| 29 |
+
"vocab_size": 250002
|
| 30 |
+
}
|
dataset_stats.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"train": {
|
| 3 |
+
"sequences": 286062,
|
| 4 |
+
"tokens": 142794676,
|
| 5 |
+
"mean_seq_len": 499.2
|
| 6 |
+
},
|
| 7 |
+
"validation": {
|
| 8 |
+
"sequences": 5568,
|
| 9 |
+
"tokens": 2812986,
|
| 10 |
+
"mean_seq_len": 505.2
|
| 11 |
+
}
|
| 12 |
+
}
|
eval_representativeness.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"split_kind": "document-level STRATIFIED group-aware",
|
| 3 |
+
"train_sequences": 286062,
|
| 4 |
+
"val_sequences": 5568,
|
| 5 |
+
"len_train": {
|
| 6 |
+
"mean": 499.173871398508,
|
| 7 |
+
"p50": 512.0,
|
| 8 |
+
"p95": 512.0
|
| 9 |
+
},
|
| 10 |
+
"len_val": {
|
| 11 |
+
"mean": 505.20581896551727,
|
| 12 |
+
"p50": 512.0,
|
| 13 |
+
"p95": 512.0
|
| 14 |
+
}
|
| 15 |
+
}
|
final_eval_results.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"epoch": 10.0,
|
| 3 |
+
"final_val_loss": 1.0483721494674683,
|
| 4 |
+
"final_val_masked_accuracy": 0.7699716443872566,
|
| 5 |
+
"final_val_perplexity": 2.853003073356264,
|
| 6 |
+
"final_val_runtime": 25.2687,
|
| 7 |
+
"final_val_samples_per_second": 220.351,
|
| 8 |
+
"final_val_steps_per_second": 3.443,
|
| 9 |
+
"num_input_tokens_seen": 1464637440
|
| 10 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c254ead4ef6c737b0586af25d64dedb5e31acbdb649243833c8ea184e63fdeb5
|
| 3 |
+
size 2240618752
|
run_metadata.json
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"config": {
|
| 3 |
+
"med_dataset": "mschonhardt/MedBerta",
|
| 4 |
+
"canon_dataset": "mschonhardt/MedBerta",
|
| 5 |
+
"dataset_split": "train",
|
| 6 |
+
"text_column": null,
|
| 7 |
+
"doc_id_column": null,
|
| 8 |
+
"order_column": null,
|
| 9 |
+
"model_name": "xlm-roberta-large",
|
| 10 |
+
"max_seq_length": 512,
|
| 11 |
+
"mlm_probability": 0.15,
|
| 12 |
+
"attn_implementation": "sdpa",
|
| 13 |
+
"min_chunk_tokens": 64,
|
| 14 |
+
"val_doc_fraction": 0.01,
|
| 15 |
+
"test_doc_fraction": 0.0,
|
| 16 |
+
"max_eval_docs_per_source": 400,
|
| 17 |
+
"stratify_column": "category",
|
| 18 |
+
"split_seed": 13,
|
| 19 |
+
"train_corpus": "canon",
|
| 20 |
+
"dedup_train": false,
|
| 21 |
+
"diagnose_tokenizer": true,
|
| 22 |
+
"tokenizer_sample_size": 20000,
|
| 23 |
+
"normalize_nfc": false,
|
| 24 |
+
"diagnose_only": false,
|
| 25 |
+
"learning_rate": 0.0001,
|
| 26 |
+
"weight_decay": 0.01,
|
| 27 |
+
"adam_beta1": 0.9,
|
| 28 |
+
"adam_beta2": 0.98,
|
| 29 |
+
"adam_epsilon": 1e-06,
|
| 30 |
+
"max_grad_norm": 1.0,
|
| 31 |
+
"warmup_ratio": 0.06,
|
| 32 |
+
"lr_scheduler_type": "linear",
|
| 33 |
+
"num_train_epochs": 10.0,
|
| 34 |
+
"max_steps": -1,
|
| 35 |
+
"optim": "adamw_torch_fused",
|
| 36 |
+
"early_stopping_patience": 5,
|
| 37 |
+
"early_stopping_threshold": 0.0001,
|
| 38 |
+
"per_device_train_batch_size": 64,
|
| 39 |
+
"per_device_eval_batch_size": 64,
|
| 40 |
+
"gradient_accumulation_steps": 16,
|
| 41 |
+
"gradient_checkpointing": false,
|
| 42 |
+
"auto_find_batch_size": true,
|
| 43 |
+
"torch_compile": false,
|
| 44 |
+
"dataloader_num_workers": 8,
|
| 45 |
+
"eval_accumulation_steps": 50,
|
| 46 |
+
"eval_steps": 500,
|
| 47 |
+
"logging_steps": 50,
|
| 48 |
+
"save_steps": 500,
|
| 49 |
+
"save_total_limit": 3,
|
| 50 |
+
"output_dir": "./MediBerta",
|
| 51 |
+
"cache_dir": ".cache_packed",
|
| 52 |
+
"seed": 42,
|
| 53 |
+
"report_to_wandb": false,
|
| 54 |
+
"preprocess_num_proc": 16,
|
| 55 |
+
"resume": true,
|
| 56 |
+
"dry_run": false
|
| 57 |
+
},
|
| 58 |
+
"environment": {
|
| 59 |
+
"timestamp": "2026-06-21T18:28:11",
|
| 60 |
+
"python": "3.11.2",
|
| 61 |
+
"platform": "Linux-6.1.0-49-amd64-x86_64-with-glibc2.36",
|
| 62 |
+
"torch": "2.12.1+cu130",
|
| 63 |
+
"transformers": "5.12.1",
|
| 64 |
+
"datasets": "5.0.0",
|
| 65 |
+
"cuda_available": true,
|
| 66 |
+
"cuda": "13.0",
|
| 67 |
+
"gpu_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 68 |
+
"gpu_total_memory_gb": 95.0,
|
| 69 |
+
"gpu_capability": "12.0",
|
| 70 |
+
"bf16_supported": true,
|
| 71 |
+
"torch_arch_list": [
|
| 72 |
+
"sm_75",
|
| 73 |
+
"sm_80",
|
| 74 |
+
"sm_86",
|
| 75 |
+
"sm_90",
|
| 76 |
+
"sm_100",
|
| 77 |
+
"sm_120"
|
| 78 |
+
],
|
| 79 |
+
"gpu_arch_supported_by_torch": true
|
| 80 |
+
}
|
| 81 |
+
}
|
runs/Jun21_18-29-32_linglit-gpu/events.out.tfevents.1782059372.linglit-gpu
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f7d560e3dbaa720ddf8771a7e267214ce871a52330b744411e47ba056f2389d7
|
| 3 |
+
size 62249
|
runs/Jun21_18-29-32_linglit-gpu/events.out.tfevents.1782093628.linglit-gpu
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1d0eb78808c93b7941e7675b6c1a1cc075a2bf61bd5c18d85fc6e5bb2b09d87c
|
| 3 |
+
size 491
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:acbd420e2269cdc1ef45332d3d5c418be4aef6b8cb5a0b7ccae0893485307153
|
| 3 |
+
size 17098086
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": true,
|
| 3 |
+
"backend": "tokenizers",
|
| 4 |
+
"bos_token": "<s>",
|
| 5 |
+
"cls_token": "<s>",
|
| 6 |
+
"eos_token": "</s>",
|
| 7 |
+
"is_local": false,
|
| 8 |
+
"local_files_only": false,
|
| 9 |
+
"mask_token": "<mask>",
|
| 10 |
+
"model_max_length": 512,
|
| 11 |
+
"pad_token": "<pad>",
|
| 12 |
+
"sep_token": "</s>",
|
| 13 |
+
"tokenizer_class": "XLMRobertaTokenizer",
|
| 14 |
+
"unk_token": "<unk>"
|
| 15 |
+
}
|
tokenizer_diagnostics.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"mode": "pretokenized",
|
| 3 |
+
"sampled_sequences": 20000,
|
| 4 |
+
"total_sequences": 291630,
|
| 5 |
+
"unk_rate": 3.7e-05,
|
| 6 |
+
"total_unk_tokens": 368,
|
| 7 |
+
"seq_len": {
|
| 8 |
+
"mean": 500.11885,
|
| 9 |
+
"p50": 512.0,
|
| 10 |
+
"p95": 512.0,
|
| 11 |
+
"max": 512
|
| 12 |
+
},
|
| 13 |
+
"share_over_max_seq_len": 0.0,
|
| 14 |
+
"starts_with_bos_rate": 1.0,
|
| 15 |
+
"ends_with_eos_rate": 1.0
|
| 16 |
+
}
|
train.log
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-06-21 18:28:11 | INFO | dapt | Environment:
|
| 2 |
+
{
|
| 3 |
+
"timestamp": "2026-06-21T18:28:11",
|
| 4 |
+
"python": "3.11.2",
|
| 5 |
+
"platform": "Linux-6.1.0-49-amd64-x86_64-with-glibc2.36",
|
| 6 |
+
"torch": "2.12.1+cu130",
|
| 7 |
+
"transformers": "5.12.1",
|
| 8 |
+
"datasets": "5.0.0",
|
| 9 |
+
"cuda_available": true,
|
| 10 |
+
"cuda": "13.0",
|
| 11 |
+
"gpu_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 12 |
+
"gpu_total_memory_gb": 95.0,
|
| 13 |
+
"gpu_capability": "12.0",
|
| 14 |
+
"bf16_supported": true,
|
| 15 |
+
"torch_arch_list": [
|
| 16 |
+
"sm_75",
|
| 17 |
+
"sm_80",
|
| 18 |
+
"sm_86",
|
| 19 |
+
"sm_90",
|
| 20 |
+
"sm_100",
|
| 21 |
+
"sm_120"
|
| 22 |
+
],
|
| 23 |
+
"gpu_arch_supported_by_torch": true
|
| 24 |
+
}
|
| 25 |
+
2026-06-21 18:28:12 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/xlm-roberta-large/resolve/main/config.json "HTTP/1.1 200 OK"
|
| 26 |
+
2026-06-21 18:28:12 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer_config.json "HTTP/1.1 200 OK"
|
| 27 |
+
2026-06-21 18:28:12 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/xlm-roberta-large/tree/main/additional_chat_templates?recursive=false&expand=false "HTTP/1.1 307 Temporary Redirect"
|
| 28 |
+
2026-06-21 18:28:12 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/FacebookAI/xlm-roberta-large/tree/main/additional_chat_templates?recursive=false&expand=false "HTTP/1.1 404 Not Found"
|
| 29 |
+
2026-06-21 18:28:12 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/xlm-roberta-large/tree/main?recursive=true&expand=false "HTTP/1.1 307 Temporary Redirect"
|
| 30 |
+
2026-06-21 18:28:12 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/FacebookAI/xlm-roberta-large/tree/main?recursive=true&expand=false "HTTP/1.1 200 OK"
|
| 31 |
+
2026-06-21 18:28:14 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/xlm-roberta-large "HTTP/1.1 307 Temporary Redirect"
|
| 32 |
+
2026-06-21 18:28:14 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/FacebookAI/xlm-roberta-large "HTTP/1.1 200 OK"
|
| 33 |
+
2026-06-21 18:28:14 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/main/README.md "HTTP/1.1 404 Not Found"
|
| 34 |
+
2026-06-21 18:28:14 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta "HTTP/1.1 200 OK"
|
| 35 |
+
2026-06-21 18:28:14 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/MedBerta.py "HTTP/1.1 404 Not Found"
|
| 36 |
+
2026-06-21 18:28:15 | INFO | httpx | HTTP Request: HEAD https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets/mschonhardt/MedBerta/mschonhardt/MedBerta.py "HTTP/1.1 404 Not Found"
|
| 37 |
+
2026-06-21 18:28:15 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/README.md "HTTP/1.1 404 Not Found"
|
| 38 |
+
2026-06-21 18:28:15 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta/revision/bb4994d1d2ac8765e4df863a5846210c8ecaacd1 "HTTP/1.1 200 OK"
|
| 39 |
+
2026-06-21 18:28:15 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/.huggingface.yaml "HTTP/1.1 404 Not Found"
|
| 40 |
+
2026-06-21 18:28:15 | INFO | httpx | HTTP Request: GET https://datasets-server.huggingface.co/info?dataset=mschonhardt/MedBerta "HTTP/1.1 501 Not Implemented"
|
| 41 |
+
2026-06-21 18:28:16 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta/tree/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/data?recursive=true&expand=false "HTTP/1.1 404 Not Found"
|
| 42 |
+
2026-06-21 18:28:16 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta/tree/bb4994d1d2ac8765e4df863a5846210c8ecaacd1?recursive=false&expand=false "HTTP/1.1 200 OK"
|
| 43 |
+
2026-06-21 18:28:16 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/dataset_infos.json "HTTP/1.1 404 Not Found"
|
| 44 |
+
2026-06-21 18:28:16 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/data-00000-of-00002.arrow "HTTP/1.1 302 Found"
|
| 45 |
+
2026-06-21 18:28:22 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/data-00001-of-00002.arrow "HTTP/1.1 302 Found"
|
| 46 |
+
2026-06-21 18:28:28 | WARNING | dapt | Dataset mschonhardt/MedBerta is PRE-TOKENIZED (columns=['input_ids', 'attention_mask', 'doc_id', 'category']). Skipping raw-text tokenization and DOC-SENTENCES packing.
|
| 47 |
+
2026-06-21 18:28:28 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/main/README.md "HTTP/1.1 404 Not Found"
|
| 48 |
+
2026-06-21 18:28:28 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta "HTTP/1.1 200 OK"
|
| 49 |
+
2026-06-21 18:28:28 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/MedBerta.py "HTTP/1.1 404 Not Found"
|
| 50 |
+
2026-06-21 18:28:28 | INFO | httpx | HTTP Request: HEAD https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets/mschonhardt/MedBerta/mschonhardt/MedBerta.py "HTTP/1.1 404 Not Found"
|
| 51 |
+
2026-06-21 18:28:28 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/README.md "HTTP/1.1 404 Not Found"
|
| 52 |
+
2026-06-21 18:28:29 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/.huggingface.yaml "HTTP/1.1 404 Not Found"
|
| 53 |
+
2026-06-21 18:28:29 | INFO | httpx | HTTP Request: GET https://datasets-server.huggingface.co/info?dataset=mschonhardt/MedBerta "HTTP/1.1 501 Not Implemented"
|
| 54 |
+
2026-06-21 18:28:29 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta/tree/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/data?recursive=true&expand=false "HTTP/1.1 404 Not Found"
|
| 55 |
+
2026-06-21 18:28:29 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/dataset_infos.json "HTTP/1.1 404 Not Found"
|
| 56 |
+
2026-06-21 18:28:29 | INFO | dapt | loaded pretokenized canon -> 291630 sequences (doc_id=doc_id)
|
| 57 |
+
2026-06-21 18:28:35 | INFO | dapt | Pretokenized diagnostics: unk_rate=0.0037% mean_len=500.1 bos=100% eos=100%
|
| 58 |
+
2026-06-21 18:28:36 | INFO | dapt | Applying STRATIFIED document split based on column 'category'.
|
| 59 |
+
2026-06-21 18:28:40 | INFO | dapt | Pretokenized split (document-level STRATIFIED group-aware) -> train=286062 val=5568 test=0
|
| 60 |
+
2026-06-21 18:29:04 | INFO | dapt | Saved pretokenized datasets to .cache_packed/packed_a891125d9987e2ce527ec74b
|
| 61 |
+
2026-06-21 18:29:28 | INFO | dapt | Packed dataset statistics:
|
| 62 |
+
{
|
| 63 |
+
"train": {
|
| 64 |
+
"sequences": 286062,
|
| 65 |
+
"tokens": 142794676,
|
| 66 |
+
"mean_seq_len": 499.2
|
| 67 |
+
},
|
| 68 |
+
"validation": {
|
| 69 |
+
"sequences": 5568,
|
| 70 |
+
"tokens": 2812986,
|
| 71 |
+
"mean_seq_len": 505.2
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
2026-06-21 18:29:28 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/xlm-roberta-large/resolve/main/config.json "HTTP/1.1 200 OK"
|
| 75 |
+
2026-06-21 18:29:28 | INFO | dapt | Model parameters: 560.1M
|
| 76 |
+
2026-06-22 04:00:28 | INFO | dapt | Final evaluation:
|
| 77 |
+
{
|
| 78 |
+
"final_val_loss": 1.0483721494674683,
|
| 79 |
+
"final_val_masked_accuracy": 0.7699716443872566,
|
| 80 |
+
"final_val_runtime": 25.2687,
|
| 81 |
+
"final_val_samples_per_second": 220.351,
|
| 82 |
+
"final_val_steps_per_second": 3.443,
|
| 83 |
+
"epoch": 10.0,
|
| 84 |
+
"num_input_tokens_seen": 1464637440,
|
| 85 |
+
"final_val_perplexity": 2.853003073356264
|
| 86 |
+
}
|
| 87 |
+
2026-06-22 04:00:28 | INFO | dapt | Done. Best model + tokenizer saved to ./MediBerta
|
train.py
ADDED
|
@@ -0,0 +1,1178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
# -*- coding: utf-8 -*-
|
| 3 |
+
"""
|
| 4 |
+
Domain-Adaptive Pretraining (DAPT) of xlm-roberta-large for Medieval Latin.
|
| 5 |
+
|
| 6 |
+
Pipeline
|
| 7 |
+
--------
|
| 8 |
+
1. Load MedBerta (baseline) + CanonBerta (already upsampled) from the HF Hub.
|
| 9 |
+
2. Carve a *group-aware* held-out set: whole documents are held out (by
|
| 10 |
+
``document_id``) so no document leaks across train / validation / test.
|
| 11 |
+
This matters for an honest perplexity and for downstream Loci-Similes /
|
| 12 |
+
text-reuse evaluation.
|
| 13 |
+
3. DOC-SENTENCES packing: paragraphs are grouped by ``document_id`` (kept in
|
| 14 |
+
document order), tokenized, and greedily packed into fixed 512-token
|
| 15 |
+
sequences that never cross a document boundary.
|
| 16 |
+
4. MLM continued pretraining with the HF Trainer, bf16, SDPA attention,
|
| 17 |
+
in-training evaluation (loss + perplexity + masked-token accuracy),
|
| 18 |
+
best-model selection, and rich logging for later write-up.
|
| 19 |
+
|
| 20 |
+
The script is built for a single NVIDIA RTX PRO 6000 Blackwell (96 GB) and is
|
| 21 |
+
deliberately defensive about OOM (auto batch-size search, expandable CUDA
|
| 22 |
+
segments, optional gradient checkpointing, eval logit reduction).
|
| 23 |
+
|
| 24 |
+
Authentication
|
| 25 |
+
--------------
|
| 26 |
+
The datasets are private. Log in once before running:
|
| 27 |
+
huggingface-cli login # or: export HF_TOKEN=hf_...
|
| 28 |
+
|
| 29 |
+
Quick check before spending GPU time (builds + caches datasets, prints stats,
|
| 30 |
+
no training):
|
| 31 |
+
python dapt_xlmr_pretrain.py --dry-run
|
| 32 |
+
|
| 33 |
+
Typical run:
|
| 34 |
+
python dapt_xlmr_pretrain.py \
|
| 35 |
+
--output-dir runs/dapt_xlmr_medlatin_v1 \
|
| 36 |
+
--num-train-epochs 3 \
|
| 37 |
+
--per-device-train-batch-size 32 \
|
| 38 |
+
--gradient-accumulation-steps 8
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
import argparse
|
| 42 |
+
import hashlib
|
| 43 |
+
import inspect
|
| 44 |
+
import json
|
| 45 |
+
import logging
|
| 46 |
+
import math
|
| 47 |
+
import os
|
| 48 |
+
import platform
|
| 49 |
+
import sys
|
| 50 |
+
import traceback
|
| 51 |
+
import unicodedata
|
| 52 |
+
from collections import Counter
|
| 53 |
+
from dataclasses import dataclass, field
|
| 54 |
+
from datetime import datetime
|
| 55 |
+
from typing import Dict, List, Optional
|
| 56 |
+
|
| 57 |
+
# Must be set BEFORE torch is imported to take effect: reduces fragmentation,
|
| 58 |
+
# which is the most common cause of "phantom" OOM on long runs.
|
| 59 |
+
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
| 60 |
+
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") # avoid fork warnings
|
| 61 |
+
|
| 62 |
+
import numpy as np
|
| 63 |
+
import torch
|
| 64 |
+
from datasets import Dataset, concatenate_datasets, load_dataset
|
| 65 |
+
from transformers import (
|
| 66 |
+
AutoModelForMaskedLM,
|
| 67 |
+
AutoTokenizer,
|
| 68 |
+
DataCollatorForLanguageModeling,
|
| 69 |
+
EarlyStoppingCallback,
|
| 70 |
+
Trainer,
|
| 71 |
+
TrainerCallback,
|
| 72 |
+
TrainingArguments,
|
| 73 |
+
set_seed,
|
| 74 |
+
)
|
| 75 |
+
from transformers.trainer_utils import get_last_checkpoint
|
| 76 |
+
|
| 77 |
+
try:
|
| 78 |
+
_USE_PROCESSING_CLASS = "processing_class" in inspect.signature(Trainer.__init__).parameters
|
| 79 |
+
except Exception: # extremely defensive; fall back to legacy name
|
| 80 |
+
_USE_PROCESSING_CLASS = False
|
| 81 |
+
|
| 82 |
+
logging.basicConfig(
|
| 83 |
+
format="%(asctime)s | %(levelname)-7s | %(name)s | %(message)s",
|
| 84 |
+
datefmt="%Y-%m-%d %H:%M:%S",
|
| 85 |
+
level=logging.INFO,
|
| 86 |
+
handlers=[logging.StreamHandler(sys.stdout)],
|
| 87 |
+
)
|
| 88 |
+
logger = logging.getLogger("dapt")
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
# --------------------------------------------------------------------------- #
|
| 92 |
+
# Config
|
| 93 |
+
# --------------------------------------------------------------------------- #
|
| 94 |
+
@dataclass
|
| 95 |
+
class Config:
|
| 96 |
+
# Data
|
| 97 |
+
med_dataset: str = "mschonhardt/MedBerta"
|
| 98 |
+
canon_dataset: str = "mschonhardt/CanonBerta"
|
| 99 |
+
dataset_split: str = "train"
|
| 100 |
+
text_column: Optional[str] = None # auto-detected if None
|
| 101 |
+
doc_id_column: Optional[str] = None # auto-detected if None
|
| 102 |
+
order_column: Optional[str] = None # optional within-doc ordering key
|
| 103 |
+
|
| 104 |
+
# Model / tokenizer
|
| 105 |
+
model_name: str = "xlm-roberta-large"
|
| 106 |
+
max_seq_length: int = 512
|
| 107 |
+
mlm_probability: float = 0.15
|
| 108 |
+
attn_implementation: str = "sdpa" # robust on Blackwell; "flash_attention_2" optional
|
| 109 |
+
min_chunk_tokens: int = 64 # drop doc-final fragments shorter than this
|
| 110 |
+
|
| 111 |
+
# Held-out construction (group-aware, by document_id)
|
| 112 |
+
val_doc_fraction: float = 0.01
|
| 113 |
+
test_doc_fraction: float = 0.0 # set >0 to also hold out a test set
|
| 114 |
+
max_eval_docs_per_source: int = 400 # cap eval size to keep eval fast
|
| 115 |
+
stratify_column: Optional[str] = "category" # e.g. "genre"/"subcorpus": stratified held-out
|
| 116 |
+
split_seed: int = 13
|
| 117 |
+
|
| 118 |
+
# Which corpus to train ON. "combined" = ONE model on MedBerta + upsampled
|
| 119 |
+
# CanonBerta (the methodology you described). "med"/"canon" = a single-corpus
|
| 120 |
+
# model, e.g. for an ablation or two separate deliverables. Held-out eval is
|
| 121 |
+
# still reported per-source where both are present.
|
| 122 |
+
train_corpus: str = "combined" # {"combined", "med", "canon"}
|
| 123 |
+
dedup_train: bool = False # collapse upsampling duplicates in TRAIN
|
| 124 |
+
# (recommended for a standalone canon model)
|
| 125 |
+
|
| 126 |
+
# Pre-flight diagnostics & text handling
|
| 127 |
+
diagnose_tokenizer: bool = True # run UNK / fertility report before training
|
| 128 |
+
tokenizer_sample_size: int = 20000 # paragraphs sampled for the diagnostic
|
| 129 |
+
normalize_nfc: bool = False # apply Unicode NFC before tokenizing
|
| 130 |
+
diagnose_only: bool = False # run diagnostics + reports, then exit
|
| 131 |
+
|
| 132 |
+
# Optimisation (RoBERTa-style defaults, tuned for *continued* pretraining)
|
| 133 |
+
learning_rate: float = 1e-4 # peak LR for the combined DAPT corpus
|
| 134 |
+
weight_decay: float = 0.01
|
| 135 |
+
adam_beta1: float = 0.9
|
| 136 |
+
adam_beta2: float = 0.98
|
| 137 |
+
adam_epsilon: float = 1e-6
|
| 138 |
+
max_grad_norm: float = 1.0
|
| 139 |
+
warmup_ratio: float = 0.06
|
| 140 |
+
lr_scheduler_type: str = "linear"
|
| 141 |
+
num_train_epochs: float = 10.0 # ceiling; early stopping ends training
|
| 142 |
+
max_steps: int = -1
|
| 143 |
+
optim: str = "adamw_torch_fused" # fused AdamW: faster on Blackwell
|
| 144 |
+
early_stopping_patience: int = 5 # eval rounds w/o improvement before stop
|
| 145 |
+
early_stopping_threshold: float = 1e-4 # min eval_loss delta to count as improvement
|
| 146 |
+
|
| 147 |
+
# Throughput / memory (96 GB GDDR7 -> large effective batch)
|
| 148 |
+
per_device_train_batch_size: int = 64
|
| 149 |
+
per_device_eval_batch_size: int = 64
|
| 150 |
+
gradient_accumulation_steps: int = 16 # effective batch ~1024 sequences
|
| 151 |
+
gradient_checkpointing: bool = False
|
| 152 |
+
auto_find_batch_size: bool = True
|
| 153 |
+
torch_compile: bool = False # enable for the final fixed-batch run
|
| 154 |
+
dataloader_num_workers: int = 8
|
| 155 |
+
eval_accumulation_steps: int = 50 # offload eval tensors to CPU periodically
|
| 156 |
+
|
| 157 |
+
# Schedule of eval / logging / checkpointing
|
| 158 |
+
eval_steps: int = 500
|
| 159 |
+
logging_steps: int = 50
|
| 160 |
+
save_steps: int = 500
|
| 161 |
+
save_total_limit: int = 3
|
| 162 |
+
|
| 163 |
+
# Bookkeeping
|
| 164 |
+
output_dir: str = "runs/dapt_xlmr_medlatin"
|
| 165 |
+
cache_dir: str = ".cache_packed"
|
| 166 |
+
seed: int = 42
|
| 167 |
+
report_to_wandb: bool = False
|
| 168 |
+
preprocess_num_proc: int = 16
|
| 169 |
+
resume: bool = True
|
| 170 |
+
dry_run: bool = False
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
# --------------------------------------------------------------------------- #
|
| 174 |
+
# Column auto-detection
|
| 175 |
+
# --------------------------------------------------------------------------- #
|
| 176 |
+
TEXT_CANDIDATES = ["text", "paragraph", "content", "sentence", "passage"]
|
| 177 |
+
DOC_CANDIDATES = ["document_id", "doc_id", "docid", "document", "work_id", "work"]
|
| 178 |
+
ORDER_CANDIDATES = ["order", "paragraph_id", "par_id", "index", "idx", "n", "position", "seq"]
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def _pick(columns: List[str], candidates: List[str], explicit: Optional[str], kind: str) -> Optional[str]:
|
| 182 |
+
if explicit is not None:
|
| 183 |
+
if explicit not in columns:
|
| 184 |
+
raise ValueError(f"Requested {kind} column '{explicit}' not in dataset columns {columns}")
|
| 185 |
+
return explicit
|
| 186 |
+
for c in candidates:
|
| 187 |
+
if c in columns:
|
| 188 |
+
return c
|
| 189 |
+
return None
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
# --------------------------------------------------------------------------- #
|
| 193 |
+
# Dataset loading & group-aware split
|
| 194 |
+
# --------------------------------------------------------------------------- #
|
| 195 |
+
def load_source(repo: str, split: str, source_tag: str, cfg: Config):
|
| 196 |
+
logger.info("Loading %s (split=%s) ...", repo, split)
|
| 197 |
+
ds = load_dataset(repo, split=split, token=True)
|
| 198 |
+
cols = ds.column_names
|
| 199 |
+
|
| 200 |
+
text_col = _pick(cols, TEXT_CANDIDATES, cfg.text_column, "text")
|
| 201 |
+
doc_col = _pick(cols, DOC_CANDIDATES, cfg.doc_id_column, "document_id")
|
| 202 |
+
order_col = _pick(cols, ORDER_CANDIDATES, cfg.order_column, "order") # may be None
|
| 203 |
+
|
| 204 |
+
if text_col is None or doc_col is None:
|
| 205 |
+
raise ValueError(
|
| 206 |
+
f"Could not resolve text/doc_id columns for {repo}. "
|
| 207 |
+
f"Available columns: {cols}. "
|
| 208 |
+
f"Pass --text-column / --doc-id-column explicitly."
|
| 209 |
+
)
|
| 210 |
+
logger.info(" %s -> text='%s', doc_id='%s', order='%s', rows=%d",
|
| 211 |
+
source_tag, text_col, doc_col, order_col, len(ds))
|
| 212 |
+
|
| 213 |
+
# Normalise to a small canonical schema and stamp the source.
|
| 214 |
+
keep = {text_col: "text", doc_col: "document_id"}
|
| 215 |
+
if order_col:
|
| 216 |
+
keep[order_col] = "order"
|
| 217 |
+
strat_col = cfg.stratify_column
|
| 218 |
+
if strat_col is not None:
|
| 219 |
+
if strat_col not in cols:
|
| 220 |
+
raise ValueError(f"--stratify-column '{strat_col}' not in {repo} columns {cols}")
|
| 221 |
+
keep[strat_col] = "stratum"
|
| 222 |
+
ds = ds.rename_columns(keep)
|
| 223 |
+
# The corpora already ship a `source` column ("med"/"canon"). We deliberately
|
| 224 |
+
# drop it here and re-stamp an authoritative per-dataset tag below, so
|
| 225 |
+
# provenance is guaranteed consistent regardless of the incoming values.
|
| 226 |
+
canonical = {"text", "document_id", "order", "stratum"}
|
| 227 |
+
drop = [c for c in ds.column_names if c not in canonical]
|
| 228 |
+
if drop:
|
| 229 |
+
ds = ds.remove_columns(drop)
|
| 230 |
+
if "order" not in ds.column_names:
|
| 231 |
+
# Fallback only (your schema always has `order`): stable within-doc order.
|
| 232 |
+
ds = ds.add_column("order", list(range(len(ds))))
|
| 233 |
+
if "source" in ds.column_names: # safety: never collide with add_column
|
| 234 |
+
ds = ds.remove_columns(["source"])
|
| 235 |
+
ds = ds.add_column("source", [source_tag] * len(ds))
|
| 236 |
+
# document_id is namespaced per source so identical ids in both corpora
|
| 237 |
+
# never collide during splitting/packing.
|
| 238 |
+
ds = ds.map(lambda b: {"document_id": [f"{source_tag}:{d}" for d in b["document_id"]]},
|
| 239 |
+
batched=True, desc=f"namespace doc ids ({source_tag})")
|
| 240 |
+
return ds
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def choose_heldout_docs(ds, cfg: Config):
|
| 244 |
+
"""Pick whole documents to hold out, capped per source, deterministic.
|
| 245 |
+
|
| 246 |
+
If a ``stratum`` column is present (via --stratify-column), documents are
|
| 247 |
+
sampled proportionally per stratum so rare genres are not lost from the
|
| 248 |
+
held-out set; otherwise sampling is uniform-random over document ids.
|
| 249 |
+
"""
|
| 250 |
+
rng = np.random.default_rng(cfg.split_seed)
|
| 251 |
+
n_total_docs = len(set(ds["document_id"]))
|
| 252 |
+
n_val = min(int(round(n_total_docs * cfg.val_doc_fraction)), cfg.max_eval_docs_per_source)
|
| 253 |
+
n_test = min(int(round(n_total_docs * cfg.test_doc_fraction)), cfg.max_eval_docs_per_source)
|
| 254 |
+
n_pick = n_val + n_test
|
| 255 |
+
|
| 256 |
+
if "stratum" in ds.column_names:
|
| 257 |
+
# Map each document to its (majority) stratum.
|
| 258 |
+
doc_stratum: Dict[str, str] = {}
|
| 259 |
+
for d, s in zip(ds["document_id"], ds["stratum"]):
|
| 260 |
+
doc_stratum.setdefault(d, str(s))
|
| 261 |
+
by_stratum: Dict[str, List[str]] = {}
|
| 262 |
+
for d, s in doc_stratum.items():
|
| 263 |
+
by_stratum.setdefault(s, []).append(d)
|
| 264 |
+
picked: List[str] = []
|
| 265 |
+
for s, docs in sorted(by_stratum.items()):
|
| 266 |
+
docs = sorted(docs)
|
| 267 |
+
rng.shuffle(docs)
|
| 268 |
+
# proportional allocation, at least 1 doc per non-empty stratum
|
| 269 |
+
k = max(1, int(round(n_pick * len(docs) / len(doc_stratum))))
|
| 270 |
+
picked.extend(docs[:min(k, len(docs))])
|
| 271 |
+
rng.shuffle(picked)
|
| 272 |
+
picked = picked[:n_pick]
|
| 273 |
+
else:
|
| 274 |
+
doc_ids = sorted(set(ds["document_id"]))
|
| 275 |
+
rng.shuffle(doc_ids)
|
| 276 |
+
picked = doc_ids[:n_pick]
|
| 277 |
+
|
| 278 |
+
val_docs = set(picked[:n_val])
|
| 279 |
+
test_docs = set(picked[n_val:n_val + n_test])
|
| 280 |
+
return val_docs, test_docs
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def representativeness_report(train_ds, val_ds, cfg: Config, out_dir: str):
|
| 284 |
+
"""Compare the held-out validation set against train on cheap, honest
|
| 285 |
+
signals (paragraph char-length distribution, and stratum proportions if
|
| 286 |
+
available) so a skewed eval set is caught before training."""
|
| 287 |
+
def char_lengths(ds):
|
| 288 |
+
out: List[int] = []
|
| 289 |
+
for batch in ds.iter(batch_size=10000):
|
| 290 |
+
out.extend(len(t) for t in batch["text"])
|
| 291 |
+
return np.asarray(out, dtype=np.int64) if out else np.array([0], dtype=np.int64)
|
| 292 |
+
|
| 293 |
+
def pct(a):
|
| 294 |
+
if len(a) == 0:
|
| 295 |
+
return {}
|
| 296 |
+
return {f"p{p}": float(np.percentile(a, p)) for p in (5, 25, 50, 75, 95)}
|
| 297 |
+
|
| 298 |
+
tr_len, va_len = char_lengths(train_ds), char_lengths(val_ds)
|
| 299 |
+
report = {
|
| 300 |
+
"train_paragraphs": len(train_ds),
|
| 301 |
+
"val_paragraphs": len(val_ds),
|
| 302 |
+
"char_len_train": {"mean": float(tr_len.mean()), **pct(tr_len)},
|
| 303 |
+
"char_len_val": {"mean": float(va_len.mean()), **pct(va_len)},
|
| 304 |
+
}
|
| 305 |
+
if "stratum" in train_ds.column_names and "stratum" in val_ds.column_names:
|
| 306 |
+
def props(ds):
|
| 307 |
+
c = Counter(str(s) for s in ds["stratum"])
|
| 308 |
+
tot = sum(c.values()) or 1
|
| 309 |
+
return {k: v / tot for k, v in c.items()}
|
| 310 |
+
tr_p, va_p = props(train_ds), props(val_ds)
|
| 311 |
+
keys = sorted(set(tr_p) | set(va_p))
|
| 312 |
+
# total variation distance between the two stratum distributions
|
| 313 |
+
tvd = 0.5 * sum(abs(tr_p.get(k, 0.0) - va_p.get(k, 0.0)) for k in keys)
|
| 314 |
+
report["stratum_proportions_train"] = {k: round(tr_p.get(k, 0.0), 4) for k in keys}
|
| 315 |
+
report["stratum_proportions_val"] = {k: round(va_p.get(k, 0.0), 4) for k in keys}
|
| 316 |
+
report["stratum_total_variation_distance"] = round(tvd, 4)
|
| 317 |
+
missing = [k for k in tr_p if k not in va_p]
|
| 318 |
+
if missing:
|
| 319 |
+
report["strata_absent_from_val"] = missing
|
| 320 |
+
|
| 321 |
+
os.makedirs(out_dir, exist_ok=True)
|
| 322 |
+
with open(os.path.join(out_dir, "eval_representativeness.json"), "w") as fh:
|
| 323 |
+
json.dump(report, fh, indent=2, ensure_ascii=False)
|
| 324 |
+
logger.info("Eval representativeness:\n%s", json.dumps(report, indent=2, ensure_ascii=False))
|
| 325 |
+
if report.get("strata_absent_from_val"):
|
| 326 |
+
logger.warning("Strata present in train but MISSING from validation: %s",
|
| 327 |
+
report["strata_absent_from_val"])
|
| 328 |
+
if report.get("stratum_total_variation_distance", 0.0) > 0.15:
|
| 329 |
+
logger.warning("Validation stratum distribution differs notably from train "
|
| 330 |
+
"(TVD=%.3f); consider --stratify-column or a larger eval cap.",
|
| 331 |
+
report["stratum_total_variation_distance"])
|
| 332 |
+
return report
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def dedup_rows(ds):
|
| 336 |
+
"""Eval honesty: collapse exact (document_id, order, text) duplicates that
|
| 337 |
+
upsampling introduced, so every held-out paragraph is counted once."""
|
| 338 |
+
seen = set()
|
| 339 |
+
keep_idx = []
|
| 340 |
+
for i, (d, o, t) in enumerate(zip(ds["document_id"], ds["order"], ds["text"])):
|
| 341 |
+
key = hashlib.blake2b(f"{d}\x1f{o}\x1f{t}".encode("utf-8"), digest_size=16).digest()
|
| 342 |
+
if key not in seen:
|
| 343 |
+
seen.add(key)
|
| 344 |
+
keep_idx.append(i)
|
| 345 |
+
return ds.select(keep_idx)
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
# --------------------------------------------------------------------------- #
|
| 349 |
+
# Tokenization + DOC-SENTENCES packing
|
| 350 |
+
# --------------------------------------------------------------------------- #
|
| 351 |
+
def maybe_nfc(text: str, cfg: Config) -> str:
|
| 352 |
+
return unicodedata.normalize("NFC", text) if cfg.normalize_nfc else text
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
def diagnose_tokenizer(texts: List[str], tokenizer, cfg: Config, out_dir: str):
|
| 356 |
+
"""Pre-flight check for SentencePiece behaviour on historical text."""
|
| 357 |
+
unk = tokenizer.unk_token
|
| 358 |
+
unk_id = tokenizer.unk_token_id
|
| 359 |
+
rng = np.random.default_rng(cfg.split_seed)
|
| 360 |
+
if len(texts) > cfg.tokenizer_sample_size:
|
| 361 |
+
idx = rng.choice(len(texts), size=cfg.tokenizer_sample_size, replace=False)
|
| 362 |
+
texts = [texts[i] for i in idx]
|
| 363 |
+
|
| 364 |
+
total_tokens = 0
|
| 365 |
+
total_unk = 0
|
| 366 |
+
total_chars = 0
|
| 367 |
+
total_words = 0
|
| 368 |
+
tok_lengths: List[int] = []
|
| 369 |
+
unk_char_counter: Counter = Counter()
|
| 370 |
+
pathological: List[dict] = []
|
| 371 |
+
|
| 372 |
+
for t in texts:
|
| 373 |
+
t = maybe_nfc(t, cfg)
|
| 374 |
+
toks = tokenizer.tokenize(t)
|
| 375 |
+
n = len(toks)
|
| 376 |
+
n_unk = sum(1 for x in toks if x == unk)
|
| 377 |
+
total_tokens += n
|
| 378 |
+
total_unk += n_unk
|
| 379 |
+
total_chars += len(t)
|
| 380 |
+
total_words += max(1, len(t.split()))
|
| 381 |
+
tok_lengths.append(n)
|
| 382 |
+
if len(t) >= 20 and n / max(1, len(t)) > 1.5:
|
| 383 |
+
if len(pathological) < 25:
|
| 384 |
+
pathological.append({"chars": len(t), "tokens": n, "preview": t[:80]})
|
| 385 |
+
|
| 386 |
+
uniq_chars = set()
|
| 387 |
+
for t in texts[:5000]:
|
| 388 |
+
uniq_chars.update(maybe_nfc(t, cfg))
|
| 389 |
+
risky_chars = {}
|
| 390 |
+
for ch in uniq_chars:
|
| 391 |
+
if ch.isspace():
|
| 392 |
+
continue
|
| 393 |
+
ids = tokenizer(ch, add_special_tokens=False)["input_ids"]
|
| 394 |
+
if unk_id is not None and unk_id in ids:
|
| 395 |
+
name = unicodedata.name(ch, "UNNAMED")
|
| 396 |
+
risky_chars[ch] = {"codepoint": f"U+{ord(ch):04X}", "name": name}
|
| 397 |
+
|
| 398 |
+
tok_lengths_arr = np.asarray(tok_lengths) if tok_lengths else np.array([0])
|
| 399 |
+
report = {
|
| 400 |
+
"sampled_paragraphs": len(texts),
|
| 401 |
+
"normalize_nfc": cfg.normalize_nfc,
|
| 402 |
+
"unk_token": unk,
|
| 403 |
+
"unk_rate": round(total_unk / max(1, total_tokens), 6),
|
| 404 |
+
"total_unk_tokens": total_unk,
|
| 405 |
+
"fertility_tokens_per_word": round(total_tokens / max(1, total_words), 3),
|
| 406 |
+
"fertility_tokens_per_char": round(total_tokens / max(1, total_chars), 3),
|
| 407 |
+
"tokens_per_paragraph": {
|
| 408 |
+
"mean": float(tok_lengths_arr.mean()),
|
| 409 |
+
"p50": float(np.percentile(tok_lengths_arr, 50)),
|
| 410 |
+
"p95": float(np.percentile(tok_lengths_arr, 95)),
|
| 411 |
+
"max": int(tok_lengths_arr.max()),
|
| 412 |
+
"share_over_max_seq_len": round(
|
| 413 |
+
float((tok_lengths_arr > cfg.max_seq_length).mean()), 4),
|
| 414 |
+
},
|
| 415 |
+
"num_risky_unk_characters": len(risky_chars),
|
| 416 |
+
"risky_unk_characters": dict(sorted(risky_chars.items())[:100]),
|
| 417 |
+
"pathological_examples": pathological,
|
| 418 |
+
}
|
| 419 |
+
os.makedirs(out_dir, exist_ok=True)
|
| 420 |
+
with open(os.path.join(out_dir, "tokenizer_diagnostics.json"), "w") as fh:
|
| 421 |
+
json.dump(report, fh, indent=2, ensure_ascii=False)
|
| 422 |
+
|
| 423 |
+
logger.info("Tokenizer diagnostics: unk_rate=%.4f%% fertility=%.2f tok/word "
|
| 424 |
+
"risky_chars=%d", report["unk_rate"] * 100,
|
| 425 |
+
report["fertility_tokens_per_word"], report["num_risky_unk_characters"])
|
| 426 |
+
if report["unk_rate"] > 0.005:
|
| 427 |
+
logger.warning("High <unk> rate (%.3f%%). Inspect risky_unk_characters in "
|
| 428 |
+
"tokenizer_diagnostics.json; consider --normalize-nfc or a "
|
| 429 |
+
"transliteration/cleanup pass for medieval glyphs.",
|
| 430 |
+
report["unk_rate"] * 100)
|
| 431 |
+
if risky_chars:
|
| 432 |
+
sample = ", ".join(f"{c} ({m['codepoint']})" for c, m in list(risky_chars.items())[:15])
|
| 433 |
+
logger.warning("Characters mapping to <unk> (sample): %s", sample)
|
| 434 |
+
return report
|
| 435 |
+
|
| 436 |
+
|
| 437 |
+
def tokenize_paragraphs(ds, tokenizer, cfg: Config):
|
| 438 |
+
def _tok(batch):
|
| 439 |
+
texts = [maybe_nfc(t, cfg) for t in batch["text"]] if cfg.normalize_nfc else batch["text"]
|
| 440 |
+
enc = tokenizer(texts, add_special_tokens=False,
|
| 441 |
+
truncation=False, return_attention_mask=False)
|
| 442 |
+
return {"input_ids": enc["input_ids"]}
|
| 443 |
+
|
| 444 |
+
return ds.map(
|
| 445 |
+
_tok,
|
| 446 |
+
batched=True,
|
| 447 |
+
num_proc=cfg.preprocess_num_proc,
|
| 448 |
+
remove_columns=["text"],
|
| 449 |
+
desc="tokenize paragraphs",
|
| 450 |
+
)
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
def pack_doc_sentences(ds, tokenizer, cfg: Config, desc: str) -> Dataset:
|
| 454 |
+
bos = tokenizer.bos_token_id if tokenizer.bos_token_id is not None else tokenizer.cls_token_id
|
| 455 |
+
eos = tokenizer.eos_token_id if tokenizer.eos_token_id is not None else tokenizer.sep_token_id
|
| 456 |
+
if bos is None or eos is None:
|
| 457 |
+
raise ValueError("Tokenizer has no bos/cls or eos/sep token; cannot pack.")
|
| 458 |
+
max_content = cfg.max_seq_length - 2
|
| 459 |
+
|
| 460 |
+
sort_keys = ["document_id", "order"] if "order" in ds.column_names else ["document_id"]
|
| 461 |
+
ds = ds.sort(sort_keys)
|
| 462 |
+
|
| 463 |
+
def generator():
|
| 464 |
+
buffer: List[int] = []
|
| 465 |
+
cur_doc = None
|
| 466 |
+
cur_src = None
|
| 467 |
+
|
| 468 |
+
def flush_full():
|
| 469 |
+
nonlocal buffer
|
| 470 |
+
while len(buffer) >= max_content:
|
| 471 |
+
chunk = buffer[:max_content]
|
| 472 |
+
buffer = buffer[max_content:]
|
| 473 |
+
yield {"input_ids": [bos] + chunk + [eos],
|
| 474 |
+
"document_id": cur_doc, "source": cur_src}
|
| 475 |
+
|
| 476 |
+
for batch in ds.iter(batch_size=2000):
|
| 477 |
+
ids_col = batch["input_ids"]
|
| 478 |
+
doc_col = batch["document_id"]
|
| 479 |
+
src_col = batch["source"]
|
| 480 |
+
for ids, doc, src in zip(ids_col, doc_col, src_col):
|
| 481 |
+
if doc != cur_doc:
|
| 482 |
+
if buffer and len(buffer) >= cfg.min_chunk_tokens:
|
| 483 |
+
yield {"input_ids": [bos] + buffer + [eos],
|
| 484 |
+
"document_id": cur_doc, "source": cur_src}
|
| 485 |
+
buffer = []
|
| 486 |
+
cur_doc, cur_src = doc, src
|
| 487 |
+
buffer.extend(ids)
|
| 488 |
+
yield from flush_full()
|
| 489 |
+
if buffer and len(buffer) >= cfg.min_chunk_tokens:
|
| 490 |
+
yield {"input_ids": [bos] + buffer + [eos],
|
| 491 |
+
"document_id": cur_doc, "source": cur_src}
|
| 492 |
+
|
| 493 |
+
packed = Dataset.from_generator(generator, cache_dir=cfg.cache_dir)
|
| 494 |
+
logger.info(" packed %s -> %d sequences (max_len=%d)", desc, len(packed), cfg.max_seq_length)
|
| 495 |
+
return packed
|
| 496 |
+
|
| 497 |
+
|
| 498 |
+
def diagnose_pretokenized(ds, tokenizer, cfg: Config, out_dir: str):
|
| 499 |
+
unk_id = tokenizer.unk_token_id
|
| 500 |
+
bos_id = tokenizer.bos_token_id if tokenizer.bos_token_id is not None else tokenizer.cls_token_id
|
| 501 |
+
eos_id = tokenizer.eos_token_id if tokenizer.eos_token_id is not None else tokenizer.sep_token_id
|
| 502 |
+
n = len(ds)
|
| 503 |
+
m = min(cfg.tokenizer_sample_size, n)
|
| 504 |
+
idx = np.random.default_rng(cfg.split_seed).choice(n, size=m, replace=False)
|
| 505 |
+
sample = ds.select(idx)
|
| 506 |
+
|
| 507 |
+
total_tokens = 0
|
| 508 |
+
total_unk = 0
|
| 509 |
+
lengths: List[int] = []
|
| 510 |
+
bos_ok = 0
|
| 511 |
+
eos_ok = 0
|
| 512 |
+
over_len = 0
|
| 513 |
+
for ids in sample["input_ids"]:
|
| 514 |
+
L = len(ids)
|
| 515 |
+
lengths.append(L)
|
| 516 |
+
total_tokens += L
|
| 517 |
+
if unk_id is not None:
|
| 518 |
+
total_unk += ids.count(unk_id)
|
| 519 |
+
if L and ids[0] == bos_id:
|
| 520 |
+
bos_ok += 1
|
| 521 |
+
if L and ids[-1] == eos_id:
|
| 522 |
+
eos_ok += 1
|
| 523 |
+
if L > cfg.max_seq_length:
|
| 524 |
+
over_len += 1
|
| 525 |
+
|
| 526 |
+
arr = np.asarray(lengths) if lengths else np.array([0])
|
| 527 |
+
report = {
|
| 528 |
+
"mode": "pretokenized",
|
| 529 |
+
"sampled_sequences": int(m),
|
| 530 |
+
"total_sequences": int(n),
|
| 531 |
+
"unk_rate": round(total_unk / max(1, total_tokens), 6),
|
| 532 |
+
"total_unk_tokens": int(total_unk),
|
| 533 |
+
"seq_len": {"mean": float(arr.mean()), "p50": float(np.percentile(arr, 50)),
|
| 534 |
+
"p95": float(np.percentile(arr, 95)), "max": int(arr.max())},
|
| 535 |
+
"share_over_max_seq_len": round(float(over_len) / max(1, m), 4),
|
| 536 |
+
"starts_with_bos_rate": round(bos_ok / max(1, m), 4),
|
| 537 |
+
"ends_with_eos_rate": round(eos_ok / max(1, m), 4),
|
| 538 |
+
}
|
| 539 |
+
os.makedirs(out_dir, exist_ok=True)
|
| 540 |
+
with open(os.path.join(out_dir, "tokenizer_diagnostics.json"), "w") as fh:
|
| 541 |
+
json.dump(report, fh, indent=2, ensure_ascii=False)
|
| 542 |
+
logger.info("Pretokenized diagnostics: unk_rate=%.4f%% mean_len=%.1f "
|
| 543 |
+
"bos=%.0f%% eos=%.0f%%", report["unk_rate"] * 100, report["seq_len"]["mean"],
|
| 544 |
+
report["starts_with_bos_rate"] * 100, report["ends_with_eos_rate"] * 100)
|
| 545 |
+
if report["unk_rate"] > 0.005:
|
| 546 |
+
logger.warning("High <unk> rate (%.3f%%) in the pre-tokenized data — verify it was "
|
| 547 |
+
"tokenized with the SAME tokenizer (%s).", report["unk_rate"] * 100, cfg.model_name)
|
| 548 |
+
if report["starts_with_bos_rate"] < 0.5 or report["ends_with_eos_rate"] < 0.5:
|
| 549 |
+
logger.warning("Many sequences lack bos/eos boundaries; if your packing omitted special "
|
| 550 |
+
"tokens, MLM quality may suffer.")
|
| 551 |
+
if report["share_over_max_seq_len"] > 0:
|
| 552 |
+
logger.warning("%.2f%% of sequences exceed max_seq_length=%d and will be truncated.",
|
| 553 |
+
report["share_over_max_seq_len"] * 100, cfg.max_seq_length)
|
| 554 |
+
return report
|
| 555 |
+
|
| 556 |
+
|
| 557 |
+
def _load_dataset_any(name: str, split: str, token: bool):
|
| 558 |
+
if os.path.isdir(name):
|
| 559 |
+
from datasets import load_from_disk
|
| 560 |
+
d = load_from_disk(name)
|
| 561 |
+
if hasattr(d, "keys") and not hasattr(d, "column_names"): # DatasetDict
|
| 562 |
+
d = d[split]
|
| 563 |
+
return d
|
| 564 |
+
return load_dataset(name, split=split, token=token)
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def _ids_hash(ids) -> bytes:
|
| 568 |
+
return hashlib.blake2b(np.asarray(ids, dtype=np.int32).tobytes(), digest_size=12).digest()
|
| 569 |
+
|
| 570 |
+
|
| 571 |
+
def build_pretokenized_bundle(cfg: Config, tokenizer, selected, cache_root: str):
|
| 572 |
+
max_len = cfg.max_seq_length
|
| 573 |
+
DOCID_CANDS = ("doc_id", "document_id", "docid")
|
| 574 |
+
parts = []
|
| 575 |
+
|
| 576 |
+
for repo, tag in selected:
|
| 577 |
+
ds = _load_dataset_any(repo, cfg.dataset_split, True)
|
| 578 |
+
docid_col = next((c for c in DOCID_CANDS if c in ds.column_names), None)
|
| 579 |
+
|
| 580 |
+
# ----- WISSENSCHAFTLICHE KORREKTUR: Metadaten beibehalten -----
|
| 581 |
+
keep = ["input_ids"]
|
| 582 |
+
if "attention_mask" in ds.column_names:
|
| 583 |
+
keep.append("attention_mask")
|
| 584 |
+
if docid_col:
|
| 585 |
+
keep.append(docid_col)
|
| 586 |
+
|
| 587 |
+
strat_col = cfg.stratify_column
|
| 588 |
+
if strat_col and strat_col in ds.column_names:
|
| 589 |
+
keep.append(strat_col)
|
| 590 |
+
|
| 591 |
+
ds = ds.remove_columns([c for c in ds.column_names if c not in keep])
|
| 592 |
+
|
| 593 |
+
if docid_col and docid_col != "document_id":
|
| 594 |
+
ds = ds.rename_column(docid_col, "document_id")
|
| 595 |
+
if strat_col and strat_col in ds.column_names and strat_col != "stratum":
|
| 596 |
+
ds = ds.rename_column(strat_col, "stratum")
|
| 597 |
+
# --------------------------------------------------------------
|
| 598 |
+
|
| 599 |
+
ds = ds.add_column("source", [tag] * len(ds))
|
| 600 |
+
parts.append(ds)
|
| 601 |
+
logger.info(" loaded pretokenized %s -> %d sequences (doc_id=%s)",
|
| 602 |
+
tag, len(ds), docid_col)
|
| 603 |
+
|
| 604 |
+
full = concatenate_datasets(parts) if len(parts) > 1 else parts[0]
|
| 605 |
+
has_doc = "document_id" in full.column_names
|
| 606 |
+
|
| 607 |
+
if has_doc:
|
| 608 |
+
full = full.map(
|
| 609 |
+
lambda b: {"document_id": [f"{s}:{d}" for s, d in zip(b["source"], b["document_id"])]},
|
| 610 |
+
batched=True, desc="namespace doc ids")
|
| 611 |
+
|
| 612 |
+
has_mask = "attention_mask" in full.column_names
|
| 613 |
+
|
| 614 |
+
def _fix(batch):
|
| 615 |
+
out = []
|
| 616 |
+
masks = batch["attention_mask"] if has_mask else [None] * len(batch["input_ids"])
|
| 617 |
+
for ids, am in zip(batch["input_ids"], masks):
|
| 618 |
+
if am is not None:
|
| 619 |
+
ids = [t for t, a in zip(ids, am) if a == 1]
|
| 620 |
+
if len(ids) > max_len:
|
| 621 |
+
ids = ids[:max_len]
|
| 622 |
+
out.append(ids)
|
| 623 |
+
return {"input_ids": out}
|
| 624 |
+
|
| 625 |
+
full = full.map(_fix, batched=True, num_proc=cfg.preprocess_num_proc,
|
| 626 |
+
remove_columns=(["attention_mask"] if has_mask else []),
|
| 627 |
+
desc="normalize pretokenized")
|
| 628 |
+
|
| 629 |
+
out_dir = cfg.output_dir
|
| 630 |
+
if cfg.diagnose_tokenizer:
|
| 631 |
+
diagnose_pretokenized(full, tokenizer, cfg, out_dir)
|
| 632 |
+
|
| 633 |
+
rng = np.random.default_rng(cfg.split_seed)
|
| 634 |
+
train_idx: List[int] = []
|
| 635 |
+
val_idx: List[int] = []
|
| 636 |
+
test_idx: List[int] = []
|
| 637 |
+
|
| 638 |
+
if has_doc:
|
| 639 |
+
docs = full["document_id"]
|
| 640 |
+
unique_docs = sorted(set(docs))
|
| 641 |
+
n_val = min(int(round(len(unique_docs) * cfg.val_doc_fraction)), cfg.max_eval_docs_per_source)
|
| 642 |
+
n_test = min(int(round(len(unique_docs) * cfg.test_doc_fraction)), cfg.max_eval_docs_per_source)
|
| 643 |
+
n_pick = n_val + n_test
|
| 644 |
+
|
| 645 |
+
# ----- WISSENSCHAFTLICHE KORREKTUR: Stratified Split -----
|
| 646 |
+
if "stratum" in full.column_names:
|
| 647 |
+
logger.info(f"Applying STRATIFIED document split based on column '{cfg.stratify_column}'.")
|
| 648 |
+
strata = full["stratum"]
|
| 649 |
+
doc_stratum: Dict[str, str] = {}
|
| 650 |
+
for d, s in zip(docs, strata):
|
| 651 |
+
doc_stratum.setdefault(d, str(s))
|
| 652 |
+
by_stratum: Dict[str, List[str]] = {}
|
| 653 |
+
for d, s in doc_stratum.items():
|
| 654 |
+
by_stratum.setdefault(s, []).append(d)
|
| 655 |
+
|
| 656 |
+
picked: List[str] = []
|
| 657 |
+
for s, d_list in sorted(by_stratum.items()):
|
| 658 |
+
d_list = sorted(d_list)
|
| 659 |
+
rng.shuffle(d_list)
|
| 660 |
+
k = max(1, int(round(n_pick * len(d_list) / len(doc_stratum))))
|
| 661 |
+
picked.extend(d_list[:min(k, len(d_list))])
|
| 662 |
+
rng.shuffle(picked)
|
| 663 |
+
val_docs = set(picked[:n_val])
|
| 664 |
+
test_docs = set(picked[n_val:n_val + n_test])
|
| 665 |
+
split_kind = "document-level STRATIFIED group-aware"
|
| 666 |
+
else:
|
| 667 |
+
rng.shuffle(unique_docs)
|
| 668 |
+
val_docs = set(unique_docs[:n_val])
|
| 669 |
+
test_docs = set(unique_docs[n_val:n_val + n_test])
|
| 670 |
+
split_kind = "document-level group-aware"
|
| 671 |
+
# ---------------------------------------------------------
|
| 672 |
+
|
| 673 |
+
val_cand, test_cand = [], []
|
| 674 |
+
for i, d in enumerate(docs):
|
| 675 |
+
if d in val_docs:
|
| 676 |
+
val_cand.append((i, d))
|
| 677 |
+
elif d in test_docs:
|
| 678 |
+
test_cand.append((i, d))
|
| 679 |
+
else:
|
| 680 |
+
train_idx.append(i)
|
| 681 |
+
|
| 682 |
+
for cand, dst in ((val_cand, val_idx), (test_cand, test_idx)):
|
| 683 |
+
seen = set()
|
| 684 |
+
for i, d in cand:
|
| 685 |
+
key = (d, _ids_hash(full[i]["input_ids"]))
|
| 686 |
+
if key not in seen:
|
| 687 |
+
seen.add(key)
|
| 688 |
+
dst.append(i)
|
| 689 |
+
else:
|
| 690 |
+
logger.warning("No doc_id/document_id column found in the pre-tokenized data; "
|
| 691 |
+
"falling back to a weaker sequence-level holdout.")
|
| 692 |
+
hashes = [_ids_hash(ids) for ids in full["input_ids"]]
|
| 693 |
+
uniq = sorted(set(hashes))
|
| 694 |
+
rng.shuffle(uniq)
|
| 695 |
+
n_val = min(int(round(len(uniq) * cfg.val_doc_fraction)), cfg.max_eval_docs_per_source * 4)
|
| 696 |
+
n_test = min(int(round(len(uniq) * cfg.test_doc_fraction)), cfg.max_eval_docs_per_source * 4)
|
| 697 |
+
val_h = set(uniq[:n_val])
|
| 698 |
+
test_h = set(uniq[n_val:n_val + n_test])
|
| 699 |
+
seen_v, seen_t = set(), set()
|
| 700 |
+
for i, h in enumerate(hashes):
|
| 701 |
+
if h in val_h:
|
| 702 |
+
if h not in seen_v:
|
| 703 |
+
seen_v.add(h); val_idx.append(i)
|
| 704 |
+
elif h in test_h:
|
| 705 |
+
if h not in seen_t:
|
| 706 |
+
seen_t.add(h); test_idx.append(i)
|
| 707 |
+
else:
|
| 708 |
+
train_idx.append(i)
|
| 709 |
+
split_kind = "sequence-level dedup"
|
| 710 |
+
|
| 711 |
+
bundle: Dict[str, Dataset] = {"train": full.select(train_idx)}
|
| 712 |
+
if val_idx:
|
| 713 |
+
bundle["validation"] = full.select(val_idx)
|
| 714 |
+
if len(set(bundle["validation"]["source"])) > 1:
|
| 715 |
+
for tag in ("med", "canon"):
|
| 716 |
+
sub = bundle["validation"].filter(lambda b: [s == tag for s in b["source"]], batched=True)
|
| 717 |
+
if len(sub):
|
| 718 |
+
bundle[f"validation_{tag}"] = sub
|
| 719 |
+
if test_idx:
|
| 720 |
+
bundle["test"] = full.select(test_idx)
|
| 721 |
+
|
| 722 |
+
logger.info("Pretokenized split (%s) -> train=%d val=%d test=%d",
|
| 723 |
+
split_kind, len(bundle["train"]), len(val_idx), len(test_idx))
|
| 724 |
+
|
| 725 |
+
def len_stats(d):
|
| 726 |
+
ls = np.asarray([len(x) for x in d["input_ids"]]) if len(d) else np.array([0])
|
| 727 |
+
return {"mean": float(ls.mean()), "p50": float(np.percentile(ls, 50)),
|
| 728 |
+
"p95": float(np.percentile(ls, 95))}
|
| 729 |
+
rep = {"split_kind": split_kind,
|
| 730 |
+
"train_sequences": len(bundle["train"]), "val_sequences": len(val_idx),
|
| 731 |
+
"len_train": len_stats(bundle["train"])}
|
| 732 |
+
if val_idx:
|
| 733 |
+
rep["len_val"] = len_stats(bundle["validation"])
|
| 734 |
+
with open(os.path.join(out_dir, "eval_representativeness.json"), "w") as fh:
|
| 735 |
+
json.dump(rep, fh, indent=2)
|
| 736 |
+
|
| 737 |
+
if cfg.diagnose_only:
|
| 738 |
+
logger.info("--diagnose-only set: wrote diagnostics to %s, exiting before training.", out_dir)
|
| 739 |
+
sys.exit(0)
|
| 740 |
+
|
| 741 |
+
os.makedirs(cache_root, exist_ok=True)
|
| 742 |
+
for name, d in bundle.items():
|
| 743 |
+
d.save_to_disk(os.path.join(cache_root, name))
|
| 744 |
+
logger.info("Saved pretokenized datasets to %s", cache_root)
|
| 745 |
+
return bundle
|
| 746 |
+
|
| 747 |
+
|
| 748 |
+
def build_or_load_packed(cfg: Config, tokenizer):
|
| 749 |
+
sig = json.dumps({
|
| 750 |
+
"med": cfg.med_dataset, "canon": cfg.canon_dataset, "split": cfg.dataset_split,
|
| 751 |
+
"model": cfg.model_name, "max_len": cfg.max_seq_length, "min_chunk": cfg.min_chunk_tokens,
|
| 752 |
+
"val_frac": cfg.val_doc_fraction, "test_frac": cfg.test_doc_fraction,
|
| 753 |
+
"max_eval": cfg.max_eval_docs_per_source, "split_seed": cfg.split_seed,
|
| 754 |
+
"text_col": cfg.text_column, "doc_col": cfg.doc_id_column, "order_col": cfg.order_column,
|
| 755 |
+
"stratify": cfg.stratify_column, "nfc": cfg.normalize_nfc,
|
| 756 |
+
"train_corpus": cfg.train_corpus, "dedup_train": cfg.dedup_train,
|
| 757 |
+
}, sort_keys=True)
|
| 758 |
+
key = hashlib.blake2b(sig.encode("utf-8"), digest_size=12).hexdigest()
|
| 759 |
+
cache_root = os.path.join(cfg.cache_dir, f"packed_{key}")
|
| 760 |
+
|
| 761 |
+
if os.path.isdir(cache_root) and not cfg.diagnose_only:
|
| 762 |
+
from datasets import load_from_disk
|
| 763 |
+
logger.info("Loading cached packed datasets from %s", cache_root)
|
| 764 |
+
bundle = {name: load_from_disk(os.path.join(cache_root, name))
|
| 765 |
+
for name in os.listdir(cache_root)
|
| 766 |
+
if os.path.isdir(os.path.join(cache_root, name))}
|
| 767 |
+
return bundle
|
| 768 |
+
|
| 769 |
+
if cfg.train_corpus not in {"combined", "med", "canon"}:
|
| 770 |
+
raise ValueError(f"--train-corpus must be combined|med|canon, got {cfg.train_corpus!r}")
|
| 771 |
+
|
| 772 |
+
selected = []
|
| 773 |
+
if cfg.train_corpus in {"combined", "med"}:
|
| 774 |
+
selected.append((cfg.med_dataset, "med"))
|
| 775 |
+
if cfg.train_corpus in {"combined", "canon"}:
|
| 776 |
+
selected.append((cfg.canon_dataset, "canon"))
|
| 777 |
+
|
| 778 |
+
probe = _load_dataset_any(selected[0][0], cfg.dataset_split, True)
|
| 779 |
+
if "input_ids" in probe.column_names:
|
| 780 |
+
logger.warning(
|
| 781 |
+
"Dataset %s is PRE-TOKENIZED (columns=%s). Skipping raw-text tokenization "
|
| 782 |
+
"and DOC-SENTENCES packing.", selected[0][0], probe.column_names)
|
| 783 |
+
del probe
|
| 784 |
+
return build_pretokenized_bundle(cfg, tokenizer, selected, cache_root)
|
| 785 |
+
del probe
|
| 786 |
+
|
| 787 |
+
sources = []
|
| 788 |
+
if cfg.train_corpus in {"combined", "med"}:
|
| 789 |
+
sources.append(load_source(cfg.med_dataset, cfg.dataset_split, "med", cfg))
|
| 790 |
+
if cfg.train_corpus in {"combined", "canon"}:
|
| 791 |
+
sources.append(load_source(cfg.canon_dataset, cfg.dataset_split, "canon", cfg))
|
| 792 |
+
logger.info("Training corpus = %s (%d source dataset(s))", cfg.train_corpus, len(sources))
|
| 793 |
+
|
| 794 |
+
train_parts, val_parts, test_parts = [], [], []
|
| 795 |
+
for src in sources:
|
| 796 |
+
tag = src["source"][0]
|
| 797 |
+
val_docs, test_docs = choose_heldout_docs(src, cfg)
|
| 798 |
+
in_val = src.filter(lambda b: [d in val_docs for d in b["document_id"]],
|
| 799 |
+
batched=True, desc=f"select val ({tag})")
|
| 800 |
+
in_test = src.filter(lambda b: [d in test_docs for d in b["document_id"]],
|
| 801 |
+
batched=True, desc=f"select test ({tag})")
|
| 802 |
+
held = val_docs | test_docs
|
| 803 |
+
in_train = src.filter(lambda b: [d not in held for d in b["document_id"]],
|
| 804 |
+
batched=True, desc=f"select train ({tag})")
|
| 805 |
+
train_parts.append(in_train)
|
| 806 |
+
if len(in_val):
|
| 807 |
+
val_parts.append(dedup_rows(in_val))
|
| 808 |
+
if len(in_test):
|
| 809 |
+
test_parts.append(dedup_rows(in_test))
|
| 810 |
+
|
| 811 |
+
train_raw = concatenate_datasets(train_parts)
|
| 812 |
+
if cfg.dedup_train:
|
| 813 |
+
before = len(train_raw)
|
| 814 |
+
train_raw = dedup_rows(train_raw)
|
| 815 |
+
logger.info("dedup_train: collapsed %d -> %d training paragraphs "
|
| 816 |
+
"(upsampling duplicates removed).", before, len(train_raw))
|
| 817 |
+
val_raw = concatenate_datasets(val_parts) if val_parts else None
|
| 818 |
+
logger.info("Raw paragraph counts -> train=%d val=%d test=%d",
|
| 819 |
+
len(train_raw),
|
| 820 |
+
sum(len(p) for p in val_parts),
|
| 821 |
+
sum(len(p) for p in test_parts))
|
| 822 |
+
|
| 823 |
+
out_dir = cfg.output_dir
|
| 824 |
+
if cfg.diagnose_tokenizer:
|
| 825 |
+
n = len(train_raw)
|
| 826 |
+
m = min(cfg.tokenizer_sample_size, n)
|
| 827 |
+
idx = np.random.default_rng(cfg.split_seed).choice(n, size=m, replace=False)
|
| 828 |
+
sample_texts = train_raw.select(idx)["text"]
|
| 829 |
+
diagnose_tokenizer(sample_texts, tokenizer, cfg, out_dir)
|
| 830 |
+
if val_raw is not None:
|
| 831 |
+
representativeness_report(train_raw, val_raw, cfg, out_dir)
|
| 832 |
+
|
| 833 |
+
if cfg.diagnose_only:
|
| 834 |
+
logger.info("--diagnose-only set: wrote diagnostics to %s, exiting before "
|
| 835 |
+
"packing/training.", out_dir)
|
| 836 |
+
sys.exit(0)
|
| 837 |
+
|
| 838 |
+
bundle: Dict[str, Dataset] = {}
|
| 839 |
+
bundle["train"] = pack_doc_sentences(tokenize_paragraphs(train_raw, tokenizer, cfg),
|
| 840 |
+
tokenizer, cfg, "train")
|
| 841 |
+
if val_raw is not None:
|
| 842 |
+
bundle["validation"] = pack_doc_sentences(tokenize_paragraphs(val_raw, tokenizer, cfg),
|
| 843 |
+
tokenizer, cfg, "validation")
|
| 844 |
+
if len(sources) > 1:
|
| 845 |
+
for tag in ("med", "canon"):
|
| 846 |
+
sub = val_raw.filter(lambda b: [s == tag for s in b["source"]], batched=True)
|
| 847 |
+
if len(sub):
|
| 848 |
+
bundle[f"validation_{tag}"] = pack_doc_sentences(
|
| 849 |
+
tokenize_paragraphs(sub, tokenizer, cfg), tokenizer, cfg, f"validation_{tag}")
|
| 850 |
+
if test_parts:
|
| 851 |
+
test_raw = concatenate_datasets(test_parts)
|
| 852 |
+
bundle["test"] = pack_doc_sentences(tokenize_paragraphs(test_raw, tokenizer, cfg),
|
| 853 |
+
tokenizer, cfg, "test")
|
| 854 |
+
|
| 855 |
+
os.makedirs(cache_root, exist_ok=True)
|
| 856 |
+
for name, d in bundle.items():
|
| 857 |
+
d.save_to_disk(os.path.join(cache_root, name))
|
| 858 |
+
logger.info("Saved packed datasets to %s", cache_root)
|
| 859 |
+
return bundle
|
| 860 |
+
|
| 861 |
+
|
| 862 |
+
# --------------------------------------------------------------------------- #
|
| 863 |
+
# Metrics
|
| 864 |
+
# --------------------------------------------------------------------------- #
|
| 865 |
+
def preprocess_logits_for_metrics(logits, labels):
|
| 866 |
+
if isinstance(logits, tuple):
|
| 867 |
+
logits = logits[0]
|
| 868 |
+
return logits.argmax(dim=-1)
|
| 869 |
+
|
| 870 |
+
|
| 871 |
+
def compute_metrics(eval_pred):
|
| 872 |
+
preds, labels = eval_pred
|
| 873 |
+
labels = labels.reshape(-1)
|
| 874 |
+
preds = preds.reshape(-1)
|
| 875 |
+
mask = labels != -100
|
| 876 |
+
if mask.sum() == 0:
|
| 877 |
+
return {"masked_accuracy": 0.0}
|
| 878 |
+
correct = (preds[mask] == labels[mask]).sum()
|
| 879 |
+
return {"masked_accuracy": float(correct) / float(mask.sum())}
|
| 880 |
+
|
| 881 |
+
|
| 882 |
+
class PerplexityCallback(TrainerCallback):
|
| 883 |
+
def on_evaluate(self, args, state, control, metrics=None, **kwargs):
|
| 884 |
+
if not metrics:
|
| 885 |
+
return
|
| 886 |
+
for k in list(metrics.keys()):
|
| 887 |
+
if k.endswith("loss") and ("eval" in k):
|
| 888 |
+
try:
|
| 889 |
+
metrics[k.replace("loss", "perplexity")] = math.exp(min(metrics[k], 20))
|
| 890 |
+
except OverflowError:
|
| 891 |
+
metrics[k.replace("loss", "perplexity")] = float("inf")
|
| 892 |
+
|
| 893 |
+
|
| 894 |
+
class JsonlLoggingCallback(TrainerCallback):
|
| 895 |
+
def __init__(self, path: str):
|
| 896 |
+
self.path = path
|
| 897 |
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
| 898 |
+
|
| 899 |
+
def on_log(self, args, state, control, logs=None, **kwargs):
|
| 900 |
+
if not logs:
|
| 901 |
+
return
|
| 902 |
+
record = dict(logs)
|
| 903 |
+
record["step"] = state.global_step
|
| 904 |
+
record["epoch"] = state.epoch
|
| 905 |
+
record["wall_time"] = datetime.now().isoformat(timespec="seconds")
|
| 906 |
+
with open(self.path, "a", encoding="utf-8") as fh:
|
| 907 |
+
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
|
| 908 |
+
fh.flush()
|
| 909 |
+
os.fsync(fh.fileno())
|
| 910 |
+
|
| 911 |
+
|
| 912 |
+
# --------------------------------------------------------------------------- #
|
| 913 |
+
# Environment / run metadata
|
| 914 |
+
# --------------------------------------------------------------------------- #
|
| 915 |
+
def capture_environment(cfg: Config) -> dict:
|
| 916 |
+
import transformers
|
| 917 |
+
import datasets as ds_lib
|
| 918 |
+
info = {
|
| 919 |
+
"timestamp": datetime.now().isoformat(timespec="seconds"),
|
| 920 |
+
"python": platform.python_version(),
|
| 921 |
+
"platform": platform.platform(),
|
| 922 |
+
"torch": torch.__version__,
|
| 923 |
+
"transformers": transformers.__version__,
|
| 924 |
+
"datasets": ds_lib.__version__,
|
| 925 |
+
"cuda_available": torch.cuda.is_available(),
|
| 926 |
+
}
|
| 927 |
+
if torch.cuda.is_available():
|
| 928 |
+
info["cuda"] = torch.version.cuda
|
| 929 |
+
info["gpu_name"] = torch.cuda.get_device_name(0)
|
| 930 |
+
props = torch.cuda.get_device_properties(0)
|
| 931 |
+
info["gpu_total_memory_gb"] = round(props.total_memory / 1024**3, 1)
|
| 932 |
+
cap = f"{props.major}.{props.minor}"
|
| 933 |
+
info["gpu_capability"] = cap
|
| 934 |
+
info["bf16_supported"] = torch.cuda.is_bf16_supported()
|
| 935 |
+
try:
|
| 936 |
+
arch_list = torch.cuda.get_arch_list()
|
| 937 |
+
except Exception:
|
| 938 |
+
arch_list = []
|
| 939 |
+
info["torch_arch_list"] = arch_list
|
| 940 |
+
sm_tag = f"sm_{props.major}{props.minor}"
|
| 941 |
+
info["gpu_arch_supported_by_torch"] = any(sm_tag == a for a in arch_list)
|
| 942 |
+
return info
|
| 943 |
+
|
| 944 |
+
|
| 945 |
+
# --------------------------------------------------------------------------- #
|
| 946 |
+
# Argparse
|
| 947 |
+
# --------------------------------------------------------------------------- #
|
| 948 |
+
def parse_args() -> Config:
|
| 949 |
+
cfg = Config()
|
| 950 |
+
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
| 951 |
+
for f in cfg.__dataclass_fields__.values():
|
| 952 |
+
name = "--" + f.name.replace("_", "-")
|
| 953 |
+
default = getattr(cfg, f.name)
|
| 954 |
+
if isinstance(default, bool):
|
| 955 |
+
if default:
|
| 956 |
+
p.add_argument("--no-" + f.name.replace("_", "-"), dest=f.name, action="store_false")
|
| 957 |
+
else:
|
| 958 |
+
p.add_argument(name, dest=f.name, action="store_true")
|
| 959 |
+
elif default is None:
|
| 960 |
+
p.add_argument(name, dest=f.name, default=None, type=str)
|
| 961 |
+
else:
|
| 962 |
+
p.add_argument(name, dest=f.name, default=default, type=type(default))
|
| 963 |
+
args = p.parse_args()
|
| 964 |
+
return Config(**vars(args))
|
| 965 |
+
|
| 966 |
+
|
| 967 |
+
# --------------------------------------------------------------------------- #
|
| 968 |
+
# Main
|
| 969 |
+
# --------------------------------------------------------------------------- #
|
| 970 |
+
def main():
|
| 971 |
+
cfg = parse_args()
|
| 972 |
+
set_seed(cfg.seed)
|
| 973 |
+
|
| 974 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 975 |
+
torch.backends.cudnn.allow_tf32 = True
|
| 976 |
+
torch.set_float32_matmul_precision("high")
|
| 977 |
+
|
| 978 |
+
os.makedirs(cfg.output_dir, exist_ok=True)
|
| 979 |
+
|
| 980 |
+
file_handler = logging.FileHandler(os.path.join(cfg.output_dir, "train.log"))
|
| 981 |
+
file_handler.setFormatter(logging.Formatter(
|
| 982 |
+
"%(asctime)s | %(levelname)-7s | %(name)s | %(message)s", "%Y-%m-%d %H:%M:%S"))
|
| 983 |
+
logging.getLogger().addHandler(file_handler)
|
| 984 |
+
|
| 985 |
+
env = capture_environment(cfg)
|
| 986 |
+
logger.info("Environment:\n%s", json.dumps(env, indent=2))
|
| 987 |
+
with open(os.path.join(cfg.output_dir, "run_metadata.json"), "w") as fh:
|
| 988 |
+
json.dump({"config": cfg.__dict__, "environment": env}, fh, indent=2)
|
| 989 |
+
|
| 990 |
+
if not env["cuda_available"]:
|
| 991 |
+
logger.warning("CUDA not available — this script is intended to run on a GPU server.")
|
| 992 |
+
else:
|
| 993 |
+
if not env.get("bf16_supported", False):
|
| 994 |
+
logger.warning("bf16 not reported as supported on this GPU; consider fp16.")
|
| 995 |
+
if env.get("torch_arch_list") and not env.get("gpu_arch_supported_by_torch", True):
|
| 996 |
+
logger.error(
|
| 997 |
+
"This torch build (%s, arch_list=%s) has NO kernels for your GPU "
|
| 998 |
+
"(capability %s). On Blackwell (RTX PRO 6000) install a cu128+ build, e.g.:\n"
|
| 999 |
+
" pip install --upgrade torch --index-url https://download.pytorch.org/whl/cu128\n"
|
| 1000 |
+
"CUDA ops will fail until this matches.",
|
| 1001 |
+
env["torch"], env.get("torch_arch_list"), env.get("gpu_capability"))
|
| 1002 |
+
|
| 1003 |
+
if cfg.save_steps % cfg.eval_steps != 0:
|
| 1004 |
+
new_save = max(cfg.eval_steps, (cfg.save_steps // cfg.eval_steps) * cfg.eval_steps)
|
| 1005 |
+
logger.warning("save_steps (%d) not a multiple of eval_steps (%d); adjusting to %d.",
|
| 1006 |
+
cfg.save_steps, cfg.eval_steps, new_save)
|
| 1007 |
+
cfg.save_steps = new_save
|
| 1008 |
+
|
| 1009 |
+
tokenizer = AutoTokenizer.from_pretrained(cfg.model_name, use_fast=True)
|
| 1010 |
+
tokenizer.model_max_length = cfg.max_seq_length
|
| 1011 |
+
|
| 1012 |
+
bundle = build_or_load_packed(cfg, tokenizer)
|
| 1013 |
+
train_ds = bundle["train"]
|
| 1014 |
+
eval_ds = bundle.get("validation")
|
| 1015 |
+
|
| 1016 |
+
def tok_count(d):
|
| 1017 |
+
lengths = d.map(lambda b: {"len": [len(x) for x in b["input_ids"]]},
|
| 1018 |
+
batched=True, remove_columns=d.column_names, desc="count tokens")["len"]
|
| 1019 |
+
return int(sum(lengths)), float(np.mean(lengths))
|
| 1020 |
+
|
| 1021 |
+
stats = {}
|
| 1022 |
+
for name, d in bundle.items():
|
| 1023 |
+
n_tok, mean_len = tok_count(d)
|
| 1024 |
+
stats[name] = {"sequences": len(d), "tokens": n_tok, "mean_seq_len": round(mean_len, 1)}
|
| 1025 |
+
logger.info("Packed dataset statistics:\n%s", json.dumps(stats, indent=2))
|
| 1026 |
+
with open(os.path.join(cfg.output_dir, "dataset_stats.json"), "w") as fh:
|
| 1027 |
+
json.dump(stats, fh, indent=2)
|
| 1028 |
+
|
| 1029 |
+
if cfg.dry_run:
|
| 1030 |
+
logger.info("--dry-run set: datasets built and cached, exiting before training.")
|
| 1031 |
+
return
|
| 1032 |
+
|
| 1033 |
+
train_ds = train_ds.remove_columns([c for c in train_ds.column_names if c != "input_ids"])
|
| 1034 |
+
if eval_ds is not None:
|
| 1035 |
+
eval_ds = eval_ds.remove_columns([c for c in eval_ds.column_names if c != "input_ids"])
|
| 1036 |
+
|
| 1037 |
+
model = AutoModelForMaskedLM.from_pretrained(
|
| 1038 |
+
cfg.model_name,
|
| 1039 |
+
attn_implementation=cfg.attn_implementation,
|
| 1040 |
+
)
|
| 1041 |
+
if cfg.gradient_checkpointing:
|
| 1042 |
+
model.config.use_cache = False
|
| 1043 |
+
logger.info("Model parameters: %.1fM", sum(p.numel() for p in model.parameters()) / 1e6)
|
| 1044 |
+
|
| 1045 |
+
collator = DataCollatorForLanguageModeling(
|
| 1046 |
+
tokenizer=tokenizer,
|
| 1047 |
+
mlm=True,
|
| 1048 |
+
mlm_probability=cfg.mlm_probability,
|
| 1049 |
+
pad_to_multiple_of=8,
|
| 1050 |
+
)
|
| 1051 |
+
|
| 1052 |
+
report_to = ["tensorboard"]
|
| 1053 |
+
if cfg.report_to_wandb:
|
| 1054 |
+
report_to.append("wandb")
|
| 1055 |
+
|
| 1056 |
+
do_eval = eval_ds is not None
|
| 1057 |
+
args = TrainingArguments(
|
| 1058 |
+
output_dir=cfg.output_dir,
|
| 1059 |
+
seed=cfg.seed,
|
| 1060 |
+
data_seed=cfg.seed,
|
| 1061 |
+
num_train_epochs=cfg.num_train_epochs,
|
| 1062 |
+
max_steps=cfg.max_steps,
|
| 1063 |
+
learning_rate=cfg.learning_rate,
|
| 1064 |
+
weight_decay=cfg.weight_decay,
|
| 1065 |
+
adam_beta1=cfg.adam_beta1,
|
| 1066 |
+
adam_beta2=cfg.adam_beta2,
|
| 1067 |
+
adam_epsilon=cfg.adam_epsilon,
|
| 1068 |
+
max_grad_norm=cfg.max_grad_norm,
|
| 1069 |
+
warmup_ratio=cfg.warmup_ratio,
|
| 1070 |
+
lr_scheduler_type=cfg.lr_scheduler_type,
|
| 1071 |
+
optim=cfg.optim,
|
| 1072 |
+
bf16=bool(env.get("bf16_supported")),
|
| 1073 |
+
per_device_train_batch_size=cfg.per_device_train_batch_size,
|
| 1074 |
+
per_device_eval_batch_size=cfg.per_device_eval_batch_size,
|
| 1075 |
+
gradient_accumulation_steps=cfg.gradient_accumulation_steps,
|
| 1076 |
+
gradient_checkpointing=cfg.gradient_checkpointing,
|
| 1077 |
+
auto_find_batch_size=cfg.auto_find_batch_size,
|
| 1078 |
+
torch_compile=cfg.torch_compile,
|
| 1079 |
+
dataloader_num_workers=cfg.dataloader_num_workers,
|
| 1080 |
+
dataloader_pin_memory=True,
|
| 1081 |
+
eval_accumulation_steps=cfg.eval_accumulation_steps,
|
| 1082 |
+
eval_strategy="steps" if do_eval else "no",
|
| 1083 |
+
eval_steps=cfg.eval_steps if do_eval else None,
|
| 1084 |
+
logging_strategy="steps",
|
| 1085 |
+
logging_steps=cfg.logging_steps,
|
| 1086 |
+
logging_first_step=True,
|
| 1087 |
+
save_strategy="steps",
|
| 1088 |
+
save_steps=cfg.save_steps,
|
| 1089 |
+
save_total_limit=cfg.save_total_limit,
|
| 1090 |
+
load_best_model_at_end=do_eval,
|
| 1091 |
+
metric_for_best_model="eval_loss" if do_eval else None,
|
| 1092 |
+
greater_is_better=False,
|
| 1093 |
+
report_to=report_to,
|
| 1094 |
+
run_name=os.path.basename(cfg.output_dir.rstrip("/")),
|
| 1095 |
+
include_num_input_tokens_seen=True,
|
| 1096 |
+
logging_dir=os.path.join(cfg.output_dir, "tb"),
|
| 1097 |
+
)
|
| 1098 |
+
|
| 1099 |
+
callbacks = [PerplexityCallback(),
|
| 1100 |
+
JsonlLoggingCallback(os.path.join(cfg.output_dir, "training_log.jsonl"))]
|
| 1101 |
+
if do_eval and cfg.early_stopping_patience > 0:
|
| 1102 |
+
callbacks.append(EarlyStoppingCallback(
|
| 1103 |
+
early_stopping_patience=cfg.early_stopping_patience,
|
| 1104 |
+
early_stopping_threshold=cfg.early_stopping_threshold))
|
| 1105 |
+
|
| 1106 |
+
tok_kwarg = {"processing_class": tokenizer} if _USE_PROCESSING_CLASS else {"tokenizer": tokenizer}
|
| 1107 |
+
trainer = Trainer(
|
| 1108 |
+
model=model,
|
| 1109 |
+
args=args,
|
| 1110 |
+
train_dataset=train_ds,
|
| 1111 |
+
eval_dataset=eval_ds,
|
| 1112 |
+
data_collator=collator,
|
| 1113 |
+
compute_metrics=compute_metrics if do_eval else None,
|
| 1114 |
+
preprocess_logits_for_metrics=preprocess_logits_for_metrics if do_eval else None,
|
| 1115 |
+
callbacks=callbacks,
|
| 1116 |
+
**tok_kwarg,
|
| 1117 |
+
)
|
| 1118 |
+
|
| 1119 |
+
resume_from = None
|
| 1120 |
+
if cfg.resume:
|
| 1121 |
+
last = get_last_checkpoint(cfg.output_dir)
|
| 1122 |
+
if last:
|
| 1123 |
+
logger.info("Resuming from checkpoint %s", last)
|
| 1124 |
+
resume_from = last
|
| 1125 |
+
|
| 1126 |
+
try:
|
| 1127 |
+
train_result = trainer.train(resume_from_checkpoint=resume_from)
|
| 1128 |
+
except BaseException as exc:
|
| 1129 |
+
tb_path = os.path.join(cfg.output_dir, "crash_traceback.txt")
|
| 1130 |
+
with open(tb_path, "w", encoding="utf-8") as fh:
|
| 1131 |
+
fh.write(f"Crashed at {datetime.now().isoformat()}\n\n")
|
| 1132 |
+
traceback.print_exc(file=fh)
|
| 1133 |
+
logger.error("Training crashed (%s: %s). Traceback written to %s",
|
| 1134 |
+
type(exc).__name__, exc, tb_path)
|
| 1135 |
+
try:
|
| 1136 |
+
emergency = os.path.join(cfg.output_dir, "emergency_checkpoint")
|
| 1137 |
+
trainer.save_model(emergency)
|
| 1138 |
+
trainer.save_state()
|
| 1139 |
+
logger.error("Emergency model state saved to %s", emergency)
|
| 1140 |
+
except Exception as save_exc:
|
| 1141 |
+
logger.error("Emergency save also failed: %s", save_exc)
|
| 1142 |
+
raise
|
| 1143 |
+
|
| 1144 |
+
trainer.save_model()
|
| 1145 |
+
tokenizer.save_pretrained(cfg.output_dir)
|
| 1146 |
+
trainer.save_state()
|
| 1147 |
+
|
| 1148 |
+
metrics = train_result.metrics
|
| 1149 |
+
trainer.log_metrics("train", metrics)
|
| 1150 |
+
trainer.save_metrics("train", metrics)
|
| 1151 |
+
|
| 1152 |
+
final = {}
|
| 1153 |
+
if eval_ds is not None:
|
| 1154 |
+
m = trainer.evaluate(eval_dataset=eval_ds, metric_key_prefix="final_val")
|
| 1155 |
+
m["final_val_perplexity"] = math.exp(min(m["final_val_loss"], 20))
|
| 1156 |
+
final.update(m)
|
| 1157 |
+
for tag in ("med", "canon"):
|
| 1158 |
+
sub = bundle.get(f"validation_{tag}")
|
| 1159 |
+
if sub is not None:
|
| 1160 |
+
sub = sub.remove_columns([c for c in sub.column_names if c != "input_ids"])
|
| 1161 |
+
m = trainer.evaluate(eval_dataset=sub, metric_key_prefix=f"final_val_{tag}")
|
| 1162 |
+
m[f"final_val_{tag}_perplexity"] = math.exp(min(m[f"final_val_{tag}_loss"], 20))
|
| 1163 |
+
final.update(m)
|
| 1164 |
+
if "test" in bundle:
|
| 1165 |
+
test_ds = bundle["test"].remove_columns(
|
| 1166 |
+
[c for c in bundle["test"].column_names if c != "input_ids"])
|
| 1167 |
+
m = trainer.evaluate(eval_dataset=test_ds, metric_key_prefix="final_test")
|
| 1168 |
+
m["final_test_perplexity"] = math.exp(min(m["final_test_loss"], 20))
|
| 1169 |
+
final.update(m)
|
| 1170 |
+
|
| 1171 |
+
if final:
|
| 1172 |
+
logger.info("Final evaluation:\n%s", json.dumps(final, indent=2))
|
| 1173 |
+
trainer.save_metrics("final_eval", final)
|
| 1174 |
+
logger.info("Done. Best model + tokenizer saved to %s", cfg.output_dir)
|
| 1175 |
+
|
| 1176 |
+
|
| 1177 |
+
if __name__ == "__main__":
|
| 1178 |
+
main()
|
train_results.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"epoch": 10.0,
|
| 3 |
+
"num_input_tokens_seen": 1464637440,
|
| 4 |
+
"total_flos": 4.917799558028206e+18,
|
| 5 |
+
"train_loss": 21.80173199032777,
|
| 6 |
+
"train_runtime": 34212.9191,
|
| 7 |
+
"train_samples_per_second": 83.612,
|
| 8 |
+
"train_steps_per_second": 0.131
|
| 9 |
+
}
|
trainer_state.json
ADDED
|
@@ -0,0 +1,1043 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_global_step": 4470,
|
| 3 |
+
"best_metric": 1.0481648445129395,
|
| 4 |
+
"best_model_checkpoint": "./MediBerta/checkpoint-4470",
|
| 5 |
+
"epoch": 10.0,
|
| 6 |
+
"eval_steps": 500,
|
| 7 |
+
"global_step": 4470,
|
| 8 |
+
"is_hyper_param_search": false,
|
| 9 |
+
"is_local_process_zero": true,
|
| 10 |
+
"is_world_process_zero": true,
|
| 11 |
+
"log_history": [
|
| 12 |
+
{
|
| 13 |
+
"epoch": 0.0022371364653243847,
|
| 14 |
+
"grad_norm": 32.67197036743164,
|
| 15 |
+
"learning_rate": 0.0,
|
| 16 |
+
"loss": 31.76995086669922,
|
| 17 |
+
"num_input_tokens_seen": 327680,
|
| 18 |
+
"step": 1,
|
| 19 |
+
"train_runtime": 7.8573,
|
| 20 |
+
"train_tokens_per_second": 41704.008
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"epoch": 0.11185682326621924,
|
| 24 |
+
"grad_norm": 31.965707778930664,
|
| 25 |
+
"learning_rate": 1.821561338289963e-05,
|
| 26 |
+
"loss": 32.44386509486607,
|
| 27 |
+
"num_input_tokens_seen": 16384000,
|
| 28 |
+
"step": 50,
|
| 29 |
+
"train_runtime": 380.1137,
|
| 30 |
+
"train_tokens_per_second": 43102.889
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"epoch": 0.22371364653243847,
|
| 34 |
+
"grad_norm": 34.0458984375,
|
| 35 |
+
"learning_rate": 3.680297397769517e-05,
|
| 36 |
+
"loss": 31.56456787109375,
|
| 37 |
+
"num_input_tokens_seen": 32768000,
|
| 38 |
+
"step": 100,
|
| 39 |
+
"train_runtime": 759.9744,
|
| 40 |
+
"train_tokens_per_second": 43117.243
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"epoch": 0.33557046979865773,
|
| 44 |
+
"grad_norm": 36.13215255737305,
|
| 45 |
+
"learning_rate": 5.539033457249071e-05,
|
| 46 |
+
"loss": 30.703720703125,
|
| 47 |
+
"num_input_tokens_seen": 49152000,
|
| 48 |
+
"step": 150,
|
| 49 |
+
"train_runtime": 1140.0254,
|
| 50 |
+
"train_tokens_per_second": 43114.831
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"epoch": 0.44742729306487694,
|
| 54 |
+
"grad_norm": 75.77301788330078,
|
| 55 |
+
"learning_rate": 7.397769516728626e-05,
|
| 56 |
+
"loss": 30.0342919921875,
|
| 57 |
+
"num_input_tokens_seen": 65536000,
|
| 58 |
+
"step": 200,
|
| 59 |
+
"train_runtime": 1519.7868,
|
| 60 |
+
"train_tokens_per_second": 43121.837
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"epoch": 0.5592841163310962,
|
| 64 |
+
"grad_norm": 31.4502010345459,
|
| 65 |
+
"learning_rate": 9.256505576208179e-05,
|
| 66 |
+
"loss": 29.2935986328125,
|
| 67 |
+
"num_input_tokens_seen": 81920000,
|
| 68 |
+
"step": 250,
|
| 69 |
+
"train_runtime": 1899.4384,
|
| 70 |
+
"train_tokens_per_second": 43128.538
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"epoch": 0.6711409395973155,
|
| 74 |
+
"grad_norm": 35.78462600708008,
|
| 75 |
+
"learning_rate": 9.928588431325875e-05,
|
| 76 |
+
"loss": 28.63412353515625,
|
| 77 |
+
"num_input_tokens_seen": 98304000,
|
| 78 |
+
"step": 300,
|
| 79 |
+
"train_runtime": 2279.106,
|
| 80 |
+
"train_tokens_per_second": 43132.703
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"epoch": 0.7829977628635347,
|
| 84 |
+
"grad_norm": 32.92461395263672,
|
| 85 |
+
"learning_rate": 9.809569150202333e-05,
|
| 86 |
+
"loss": 27.89616455078125,
|
| 87 |
+
"num_input_tokens_seen": 114688000,
|
| 88 |
+
"step": 350,
|
| 89 |
+
"train_runtime": 2659.6638,
|
| 90 |
+
"train_tokens_per_second": 43121.24
|
| 91 |
+
},
|
| 92 |
+
{
|
| 93 |
+
"epoch": 0.8948545861297539,
|
| 94 |
+
"grad_norm": 31.881704330444336,
|
| 95 |
+
"learning_rate": 9.69054986907879e-05,
|
| 96 |
+
"loss": 27.38276123046875,
|
| 97 |
+
"num_input_tokens_seen": 131072000,
|
| 98 |
+
"step": 400,
|
| 99 |
+
"train_runtime": 3039.907,
|
| 100 |
+
"train_tokens_per_second": 43117.108
|
| 101 |
+
},
|
| 102 |
+
{
|
| 103 |
+
"epoch": 1.0067114093959733,
|
| 104 |
+
"grad_norm": 28.32501220703125,
|
| 105 |
+
"learning_rate": 9.57153058795525e-05,
|
| 106 |
+
"loss": 26.72684814453125,
|
| 107 |
+
"num_input_tokens_seen": 147446784,
|
| 108 |
+
"step": 450,
|
| 109 |
+
"train_runtime": 3419.3158,
|
| 110 |
+
"train_tokens_per_second": 43121.721
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"epoch": 1.1185682326621924,
|
| 114 |
+
"grad_norm": 27.903852462768555,
|
| 115 |
+
"learning_rate": 9.452511306831707e-05,
|
| 116 |
+
"loss": 26.2695166015625,
|
| 117 |
+
"num_input_tokens_seen": 163830784,
|
| 118 |
+
"step": 500,
|
| 119 |
+
"train_runtime": 3799.2571,
|
| 120 |
+
"train_tokens_per_second": 43121.794
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"epoch": 1.1185682326621924,
|
| 124 |
+
"eval_loss": 1.3941264152526855,
|
| 125 |
+
"eval_masked_accuracy": 0.7140736610865701,
|
| 126 |
+
"eval_runtime": 25.352,
|
| 127 |
+
"eval_samples_per_second": 219.627,
|
| 128 |
+
"eval_steps_per_second": 3.432,
|
| 129 |
+
"num_input_tokens_seen": 163830784,
|
| 130 |
+
"step": 500
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"epoch": 1.2304250559284116,
|
| 134 |
+
"grad_norm": 25.51409339904785,
|
| 135 |
+
"learning_rate": 9.333492025708165e-05,
|
| 136 |
+
"loss": 25.86959228515625,
|
| 137 |
+
"num_input_tokens_seen": 180214784,
|
| 138 |
+
"step": 550,
|
| 139 |
+
"train_runtime": 4207.1627,
|
| 140 |
+
"train_tokens_per_second": 42835.231
|
| 141 |
+
},
|
| 142 |
+
{
|
| 143 |
+
"epoch": 1.342281879194631,
|
| 144 |
+
"grad_norm": 31.309749603271484,
|
| 145 |
+
"learning_rate": 9.214472744584624e-05,
|
| 146 |
+
"loss": 25.3655517578125,
|
| 147 |
+
"num_input_tokens_seen": 196598784,
|
| 148 |
+
"step": 600,
|
| 149 |
+
"train_runtime": 4586.3773,
|
| 150 |
+
"train_tokens_per_second": 42865.812
|
| 151 |
+
},
|
| 152 |
+
{
|
| 153 |
+
"epoch": 1.45413870246085,
|
| 154 |
+
"grad_norm": 28.270841598510742,
|
| 155 |
+
"learning_rate": 9.095453463461082e-05,
|
| 156 |
+
"loss": 25.1278271484375,
|
| 157 |
+
"num_input_tokens_seen": 212982784,
|
| 158 |
+
"step": 650,
|
| 159 |
+
"train_runtime": 4966.2335,
|
| 160 |
+
"train_tokens_per_second": 42886.18
|
| 161 |
+
},
|
| 162 |
+
{
|
| 163 |
+
"epoch": 1.5659955257270695,
|
| 164 |
+
"grad_norm": 27.522308349609375,
|
| 165 |
+
"learning_rate": 8.976434182337538e-05,
|
| 166 |
+
"loss": 24.80788818359375,
|
| 167 |
+
"num_input_tokens_seen": 229366784,
|
| 168 |
+
"step": 700,
|
| 169 |
+
"train_runtime": 5346.6697,
|
| 170 |
+
"train_tokens_per_second": 42899.0
|
| 171 |
+
},
|
| 172 |
+
{
|
| 173 |
+
"epoch": 1.6778523489932886,
|
| 174 |
+
"grad_norm": 27.093711853027344,
|
| 175 |
+
"learning_rate": 8.857414901213997e-05,
|
| 176 |
+
"loss": 24.6148046875,
|
| 177 |
+
"num_input_tokens_seen": 245750784,
|
| 178 |
+
"step": 750,
|
| 179 |
+
"train_runtime": 5726.4531,
|
| 180 |
+
"train_tokens_per_second": 42915.008
|
| 181 |
+
},
|
| 182 |
+
{
|
| 183 |
+
"epoch": 1.7897091722595078,
|
| 184 |
+
"grad_norm": 25.49102210998535,
|
| 185 |
+
"learning_rate": 8.738395620090455e-05,
|
| 186 |
+
"loss": 24.40305908203125,
|
| 187 |
+
"num_input_tokens_seen": 262134784,
|
| 188 |
+
"step": 800,
|
| 189 |
+
"train_runtime": 6106.8482,
|
| 190 |
+
"train_tokens_per_second": 42924.726
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"epoch": 1.901565995525727,
|
| 194 |
+
"grad_norm": 25.048629760742188,
|
| 195 |
+
"learning_rate": 8.619376338966914e-05,
|
| 196 |
+
"loss": 24.0977783203125,
|
| 197 |
+
"num_input_tokens_seen": 278518784,
|
| 198 |
+
"step": 850,
|
| 199 |
+
"train_runtime": 6486.6563,
|
| 200 |
+
"train_tokens_per_second": 42937.189
|
| 201 |
+
},
|
| 202 |
+
{
|
| 203 |
+
"epoch": 2.0134228187919465,
|
| 204 |
+
"grad_norm": 25.157657623291016,
|
| 205 |
+
"learning_rate": 8.500357057843372e-05,
|
| 206 |
+
"loss": 23.89361572265625,
|
| 207 |
+
"num_input_tokens_seen": 294893568,
|
| 208 |
+
"step": 900,
|
| 209 |
+
"train_runtime": 6867.068,
|
| 210 |
+
"train_tokens_per_second": 42943.155
|
| 211 |
+
},
|
| 212 |
+
{
|
| 213 |
+
"epoch": 2.1252796420581657,
|
| 214 |
+
"grad_norm": 25.902904510498047,
|
| 215 |
+
"learning_rate": 8.381337776719829e-05,
|
| 216 |
+
"loss": 23.57096435546875,
|
| 217 |
+
"num_input_tokens_seen": 311277568,
|
| 218 |
+
"step": 950,
|
| 219 |
+
"train_runtime": 7246.8643,
|
| 220 |
+
"train_tokens_per_second": 42953.415
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"epoch": 2.237136465324385,
|
| 224 |
+
"grad_norm": 24.553022384643555,
|
| 225 |
+
"learning_rate": 8.262318495596287e-05,
|
| 226 |
+
"loss": 23.42177490234375,
|
| 227 |
+
"num_input_tokens_seen": 327661568,
|
| 228 |
+
"step": 1000,
|
| 229 |
+
"train_runtime": 7626.2318,
|
| 230 |
+
"train_tokens_per_second": 42965.068
|
| 231 |
+
},
|
| 232 |
+
{
|
| 233 |
+
"epoch": 2.237136465324385,
|
| 234 |
+
"eval_loss": 1.2587531805038452,
|
| 235 |
+
"eval_masked_accuracy": 0.7357594334234474,
|
| 236 |
+
"eval_runtime": 25.3576,
|
| 237 |
+
"eval_samples_per_second": 219.579,
|
| 238 |
+
"eval_steps_per_second": 3.431,
|
| 239 |
+
"num_input_tokens_seen": 327661568,
|
| 240 |
+
"step": 1000
|
| 241 |
+
},
|
| 242 |
+
{
|
| 243 |
+
"epoch": 2.348993288590604,
|
| 244 |
+
"grad_norm": 24.940990447998047,
|
| 245 |
+
"learning_rate": 8.143299214472745e-05,
|
| 246 |
+
"loss": 23.259130859375,
|
| 247 |
+
"num_input_tokens_seen": 344045568,
|
| 248 |
+
"step": 1050,
|
| 249 |
+
"train_runtime": 8033.7212,
|
| 250 |
+
"train_tokens_per_second": 42825.182
|
| 251 |
+
},
|
| 252 |
+
{
|
| 253 |
+
"epoch": 2.460850111856823,
|
| 254 |
+
"grad_norm": 24.946720123291016,
|
| 255 |
+
"learning_rate": 8.024279933349202e-05,
|
| 256 |
+
"loss": 23.12722900390625,
|
| 257 |
+
"num_input_tokens_seen": 360429568,
|
| 258 |
+
"step": 1100,
|
| 259 |
+
"train_runtime": 8413.8773,
|
| 260 |
+
"train_tokens_per_second": 42837.512
|
| 261 |
+
},
|
| 262 |
+
{
|
| 263 |
+
"epoch": 2.5727069351230423,
|
| 264 |
+
"grad_norm": 24.135683059692383,
|
| 265 |
+
"learning_rate": 7.905260652225661e-05,
|
| 266 |
+
"loss": 22.9796826171875,
|
| 267 |
+
"num_input_tokens_seen": 376813568,
|
| 268 |
+
"step": 1150,
|
| 269 |
+
"train_runtime": 8793.1262,
|
| 270 |
+
"train_tokens_per_second": 42853.197
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"epoch": 2.684563758389262,
|
| 274 |
+
"grad_norm": 25.850025177001953,
|
| 275 |
+
"learning_rate": 7.786241371102119e-05,
|
| 276 |
+
"loss": 22.77516845703125,
|
| 277 |
+
"num_input_tokens_seen": 393197568,
|
| 278 |
+
"step": 1200,
|
| 279 |
+
"train_runtime": 9172.793,
|
| 280 |
+
"train_tokens_per_second": 42865.632
|
| 281 |
+
},
|
| 282 |
+
{
|
| 283 |
+
"epoch": 2.796420581655481,
|
| 284 |
+
"grad_norm": 22.62376594543457,
|
| 285 |
+
"learning_rate": 7.667222089978577e-05,
|
| 286 |
+
"loss": 22.6738720703125,
|
| 287 |
+
"num_input_tokens_seen": 409581568,
|
| 288 |
+
"step": 1250,
|
| 289 |
+
"train_runtime": 9553.7727,
|
| 290 |
+
"train_tokens_per_second": 42871.186
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"epoch": 2.9082774049217,
|
| 294 |
+
"grad_norm": 22.361886978149414,
|
| 295 |
+
"learning_rate": 7.548202808855035e-05,
|
| 296 |
+
"loss": 22.53384033203125,
|
| 297 |
+
"num_input_tokens_seen": 425965568,
|
| 298 |
+
"step": 1300,
|
| 299 |
+
"train_runtime": 9932.9496,
|
| 300 |
+
"train_tokens_per_second": 42884.096
|
| 301 |
+
},
|
| 302 |
+
{
|
| 303 |
+
"epoch": 3.0201342281879193,
|
| 304 |
+
"grad_norm": 24.093069076538086,
|
| 305 |
+
"learning_rate": 7.429183527731492e-05,
|
| 306 |
+
"loss": 22.38953125,
|
| 307 |
+
"num_input_tokens_seen": 442340352,
|
| 308 |
+
"step": 1350,
|
| 309 |
+
"train_runtime": 10313.033,
|
| 310 |
+
"train_tokens_per_second": 42891.393
|
| 311 |
+
},
|
| 312 |
+
{
|
| 313 |
+
"epoch": 3.131991051454139,
|
| 314 |
+
"grad_norm": 22.419540405273438,
|
| 315 |
+
"learning_rate": 7.310164246607951e-05,
|
| 316 |
+
"loss": 22.18701416015625,
|
| 317 |
+
"num_input_tokens_seen": 458724352,
|
| 318 |
+
"step": 1400,
|
| 319 |
+
"train_runtime": 10693.1337,
|
| 320 |
+
"train_tokens_per_second": 42898.964
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"epoch": 3.243847874720358,
|
| 324 |
+
"grad_norm": 24.061424255371094,
|
| 325 |
+
"learning_rate": 7.191144965484409e-05,
|
| 326 |
+
"loss": 22.1206396484375,
|
| 327 |
+
"num_input_tokens_seen": 475108352,
|
| 328 |
+
"step": 1450,
|
| 329 |
+
"train_runtime": 11073.944,
|
| 330 |
+
"train_tokens_per_second": 42903.265
|
| 331 |
+
},
|
| 332 |
+
{
|
| 333 |
+
"epoch": 3.3557046979865772,
|
| 334 |
+
"grad_norm": 21.660381317138672,
|
| 335 |
+
"learning_rate": 7.072125684360867e-05,
|
| 336 |
+
"loss": 21.99459228515625,
|
| 337 |
+
"num_input_tokens_seen": 491492352,
|
| 338 |
+
"step": 1500,
|
| 339 |
+
"train_runtime": 11453.1596,
|
| 340 |
+
"train_tokens_per_second": 42913.254
|
| 341 |
+
},
|
| 342 |
+
{
|
| 343 |
+
"epoch": 3.3557046979865772,
|
| 344 |
+
"eval_loss": 1.194376826286316,
|
| 345 |
+
"eval_masked_accuracy": 0.7462490242091618,
|
| 346 |
+
"eval_runtime": 25.3329,
|
| 347 |
+
"eval_samples_per_second": 219.794,
|
| 348 |
+
"eval_steps_per_second": 3.434,
|
| 349 |
+
"num_input_tokens_seen": 491492352,
|
| 350 |
+
"step": 1500
|
| 351 |
+
},
|
| 352 |
+
{
|
| 353 |
+
"epoch": 3.4675615212527964,
|
| 354 |
+
"grad_norm": 23.65436553955078,
|
| 355 |
+
"learning_rate": 6.953106403237324e-05,
|
| 356 |
+
"loss": 21.8618212890625,
|
| 357 |
+
"num_input_tokens_seen": 507876352,
|
| 358 |
+
"step": 1550,
|
| 359 |
+
"train_runtime": 11860.4601,
|
| 360 |
+
"train_tokens_per_second": 42820.965
|
| 361 |
+
},
|
| 362 |
+
{
|
| 363 |
+
"epoch": 3.5794183445190155,
|
| 364 |
+
"grad_norm": 23.492694854736328,
|
| 365 |
+
"learning_rate": 6.834087122113782e-05,
|
| 366 |
+
"loss": 21.84644287109375,
|
| 367 |
+
"num_input_tokens_seen": 524260352,
|
| 368 |
+
"step": 1600,
|
| 369 |
+
"train_runtime": 12240.4952,
|
| 370 |
+
"train_tokens_per_second": 42829.995
|
| 371 |
+
},
|
| 372 |
+
{
|
| 373 |
+
"epoch": 3.6912751677852347,
|
| 374 |
+
"grad_norm": 22.143146514892578,
|
| 375 |
+
"learning_rate": 6.715067840990241e-05,
|
| 376 |
+
"loss": 21.695458984375,
|
| 377 |
+
"num_input_tokens_seen": 540644352,
|
| 378 |
+
"step": 1650,
|
| 379 |
+
"train_runtime": 12620.3553,
|
| 380 |
+
"train_tokens_per_second": 42839.075
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"epoch": 3.8031319910514543,
|
| 384 |
+
"grad_norm": 22.582658767700195,
|
| 385 |
+
"learning_rate": 6.596048559866699e-05,
|
| 386 |
+
"loss": 21.49891357421875,
|
| 387 |
+
"num_input_tokens_seen": 557028352,
|
| 388 |
+
"step": 1700,
|
| 389 |
+
"train_runtime": 13000.3344,
|
| 390 |
+
"train_tokens_per_second": 42847.233
|
| 391 |
+
},
|
| 392 |
+
{
|
| 393 |
+
"epoch": 3.9149888143176734,
|
| 394 |
+
"grad_norm": 22.46770477294922,
|
| 395 |
+
"learning_rate": 6.477029278743157e-05,
|
| 396 |
+
"loss": 21.4741015625,
|
| 397 |
+
"num_input_tokens_seen": 573412352,
|
| 398 |
+
"step": 1750,
|
| 399 |
+
"train_runtime": 13380.7098,
|
| 400 |
+
"train_tokens_per_second": 42853.657
|
| 401 |
+
},
|
| 402 |
+
{
|
| 403 |
+
"epoch": 4.026845637583893,
|
| 404 |
+
"grad_norm": 22.605365753173828,
|
| 405 |
+
"learning_rate": 6.358009997619616e-05,
|
| 406 |
+
"loss": 21.39329833984375,
|
| 407 |
+
"num_input_tokens_seen": 589787136,
|
| 408 |
+
"step": 1800,
|
| 409 |
+
"train_runtime": 13759.8334,
|
| 410 |
+
"train_tokens_per_second": 42862.956
|
| 411 |
+
},
|
| 412 |
+
{
|
| 413 |
+
"epoch": 4.138702460850112,
|
| 414 |
+
"grad_norm": 21.95088005065918,
|
| 415 |
+
"learning_rate": 6.238990716496072e-05,
|
| 416 |
+
"loss": 21.18496826171875,
|
| 417 |
+
"num_input_tokens_seen": 606171136,
|
| 418 |
+
"step": 1850,
|
| 419 |
+
"train_runtime": 14139.2097,
|
| 420 |
+
"train_tokens_per_second": 42871.642
|
| 421 |
+
},
|
| 422 |
+
{
|
| 423 |
+
"epoch": 4.250559284116331,
|
| 424 |
+
"grad_norm": 22.34777069091797,
|
| 425 |
+
"learning_rate": 6.11997143537253e-05,
|
| 426 |
+
"loss": 21.2357177734375,
|
| 427 |
+
"num_input_tokens_seen": 622555136,
|
| 428 |
+
"step": 1900,
|
| 429 |
+
"train_runtime": 14518.7366,
|
| 430 |
+
"train_tokens_per_second": 42879.429
|
| 431 |
+
},
|
| 432 |
+
{
|
| 433 |
+
"epoch": 4.3624161073825505,
|
| 434 |
+
"grad_norm": 22.553071975708008,
|
| 435 |
+
"learning_rate": 6.000952154248989e-05,
|
| 436 |
+
"loss": 21.16220458984375,
|
| 437 |
+
"num_input_tokens_seen": 638939136,
|
| 438 |
+
"step": 1950,
|
| 439 |
+
"train_runtime": 14898.759,
|
| 440 |
+
"train_tokens_per_second": 42885.393
|
| 441 |
+
},
|
| 442 |
+
{
|
| 443 |
+
"epoch": 4.47427293064877,
|
| 444 |
+
"grad_norm": 22.87005615234375,
|
| 445 |
+
"learning_rate": 5.8819328731254464e-05,
|
| 446 |
+
"loss": 21.01521240234375,
|
| 447 |
+
"num_input_tokens_seen": 655323136,
|
| 448 |
+
"step": 2000,
|
| 449 |
+
"train_runtime": 15278.457,
|
| 450 |
+
"train_tokens_per_second": 42891.971
|
| 451 |
+
},
|
| 452 |
+
{
|
| 453 |
+
"epoch": 4.47427293064877,
|
| 454 |
+
"eval_loss": 1.14796781539917,
|
| 455 |
+
"eval_masked_accuracy": 0.7536936821619982,
|
| 456 |
+
"eval_runtime": 25.3387,
|
| 457 |
+
"eval_samples_per_second": 219.743,
|
| 458 |
+
"eval_steps_per_second": 3.433,
|
| 459 |
+
"num_input_tokens_seen": 655323136,
|
| 460 |
+
"step": 2000
|
| 461 |
+
},
|
| 462 |
+
{
|
| 463 |
+
"epoch": 4.586129753914989,
|
| 464 |
+
"grad_norm": 22.233734130859375,
|
| 465 |
+
"learning_rate": 5.762913592001905e-05,
|
| 466 |
+
"loss": 20.9041943359375,
|
| 467 |
+
"num_input_tokens_seen": 671707136,
|
| 468 |
+
"step": 2050,
|
| 469 |
+
"train_runtime": 15686.3972,
|
| 470 |
+
"train_tokens_per_second": 42820.995
|
| 471 |
+
},
|
| 472 |
+
{
|
| 473 |
+
"epoch": 4.697986577181208,
|
| 474 |
+
"grad_norm": 21.8343448638916,
|
| 475 |
+
"learning_rate": 5.6438943108783625e-05,
|
| 476 |
+
"loss": 20.91035888671875,
|
| 477 |
+
"num_input_tokens_seen": 688091136,
|
| 478 |
+
"step": 2100,
|
| 479 |
+
"train_runtime": 16066.5233,
|
| 480 |
+
"train_tokens_per_second": 42827.631
|
| 481 |
+
},
|
| 482 |
+
{
|
| 483 |
+
"epoch": 4.809843400447427,
|
| 484 |
+
"grad_norm": 21.778446197509766,
|
| 485 |
+
"learning_rate": 5.52487502975482e-05,
|
| 486 |
+
"loss": 20.8124365234375,
|
| 487 |
+
"num_input_tokens_seen": 704475136,
|
| 488 |
+
"step": 2150,
|
| 489 |
+
"train_runtime": 16446.1615,
|
| 490 |
+
"train_tokens_per_second": 42835.231
|
| 491 |
+
},
|
| 492 |
+
{
|
| 493 |
+
"epoch": 4.921700223713646,
|
| 494 |
+
"grad_norm": 21.686033248901367,
|
| 495 |
+
"learning_rate": 5.4058557486312786e-05,
|
| 496 |
+
"loss": 20.7349560546875,
|
| 497 |
+
"num_input_tokens_seen": 720859136,
|
| 498 |
+
"step": 2200,
|
| 499 |
+
"train_runtime": 16825.7207,
|
| 500 |
+
"train_tokens_per_second": 42842.69
|
| 501 |
+
},
|
| 502 |
+
{
|
| 503 |
+
"epoch": 5.033557046979865,
|
| 504 |
+
"grad_norm": 21.80782127380371,
|
| 505 |
+
"learning_rate": 5.286836467507736e-05,
|
| 506 |
+
"loss": 20.7165283203125,
|
| 507 |
+
"num_input_tokens_seen": 737233920,
|
| 508 |
+
"step": 2250,
|
| 509 |
+
"train_runtime": 17206.3274,
|
| 510 |
+
"train_tokens_per_second": 42846.675
|
| 511 |
+
},
|
| 512 |
+
{
|
| 513 |
+
"epoch": 5.145413870246085,
|
| 514 |
+
"grad_norm": 23.681589126586914,
|
| 515 |
+
"learning_rate": 5.167817186384194e-05,
|
| 516 |
+
"loss": 20.5538916015625,
|
| 517 |
+
"num_input_tokens_seen": 753617920,
|
| 518 |
+
"step": 2300,
|
| 519 |
+
"train_runtime": 17586.976,
|
| 520 |
+
"train_tokens_per_second": 42850.91
|
| 521 |
+
},
|
| 522 |
+
{
|
| 523 |
+
"epoch": 5.257270693512305,
|
| 524 |
+
"grad_norm": 23.062753677368164,
|
| 525 |
+
"learning_rate": 5.0487979052606524e-05,
|
| 526 |
+
"loss": 20.505771484375,
|
| 527 |
+
"num_input_tokens_seen": 770001920,
|
| 528 |
+
"step": 2350,
|
| 529 |
+
"train_runtime": 17966.3576,
|
| 530 |
+
"train_tokens_per_second": 42857.987
|
| 531 |
+
},
|
| 532 |
+
{
|
| 533 |
+
"epoch": 5.369127516778524,
|
| 534 |
+
"grad_norm": 22.336442947387695,
|
| 535 |
+
"learning_rate": 4.92977862413711e-05,
|
| 536 |
+
"loss": 20.479666748046874,
|
| 537 |
+
"num_input_tokens_seen": 786385920,
|
| 538 |
+
"step": 2400,
|
| 539 |
+
"train_runtime": 18346.1922,
|
| 540 |
+
"train_tokens_per_second": 42863.713
|
| 541 |
+
},
|
| 542 |
+
{
|
| 543 |
+
"epoch": 5.480984340044743,
|
| 544 |
+
"grad_norm": 22.454973220825195,
|
| 545 |
+
"learning_rate": 4.8107593430135685e-05,
|
| 546 |
+
"loss": 20.462833251953125,
|
| 547 |
+
"num_input_tokens_seen": 802769920,
|
| 548 |
+
"step": 2450,
|
| 549 |
+
"train_runtime": 18726.6308,
|
| 550 |
+
"train_tokens_per_second": 42867.824
|
| 551 |
+
},
|
| 552 |
+
{
|
| 553 |
+
"epoch": 5.592841163310962,
|
| 554 |
+
"grad_norm": 21.835525512695312,
|
| 555 |
+
"learning_rate": 4.691740061890026e-05,
|
| 556 |
+
"loss": 20.33416748046875,
|
| 557 |
+
"num_input_tokens_seen": 819153920,
|
| 558 |
+
"step": 2500,
|
| 559 |
+
"train_runtime": 19106.2918,
|
| 560 |
+
"train_tokens_per_second": 42873.517
|
| 561 |
+
},
|
| 562 |
+
{
|
| 563 |
+
"epoch": 5.592841163310962,
|
| 564 |
+
"eval_loss": 1.1132277250289917,
|
| 565 |
+
"eval_masked_accuracy": 0.7584756667111459,
|
| 566 |
+
"eval_runtime": 25.3527,
|
| 567 |
+
"eval_samples_per_second": 219.621,
|
| 568 |
+
"eval_steps_per_second": 3.432,
|
| 569 |
+
"num_input_tokens_seen": 819153920,
|
| 570 |
+
"step": 2500
|
| 571 |
+
},
|
| 572 |
+
{
|
| 573 |
+
"epoch": 5.704697986577181,
|
| 574 |
+
"grad_norm": 23.234500885009766,
|
| 575 |
+
"learning_rate": 4.5727207807664845e-05,
|
| 576 |
+
"loss": 20.25551513671875,
|
| 577 |
+
"num_input_tokens_seen": 835537920,
|
| 578 |
+
"step": 2550,
|
| 579 |
+
"train_runtime": 19513.5356,
|
| 580 |
+
"train_tokens_per_second": 42818.377
|
| 581 |
+
},
|
| 582 |
+
{
|
| 583 |
+
"epoch": 5.8165548098434,
|
| 584 |
+
"grad_norm": 21.292207717895508,
|
| 585 |
+
"learning_rate": 4.453701499642942e-05,
|
| 586 |
+
"loss": 20.156134033203124,
|
| 587 |
+
"num_input_tokens_seen": 851921920,
|
| 588 |
+
"step": 2600,
|
| 589 |
+
"train_runtime": 19893.1597,
|
| 590 |
+
"train_tokens_per_second": 42824.867
|
| 591 |
+
},
|
| 592 |
+
{
|
| 593 |
+
"epoch": 5.9284116331096195,
|
| 594 |
+
"grad_norm": 20.622861862182617,
|
| 595 |
+
"learning_rate": 4.3346822185194e-05,
|
| 596 |
+
"loss": 20.159989013671876,
|
| 597 |
+
"num_input_tokens_seen": 868305920,
|
| 598 |
+
"step": 2650,
|
| 599 |
+
"train_runtime": 20273.9368,
|
| 600 |
+
"train_tokens_per_second": 42828.679
|
| 601 |
+
},
|
| 602 |
+
{
|
| 603 |
+
"epoch": 6.040268456375839,
|
| 604 |
+
"grad_norm": 23.077646255493164,
|
| 605 |
+
"learning_rate": 4.215662937395858e-05,
|
| 606 |
+
"loss": 20.13439697265625,
|
| 607 |
+
"num_input_tokens_seen": 884680704,
|
| 608 |
+
"step": 2700,
|
| 609 |
+
"train_runtime": 20654.2855,
|
| 610 |
+
"train_tokens_per_second": 42832.791
|
| 611 |
+
},
|
| 612 |
+
{
|
| 613 |
+
"epoch": 6.152125279642058,
|
| 614 |
+
"grad_norm": 21.280527114868164,
|
| 615 |
+
"learning_rate": 4.096643656272317e-05,
|
| 616 |
+
"loss": 19.994425048828123,
|
| 617 |
+
"num_input_tokens_seen": 901064704,
|
| 618 |
+
"step": 2750,
|
| 619 |
+
"train_runtime": 21033.1909,
|
| 620 |
+
"train_tokens_per_second": 42840.133
|
| 621 |
+
},
|
| 622 |
+
{
|
| 623 |
+
"epoch": 6.263982102908278,
|
| 624 |
+
"grad_norm": 23.213178634643555,
|
| 625 |
+
"learning_rate": 3.977624375148774e-05,
|
| 626 |
+
"loss": 19.980313720703126,
|
| 627 |
+
"num_input_tokens_seen": 917448704,
|
| 628 |
+
"step": 2800,
|
| 629 |
+
"train_runtime": 21412.6494,
|
| 630 |
+
"train_tokens_per_second": 42846.109
|
| 631 |
+
},
|
| 632 |
+
{
|
| 633 |
+
"epoch": 6.375838926174497,
|
| 634 |
+
"grad_norm": 22.994335174560547,
|
| 635 |
+
"learning_rate": 3.858605094025232e-05,
|
| 636 |
+
"loss": 19.9128857421875,
|
| 637 |
+
"num_input_tokens_seen": 933832704,
|
| 638 |
+
"step": 2850,
|
| 639 |
+
"train_runtime": 21792.7904,
|
| 640 |
+
"train_tokens_per_second": 42850.534
|
| 641 |
+
},
|
| 642 |
+
{
|
| 643 |
+
"epoch": 6.487695749440716,
|
| 644 |
+
"grad_norm": 25.134178161621094,
|
| 645 |
+
"learning_rate": 3.7395858129016905e-05,
|
| 646 |
+
"loss": 19.874437255859377,
|
| 647 |
+
"num_input_tokens_seen": 950216704,
|
| 648 |
+
"step": 2900,
|
| 649 |
+
"train_runtime": 22173.1567,
|
| 650 |
+
"train_tokens_per_second": 42854.372
|
| 651 |
+
},
|
| 652 |
+
{
|
| 653 |
+
"epoch": 6.599552572706935,
|
| 654 |
+
"grad_norm": 20.69732093811035,
|
| 655 |
+
"learning_rate": 3.620566531778148e-05,
|
| 656 |
+
"loss": 19.910538330078126,
|
| 657 |
+
"num_input_tokens_seen": 966600704,
|
| 658 |
+
"step": 2950,
|
| 659 |
+
"train_runtime": 22553.2542,
|
| 660 |
+
"train_tokens_per_second": 42858.591
|
| 661 |
+
},
|
| 662 |
+
{
|
| 663 |
+
"epoch": 6.7114093959731544,
|
| 664 |
+
"grad_norm": 21.227550506591797,
|
| 665 |
+
"learning_rate": 3.501547250654606e-05,
|
| 666 |
+
"loss": 19.87051025390625,
|
| 667 |
+
"num_input_tokens_seen": 982984704,
|
| 668 |
+
"step": 3000,
|
| 669 |
+
"train_runtime": 22933.4022,
|
| 670 |
+
"train_tokens_per_second": 42862.576
|
| 671 |
+
},
|
| 672 |
+
{
|
| 673 |
+
"epoch": 6.7114093959731544,
|
| 674 |
+
"eval_loss": 1.0944103002548218,
|
| 675 |
+
"eval_masked_accuracy": 0.7619272139314145,
|
| 676 |
+
"eval_runtime": 25.3399,
|
| 677 |
+
"eval_samples_per_second": 219.732,
|
| 678 |
+
"eval_steps_per_second": 3.433,
|
| 679 |
+
"num_input_tokens_seen": 982984704,
|
| 680 |
+
"step": 3000
|
| 681 |
+
},
|
| 682 |
+
{
|
| 683 |
+
"epoch": 6.823266219239374,
|
| 684 |
+
"grad_norm": 21.8638858795166,
|
| 685 |
+
"learning_rate": 3.382527969531064e-05,
|
| 686 |
+
"loss": 19.65084228515625,
|
| 687 |
+
"num_input_tokens_seen": 999368704,
|
| 688 |
+
"step": 3050,
|
| 689 |
+
"train_runtime": 23340.7393,
|
| 690 |
+
"train_tokens_per_second": 42816.497
|
| 691 |
+
},
|
| 692 |
+
{
|
| 693 |
+
"epoch": 6.935123042505593,
|
| 694 |
+
"grad_norm": 25.967254638671875,
|
| 695 |
+
"learning_rate": 3.263508688407522e-05,
|
| 696 |
+
"loss": 19.7602978515625,
|
| 697 |
+
"num_input_tokens_seen": 1015752704,
|
| 698 |
+
"step": 3100,
|
| 699 |
+
"train_runtime": 23721.0833,
|
| 700 |
+
"train_tokens_per_second": 42820.671
|
| 701 |
+
},
|
| 702 |
+
{
|
| 703 |
+
"epoch": 7.046979865771812,
|
| 704 |
+
"grad_norm": 29.288837432861328,
|
| 705 |
+
"learning_rate": 3.1444894072839803e-05,
|
| 706 |
+
"loss": 19.64591064453125,
|
| 707 |
+
"num_input_tokens_seen": 1032127488,
|
| 708 |
+
"step": 3150,
|
| 709 |
+
"train_runtime": 24100.8116,
|
| 710 |
+
"train_tokens_per_second": 42825.425
|
| 711 |
+
},
|
| 712 |
+
{
|
| 713 |
+
"epoch": 7.158836689038031,
|
| 714 |
+
"grad_norm": 20.559537887573242,
|
| 715 |
+
"learning_rate": 3.025470126160438e-05,
|
| 716 |
+
"loss": 19.643092041015624,
|
| 717 |
+
"num_input_tokens_seen": 1048511488,
|
| 718 |
+
"step": 3200,
|
| 719 |
+
"train_runtime": 24479.6774,
|
| 720 |
+
"train_tokens_per_second": 42831.916
|
| 721 |
+
},
|
| 722 |
+
{
|
| 723 |
+
"epoch": 7.27069351230425,
|
| 724 |
+
"grad_norm": 21.216949462890625,
|
| 725 |
+
"learning_rate": 2.906450845036896e-05,
|
| 726 |
+
"loss": 19.5406787109375,
|
| 727 |
+
"num_input_tokens_seen": 1064895488,
|
| 728 |
+
"step": 3250,
|
| 729 |
+
"train_runtime": 24859.9356,
|
| 730 |
+
"train_tokens_per_second": 42835.81
|
| 731 |
+
},
|
| 732 |
+
{
|
| 733 |
+
"epoch": 7.382550335570469,
|
| 734 |
+
"grad_norm": 19.948974609375,
|
| 735 |
+
"learning_rate": 2.787431563913354e-05,
|
| 736 |
+
"loss": 19.52265625,
|
| 737 |
+
"num_input_tokens_seen": 1081279488,
|
| 738 |
+
"step": 3300,
|
| 739 |
+
"train_runtime": 25239.4624,
|
| 740 |
+
"train_tokens_per_second": 42840.829
|
| 741 |
+
},
|
| 742 |
+
{
|
| 743 |
+
"epoch": 7.494407158836689,
|
| 744 |
+
"grad_norm": 24.806529998779297,
|
| 745 |
+
"learning_rate": 2.668412282789812e-05,
|
| 746 |
+
"loss": 19.483994140625,
|
| 747 |
+
"num_input_tokens_seen": 1097663488,
|
| 748 |
+
"step": 3350,
|
| 749 |
+
"train_runtime": 25619.1507,
|
| 750 |
+
"train_tokens_per_second": 42845.428
|
| 751 |
+
},
|
| 752 |
+
{
|
| 753 |
+
"epoch": 7.6062639821029085,
|
| 754 |
+
"grad_norm": 22.306764602661133,
|
| 755 |
+
"learning_rate": 2.54939300166627e-05,
|
| 756 |
+
"loss": 19.500614013671875,
|
| 757 |
+
"num_input_tokens_seen": 1114047488,
|
| 758 |
+
"step": 3400,
|
| 759 |
+
"train_runtime": 25998.4643,
|
| 760 |
+
"train_tokens_per_second": 42850.511
|
| 761 |
+
},
|
| 762 |
+
{
|
| 763 |
+
"epoch": 7.718120805369128,
|
| 764 |
+
"grad_norm": 20.259811401367188,
|
| 765 |
+
"learning_rate": 2.430373720542728e-05,
|
| 766 |
+
"loss": 19.44253173828125,
|
| 767 |
+
"num_input_tokens_seen": 1130431488,
|
| 768 |
+
"step": 3450,
|
| 769 |
+
"train_runtime": 26378.2281,
|
| 770 |
+
"train_tokens_per_second": 42854.716
|
| 771 |
+
},
|
| 772 |
+
{
|
| 773 |
+
"epoch": 7.829977628635347,
|
| 774 |
+
"grad_norm": 31.171615600585938,
|
| 775 |
+
"learning_rate": 2.311354439419186e-05,
|
| 776 |
+
"loss": 19.37197021484375,
|
| 777 |
+
"num_input_tokens_seen": 1146815488,
|
| 778 |
+
"step": 3500,
|
| 779 |
+
"train_runtime": 26757.8966,
|
| 780 |
+
"train_tokens_per_second": 42858.955
|
| 781 |
+
},
|
| 782 |
+
{
|
| 783 |
+
"epoch": 7.829977628635347,
|
| 784 |
+
"eval_loss": 1.0732760429382324,
|
| 785 |
+
"eval_masked_accuracy": 0.7661423014370904,
|
| 786 |
+
"eval_runtime": 25.3349,
|
| 787 |
+
"eval_samples_per_second": 219.775,
|
| 788 |
+
"eval_steps_per_second": 3.434,
|
| 789 |
+
"num_input_tokens_seen": 1146815488,
|
| 790 |
+
"step": 3500
|
| 791 |
+
},
|
| 792 |
+
{
|
| 793 |
+
"epoch": 7.941834451901566,
|
| 794 |
+
"grad_norm": 19.443883895874023,
|
| 795 |
+
"learning_rate": 2.192335158295644e-05,
|
| 796 |
+
"loss": 19.302330322265625,
|
| 797 |
+
"num_input_tokens_seen": 1163199488,
|
| 798 |
+
"step": 3550,
|
| 799 |
+
"train_runtime": 27165.3477,
|
| 800 |
+
"train_tokens_per_second": 42819.238
|
| 801 |
+
},
|
| 802 |
+
{
|
| 803 |
+
"epoch": 8.053691275167786,
|
| 804 |
+
"grad_norm": 19.65326690673828,
|
| 805 |
+
"learning_rate": 2.073315877172102e-05,
|
| 806 |
+
"loss": 19.334464111328124,
|
| 807 |
+
"num_input_tokens_seen": 1179574272,
|
| 808 |
+
"step": 3600,
|
| 809 |
+
"train_runtime": 27545.6424,
|
| 810 |
+
"train_tokens_per_second": 42822.536
|
| 811 |
+
},
|
| 812 |
+
{
|
| 813 |
+
"epoch": 8.165548098434005,
|
| 814 |
+
"grad_norm": 20.760732650756836,
|
| 815 |
+
"learning_rate": 1.9542965960485597e-05,
|
| 816 |
+
"loss": 19.248583984375,
|
| 817 |
+
"num_input_tokens_seen": 1195958272,
|
| 818 |
+
"step": 3650,
|
| 819 |
+
"train_runtime": 27925.9284,
|
| 820 |
+
"train_tokens_per_second": 42826.088
|
| 821 |
+
},
|
| 822 |
+
{
|
| 823 |
+
"epoch": 8.277404921700224,
|
| 824 |
+
"grad_norm": 19.925457000732422,
|
| 825 |
+
"learning_rate": 1.835277314925018e-05,
|
| 826 |
+
"loss": 19.22830322265625,
|
| 827 |
+
"num_input_tokens_seen": 1212342272,
|
| 828 |
+
"step": 3700,
|
| 829 |
+
"train_runtime": 28305.1518,
|
| 830 |
+
"train_tokens_per_second": 42831.152
|
| 831 |
+
},
|
| 832 |
+
{
|
| 833 |
+
"epoch": 8.389261744966444,
|
| 834 |
+
"grad_norm": 20.903831481933594,
|
| 835 |
+
"learning_rate": 1.7162580338014758e-05,
|
| 836 |
+
"loss": 19.20845458984375,
|
| 837 |
+
"num_input_tokens_seen": 1228726272,
|
| 838 |
+
"step": 3750,
|
| 839 |
+
"train_runtime": 28684.7612,
|
| 840 |
+
"train_tokens_per_second": 42835.506
|
| 841 |
+
},
|
| 842 |
+
{
|
| 843 |
+
"epoch": 8.501118568232663,
|
| 844 |
+
"grad_norm": 25.746238708496094,
|
| 845 |
+
"learning_rate": 1.597238752677934e-05,
|
| 846 |
+
"loss": 19.181019287109375,
|
| 847 |
+
"num_input_tokens_seen": 1245110272,
|
| 848 |
+
"step": 3800,
|
| 849 |
+
"train_runtime": 29065.5625,
|
| 850 |
+
"train_tokens_per_second": 42837.99
|
| 851 |
+
},
|
| 852 |
+
{
|
| 853 |
+
"epoch": 8.612975391498882,
|
| 854 |
+
"grad_norm": 19.735681533813477,
|
| 855 |
+
"learning_rate": 1.4782194715543917e-05,
|
| 856 |
+
"loss": 19.200714111328125,
|
| 857 |
+
"num_input_tokens_seen": 1261494272,
|
| 858 |
+
"step": 3850,
|
| 859 |
+
"train_runtime": 29444.9518,
|
| 860 |
+
"train_tokens_per_second": 42842.463
|
| 861 |
+
},
|
| 862 |
+
{
|
| 863 |
+
"epoch": 8.724832214765101,
|
| 864 |
+
"grad_norm": 20.348718643188477,
|
| 865 |
+
"learning_rate": 1.35920019043085e-05,
|
| 866 |
+
"loss": 19.14758056640625,
|
| 867 |
+
"num_input_tokens_seen": 1277878272,
|
| 868 |
+
"step": 3900,
|
| 869 |
+
"train_runtime": 29825.2862,
|
| 870 |
+
"train_tokens_per_second": 42845.466
|
| 871 |
+
},
|
| 872 |
+
{
|
| 873 |
+
"epoch": 8.83668903803132,
|
| 874 |
+
"grad_norm": 19.208051681518555,
|
| 875 |
+
"learning_rate": 1.2401809093073078e-05,
|
| 876 |
+
"loss": 19.07127685546875,
|
| 877 |
+
"num_input_tokens_seen": 1294262272,
|
| 878 |
+
"step": 3950,
|
| 879 |
+
"train_runtime": 30205.5851,
|
| 880 |
+
"train_tokens_per_second": 42848.442
|
| 881 |
+
},
|
| 882 |
+
{
|
| 883 |
+
"epoch": 8.94854586129754,
|
| 884 |
+
"grad_norm": 20.35933494567871,
|
| 885 |
+
"learning_rate": 1.1211616281837658e-05,
|
| 886 |
+
"loss": 19.101888427734377,
|
| 887 |
+
"num_input_tokens_seen": 1310646272,
|
| 888 |
+
"step": 4000,
|
| 889 |
+
"train_runtime": 30585.6001,
|
| 890 |
+
"train_tokens_per_second": 42851.743
|
| 891 |
+
},
|
| 892 |
+
{
|
| 893 |
+
"epoch": 8.94854586129754,
|
| 894 |
+
"eval_loss": 1.0551483631134033,
|
| 895 |
+
"eval_masked_accuracy": 0.7687605175839743,
|
| 896 |
+
"eval_runtime": 25.3745,
|
| 897 |
+
"eval_samples_per_second": 219.433,
|
| 898 |
+
"eval_steps_per_second": 3.429,
|
| 899 |
+
"num_input_tokens_seen": 1310646272,
|
| 900 |
+
"step": 4000
|
| 901 |
+
},
|
| 902 |
+
{
|
| 903 |
+
"epoch": 9.060402684563758,
|
| 904 |
+
"grad_norm": 19.61882972717285,
|
| 905 |
+
"learning_rate": 1.0021423470602239e-05,
|
| 906 |
+
"loss": 19.01905517578125,
|
| 907 |
+
"num_input_tokens_seen": 1327021056,
|
| 908 |
+
"step": 4050,
|
| 909 |
+
"train_runtime": 30993.7338,
|
| 910 |
+
"train_tokens_per_second": 42815.785
|
| 911 |
+
},
|
| 912 |
+
{
|
| 913 |
+
"epoch": 9.172259507829978,
|
| 914 |
+
"grad_norm": 20.218429565429688,
|
| 915 |
+
"learning_rate": 8.831230659366818e-06,
|
| 916 |
+
"loss": 19.05180419921875,
|
| 917 |
+
"num_input_tokens_seen": 1343405056,
|
| 918 |
+
"step": 4100,
|
| 919 |
+
"train_runtime": 31373.034,
|
| 920 |
+
"train_tokens_per_second": 42820.374
|
| 921 |
+
},
|
| 922 |
+
{
|
| 923 |
+
"epoch": 9.284116331096197,
|
| 924 |
+
"grad_norm": 19.39629554748535,
|
| 925 |
+
"learning_rate": 7.641037848131398e-06,
|
| 926 |
+
"loss": 19.03988525390625,
|
| 927 |
+
"num_input_tokens_seen": 1359789056,
|
| 928 |
+
"step": 4150,
|
| 929 |
+
"train_runtime": 31753.5802,
|
| 930 |
+
"train_tokens_per_second": 42823.173
|
| 931 |
+
},
|
| 932 |
+
{
|
| 933 |
+
"epoch": 9.395973154362416,
|
| 934 |
+
"grad_norm": 20.272720336914062,
|
| 935 |
+
"learning_rate": 6.4508450368959775e-06,
|
| 936 |
+
"loss": 18.968330078125,
|
| 937 |
+
"num_input_tokens_seen": 1376173056,
|
| 938 |
+
"step": 4200,
|
| 939 |
+
"train_runtime": 32133.9665,
|
| 940 |
+
"train_tokens_per_second": 42826.119
|
| 941 |
+
},
|
| 942 |
+
{
|
| 943 |
+
"epoch": 9.507829977628635,
|
| 944 |
+
"grad_norm": 19.632165908813477,
|
| 945 |
+
"learning_rate": 5.260652225660558e-06,
|
| 946 |
+
"loss": 18.9827783203125,
|
| 947 |
+
"num_input_tokens_seen": 1392557056,
|
| 948 |
+
"step": 4250,
|
| 949 |
+
"train_runtime": 32514.0065,
|
| 950 |
+
"train_tokens_per_second": 42829.451
|
| 951 |
+
},
|
| 952 |
+
{
|
| 953 |
+
"epoch": 9.619686800894854,
|
| 954 |
+
"grad_norm": 19.80462646484375,
|
| 955 |
+
"learning_rate": 4.0704594144251375e-06,
|
| 956 |
+
"loss": 18.956285400390627,
|
| 957 |
+
"num_input_tokens_seen": 1408941056,
|
| 958 |
+
"step": 4300,
|
| 959 |
+
"train_runtime": 32894.5758,
|
| 960 |
+
"train_tokens_per_second": 42832.018
|
| 961 |
+
},
|
| 962 |
+
{
|
| 963 |
+
"epoch": 9.731543624161073,
|
| 964 |
+
"grad_norm": 20.74568748474121,
|
| 965 |
+
"learning_rate": 2.8802666031897166e-06,
|
| 966 |
+
"loss": 18.866341552734376,
|
| 967 |
+
"num_input_tokens_seen": 1425325056,
|
| 968 |
+
"step": 4350,
|
| 969 |
+
"train_runtime": 33275.028,
|
| 970 |
+
"train_tokens_per_second": 42834.676
|
| 971 |
+
},
|
| 972 |
+
{
|
| 973 |
+
"epoch": 9.843400447427292,
|
| 974 |
+
"grad_norm": 19.217836380004883,
|
| 975 |
+
"learning_rate": 1.6900737919542966e-06,
|
| 976 |
+
"loss": 18.920970458984375,
|
| 977 |
+
"num_input_tokens_seen": 1441709056,
|
| 978 |
+
"step": 4400,
|
| 979 |
+
"train_runtime": 33654.5878,
|
| 980 |
+
"train_tokens_per_second": 42838.411
|
| 981 |
+
},
|
| 982 |
+
{
|
| 983 |
+
"epoch": 9.955257270693512,
|
| 984 |
+
"grad_norm": 19.824554443359375,
|
| 985 |
+
"learning_rate": 4.998809807188764e-07,
|
| 986 |
+
"loss": 18.913228759765627,
|
| 987 |
+
"num_input_tokens_seen": 1458093056,
|
| 988 |
+
"step": 4450,
|
| 989 |
+
"train_runtime": 34033.2944,
|
| 990 |
+
"train_tokens_per_second": 42843.136
|
| 991 |
+
},
|
| 992 |
+
{
|
| 993 |
+
"epoch": 10.0,
|
| 994 |
+
"eval_loss": 1.0481648445129395,
|
| 995 |
+
"eval_masked_accuracy": 0.7704025727385737,
|
| 996 |
+
"eval_runtime": 25.3462,
|
| 997 |
+
"eval_samples_per_second": 219.678,
|
| 998 |
+
"eval_steps_per_second": 3.432,
|
| 999 |
+
"num_input_tokens_seen": 1464637440,
|
| 1000 |
+
"step": 4470
|
| 1001 |
+
},
|
| 1002 |
+
{
|
| 1003 |
+
"epoch": 10.0,
|
| 1004 |
+
"num_input_tokens_seen": 1464637440,
|
| 1005 |
+
"step": 4470,
|
| 1006 |
+
"total_flos": 4.917799558028206e+18,
|
| 1007 |
+
"train_loss": 21.80173199032777,
|
| 1008 |
+
"train_runtime": 34212.9191,
|
| 1009 |
+
"train_samples_per_second": 83.612,
|
| 1010 |
+
"train_steps_per_second": 0.131
|
| 1011 |
+
}
|
| 1012 |
+
],
|
| 1013 |
+
"logging_steps": 50,
|
| 1014 |
+
"max_steps": 4470,
|
| 1015 |
+
"num_input_tokens_seen": 1464637440,
|
| 1016 |
+
"num_train_epochs": 10,
|
| 1017 |
+
"save_steps": 500,
|
| 1018 |
+
"stateful_callbacks": {
|
| 1019 |
+
"EarlyStoppingCallback": {
|
| 1020 |
+
"args": {
|
| 1021 |
+
"early_stopping_patience": 5,
|
| 1022 |
+
"early_stopping_threshold": 0.0001
|
| 1023 |
+
},
|
| 1024 |
+
"attributes": {
|
| 1025 |
+
"early_stopping_patience_counter": 0
|
| 1026 |
+
}
|
| 1027 |
+
},
|
| 1028 |
+
"TrainerControl": {
|
| 1029 |
+
"args": {
|
| 1030 |
+
"should_epoch_stop": false,
|
| 1031 |
+
"should_evaluate": false,
|
| 1032 |
+
"should_log": false,
|
| 1033 |
+
"should_save": true,
|
| 1034 |
+
"should_training_stop": true
|
| 1035 |
+
},
|
| 1036 |
+
"attributes": {}
|
| 1037 |
+
}
|
| 1038 |
+
},
|
| 1039 |
+
"total_flos": 4.917799558028206e+18,
|
| 1040 |
+
"train_batch_size": 40,
|
| 1041 |
+
"trial_name": null,
|
| 1042 |
+
"trial_params": null
|
| 1043 |
+
}
|
training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6df1f2af59cfef522294063f953ccb9171365cb47145a67c1538a269dde99f1b
|
| 3 |
+
size 5265
|
training_log.jsonl
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"loss": 32.84220504760742, "grad_norm": 30.264930725097656, "learning_rate": 0.0, "epoch": 0.002516910492370615, "num_input_tokens_seen": 368640, "train_runtime": 8.8205, "train_tokens_per_second": 41793.643, "step": 1, "wall_time": "2026-06-21T18:29:45"}
|
| 2 |
+
{"loss": 31.76995086669922, "grad_norm": 32.67197036743164, "learning_rate": 0.0, "epoch": 0.0022371364653243847, "num_input_tokens_seen": 327680, "train_runtime": 7.8573, "train_tokens_per_second": 41704.008, "step": 1, "wall_time": "2026-06-21T18:29:54"}
|
| 3 |
+
{"loss": 32.44386509486607, "grad_norm": 31.965707778930664, "learning_rate": 1.821561338289963e-05, "epoch": 0.11185682326621924, "num_input_tokens_seen": 16384000, "train_runtime": 380.1137, "train_tokens_per_second": 43102.889, "step": 50, "wall_time": "2026-06-21T18:36:06"}
|
| 4 |
+
{"loss": 31.56456787109375, "grad_norm": 34.0458984375, "learning_rate": 3.680297397769517e-05, "epoch": 0.22371364653243847, "num_input_tokens_seen": 32768000, "train_runtime": 759.9744, "train_tokens_per_second": 43117.243, "step": 100, "wall_time": "2026-06-21T18:42:26"}
|
| 5 |
+
{"loss": 30.703720703125, "grad_norm": 36.13215255737305, "learning_rate": 5.539033457249071e-05, "epoch": 0.33557046979865773, "num_input_tokens_seen": 49152000, "train_runtime": 1140.0254, "train_tokens_per_second": 43114.831, "step": 150, "wall_time": "2026-06-21T18:48:46"}
|
| 6 |
+
{"loss": 30.0342919921875, "grad_norm": 75.77301788330078, "learning_rate": 7.397769516728626e-05, "epoch": 0.44742729306487694, "num_input_tokens_seen": 65536000, "train_runtime": 1519.7868, "train_tokens_per_second": 43121.837, "step": 200, "wall_time": "2026-06-21T18:55:06"}
|
| 7 |
+
{"loss": 29.2935986328125, "grad_norm": 31.4502010345459, "learning_rate": 9.256505576208179e-05, "epoch": 0.5592841163310962, "num_input_tokens_seen": 81920000, "train_runtime": 1899.4384, "train_tokens_per_second": 43128.538, "step": 250, "wall_time": "2026-06-21T19:01:26"}
|
| 8 |
+
{"loss": 28.63412353515625, "grad_norm": 35.78462600708008, "learning_rate": 9.928588431325875e-05, "epoch": 0.6711409395973155, "num_input_tokens_seen": 98304000, "train_runtime": 2279.106, "train_tokens_per_second": 43132.703, "step": 300, "wall_time": "2026-06-21T19:07:45"}
|
| 9 |
+
{"loss": 27.89616455078125, "grad_norm": 32.92461395263672, "learning_rate": 9.809569150202333e-05, "epoch": 0.7829977628635347, "num_input_tokens_seen": 114688000, "train_runtime": 2659.6638, "train_tokens_per_second": 43121.24, "step": 350, "wall_time": "2026-06-21T19:14:06"}
|
| 10 |
+
{"loss": 27.38276123046875, "grad_norm": 31.881704330444336, "learning_rate": 9.69054986907879e-05, "epoch": 0.8948545861297539, "num_input_tokens_seen": 131072000, "train_runtime": 3039.907, "train_tokens_per_second": 43117.108, "step": 400, "wall_time": "2026-06-21T19:20:26"}
|
| 11 |
+
{"loss": 26.72684814453125, "grad_norm": 28.32501220703125, "learning_rate": 9.57153058795525e-05, "epoch": 1.0067114093959733, "num_input_tokens_seen": 147446784, "train_runtime": 3419.3158, "train_tokens_per_second": 43121.721, "step": 450, "wall_time": "2026-06-21T19:26:45"}
|
| 12 |
+
{"loss": 26.2695166015625, "grad_norm": 27.903852462768555, "learning_rate": 9.452511306831707e-05, "epoch": 1.1185682326621924, "num_input_tokens_seen": 163830784, "train_runtime": 3799.2571, "train_tokens_per_second": 43121.794, "step": 500, "wall_time": "2026-06-21T19:33:05"}
|
| 13 |
+
{"eval_loss": 1.3941264152526855, "eval_masked_accuracy": 0.7140736610865701, "eval_runtime": 25.352, "eval_samples_per_second": 219.627, "eval_steps_per_second": 3.432, "epoch": 1.1185682326621924, "num_input_tokens_seen": 163830784, "step": 500, "wall_time": "2026-06-21T19:33:31"}
|
| 14 |
+
{"loss": 25.86959228515625, "grad_norm": 25.51409339904785, "learning_rate": 9.333492025708165e-05, "epoch": 1.2304250559284116, "num_input_tokens_seen": 180214784, "train_runtime": 4207.1627, "train_tokens_per_second": 42835.231, "step": 550, "wall_time": "2026-06-21T19:39:53"}
|
| 15 |
+
{"loss": 25.3655517578125, "grad_norm": 31.309749603271484, "learning_rate": 9.214472744584624e-05, "epoch": 1.342281879194631, "num_input_tokens_seen": 196598784, "train_runtime": 4586.3773, "train_tokens_per_second": 42865.812, "step": 600, "wall_time": "2026-06-21T19:46:12"}
|
| 16 |
+
{"loss": 25.1278271484375, "grad_norm": 28.270841598510742, "learning_rate": 9.095453463461082e-05, "epoch": 1.45413870246085, "num_input_tokens_seen": 212982784, "train_runtime": 4966.2335, "train_tokens_per_second": 42886.18, "step": 650, "wall_time": "2026-06-21T19:52:32"}
|
| 17 |
+
{"loss": 24.80788818359375, "grad_norm": 27.522308349609375, "learning_rate": 8.976434182337538e-05, "epoch": 1.5659955257270695, "num_input_tokens_seen": 229366784, "train_runtime": 5346.6697, "train_tokens_per_second": 42899.0, "step": 700, "wall_time": "2026-06-21T19:58:53"}
|
| 18 |
+
{"loss": 24.6148046875, "grad_norm": 27.093711853027344, "learning_rate": 8.857414901213997e-05, "epoch": 1.6778523489932886, "num_input_tokens_seen": 245750784, "train_runtime": 5726.4531, "train_tokens_per_second": 42915.008, "step": 750, "wall_time": "2026-06-21T20:05:13"}
|
| 19 |
+
{"loss": 24.40305908203125, "grad_norm": 25.49102210998535, "learning_rate": 8.738395620090455e-05, "epoch": 1.7897091722595078, "num_input_tokens_seen": 262134784, "train_runtime": 6106.8482, "train_tokens_per_second": 42924.726, "step": 800, "wall_time": "2026-06-21T20:11:33"}
|
| 20 |
+
{"loss": 24.0977783203125, "grad_norm": 25.048629760742188, "learning_rate": 8.619376338966914e-05, "epoch": 1.901565995525727, "num_input_tokens_seen": 278518784, "train_runtime": 6486.6563, "train_tokens_per_second": 42937.189, "step": 850, "wall_time": "2026-06-21T20:17:53"}
|
| 21 |
+
{"loss": 23.89361572265625, "grad_norm": 25.157657623291016, "learning_rate": 8.500357057843372e-05, "epoch": 2.0134228187919465, "num_input_tokens_seen": 294893568, "train_runtime": 6867.068, "train_tokens_per_second": 42943.155, "step": 900, "wall_time": "2026-06-21T20:24:13"}
|
| 22 |
+
{"loss": 23.57096435546875, "grad_norm": 25.902904510498047, "learning_rate": 8.381337776719829e-05, "epoch": 2.1252796420581657, "num_input_tokens_seen": 311277568, "train_runtime": 7246.8643, "train_tokens_per_second": 42953.415, "step": 950, "wall_time": "2026-06-21T20:30:33"}
|
| 23 |
+
{"loss": 23.42177490234375, "grad_norm": 24.553022384643555, "learning_rate": 8.262318495596287e-05, "epoch": 2.237136465324385, "num_input_tokens_seen": 327661568, "train_runtime": 7626.2318, "train_tokens_per_second": 42965.068, "step": 1000, "wall_time": "2026-06-21T20:36:52"}
|
| 24 |
+
{"eval_loss": 1.2587531805038452, "eval_masked_accuracy": 0.7357594334234474, "eval_runtime": 25.3576, "eval_samples_per_second": 219.579, "eval_steps_per_second": 3.431, "epoch": 2.237136465324385, "num_input_tokens_seen": 327661568, "step": 1000, "wall_time": "2026-06-21T20:37:18"}
|
| 25 |
+
{"loss": 23.259130859375, "grad_norm": 24.940990447998047, "learning_rate": 8.143299214472745e-05, "epoch": 2.348993288590604, "num_input_tokens_seen": 344045568, "train_runtime": 8033.7212, "train_tokens_per_second": 42825.182, "step": 1050, "wall_time": "2026-06-21T20:43:40"}
|
| 26 |
+
{"loss": 23.12722900390625, "grad_norm": 24.946720123291016, "learning_rate": 8.024279933349202e-05, "epoch": 2.460850111856823, "num_input_tokens_seen": 360429568, "train_runtime": 8413.8773, "train_tokens_per_second": 42837.512, "step": 1100, "wall_time": "2026-06-21T20:50:00"}
|
| 27 |
+
{"loss": 22.9796826171875, "grad_norm": 24.135683059692383, "learning_rate": 7.905260652225661e-05, "epoch": 2.5727069351230423, "num_input_tokens_seen": 376813568, "train_runtime": 8793.1262, "train_tokens_per_second": 42853.197, "step": 1150, "wall_time": "2026-06-21T20:56:19"}
|
| 28 |
+
{"loss": 22.77516845703125, "grad_norm": 25.850025177001953, "learning_rate": 7.786241371102119e-05, "epoch": 2.684563758389262, "num_input_tokens_seen": 393197568, "train_runtime": 9172.793, "train_tokens_per_second": 42865.632, "step": 1200, "wall_time": "2026-06-21T21:02:39"}
|
| 29 |
+
{"loss": 22.6738720703125, "grad_norm": 22.62376594543457, "learning_rate": 7.667222089978577e-05, "epoch": 2.796420581655481, "num_input_tokens_seen": 409581568, "train_runtime": 9553.7727, "train_tokens_per_second": 42871.186, "step": 1250, "wall_time": "2026-06-21T21:09:00"}
|
| 30 |
+
{"loss": 22.53384033203125, "grad_norm": 22.361886978149414, "learning_rate": 7.548202808855035e-05, "epoch": 2.9082774049217, "num_input_tokens_seen": 425965568, "train_runtime": 9932.9496, "train_tokens_per_second": 42884.096, "step": 1300, "wall_time": "2026-06-21T21:15:19"}
|
| 31 |
+
{"loss": 22.38953125, "grad_norm": 24.093069076538086, "learning_rate": 7.429183527731492e-05, "epoch": 3.0201342281879193, "num_input_tokens_seen": 442340352, "train_runtime": 10313.033, "train_tokens_per_second": 42891.393, "step": 1350, "wall_time": "2026-06-21T21:21:39"}
|
| 32 |
+
{"loss": 22.18701416015625, "grad_norm": 22.419540405273438, "learning_rate": 7.310164246607951e-05, "epoch": 3.131991051454139, "num_input_tokens_seen": 458724352, "train_runtime": 10693.1337, "train_tokens_per_second": 42898.964, "step": 1400, "wall_time": "2026-06-21T21:27:59"}
|
| 33 |
+
{"loss": 22.1206396484375, "grad_norm": 24.061424255371094, "learning_rate": 7.191144965484409e-05, "epoch": 3.243847874720358, "num_input_tokens_seen": 475108352, "train_runtime": 11073.944, "train_tokens_per_second": 42903.265, "step": 1450, "wall_time": "2026-06-21T21:34:20"}
|
| 34 |
+
{"loss": 21.99459228515625, "grad_norm": 21.660381317138672, "learning_rate": 7.072125684360867e-05, "epoch": 3.3557046979865772, "num_input_tokens_seen": 491492352, "train_runtime": 11453.1596, "train_tokens_per_second": 42913.254, "step": 1500, "wall_time": "2026-06-21T21:40:39"}
|
| 35 |
+
{"eval_loss": 1.194376826286316, "eval_masked_accuracy": 0.7462490242091618, "eval_runtime": 25.3329, "eval_samples_per_second": 219.794, "eval_steps_per_second": 3.434, "epoch": 3.3557046979865772, "num_input_tokens_seen": 491492352, "step": 1500, "wall_time": "2026-06-21T21:41:05"}
|
| 36 |
+
{"loss": 21.8618212890625, "grad_norm": 23.65436553955078, "learning_rate": 6.953106403237324e-05, "epoch": 3.4675615212527964, "num_input_tokens_seen": 507876352, "train_runtime": 11860.4601, "train_tokens_per_second": 42820.965, "step": 1550, "wall_time": "2026-06-21T21:47:27"}
|
| 37 |
+
{"loss": 21.84644287109375, "grad_norm": 23.492694854736328, "learning_rate": 6.834087122113782e-05, "epoch": 3.5794183445190155, "num_input_tokens_seen": 524260352, "train_runtime": 12240.4952, "train_tokens_per_second": 42829.995, "step": 1600, "wall_time": "2026-06-21T21:53:47"}
|
| 38 |
+
{"loss": 21.695458984375, "grad_norm": 22.143146514892578, "learning_rate": 6.715067840990241e-05, "epoch": 3.6912751677852347, "num_input_tokens_seen": 540644352, "train_runtime": 12620.3553, "train_tokens_per_second": 42839.075, "step": 1650, "wall_time": "2026-06-21T22:00:06"}
|
| 39 |
+
{"loss": 21.49891357421875, "grad_norm": 22.582658767700195, "learning_rate": 6.596048559866699e-05, "epoch": 3.8031319910514543, "num_input_tokens_seen": 557028352, "train_runtime": 13000.3344, "train_tokens_per_second": 42847.233, "step": 1700, "wall_time": "2026-06-21T22:06:26"}
|
| 40 |
+
{"loss": 21.4741015625, "grad_norm": 22.46770477294922, "learning_rate": 6.477029278743157e-05, "epoch": 3.9149888143176734, "num_input_tokens_seen": 573412352, "train_runtime": 13380.7098, "train_tokens_per_second": 42853.657, "step": 1750, "wall_time": "2026-06-21T22:12:47"}
|
| 41 |
+
{"loss": 21.39329833984375, "grad_norm": 22.605365753173828, "learning_rate": 6.358009997619616e-05, "epoch": 4.026845637583893, "num_input_tokens_seen": 589787136, "train_runtime": 13759.8334, "train_tokens_per_second": 42862.956, "step": 1800, "wall_time": "2026-06-21T22:19:06"}
|
| 42 |
+
{"loss": 21.18496826171875, "grad_norm": 21.95088005065918, "learning_rate": 6.238990716496072e-05, "epoch": 4.138702460850112, "num_input_tokens_seen": 606171136, "train_runtime": 14139.2097, "train_tokens_per_second": 42871.642, "step": 1850, "wall_time": "2026-06-21T22:25:25"}
|
| 43 |
+
{"loss": 21.2357177734375, "grad_norm": 22.34777069091797, "learning_rate": 6.11997143537253e-05, "epoch": 4.250559284116331, "num_input_tokens_seen": 622555136, "train_runtime": 14518.7366, "train_tokens_per_second": 42879.429, "step": 1900, "wall_time": "2026-06-21T22:31:45"}
|
| 44 |
+
{"loss": 21.16220458984375, "grad_norm": 22.553071975708008, "learning_rate": 6.000952154248989e-05, "epoch": 4.3624161073825505, "num_input_tokens_seen": 638939136, "train_runtime": 14898.759, "train_tokens_per_second": 42885.393, "step": 1950, "wall_time": "2026-06-21T22:38:05"}
|
| 45 |
+
{"loss": 21.01521240234375, "grad_norm": 22.87005615234375, "learning_rate": 5.8819328731254464e-05, "epoch": 4.47427293064877, "num_input_tokens_seen": 655323136, "train_runtime": 15278.457, "train_tokens_per_second": 42891.971, "step": 2000, "wall_time": "2026-06-21T22:44:25"}
|
| 46 |
+
{"eval_loss": 1.14796781539917, "eval_masked_accuracy": 0.7536936821619982, "eval_runtime": 25.3387, "eval_samples_per_second": 219.743, "eval_steps_per_second": 3.433, "epoch": 4.47427293064877, "num_input_tokens_seen": 655323136, "step": 2000, "wall_time": "2026-06-21T22:44:50"}
|
| 47 |
+
{"loss": 20.9041943359375, "grad_norm": 22.233734130859375, "learning_rate": 5.762913592001905e-05, "epoch": 4.586129753914989, "num_input_tokens_seen": 671707136, "train_runtime": 15686.3972, "train_tokens_per_second": 42820.995, "step": 2050, "wall_time": "2026-06-21T22:51:12"}
|
| 48 |
+
{"loss": 20.91035888671875, "grad_norm": 21.8343448638916, "learning_rate": 5.6438943108783625e-05, "epoch": 4.697986577181208, "num_input_tokens_seen": 688091136, "train_runtime": 16066.5233, "train_tokens_per_second": 42827.631, "step": 2100, "wall_time": "2026-06-21T22:57:33"}
|
| 49 |
+
{"loss": 20.8124365234375, "grad_norm": 21.778446197509766, "learning_rate": 5.52487502975482e-05, "epoch": 4.809843400447427, "num_input_tokens_seen": 704475136, "train_runtime": 16446.1615, "train_tokens_per_second": 42835.231, "step": 2150, "wall_time": "2026-06-21T23:03:52"}
|
| 50 |
+
{"loss": 20.7349560546875, "grad_norm": 21.686033248901367, "learning_rate": 5.4058557486312786e-05, "epoch": 4.921700223713646, "num_input_tokens_seen": 720859136, "train_runtime": 16825.7207, "train_tokens_per_second": 42842.69, "step": 2200, "wall_time": "2026-06-21T23:10:12"}
|
| 51 |
+
{"loss": 20.7165283203125, "grad_norm": 21.80782127380371, "learning_rate": 5.286836467507736e-05, "epoch": 5.033557046979865, "num_input_tokens_seen": 737233920, "train_runtime": 17206.3274, "train_tokens_per_second": 42846.675, "step": 2250, "wall_time": "2026-06-21T23:16:32"}
|
| 52 |
+
{"loss": 20.5538916015625, "grad_norm": 23.681589126586914, "learning_rate": 5.167817186384194e-05, "epoch": 5.145413870246085, "num_input_tokens_seen": 753617920, "train_runtime": 17586.976, "train_tokens_per_second": 42850.91, "step": 2300, "wall_time": "2026-06-21T23:22:53"}
|
| 53 |
+
{"loss": 20.505771484375, "grad_norm": 23.062753677368164, "learning_rate": 5.0487979052606524e-05, "epoch": 5.257270693512305, "num_input_tokens_seen": 770001920, "train_runtime": 17966.3576, "train_tokens_per_second": 42857.987, "step": 2350, "wall_time": "2026-06-21T23:29:12"}
|
| 54 |
+
{"loss": 20.479666748046874, "grad_norm": 22.336442947387695, "learning_rate": 4.92977862413711e-05, "epoch": 5.369127516778524, "num_input_tokens_seen": 786385920, "train_runtime": 18346.1922, "train_tokens_per_second": 42863.713, "step": 2400, "wall_time": "2026-06-21T23:35:32"}
|
| 55 |
+
{"loss": 20.462833251953125, "grad_norm": 22.454973220825195, "learning_rate": 4.8107593430135685e-05, "epoch": 5.480984340044743, "num_input_tokens_seen": 802769920, "train_runtime": 18726.6308, "train_tokens_per_second": 42867.824, "step": 2450, "wall_time": "2026-06-21T23:41:53"}
|
| 56 |
+
{"loss": 20.33416748046875, "grad_norm": 21.835525512695312, "learning_rate": 4.691740061890026e-05, "epoch": 5.592841163310962, "num_input_tokens_seen": 819153920, "train_runtime": 19106.2918, "train_tokens_per_second": 42873.517, "step": 2500, "wall_time": "2026-06-21T23:48:12"}
|
| 57 |
+
{"eval_loss": 1.1132277250289917, "eval_masked_accuracy": 0.7584756667111459, "eval_runtime": 25.3527, "eval_samples_per_second": 219.621, "eval_steps_per_second": 3.432, "epoch": 5.592841163310962, "num_input_tokens_seen": 819153920, "step": 2500, "wall_time": "2026-06-21T23:48:38"}
|
| 58 |
+
{"loss": 20.25551513671875, "grad_norm": 23.234500885009766, "learning_rate": 4.5727207807664845e-05, "epoch": 5.704697986577181, "num_input_tokens_seen": 835537920, "train_runtime": 19513.5356, "train_tokens_per_second": 42818.377, "step": 2550, "wall_time": "2026-06-21T23:55:00"}
|
| 59 |
+
{"loss": 20.156134033203124, "grad_norm": 21.292207717895508, "learning_rate": 4.453701499642942e-05, "epoch": 5.8165548098434, "num_input_tokens_seen": 851921920, "train_runtime": 19893.1597, "train_tokens_per_second": 42824.867, "step": 2600, "wall_time": "2026-06-22T00:01:19"}
|
| 60 |
+
{"loss": 20.159989013671876, "grad_norm": 20.622861862182617, "learning_rate": 4.3346822185194e-05, "epoch": 5.9284116331096195, "num_input_tokens_seen": 868305920, "train_runtime": 20273.9368, "train_tokens_per_second": 42828.679, "step": 2650, "wall_time": "2026-06-22T00:07:40"}
|
| 61 |
+
{"loss": 20.13439697265625, "grad_norm": 23.077646255493164, "learning_rate": 4.215662937395858e-05, "epoch": 6.040268456375839, "num_input_tokens_seen": 884680704, "train_runtime": 20654.2855, "train_tokens_per_second": 42832.791, "step": 2700, "wall_time": "2026-06-22T00:14:00"}
|
| 62 |
+
{"loss": 19.994425048828123, "grad_norm": 21.280527114868164, "learning_rate": 4.096643656272317e-05, "epoch": 6.152125279642058, "num_input_tokens_seen": 901064704, "train_runtime": 21033.1909, "train_tokens_per_second": 42840.133, "step": 2750, "wall_time": "2026-06-22T00:20:19"}
|
| 63 |
+
{"loss": 19.980313720703126, "grad_norm": 23.213178634643555, "learning_rate": 3.977624375148774e-05, "epoch": 6.263982102908278, "num_input_tokens_seen": 917448704, "train_runtime": 21412.6494, "train_tokens_per_second": 42846.109, "step": 2800, "wall_time": "2026-06-22T00:26:39"}
|
| 64 |
+
{"loss": 19.9128857421875, "grad_norm": 22.994335174560547, "learning_rate": 3.858605094025232e-05, "epoch": 6.375838926174497, "num_input_tokens_seen": 933832704, "train_runtime": 21792.7904, "train_tokens_per_second": 42850.534, "step": 2850, "wall_time": "2026-06-22T00:32:59"}
|
| 65 |
+
{"loss": 19.874437255859377, "grad_norm": 25.134178161621094, "learning_rate": 3.7395858129016905e-05, "epoch": 6.487695749440716, "num_input_tokens_seen": 950216704, "train_runtime": 22173.1567, "train_tokens_per_second": 42854.372, "step": 2900, "wall_time": "2026-06-22T00:39:19"}
|
| 66 |
+
{"loss": 19.910538330078126, "grad_norm": 20.69732093811035, "learning_rate": 3.620566531778148e-05, "epoch": 6.599552572706935, "num_input_tokens_seen": 966600704, "train_runtime": 22553.2542, "train_tokens_per_second": 42858.591, "step": 2950, "wall_time": "2026-06-22T00:45:39"}
|
| 67 |
+
{"loss": 19.87051025390625, "grad_norm": 21.227550506591797, "learning_rate": 3.501547250654606e-05, "epoch": 6.7114093959731544, "num_input_tokens_seen": 982984704, "train_runtime": 22933.4022, "train_tokens_per_second": 42862.576, "step": 3000, "wall_time": "2026-06-22T00:51:59"}
|
| 68 |
+
{"eval_loss": 1.0944103002548218, "eval_masked_accuracy": 0.7619272139314145, "eval_runtime": 25.3399, "eval_samples_per_second": 219.732, "eval_steps_per_second": 3.433, "epoch": 6.7114093959731544, "num_input_tokens_seen": 982984704, "step": 3000, "wall_time": "2026-06-22T00:52:25"}
|
| 69 |
+
{"loss": 19.65084228515625, "grad_norm": 21.8638858795166, "learning_rate": 3.382527969531064e-05, "epoch": 6.823266219239374, "num_input_tokens_seen": 999368704, "train_runtime": 23340.7393, "train_tokens_per_second": 42816.497, "step": 3050, "wall_time": "2026-06-22T00:58:47"}
|
| 70 |
+
{"loss": 19.7602978515625, "grad_norm": 25.967254638671875, "learning_rate": 3.263508688407522e-05, "epoch": 6.935123042505593, "num_input_tokens_seen": 1015752704, "train_runtime": 23721.0833, "train_tokens_per_second": 42820.671, "step": 3100, "wall_time": "2026-06-22T01:05:07"}
|
| 71 |
+
{"loss": 19.64591064453125, "grad_norm": 29.288837432861328, "learning_rate": 3.1444894072839803e-05, "epoch": 7.046979865771812, "num_input_tokens_seen": 1032127488, "train_runtime": 24100.8116, "train_tokens_per_second": 42825.425, "step": 3150, "wall_time": "2026-06-22T01:11:27"}
|
| 72 |
+
{"loss": 19.643092041015624, "grad_norm": 20.559537887573242, "learning_rate": 3.025470126160438e-05, "epoch": 7.158836689038031, "num_input_tokens_seen": 1048511488, "train_runtime": 24479.6774, "train_tokens_per_second": 42831.916, "step": 3200, "wall_time": "2026-06-22T01:17:46"}
|
| 73 |
+
{"loss": 19.5406787109375, "grad_norm": 21.216949462890625, "learning_rate": 2.906450845036896e-05, "epoch": 7.27069351230425, "num_input_tokens_seen": 1064895488, "train_runtime": 24859.9356, "train_tokens_per_second": 42835.81, "step": 3250, "wall_time": "2026-06-22T01:24:06"}
|
| 74 |
+
{"loss": 19.52265625, "grad_norm": 19.948974609375, "learning_rate": 2.787431563913354e-05, "epoch": 7.382550335570469, "num_input_tokens_seen": 1081279488, "train_runtime": 25239.4624, "train_tokens_per_second": 42840.829, "step": 3300, "wall_time": "2026-06-22T01:30:26"}
|
| 75 |
+
{"loss": 19.483994140625, "grad_norm": 24.806529998779297, "learning_rate": 2.668412282789812e-05, "epoch": 7.494407158836689, "num_input_tokens_seen": 1097663488, "train_runtime": 25619.1507, "train_tokens_per_second": 42845.428, "step": 3350, "wall_time": "2026-06-22T01:36:45"}
|
| 76 |
+
{"loss": 19.500614013671875, "grad_norm": 22.306764602661133, "learning_rate": 2.54939300166627e-05, "epoch": 7.6062639821029085, "num_input_tokens_seen": 1114047488, "train_runtime": 25998.4643, "train_tokens_per_second": 42850.511, "step": 3400, "wall_time": "2026-06-22T01:43:05"}
|
| 77 |
+
{"loss": 19.44253173828125, "grad_norm": 20.259811401367188, "learning_rate": 2.430373720542728e-05, "epoch": 7.718120805369128, "num_input_tokens_seen": 1130431488, "train_runtime": 26378.2281, "train_tokens_per_second": 42854.716, "step": 3450, "wall_time": "2026-06-22T01:49:24"}
|
| 78 |
+
{"loss": 19.37197021484375, "grad_norm": 31.171615600585938, "learning_rate": 2.311354439419186e-05, "epoch": 7.829977628635347, "num_input_tokens_seen": 1146815488, "train_runtime": 26757.8966, "train_tokens_per_second": 42858.955, "step": 3500, "wall_time": "2026-06-22T01:55:44"}
|
| 79 |
+
{"eval_loss": 1.0732760429382324, "eval_masked_accuracy": 0.7661423014370904, "eval_runtime": 25.3349, "eval_samples_per_second": 219.775, "eval_steps_per_second": 3.434, "epoch": 7.829977628635347, "num_input_tokens_seen": 1146815488, "step": 3500, "wall_time": "2026-06-22T01:56:09"}
|
| 80 |
+
{"loss": 19.302330322265625, "grad_norm": 19.443883895874023, "learning_rate": 2.192335158295644e-05, "epoch": 7.941834451901566, "num_input_tokens_seen": 1163199488, "train_runtime": 27165.3477, "train_tokens_per_second": 42819.238, "step": 3550, "wall_time": "2026-06-22T02:02:31"}
|
| 81 |
+
{"loss": 19.334464111328124, "grad_norm": 19.65326690673828, "learning_rate": 2.073315877172102e-05, "epoch": 8.053691275167786, "num_input_tokens_seen": 1179574272, "train_runtime": 27545.6424, "train_tokens_per_second": 42822.536, "step": 3600, "wall_time": "2026-06-22T02:08:52"}
|
| 82 |
+
{"loss": 19.248583984375, "grad_norm": 20.760732650756836, "learning_rate": 1.9542965960485597e-05, "epoch": 8.165548098434005, "num_input_tokens_seen": 1195958272, "train_runtime": 27925.9284, "train_tokens_per_second": 42826.088, "step": 3650, "wall_time": "2026-06-22T02:15:12"}
|
| 83 |
+
{"loss": 19.22830322265625, "grad_norm": 19.925457000732422, "learning_rate": 1.835277314925018e-05, "epoch": 8.277404921700224, "num_input_tokens_seen": 1212342272, "train_runtime": 28305.1518, "train_tokens_per_second": 42831.152, "step": 3700, "wall_time": "2026-06-22T02:21:31"}
|
| 84 |
+
{"loss": 19.20845458984375, "grad_norm": 20.903831481933594, "learning_rate": 1.7162580338014758e-05, "epoch": 8.389261744966444, "num_input_tokens_seen": 1228726272, "train_runtime": 28684.7612, "train_tokens_per_second": 42835.506, "step": 3750, "wall_time": "2026-06-22T02:27:51"}
|
| 85 |
+
{"loss": 19.181019287109375, "grad_norm": 25.746238708496094, "learning_rate": 1.597238752677934e-05, "epoch": 8.501118568232663, "num_input_tokens_seen": 1245110272, "train_runtime": 29065.5625, "train_tokens_per_second": 42837.99, "step": 3800, "wall_time": "2026-06-22T02:34:12"}
|
| 86 |
+
{"loss": 19.200714111328125, "grad_norm": 19.735681533813477, "learning_rate": 1.4782194715543917e-05, "epoch": 8.612975391498882, "num_input_tokens_seen": 1261494272, "train_runtime": 29444.9518, "train_tokens_per_second": 42842.463, "step": 3850, "wall_time": "2026-06-22T02:40:31"}
|
| 87 |
+
{"loss": 19.14758056640625, "grad_norm": 20.348718643188477, "learning_rate": 1.35920019043085e-05, "epoch": 8.724832214765101, "num_input_tokens_seen": 1277878272, "train_runtime": 29825.2862, "train_tokens_per_second": 42845.466, "step": 3900, "wall_time": "2026-06-22T02:46:51"}
|
| 88 |
+
{"loss": 19.07127685546875, "grad_norm": 19.208051681518555, "learning_rate": 1.2401809093073078e-05, "epoch": 8.83668903803132, "num_input_tokens_seen": 1294262272, "train_runtime": 30205.5851, "train_tokens_per_second": 42848.442, "step": 3950, "wall_time": "2026-06-22T02:53:12"}
|
| 89 |
+
{"loss": 19.101888427734377, "grad_norm": 20.35933494567871, "learning_rate": 1.1211616281837658e-05, "epoch": 8.94854586129754, "num_input_tokens_seen": 1310646272, "train_runtime": 30585.6001, "train_tokens_per_second": 42851.743, "step": 4000, "wall_time": "2026-06-22T02:59:32"}
|
| 90 |
+
{"eval_loss": 1.0551483631134033, "eval_masked_accuracy": 0.7687605175839743, "eval_runtime": 25.3745, "eval_samples_per_second": 219.433, "eval_steps_per_second": 3.429, "epoch": 8.94854586129754, "num_input_tokens_seen": 1310646272, "step": 4000, "wall_time": "2026-06-22T02:59:57"}
|
| 91 |
+
{"loss": 19.01905517578125, "grad_norm": 19.61882972717285, "learning_rate": 1.0021423470602239e-05, "epoch": 9.060402684563758, "num_input_tokens_seen": 1327021056, "train_runtime": 30993.7338, "train_tokens_per_second": 42815.785, "step": 4050, "wall_time": "2026-06-22T03:06:20"}
|
| 92 |
+
{"loss": 19.05180419921875, "grad_norm": 20.218429565429688, "learning_rate": 8.831230659366818e-06, "epoch": 9.172259507829978, "num_input_tokens_seen": 1343405056, "train_runtime": 31373.034, "train_tokens_per_second": 42820.374, "step": 4100, "wall_time": "2026-06-22T03:12:39"}
|
| 93 |
+
{"loss": 19.03988525390625, "grad_norm": 19.39629554748535, "learning_rate": 7.641037848131398e-06, "epoch": 9.284116331096197, "num_input_tokens_seen": 1359789056, "train_runtime": 31753.5802, "train_tokens_per_second": 42823.173, "step": 4150, "wall_time": "2026-06-22T03:19:00"}
|
| 94 |
+
{"loss": 18.968330078125, "grad_norm": 20.272720336914062, "learning_rate": 6.4508450368959775e-06, "epoch": 9.395973154362416, "num_input_tokens_seen": 1376173056, "train_runtime": 32133.9665, "train_tokens_per_second": 42826.119, "step": 4200, "wall_time": "2026-06-22T03:25:20"}
|
| 95 |
+
{"loss": 18.9827783203125, "grad_norm": 19.632165908813477, "learning_rate": 5.260652225660558e-06, "epoch": 9.507829977628635, "num_input_tokens_seen": 1392557056, "train_runtime": 32514.0065, "train_tokens_per_second": 42829.451, "step": 4250, "wall_time": "2026-06-22T03:31:40"}
|
| 96 |
+
{"loss": 18.956285400390627, "grad_norm": 19.80462646484375, "learning_rate": 4.0704594144251375e-06, "epoch": 9.619686800894854, "num_input_tokens_seen": 1408941056, "train_runtime": 32894.5758, "train_tokens_per_second": 42832.018, "step": 4300, "wall_time": "2026-06-22T03:38:01"}
|
| 97 |
+
{"loss": 18.866341552734376, "grad_norm": 20.74568748474121, "learning_rate": 2.8802666031897166e-06, "epoch": 9.731543624161073, "num_input_tokens_seen": 1425325056, "train_runtime": 33275.028, "train_tokens_per_second": 42834.676, "step": 4350, "wall_time": "2026-06-22T03:44:21"}
|
| 98 |
+
{"loss": 18.920970458984375, "grad_norm": 19.217836380004883, "learning_rate": 1.6900737919542966e-06, "epoch": 9.843400447427292, "num_input_tokens_seen": 1441709056, "train_runtime": 33654.5878, "train_tokens_per_second": 42838.411, "step": 4400, "wall_time": "2026-06-22T03:50:41"}
|
| 99 |
+
{"loss": 18.913228759765627, "grad_norm": 19.824554443359375, "learning_rate": 4.998809807188764e-07, "epoch": 9.955257270693512, "num_input_tokens_seen": 1458093056, "train_runtime": 34033.2944, "train_tokens_per_second": 42843.136, "step": 4450, "wall_time": "2026-06-22T03:56:59"}
|
| 100 |
+
{"eval_loss": 1.0481648445129395, "eval_masked_accuracy": 0.7704025727385737, "eval_runtime": 25.3462, "eval_samples_per_second": 219.678, "eval_steps_per_second": 3.432, "epoch": 10.0, "num_input_tokens_seen": 1464637440, "step": 4470, "wall_time": "2026-06-22T03:59:57"}
|
| 101 |
+
{"train_runtime": 34212.9191, "train_samples_per_second": 83.612, "train_steps_per_second": 0.131, "total_flos": 4.917799558028206e+18, "train_loss": 21.80173199032777, "epoch": 10.0, "num_input_tokens_seen": 1464637440, "step": 4470, "wall_time": "2026-06-22T03:59:59"}
|
| 102 |
+
{"final_val_loss": 1.0483721494674683, "final_val_masked_accuracy": 0.7699716443872566, "final_val_runtime": 25.2687, "final_val_samples_per_second": 220.351, "final_val_steps_per_second": 3.443, "epoch": 10.0, "num_input_tokens_seen": 1464637440, "step": 4470, "wall_time": "2026-06-22T04:00:28"}
|