mschonhardt commited on
Commit
6b4288c
·
verified ·
1 Parent(s): 1dd2574

Add files using upload-large-folder tool

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
all_results.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "final_val_loss": 1.0483721494674683,
4
+ "final_val_masked_accuracy": 0.7699716443872566,
5
+ "final_val_perplexity": 2.853003073356264,
6
+ "final_val_runtime": 25.2687,
7
+ "final_val_samples_per_second": 220.351,
8
+ "final_val_steps_per_second": 3.443,
9
+ "num_input_tokens_seen": 1464637440,
10
+ "total_flos": 4.917799558028206e+18,
11
+ "train_loss": 21.80173199032777,
12
+ "train_runtime": 34212.9191,
13
+ "train_samples_per_second": 83.612,
14
+ "train_steps_per_second": 0.131
15
+ }
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "XLMRobertaForMaskedLM"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": 0,
8
+ "classifier_dropout": null,
9
+ "dtype": "float32",
10
+ "eos_token_id": 2,
11
+ "hidden_act": "gelu",
12
+ "hidden_dropout_prob": 0.1,
13
+ "hidden_size": 1024,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 4096,
16
+ "is_decoder": false,
17
+ "layer_norm_eps": 1e-05,
18
+ "max_position_embeddings": 514,
19
+ "model_type": "xlm-roberta",
20
+ "num_attention_heads": 16,
21
+ "num_hidden_layers": 24,
22
+ "output_past": true,
23
+ "pad_token_id": 1,
24
+ "position_embedding_type": "absolute",
25
+ "tie_word_embeddings": true,
26
+ "transformers_version": "5.12.1",
27
+ "type_vocab_size": 1,
28
+ "use_cache": false,
29
+ "vocab_size": 250002
30
+ }
dataset_stats.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train": {
3
+ "sequences": 286062,
4
+ "tokens": 142794676,
5
+ "mean_seq_len": 499.2
6
+ },
7
+ "validation": {
8
+ "sequences": 5568,
9
+ "tokens": 2812986,
10
+ "mean_seq_len": 505.2
11
+ }
12
+ }
eval_representativeness.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "split_kind": "document-level STRATIFIED group-aware",
3
+ "train_sequences": 286062,
4
+ "val_sequences": 5568,
5
+ "len_train": {
6
+ "mean": 499.173871398508,
7
+ "p50": 512.0,
8
+ "p95": 512.0
9
+ },
10
+ "len_val": {
11
+ "mean": 505.20581896551727,
12
+ "p50": 512.0,
13
+ "p95": 512.0
14
+ }
15
+ }
final_eval_results.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "final_val_loss": 1.0483721494674683,
4
+ "final_val_masked_accuracy": 0.7699716443872566,
5
+ "final_val_perplexity": 2.853003073356264,
6
+ "final_val_runtime": 25.2687,
7
+ "final_val_samples_per_second": 220.351,
8
+ "final_val_steps_per_second": 3.443,
9
+ "num_input_tokens_seen": 1464637440
10
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c254ead4ef6c737b0586af25d64dedb5e31acbdb649243833c8ea184e63fdeb5
3
+ size 2240618752
run_metadata.json ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "med_dataset": "mschonhardt/MedBerta",
4
+ "canon_dataset": "mschonhardt/MedBerta",
5
+ "dataset_split": "train",
6
+ "text_column": null,
7
+ "doc_id_column": null,
8
+ "order_column": null,
9
+ "model_name": "xlm-roberta-large",
10
+ "max_seq_length": 512,
11
+ "mlm_probability": 0.15,
12
+ "attn_implementation": "sdpa",
13
+ "min_chunk_tokens": 64,
14
+ "val_doc_fraction": 0.01,
15
+ "test_doc_fraction": 0.0,
16
+ "max_eval_docs_per_source": 400,
17
+ "stratify_column": "category",
18
+ "split_seed": 13,
19
+ "train_corpus": "canon",
20
+ "dedup_train": false,
21
+ "diagnose_tokenizer": true,
22
+ "tokenizer_sample_size": 20000,
23
+ "normalize_nfc": false,
24
+ "diagnose_only": false,
25
+ "learning_rate": 0.0001,
26
+ "weight_decay": 0.01,
27
+ "adam_beta1": 0.9,
28
+ "adam_beta2": 0.98,
29
+ "adam_epsilon": 1e-06,
30
+ "max_grad_norm": 1.0,
31
+ "warmup_ratio": 0.06,
32
+ "lr_scheduler_type": "linear",
33
+ "num_train_epochs": 10.0,
34
+ "max_steps": -1,
35
+ "optim": "adamw_torch_fused",
36
+ "early_stopping_patience": 5,
37
+ "early_stopping_threshold": 0.0001,
38
+ "per_device_train_batch_size": 64,
39
+ "per_device_eval_batch_size": 64,
40
+ "gradient_accumulation_steps": 16,
41
+ "gradient_checkpointing": false,
42
+ "auto_find_batch_size": true,
43
+ "torch_compile": false,
44
+ "dataloader_num_workers": 8,
45
+ "eval_accumulation_steps": 50,
46
+ "eval_steps": 500,
47
+ "logging_steps": 50,
48
+ "save_steps": 500,
49
+ "save_total_limit": 3,
50
+ "output_dir": "./MediBerta",
51
+ "cache_dir": ".cache_packed",
52
+ "seed": 42,
53
+ "report_to_wandb": false,
54
+ "preprocess_num_proc": 16,
55
+ "resume": true,
56
+ "dry_run": false
57
+ },
58
+ "environment": {
59
+ "timestamp": "2026-06-21T18:28:11",
60
+ "python": "3.11.2",
61
+ "platform": "Linux-6.1.0-49-amd64-x86_64-with-glibc2.36",
62
+ "torch": "2.12.1+cu130",
63
+ "transformers": "5.12.1",
64
+ "datasets": "5.0.0",
65
+ "cuda_available": true,
66
+ "cuda": "13.0",
67
+ "gpu_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
68
+ "gpu_total_memory_gb": 95.0,
69
+ "gpu_capability": "12.0",
70
+ "bf16_supported": true,
71
+ "torch_arch_list": [
72
+ "sm_75",
73
+ "sm_80",
74
+ "sm_86",
75
+ "sm_90",
76
+ "sm_100",
77
+ "sm_120"
78
+ ],
79
+ "gpu_arch_supported_by_torch": true
80
+ }
81
+ }
runs/Jun21_18-29-32_linglit-gpu/events.out.tfevents.1782059372.linglit-gpu ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7d560e3dbaa720ddf8771a7e267214ce871a52330b744411e47ba056f2389d7
3
+ size 62249
runs/Jun21_18-29-32_linglit-gpu/events.out.tfevents.1782093628.linglit-gpu ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d0eb78808c93b7941e7675b6c1a1cc075a2bf61bd5c18d85fc6e5bb2b09d87c
3
+ size 491
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:acbd420e2269cdc1ef45332d3d5c418be4aef6b8cb5a0b7ccae0893485307153
3
+ size 17098086
tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": true,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "cls_token": "<s>",
6
+ "eos_token": "</s>",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "mask_token": "<mask>",
10
+ "model_max_length": 512,
11
+ "pad_token": "<pad>",
12
+ "sep_token": "</s>",
13
+ "tokenizer_class": "XLMRobertaTokenizer",
14
+ "unk_token": "<unk>"
15
+ }
tokenizer_diagnostics.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "mode": "pretokenized",
3
+ "sampled_sequences": 20000,
4
+ "total_sequences": 291630,
5
+ "unk_rate": 3.7e-05,
6
+ "total_unk_tokens": 368,
7
+ "seq_len": {
8
+ "mean": 500.11885,
9
+ "p50": 512.0,
10
+ "p95": 512.0,
11
+ "max": 512
12
+ },
13
+ "share_over_max_seq_len": 0.0,
14
+ "starts_with_bos_rate": 1.0,
15
+ "ends_with_eos_rate": 1.0
16
+ }
train.log ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-06-21 18:28:11 | INFO | dapt | Environment:
2
+ {
3
+ "timestamp": "2026-06-21T18:28:11",
4
+ "python": "3.11.2",
5
+ "platform": "Linux-6.1.0-49-amd64-x86_64-with-glibc2.36",
6
+ "torch": "2.12.1+cu130",
7
+ "transformers": "5.12.1",
8
+ "datasets": "5.0.0",
9
+ "cuda_available": true,
10
+ "cuda": "13.0",
11
+ "gpu_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
12
+ "gpu_total_memory_gb": 95.0,
13
+ "gpu_capability": "12.0",
14
+ "bf16_supported": true,
15
+ "torch_arch_list": [
16
+ "sm_75",
17
+ "sm_80",
18
+ "sm_86",
19
+ "sm_90",
20
+ "sm_100",
21
+ "sm_120"
22
+ ],
23
+ "gpu_arch_supported_by_torch": true
24
+ }
25
+ 2026-06-21 18:28:12 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/xlm-roberta-large/resolve/main/config.json "HTTP/1.1 200 OK"
26
+ 2026-06-21 18:28:12 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer_config.json "HTTP/1.1 200 OK"
27
+ 2026-06-21 18:28:12 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/xlm-roberta-large/tree/main/additional_chat_templates?recursive=false&expand=false "HTTP/1.1 307 Temporary Redirect"
28
+ 2026-06-21 18:28:12 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/FacebookAI/xlm-roberta-large/tree/main/additional_chat_templates?recursive=false&expand=false "HTTP/1.1 404 Not Found"
29
+ 2026-06-21 18:28:12 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/xlm-roberta-large/tree/main?recursive=true&expand=false "HTTP/1.1 307 Temporary Redirect"
30
+ 2026-06-21 18:28:12 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/FacebookAI/xlm-roberta-large/tree/main?recursive=true&expand=false "HTTP/1.1 200 OK"
31
+ 2026-06-21 18:28:14 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/xlm-roberta-large "HTTP/1.1 307 Temporary Redirect"
32
+ 2026-06-21 18:28:14 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/models/FacebookAI/xlm-roberta-large "HTTP/1.1 200 OK"
33
+ 2026-06-21 18:28:14 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/main/README.md "HTTP/1.1 404 Not Found"
34
+ 2026-06-21 18:28:14 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta "HTTP/1.1 200 OK"
35
+ 2026-06-21 18:28:14 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/MedBerta.py "HTTP/1.1 404 Not Found"
36
+ 2026-06-21 18:28:15 | INFO | httpx | HTTP Request: HEAD https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets/mschonhardt/MedBerta/mschonhardt/MedBerta.py "HTTP/1.1 404 Not Found"
37
+ 2026-06-21 18:28:15 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/README.md "HTTP/1.1 404 Not Found"
38
+ 2026-06-21 18:28:15 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta/revision/bb4994d1d2ac8765e4df863a5846210c8ecaacd1 "HTTP/1.1 200 OK"
39
+ 2026-06-21 18:28:15 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/.huggingface.yaml "HTTP/1.1 404 Not Found"
40
+ 2026-06-21 18:28:15 | INFO | httpx | HTTP Request: GET https://datasets-server.huggingface.co/info?dataset=mschonhardt/MedBerta "HTTP/1.1 501 Not Implemented"
41
+ 2026-06-21 18:28:16 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta/tree/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/data?recursive=true&expand=false "HTTP/1.1 404 Not Found"
42
+ 2026-06-21 18:28:16 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta/tree/bb4994d1d2ac8765e4df863a5846210c8ecaacd1?recursive=false&expand=false "HTTP/1.1 200 OK"
43
+ 2026-06-21 18:28:16 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/dataset_infos.json "HTTP/1.1 404 Not Found"
44
+ 2026-06-21 18:28:16 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/data-00000-of-00002.arrow "HTTP/1.1 302 Found"
45
+ 2026-06-21 18:28:22 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/data-00001-of-00002.arrow "HTTP/1.1 302 Found"
46
+ 2026-06-21 18:28:28 | WARNING | dapt | Dataset mschonhardt/MedBerta is PRE-TOKENIZED (columns=['input_ids', 'attention_mask', 'doc_id', 'category']). Skipping raw-text tokenization and DOC-SENTENCES packing.
47
+ 2026-06-21 18:28:28 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/main/README.md "HTTP/1.1 404 Not Found"
48
+ 2026-06-21 18:28:28 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta "HTTP/1.1 200 OK"
49
+ 2026-06-21 18:28:28 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/MedBerta.py "HTTP/1.1 404 Not Found"
50
+ 2026-06-21 18:28:28 | INFO | httpx | HTTP Request: HEAD https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets/mschonhardt/MedBerta/mschonhardt/MedBerta.py "HTTP/1.1 404 Not Found"
51
+ 2026-06-21 18:28:28 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/README.md "HTTP/1.1 404 Not Found"
52
+ 2026-06-21 18:28:29 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/.huggingface.yaml "HTTP/1.1 404 Not Found"
53
+ 2026-06-21 18:28:29 | INFO | httpx | HTTP Request: GET https://datasets-server.huggingface.co/info?dataset=mschonhardt/MedBerta "HTTP/1.1 501 Not Implemented"
54
+ 2026-06-21 18:28:29 | INFO | httpx | HTTP Request: GET https://huggingface.co/api/datasets/mschonhardt/MedBerta/tree/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/data?recursive=true&expand=false "HTTP/1.1 404 Not Found"
55
+ 2026-06-21 18:28:29 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/datasets/mschonhardt/MedBerta/resolve/bb4994d1d2ac8765e4df863a5846210c8ecaacd1/dataset_infos.json "HTTP/1.1 404 Not Found"
56
+ 2026-06-21 18:28:29 | INFO | dapt | loaded pretokenized canon -> 291630 sequences (doc_id=doc_id)
57
+ 2026-06-21 18:28:35 | INFO | dapt | Pretokenized diagnostics: unk_rate=0.0037% mean_len=500.1 bos=100% eos=100%
58
+ 2026-06-21 18:28:36 | INFO | dapt | Applying STRATIFIED document split based on column 'category'.
59
+ 2026-06-21 18:28:40 | INFO | dapt | Pretokenized split (document-level STRATIFIED group-aware) -> train=286062 val=5568 test=0
60
+ 2026-06-21 18:29:04 | INFO | dapt | Saved pretokenized datasets to .cache_packed/packed_a891125d9987e2ce527ec74b
61
+ 2026-06-21 18:29:28 | INFO | dapt | Packed dataset statistics:
62
+ {
63
+ "train": {
64
+ "sequences": 286062,
65
+ "tokens": 142794676,
66
+ "mean_seq_len": 499.2
67
+ },
68
+ "validation": {
69
+ "sequences": 5568,
70
+ "tokens": 2812986,
71
+ "mean_seq_len": 505.2
72
+ }
73
+ }
74
+ 2026-06-21 18:29:28 | INFO | httpx | HTTP Request: HEAD https://huggingface.co/xlm-roberta-large/resolve/main/config.json "HTTP/1.1 200 OK"
75
+ 2026-06-21 18:29:28 | INFO | dapt | Model parameters: 560.1M
76
+ 2026-06-22 04:00:28 | INFO | dapt | Final evaluation:
77
+ {
78
+ "final_val_loss": 1.0483721494674683,
79
+ "final_val_masked_accuracy": 0.7699716443872566,
80
+ "final_val_runtime": 25.2687,
81
+ "final_val_samples_per_second": 220.351,
82
+ "final_val_steps_per_second": 3.443,
83
+ "epoch": 10.0,
84
+ "num_input_tokens_seen": 1464637440,
85
+ "final_val_perplexity": 2.853003073356264
86
+ }
87
+ 2026-06-22 04:00:28 | INFO | dapt | Done. Best model + tokenizer saved to ./MediBerta
train.py ADDED
@@ -0,0 +1,1178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Domain-Adaptive Pretraining (DAPT) of xlm-roberta-large for Medieval Latin.
5
+
6
+ Pipeline
7
+ --------
8
+ 1. Load MedBerta (baseline) + CanonBerta (already upsampled) from the HF Hub.
9
+ 2. Carve a *group-aware* held-out set: whole documents are held out (by
10
+ ``document_id``) so no document leaks across train / validation / test.
11
+ This matters for an honest perplexity and for downstream Loci-Similes /
12
+ text-reuse evaluation.
13
+ 3. DOC-SENTENCES packing: paragraphs are grouped by ``document_id`` (kept in
14
+ document order), tokenized, and greedily packed into fixed 512-token
15
+ sequences that never cross a document boundary.
16
+ 4. MLM continued pretraining with the HF Trainer, bf16, SDPA attention,
17
+ in-training evaluation (loss + perplexity + masked-token accuracy),
18
+ best-model selection, and rich logging for later write-up.
19
+
20
+ The script is built for a single NVIDIA RTX PRO 6000 Blackwell (96 GB) and is
21
+ deliberately defensive about OOM (auto batch-size search, expandable CUDA
22
+ segments, optional gradient checkpointing, eval logit reduction).
23
+
24
+ Authentication
25
+ --------------
26
+ The datasets are private. Log in once before running:
27
+ huggingface-cli login # or: export HF_TOKEN=hf_...
28
+
29
+ Quick check before spending GPU time (builds + caches datasets, prints stats,
30
+ no training):
31
+ python dapt_xlmr_pretrain.py --dry-run
32
+
33
+ Typical run:
34
+ python dapt_xlmr_pretrain.py \
35
+ --output-dir runs/dapt_xlmr_medlatin_v1 \
36
+ --num-train-epochs 3 \
37
+ --per-device-train-batch-size 32 \
38
+ --gradient-accumulation-steps 8
39
+ """
40
+
41
+ import argparse
42
+ import hashlib
43
+ import inspect
44
+ import json
45
+ import logging
46
+ import math
47
+ import os
48
+ import platform
49
+ import sys
50
+ import traceback
51
+ import unicodedata
52
+ from collections import Counter
53
+ from dataclasses import dataclass, field
54
+ from datetime import datetime
55
+ from typing import Dict, List, Optional
56
+
57
+ # Must be set BEFORE torch is imported to take effect: reduces fragmentation,
58
+ # which is the most common cause of "phantom" OOM on long runs.
59
+ os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
60
+ os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") # avoid fork warnings
61
+
62
+ import numpy as np
63
+ import torch
64
+ from datasets import Dataset, concatenate_datasets, load_dataset
65
+ from transformers import (
66
+ AutoModelForMaskedLM,
67
+ AutoTokenizer,
68
+ DataCollatorForLanguageModeling,
69
+ EarlyStoppingCallback,
70
+ Trainer,
71
+ TrainerCallback,
72
+ TrainingArguments,
73
+ set_seed,
74
+ )
75
+ from transformers.trainer_utils import get_last_checkpoint
76
+
77
+ try:
78
+ _USE_PROCESSING_CLASS = "processing_class" in inspect.signature(Trainer.__init__).parameters
79
+ except Exception: # extremely defensive; fall back to legacy name
80
+ _USE_PROCESSING_CLASS = False
81
+
82
+ logging.basicConfig(
83
+ format="%(asctime)s | %(levelname)-7s | %(name)s | %(message)s",
84
+ datefmt="%Y-%m-%d %H:%M:%S",
85
+ level=logging.INFO,
86
+ handlers=[logging.StreamHandler(sys.stdout)],
87
+ )
88
+ logger = logging.getLogger("dapt")
89
+
90
+
91
+ # --------------------------------------------------------------------------- #
92
+ # Config
93
+ # --------------------------------------------------------------------------- #
94
+ @dataclass
95
+ class Config:
96
+ # Data
97
+ med_dataset: str = "mschonhardt/MedBerta"
98
+ canon_dataset: str = "mschonhardt/CanonBerta"
99
+ dataset_split: str = "train"
100
+ text_column: Optional[str] = None # auto-detected if None
101
+ doc_id_column: Optional[str] = None # auto-detected if None
102
+ order_column: Optional[str] = None # optional within-doc ordering key
103
+
104
+ # Model / tokenizer
105
+ model_name: str = "xlm-roberta-large"
106
+ max_seq_length: int = 512
107
+ mlm_probability: float = 0.15
108
+ attn_implementation: str = "sdpa" # robust on Blackwell; "flash_attention_2" optional
109
+ min_chunk_tokens: int = 64 # drop doc-final fragments shorter than this
110
+
111
+ # Held-out construction (group-aware, by document_id)
112
+ val_doc_fraction: float = 0.01
113
+ test_doc_fraction: float = 0.0 # set >0 to also hold out a test set
114
+ max_eval_docs_per_source: int = 400 # cap eval size to keep eval fast
115
+ stratify_column: Optional[str] = "category" # e.g. "genre"/"subcorpus": stratified held-out
116
+ split_seed: int = 13
117
+
118
+ # Which corpus to train ON. "combined" = ONE model on MedBerta + upsampled
119
+ # CanonBerta (the methodology you described). "med"/"canon" = a single-corpus
120
+ # model, e.g. for an ablation or two separate deliverables. Held-out eval is
121
+ # still reported per-source where both are present.
122
+ train_corpus: str = "combined" # {"combined", "med", "canon"}
123
+ dedup_train: bool = False # collapse upsampling duplicates in TRAIN
124
+ # (recommended for a standalone canon model)
125
+
126
+ # Pre-flight diagnostics & text handling
127
+ diagnose_tokenizer: bool = True # run UNK / fertility report before training
128
+ tokenizer_sample_size: int = 20000 # paragraphs sampled for the diagnostic
129
+ normalize_nfc: bool = False # apply Unicode NFC before tokenizing
130
+ diagnose_only: bool = False # run diagnostics + reports, then exit
131
+
132
+ # Optimisation (RoBERTa-style defaults, tuned for *continued* pretraining)
133
+ learning_rate: float = 1e-4 # peak LR for the combined DAPT corpus
134
+ weight_decay: float = 0.01
135
+ adam_beta1: float = 0.9
136
+ adam_beta2: float = 0.98
137
+ adam_epsilon: float = 1e-6
138
+ max_grad_norm: float = 1.0
139
+ warmup_ratio: float = 0.06
140
+ lr_scheduler_type: str = "linear"
141
+ num_train_epochs: float = 10.0 # ceiling; early stopping ends training
142
+ max_steps: int = -1
143
+ optim: str = "adamw_torch_fused" # fused AdamW: faster on Blackwell
144
+ early_stopping_patience: int = 5 # eval rounds w/o improvement before stop
145
+ early_stopping_threshold: float = 1e-4 # min eval_loss delta to count as improvement
146
+
147
+ # Throughput / memory (96 GB GDDR7 -> large effective batch)
148
+ per_device_train_batch_size: int = 64
149
+ per_device_eval_batch_size: int = 64
150
+ gradient_accumulation_steps: int = 16 # effective batch ~1024 sequences
151
+ gradient_checkpointing: bool = False
152
+ auto_find_batch_size: bool = True
153
+ torch_compile: bool = False # enable for the final fixed-batch run
154
+ dataloader_num_workers: int = 8
155
+ eval_accumulation_steps: int = 50 # offload eval tensors to CPU periodically
156
+
157
+ # Schedule of eval / logging / checkpointing
158
+ eval_steps: int = 500
159
+ logging_steps: int = 50
160
+ save_steps: int = 500
161
+ save_total_limit: int = 3
162
+
163
+ # Bookkeeping
164
+ output_dir: str = "runs/dapt_xlmr_medlatin"
165
+ cache_dir: str = ".cache_packed"
166
+ seed: int = 42
167
+ report_to_wandb: bool = False
168
+ preprocess_num_proc: int = 16
169
+ resume: bool = True
170
+ dry_run: bool = False
171
+
172
+
173
+ # --------------------------------------------------------------------------- #
174
+ # Column auto-detection
175
+ # --------------------------------------------------------------------------- #
176
+ TEXT_CANDIDATES = ["text", "paragraph", "content", "sentence", "passage"]
177
+ DOC_CANDIDATES = ["document_id", "doc_id", "docid", "document", "work_id", "work"]
178
+ ORDER_CANDIDATES = ["order", "paragraph_id", "par_id", "index", "idx", "n", "position", "seq"]
179
+
180
+
181
+ def _pick(columns: List[str], candidates: List[str], explicit: Optional[str], kind: str) -> Optional[str]:
182
+ if explicit is not None:
183
+ if explicit not in columns:
184
+ raise ValueError(f"Requested {kind} column '{explicit}' not in dataset columns {columns}")
185
+ return explicit
186
+ for c in candidates:
187
+ if c in columns:
188
+ return c
189
+ return None
190
+
191
+
192
+ # --------------------------------------------------------------------------- #
193
+ # Dataset loading & group-aware split
194
+ # --------------------------------------------------------------------------- #
195
+ def load_source(repo: str, split: str, source_tag: str, cfg: Config):
196
+ logger.info("Loading %s (split=%s) ...", repo, split)
197
+ ds = load_dataset(repo, split=split, token=True)
198
+ cols = ds.column_names
199
+
200
+ text_col = _pick(cols, TEXT_CANDIDATES, cfg.text_column, "text")
201
+ doc_col = _pick(cols, DOC_CANDIDATES, cfg.doc_id_column, "document_id")
202
+ order_col = _pick(cols, ORDER_CANDIDATES, cfg.order_column, "order") # may be None
203
+
204
+ if text_col is None or doc_col is None:
205
+ raise ValueError(
206
+ f"Could not resolve text/doc_id columns for {repo}. "
207
+ f"Available columns: {cols}. "
208
+ f"Pass --text-column / --doc-id-column explicitly."
209
+ )
210
+ logger.info(" %s -> text='%s', doc_id='%s', order='%s', rows=%d",
211
+ source_tag, text_col, doc_col, order_col, len(ds))
212
+
213
+ # Normalise to a small canonical schema and stamp the source.
214
+ keep = {text_col: "text", doc_col: "document_id"}
215
+ if order_col:
216
+ keep[order_col] = "order"
217
+ strat_col = cfg.stratify_column
218
+ if strat_col is not None:
219
+ if strat_col not in cols:
220
+ raise ValueError(f"--stratify-column '{strat_col}' not in {repo} columns {cols}")
221
+ keep[strat_col] = "stratum"
222
+ ds = ds.rename_columns(keep)
223
+ # The corpora already ship a `source` column ("med"/"canon"). We deliberately
224
+ # drop it here and re-stamp an authoritative per-dataset tag below, so
225
+ # provenance is guaranteed consistent regardless of the incoming values.
226
+ canonical = {"text", "document_id", "order", "stratum"}
227
+ drop = [c for c in ds.column_names if c not in canonical]
228
+ if drop:
229
+ ds = ds.remove_columns(drop)
230
+ if "order" not in ds.column_names:
231
+ # Fallback only (your schema always has `order`): stable within-doc order.
232
+ ds = ds.add_column("order", list(range(len(ds))))
233
+ if "source" in ds.column_names: # safety: never collide with add_column
234
+ ds = ds.remove_columns(["source"])
235
+ ds = ds.add_column("source", [source_tag] * len(ds))
236
+ # document_id is namespaced per source so identical ids in both corpora
237
+ # never collide during splitting/packing.
238
+ ds = ds.map(lambda b: {"document_id": [f"{source_tag}:{d}" for d in b["document_id"]]},
239
+ batched=True, desc=f"namespace doc ids ({source_tag})")
240
+ return ds
241
+
242
+
243
+ def choose_heldout_docs(ds, cfg: Config):
244
+ """Pick whole documents to hold out, capped per source, deterministic.
245
+
246
+ If a ``stratum`` column is present (via --stratify-column), documents are
247
+ sampled proportionally per stratum so rare genres are not lost from the
248
+ held-out set; otherwise sampling is uniform-random over document ids.
249
+ """
250
+ rng = np.random.default_rng(cfg.split_seed)
251
+ n_total_docs = len(set(ds["document_id"]))
252
+ n_val = min(int(round(n_total_docs * cfg.val_doc_fraction)), cfg.max_eval_docs_per_source)
253
+ n_test = min(int(round(n_total_docs * cfg.test_doc_fraction)), cfg.max_eval_docs_per_source)
254
+ n_pick = n_val + n_test
255
+
256
+ if "stratum" in ds.column_names:
257
+ # Map each document to its (majority) stratum.
258
+ doc_stratum: Dict[str, str] = {}
259
+ for d, s in zip(ds["document_id"], ds["stratum"]):
260
+ doc_stratum.setdefault(d, str(s))
261
+ by_stratum: Dict[str, List[str]] = {}
262
+ for d, s in doc_stratum.items():
263
+ by_stratum.setdefault(s, []).append(d)
264
+ picked: List[str] = []
265
+ for s, docs in sorted(by_stratum.items()):
266
+ docs = sorted(docs)
267
+ rng.shuffle(docs)
268
+ # proportional allocation, at least 1 doc per non-empty stratum
269
+ k = max(1, int(round(n_pick * len(docs) / len(doc_stratum))))
270
+ picked.extend(docs[:min(k, len(docs))])
271
+ rng.shuffle(picked)
272
+ picked = picked[:n_pick]
273
+ else:
274
+ doc_ids = sorted(set(ds["document_id"]))
275
+ rng.shuffle(doc_ids)
276
+ picked = doc_ids[:n_pick]
277
+
278
+ val_docs = set(picked[:n_val])
279
+ test_docs = set(picked[n_val:n_val + n_test])
280
+ return val_docs, test_docs
281
+
282
+
283
+ def representativeness_report(train_ds, val_ds, cfg: Config, out_dir: str):
284
+ """Compare the held-out validation set against train on cheap, honest
285
+ signals (paragraph char-length distribution, and stratum proportions if
286
+ available) so a skewed eval set is caught before training."""
287
+ def char_lengths(ds):
288
+ out: List[int] = []
289
+ for batch in ds.iter(batch_size=10000):
290
+ out.extend(len(t) for t in batch["text"])
291
+ return np.asarray(out, dtype=np.int64) if out else np.array([0], dtype=np.int64)
292
+
293
+ def pct(a):
294
+ if len(a) == 0:
295
+ return {}
296
+ return {f"p{p}": float(np.percentile(a, p)) for p in (5, 25, 50, 75, 95)}
297
+
298
+ tr_len, va_len = char_lengths(train_ds), char_lengths(val_ds)
299
+ report = {
300
+ "train_paragraphs": len(train_ds),
301
+ "val_paragraphs": len(val_ds),
302
+ "char_len_train": {"mean": float(tr_len.mean()), **pct(tr_len)},
303
+ "char_len_val": {"mean": float(va_len.mean()), **pct(va_len)},
304
+ }
305
+ if "stratum" in train_ds.column_names and "stratum" in val_ds.column_names:
306
+ def props(ds):
307
+ c = Counter(str(s) for s in ds["stratum"])
308
+ tot = sum(c.values()) or 1
309
+ return {k: v / tot for k, v in c.items()}
310
+ tr_p, va_p = props(train_ds), props(val_ds)
311
+ keys = sorted(set(tr_p) | set(va_p))
312
+ # total variation distance between the two stratum distributions
313
+ tvd = 0.5 * sum(abs(tr_p.get(k, 0.0) - va_p.get(k, 0.0)) for k in keys)
314
+ report["stratum_proportions_train"] = {k: round(tr_p.get(k, 0.0), 4) for k in keys}
315
+ report["stratum_proportions_val"] = {k: round(va_p.get(k, 0.0), 4) for k in keys}
316
+ report["stratum_total_variation_distance"] = round(tvd, 4)
317
+ missing = [k for k in tr_p if k not in va_p]
318
+ if missing:
319
+ report["strata_absent_from_val"] = missing
320
+
321
+ os.makedirs(out_dir, exist_ok=True)
322
+ with open(os.path.join(out_dir, "eval_representativeness.json"), "w") as fh:
323
+ json.dump(report, fh, indent=2, ensure_ascii=False)
324
+ logger.info("Eval representativeness:\n%s", json.dumps(report, indent=2, ensure_ascii=False))
325
+ if report.get("strata_absent_from_val"):
326
+ logger.warning("Strata present in train but MISSING from validation: %s",
327
+ report["strata_absent_from_val"])
328
+ if report.get("stratum_total_variation_distance", 0.0) > 0.15:
329
+ logger.warning("Validation stratum distribution differs notably from train "
330
+ "(TVD=%.3f); consider --stratify-column or a larger eval cap.",
331
+ report["stratum_total_variation_distance"])
332
+ return report
333
+
334
+
335
+ def dedup_rows(ds):
336
+ """Eval honesty: collapse exact (document_id, order, text) duplicates that
337
+ upsampling introduced, so every held-out paragraph is counted once."""
338
+ seen = set()
339
+ keep_idx = []
340
+ for i, (d, o, t) in enumerate(zip(ds["document_id"], ds["order"], ds["text"])):
341
+ key = hashlib.blake2b(f"{d}\x1f{o}\x1f{t}".encode("utf-8"), digest_size=16).digest()
342
+ if key not in seen:
343
+ seen.add(key)
344
+ keep_idx.append(i)
345
+ return ds.select(keep_idx)
346
+
347
+
348
+ # --------------------------------------------------------------------------- #
349
+ # Tokenization + DOC-SENTENCES packing
350
+ # --------------------------------------------------------------------------- #
351
+ def maybe_nfc(text: str, cfg: Config) -> str:
352
+ return unicodedata.normalize("NFC", text) if cfg.normalize_nfc else text
353
+
354
+
355
+ def diagnose_tokenizer(texts: List[str], tokenizer, cfg: Config, out_dir: str):
356
+ """Pre-flight check for SentencePiece behaviour on historical text."""
357
+ unk = tokenizer.unk_token
358
+ unk_id = tokenizer.unk_token_id
359
+ rng = np.random.default_rng(cfg.split_seed)
360
+ if len(texts) > cfg.tokenizer_sample_size:
361
+ idx = rng.choice(len(texts), size=cfg.tokenizer_sample_size, replace=False)
362
+ texts = [texts[i] for i in idx]
363
+
364
+ total_tokens = 0
365
+ total_unk = 0
366
+ total_chars = 0
367
+ total_words = 0
368
+ tok_lengths: List[int] = []
369
+ unk_char_counter: Counter = Counter()
370
+ pathological: List[dict] = []
371
+
372
+ for t in texts:
373
+ t = maybe_nfc(t, cfg)
374
+ toks = tokenizer.tokenize(t)
375
+ n = len(toks)
376
+ n_unk = sum(1 for x in toks if x == unk)
377
+ total_tokens += n
378
+ total_unk += n_unk
379
+ total_chars += len(t)
380
+ total_words += max(1, len(t.split()))
381
+ tok_lengths.append(n)
382
+ if len(t) >= 20 and n / max(1, len(t)) > 1.5:
383
+ if len(pathological) < 25:
384
+ pathological.append({"chars": len(t), "tokens": n, "preview": t[:80]})
385
+
386
+ uniq_chars = set()
387
+ for t in texts[:5000]:
388
+ uniq_chars.update(maybe_nfc(t, cfg))
389
+ risky_chars = {}
390
+ for ch in uniq_chars:
391
+ if ch.isspace():
392
+ continue
393
+ ids = tokenizer(ch, add_special_tokens=False)["input_ids"]
394
+ if unk_id is not None and unk_id in ids:
395
+ name = unicodedata.name(ch, "UNNAMED")
396
+ risky_chars[ch] = {"codepoint": f"U+{ord(ch):04X}", "name": name}
397
+
398
+ tok_lengths_arr = np.asarray(tok_lengths) if tok_lengths else np.array([0])
399
+ report = {
400
+ "sampled_paragraphs": len(texts),
401
+ "normalize_nfc": cfg.normalize_nfc,
402
+ "unk_token": unk,
403
+ "unk_rate": round(total_unk / max(1, total_tokens), 6),
404
+ "total_unk_tokens": total_unk,
405
+ "fertility_tokens_per_word": round(total_tokens / max(1, total_words), 3),
406
+ "fertility_tokens_per_char": round(total_tokens / max(1, total_chars), 3),
407
+ "tokens_per_paragraph": {
408
+ "mean": float(tok_lengths_arr.mean()),
409
+ "p50": float(np.percentile(tok_lengths_arr, 50)),
410
+ "p95": float(np.percentile(tok_lengths_arr, 95)),
411
+ "max": int(tok_lengths_arr.max()),
412
+ "share_over_max_seq_len": round(
413
+ float((tok_lengths_arr > cfg.max_seq_length).mean()), 4),
414
+ },
415
+ "num_risky_unk_characters": len(risky_chars),
416
+ "risky_unk_characters": dict(sorted(risky_chars.items())[:100]),
417
+ "pathological_examples": pathological,
418
+ }
419
+ os.makedirs(out_dir, exist_ok=True)
420
+ with open(os.path.join(out_dir, "tokenizer_diagnostics.json"), "w") as fh:
421
+ json.dump(report, fh, indent=2, ensure_ascii=False)
422
+
423
+ logger.info("Tokenizer diagnostics: unk_rate=%.4f%% fertility=%.2f tok/word "
424
+ "risky_chars=%d", report["unk_rate"] * 100,
425
+ report["fertility_tokens_per_word"], report["num_risky_unk_characters"])
426
+ if report["unk_rate"] > 0.005:
427
+ logger.warning("High <unk> rate (%.3f%%). Inspect risky_unk_characters in "
428
+ "tokenizer_diagnostics.json; consider --normalize-nfc or a "
429
+ "transliteration/cleanup pass for medieval glyphs.",
430
+ report["unk_rate"] * 100)
431
+ if risky_chars:
432
+ sample = ", ".join(f"{c} ({m['codepoint']})" for c, m in list(risky_chars.items())[:15])
433
+ logger.warning("Characters mapping to <unk> (sample): %s", sample)
434
+ return report
435
+
436
+
437
+ def tokenize_paragraphs(ds, tokenizer, cfg: Config):
438
+ def _tok(batch):
439
+ texts = [maybe_nfc(t, cfg) for t in batch["text"]] if cfg.normalize_nfc else batch["text"]
440
+ enc = tokenizer(texts, add_special_tokens=False,
441
+ truncation=False, return_attention_mask=False)
442
+ return {"input_ids": enc["input_ids"]}
443
+
444
+ return ds.map(
445
+ _tok,
446
+ batched=True,
447
+ num_proc=cfg.preprocess_num_proc,
448
+ remove_columns=["text"],
449
+ desc="tokenize paragraphs",
450
+ )
451
+
452
+
453
+ def pack_doc_sentences(ds, tokenizer, cfg: Config, desc: str) -> Dataset:
454
+ bos = tokenizer.bos_token_id if tokenizer.bos_token_id is not None else tokenizer.cls_token_id
455
+ eos = tokenizer.eos_token_id if tokenizer.eos_token_id is not None else tokenizer.sep_token_id
456
+ if bos is None or eos is None:
457
+ raise ValueError("Tokenizer has no bos/cls or eos/sep token; cannot pack.")
458
+ max_content = cfg.max_seq_length - 2
459
+
460
+ sort_keys = ["document_id", "order"] if "order" in ds.column_names else ["document_id"]
461
+ ds = ds.sort(sort_keys)
462
+
463
+ def generator():
464
+ buffer: List[int] = []
465
+ cur_doc = None
466
+ cur_src = None
467
+
468
+ def flush_full():
469
+ nonlocal buffer
470
+ while len(buffer) >= max_content:
471
+ chunk = buffer[:max_content]
472
+ buffer = buffer[max_content:]
473
+ yield {"input_ids": [bos] + chunk + [eos],
474
+ "document_id": cur_doc, "source": cur_src}
475
+
476
+ for batch in ds.iter(batch_size=2000):
477
+ ids_col = batch["input_ids"]
478
+ doc_col = batch["document_id"]
479
+ src_col = batch["source"]
480
+ for ids, doc, src in zip(ids_col, doc_col, src_col):
481
+ if doc != cur_doc:
482
+ if buffer and len(buffer) >= cfg.min_chunk_tokens:
483
+ yield {"input_ids": [bos] + buffer + [eos],
484
+ "document_id": cur_doc, "source": cur_src}
485
+ buffer = []
486
+ cur_doc, cur_src = doc, src
487
+ buffer.extend(ids)
488
+ yield from flush_full()
489
+ if buffer and len(buffer) >= cfg.min_chunk_tokens:
490
+ yield {"input_ids": [bos] + buffer + [eos],
491
+ "document_id": cur_doc, "source": cur_src}
492
+
493
+ packed = Dataset.from_generator(generator, cache_dir=cfg.cache_dir)
494
+ logger.info(" packed %s -> %d sequences (max_len=%d)", desc, len(packed), cfg.max_seq_length)
495
+ return packed
496
+
497
+
498
+ def diagnose_pretokenized(ds, tokenizer, cfg: Config, out_dir: str):
499
+ unk_id = tokenizer.unk_token_id
500
+ bos_id = tokenizer.bos_token_id if tokenizer.bos_token_id is not None else tokenizer.cls_token_id
501
+ eos_id = tokenizer.eos_token_id if tokenizer.eos_token_id is not None else tokenizer.sep_token_id
502
+ n = len(ds)
503
+ m = min(cfg.tokenizer_sample_size, n)
504
+ idx = np.random.default_rng(cfg.split_seed).choice(n, size=m, replace=False)
505
+ sample = ds.select(idx)
506
+
507
+ total_tokens = 0
508
+ total_unk = 0
509
+ lengths: List[int] = []
510
+ bos_ok = 0
511
+ eos_ok = 0
512
+ over_len = 0
513
+ for ids in sample["input_ids"]:
514
+ L = len(ids)
515
+ lengths.append(L)
516
+ total_tokens += L
517
+ if unk_id is not None:
518
+ total_unk += ids.count(unk_id)
519
+ if L and ids[0] == bos_id:
520
+ bos_ok += 1
521
+ if L and ids[-1] == eos_id:
522
+ eos_ok += 1
523
+ if L > cfg.max_seq_length:
524
+ over_len += 1
525
+
526
+ arr = np.asarray(lengths) if lengths else np.array([0])
527
+ report = {
528
+ "mode": "pretokenized",
529
+ "sampled_sequences": int(m),
530
+ "total_sequences": int(n),
531
+ "unk_rate": round(total_unk / max(1, total_tokens), 6),
532
+ "total_unk_tokens": int(total_unk),
533
+ "seq_len": {"mean": float(arr.mean()), "p50": float(np.percentile(arr, 50)),
534
+ "p95": float(np.percentile(arr, 95)), "max": int(arr.max())},
535
+ "share_over_max_seq_len": round(float(over_len) / max(1, m), 4),
536
+ "starts_with_bos_rate": round(bos_ok / max(1, m), 4),
537
+ "ends_with_eos_rate": round(eos_ok / max(1, m), 4),
538
+ }
539
+ os.makedirs(out_dir, exist_ok=True)
540
+ with open(os.path.join(out_dir, "tokenizer_diagnostics.json"), "w") as fh:
541
+ json.dump(report, fh, indent=2, ensure_ascii=False)
542
+ logger.info("Pretokenized diagnostics: unk_rate=%.4f%% mean_len=%.1f "
543
+ "bos=%.0f%% eos=%.0f%%", report["unk_rate"] * 100, report["seq_len"]["mean"],
544
+ report["starts_with_bos_rate"] * 100, report["ends_with_eos_rate"] * 100)
545
+ if report["unk_rate"] > 0.005:
546
+ logger.warning("High <unk> rate (%.3f%%) in the pre-tokenized data — verify it was "
547
+ "tokenized with the SAME tokenizer (%s).", report["unk_rate"] * 100, cfg.model_name)
548
+ if report["starts_with_bos_rate"] < 0.5 or report["ends_with_eos_rate"] < 0.5:
549
+ logger.warning("Many sequences lack bos/eos boundaries; if your packing omitted special "
550
+ "tokens, MLM quality may suffer.")
551
+ if report["share_over_max_seq_len"] > 0:
552
+ logger.warning("%.2f%% of sequences exceed max_seq_length=%d and will be truncated.",
553
+ report["share_over_max_seq_len"] * 100, cfg.max_seq_length)
554
+ return report
555
+
556
+
557
+ def _load_dataset_any(name: str, split: str, token: bool):
558
+ if os.path.isdir(name):
559
+ from datasets import load_from_disk
560
+ d = load_from_disk(name)
561
+ if hasattr(d, "keys") and not hasattr(d, "column_names"): # DatasetDict
562
+ d = d[split]
563
+ return d
564
+ return load_dataset(name, split=split, token=token)
565
+
566
+
567
+ def _ids_hash(ids) -> bytes:
568
+ return hashlib.blake2b(np.asarray(ids, dtype=np.int32).tobytes(), digest_size=12).digest()
569
+
570
+
571
+ def build_pretokenized_bundle(cfg: Config, tokenizer, selected, cache_root: str):
572
+ max_len = cfg.max_seq_length
573
+ DOCID_CANDS = ("doc_id", "document_id", "docid")
574
+ parts = []
575
+
576
+ for repo, tag in selected:
577
+ ds = _load_dataset_any(repo, cfg.dataset_split, True)
578
+ docid_col = next((c for c in DOCID_CANDS if c in ds.column_names), None)
579
+
580
+ # ----- WISSENSCHAFTLICHE KORREKTUR: Metadaten beibehalten -----
581
+ keep = ["input_ids"]
582
+ if "attention_mask" in ds.column_names:
583
+ keep.append("attention_mask")
584
+ if docid_col:
585
+ keep.append(docid_col)
586
+
587
+ strat_col = cfg.stratify_column
588
+ if strat_col and strat_col in ds.column_names:
589
+ keep.append(strat_col)
590
+
591
+ ds = ds.remove_columns([c for c in ds.column_names if c not in keep])
592
+
593
+ if docid_col and docid_col != "document_id":
594
+ ds = ds.rename_column(docid_col, "document_id")
595
+ if strat_col and strat_col in ds.column_names and strat_col != "stratum":
596
+ ds = ds.rename_column(strat_col, "stratum")
597
+ # --------------------------------------------------------------
598
+
599
+ ds = ds.add_column("source", [tag] * len(ds))
600
+ parts.append(ds)
601
+ logger.info(" loaded pretokenized %s -> %d sequences (doc_id=%s)",
602
+ tag, len(ds), docid_col)
603
+
604
+ full = concatenate_datasets(parts) if len(parts) > 1 else parts[0]
605
+ has_doc = "document_id" in full.column_names
606
+
607
+ if has_doc:
608
+ full = full.map(
609
+ lambda b: {"document_id": [f"{s}:{d}" for s, d in zip(b["source"], b["document_id"])]},
610
+ batched=True, desc="namespace doc ids")
611
+
612
+ has_mask = "attention_mask" in full.column_names
613
+
614
+ def _fix(batch):
615
+ out = []
616
+ masks = batch["attention_mask"] if has_mask else [None] * len(batch["input_ids"])
617
+ for ids, am in zip(batch["input_ids"], masks):
618
+ if am is not None:
619
+ ids = [t for t, a in zip(ids, am) if a == 1]
620
+ if len(ids) > max_len:
621
+ ids = ids[:max_len]
622
+ out.append(ids)
623
+ return {"input_ids": out}
624
+
625
+ full = full.map(_fix, batched=True, num_proc=cfg.preprocess_num_proc,
626
+ remove_columns=(["attention_mask"] if has_mask else []),
627
+ desc="normalize pretokenized")
628
+
629
+ out_dir = cfg.output_dir
630
+ if cfg.diagnose_tokenizer:
631
+ diagnose_pretokenized(full, tokenizer, cfg, out_dir)
632
+
633
+ rng = np.random.default_rng(cfg.split_seed)
634
+ train_idx: List[int] = []
635
+ val_idx: List[int] = []
636
+ test_idx: List[int] = []
637
+
638
+ if has_doc:
639
+ docs = full["document_id"]
640
+ unique_docs = sorted(set(docs))
641
+ n_val = min(int(round(len(unique_docs) * cfg.val_doc_fraction)), cfg.max_eval_docs_per_source)
642
+ n_test = min(int(round(len(unique_docs) * cfg.test_doc_fraction)), cfg.max_eval_docs_per_source)
643
+ n_pick = n_val + n_test
644
+
645
+ # ----- WISSENSCHAFTLICHE KORREKTUR: Stratified Split -----
646
+ if "stratum" in full.column_names:
647
+ logger.info(f"Applying STRATIFIED document split based on column '{cfg.stratify_column}'.")
648
+ strata = full["stratum"]
649
+ doc_stratum: Dict[str, str] = {}
650
+ for d, s in zip(docs, strata):
651
+ doc_stratum.setdefault(d, str(s))
652
+ by_stratum: Dict[str, List[str]] = {}
653
+ for d, s in doc_stratum.items():
654
+ by_stratum.setdefault(s, []).append(d)
655
+
656
+ picked: List[str] = []
657
+ for s, d_list in sorted(by_stratum.items()):
658
+ d_list = sorted(d_list)
659
+ rng.shuffle(d_list)
660
+ k = max(1, int(round(n_pick * len(d_list) / len(doc_stratum))))
661
+ picked.extend(d_list[:min(k, len(d_list))])
662
+ rng.shuffle(picked)
663
+ val_docs = set(picked[:n_val])
664
+ test_docs = set(picked[n_val:n_val + n_test])
665
+ split_kind = "document-level STRATIFIED group-aware"
666
+ else:
667
+ rng.shuffle(unique_docs)
668
+ val_docs = set(unique_docs[:n_val])
669
+ test_docs = set(unique_docs[n_val:n_val + n_test])
670
+ split_kind = "document-level group-aware"
671
+ # ---------------------------------------------------------
672
+
673
+ val_cand, test_cand = [], []
674
+ for i, d in enumerate(docs):
675
+ if d in val_docs:
676
+ val_cand.append((i, d))
677
+ elif d in test_docs:
678
+ test_cand.append((i, d))
679
+ else:
680
+ train_idx.append(i)
681
+
682
+ for cand, dst in ((val_cand, val_idx), (test_cand, test_idx)):
683
+ seen = set()
684
+ for i, d in cand:
685
+ key = (d, _ids_hash(full[i]["input_ids"]))
686
+ if key not in seen:
687
+ seen.add(key)
688
+ dst.append(i)
689
+ else:
690
+ logger.warning("No doc_id/document_id column found in the pre-tokenized data; "
691
+ "falling back to a weaker sequence-level holdout.")
692
+ hashes = [_ids_hash(ids) for ids in full["input_ids"]]
693
+ uniq = sorted(set(hashes))
694
+ rng.shuffle(uniq)
695
+ n_val = min(int(round(len(uniq) * cfg.val_doc_fraction)), cfg.max_eval_docs_per_source * 4)
696
+ n_test = min(int(round(len(uniq) * cfg.test_doc_fraction)), cfg.max_eval_docs_per_source * 4)
697
+ val_h = set(uniq[:n_val])
698
+ test_h = set(uniq[n_val:n_val + n_test])
699
+ seen_v, seen_t = set(), set()
700
+ for i, h in enumerate(hashes):
701
+ if h in val_h:
702
+ if h not in seen_v:
703
+ seen_v.add(h); val_idx.append(i)
704
+ elif h in test_h:
705
+ if h not in seen_t:
706
+ seen_t.add(h); test_idx.append(i)
707
+ else:
708
+ train_idx.append(i)
709
+ split_kind = "sequence-level dedup"
710
+
711
+ bundle: Dict[str, Dataset] = {"train": full.select(train_idx)}
712
+ if val_idx:
713
+ bundle["validation"] = full.select(val_idx)
714
+ if len(set(bundle["validation"]["source"])) > 1:
715
+ for tag in ("med", "canon"):
716
+ sub = bundle["validation"].filter(lambda b: [s == tag for s in b["source"]], batched=True)
717
+ if len(sub):
718
+ bundle[f"validation_{tag}"] = sub
719
+ if test_idx:
720
+ bundle["test"] = full.select(test_idx)
721
+
722
+ logger.info("Pretokenized split (%s) -> train=%d val=%d test=%d",
723
+ split_kind, len(bundle["train"]), len(val_idx), len(test_idx))
724
+
725
+ def len_stats(d):
726
+ ls = np.asarray([len(x) for x in d["input_ids"]]) if len(d) else np.array([0])
727
+ return {"mean": float(ls.mean()), "p50": float(np.percentile(ls, 50)),
728
+ "p95": float(np.percentile(ls, 95))}
729
+ rep = {"split_kind": split_kind,
730
+ "train_sequences": len(bundle["train"]), "val_sequences": len(val_idx),
731
+ "len_train": len_stats(bundle["train"])}
732
+ if val_idx:
733
+ rep["len_val"] = len_stats(bundle["validation"])
734
+ with open(os.path.join(out_dir, "eval_representativeness.json"), "w") as fh:
735
+ json.dump(rep, fh, indent=2)
736
+
737
+ if cfg.diagnose_only:
738
+ logger.info("--diagnose-only set: wrote diagnostics to %s, exiting before training.", out_dir)
739
+ sys.exit(0)
740
+
741
+ os.makedirs(cache_root, exist_ok=True)
742
+ for name, d in bundle.items():
743
+ d.save_to_disk(os.path.join(cache_root, name))
744
+ logger.info("Saved pretokenized datasets to %s", cache_root)
745
+ return bundle
746
+
747
+
748
+ def build_or_load_packed(cfg: Config, tokenizer):
749
+ sig = json.dumps({
750
+ "med": cfg.med_dataset, "canon": cfg.canon_dataset, "split": cfg.dataset_split,
751
+ "model": cfg.model_name, "max_len": cfg.max_seq_length, "min_chunk": cfg.min_chunk_tokens,
752
+ "val_frac": cfg.val_doc_fraction, "test_frac": cfg.test_doc_fraction,
753
+ "max_eval": cfg.max_eval_docs_per_source, "split_seed": cfg.split_seed,
754
+ "text_col": cfg.text_column, "doc_col": cfg.doc_id_column, "order_col": cfg.order_column,
755
+ "stratify": cfg.stratify_column, "nfc": cfg.normalize_nfc,
756
+ "train_corpus": cfg.train_corpus, "dedup_train": cfg.dedup_train,
757
+ }, sort_keys=True)
758
+ key = hashlib.blake2b(sig.encode("utf-8"), digest_size=12).hexdigest()
759
+ cache_root = os.path.join(cfg.cache_dir, f"packed_{key}")
760
+
761
+ if os.path.isdir(cache_root) and not cfg.diagnose_only:
762
+ from datasets import load_from_disk
763
+ logger.info("Loading cached packed datasets from %s", cache_root)
764
+ bundle = {name: load_from_disk(os.path.join(cache_root, name))
765
+ for name in os.listdir(cache_root)
766
+ if os.path.isdir(os.path.join(cache_root, name))}
767
+ return bundle
768
+
769
+ if cfg.train_corpus not in {"combined", "med", "canon"}:
770
+ raise ValueError(f"--train-corpus must be combined|med|canon, got {cfg.train_corpus!r}")
771
+
772
+ selected = []
773
+ if cfg.train_corpus in {"combined", "med"}:
774
+ selected.append((cfg.med_dataset, "med"))
775
+ if cfg.train_corpus in {"combined", "canon"}:
776
+ selected.append((cfg.canon_dataset, "canon"))
777
+
778
+ probe = _load_dataset_any(selected[0][0], cfg.dataset_split, True)
779
+ if "input_ids" in probe.column_names:
780
+ logger.warning(
781
+ "Dataset %s is PRE-TOKENIZED (columns=%s). Skipping raw-text tokenization "
782
+ "and DOC-SENTENCES packing.", selected[0][0], probe.column_names)
783
+ del probe
784
+ return build_pretokenized_bundle(cfg, tokenizer, selected, cache_root)
785
+ del probe
786
+
787
+ sources = []
788
+ if cfg.train_corpus in {"combined", "med"}:
789
+ sources.append(load_source(cfg.med_dataset, cfg.dataset_split, "med", cfg))
790
+ if cfg.train_corpus in {"combined", "canon"}:
791
+ sources.append(load_source(cfg.canon_dataset, cfg.dataset_split, "canon", cfg))
792
+ logger.info("Training corpus = %s (%d source dataset(s))", cfg.train_corpus, len(sources))
793
+
794
+ train_parts, val_parts, test_parts = [], [], []
795
+ for src in sources:
796
+ tag = src["source"][0]
797
+ val_docs, test_docs = choose_heldout_docs(src, cfg)
798
+ in_val = src.filter(lambda b: [d in val_docs for d in b["document_id"]],
799
+ batched=True, desc=f"select val ({tag})")
800
+ in_test = src.filter(lambda b: [d in test_docs for d in b["document_id"]],
801
+ batched=True, desc=f"select test ({tag})")
802
+ held = val_docs | test_docs
803
+ in_train = src.filter(lambda b: [d not in held for d in b["document_id"]],
804
+ batched=True, desc=f"select train ({tag})")
805
+ train_parts.append(in_train)
806
+ if len(in_val):
807
+ val_parts.append(dedup_rows(in_val))
808
+ if len(in_test):
809
+ test_parts.append(dedup_rows(in_test))
810
+
811
+ train_raw = concatenate_datasets(train_parts)
812
+ if cfg.dedup_train:
813
+ before = len(train_raw)
814
+ train_raw = dedup_rows(train_raw)
815
+ logger.info("dedup_train: collapsed %d -> %d training paragraphs "
816
+ "(upsampling duplicates removed).", before, len(train_raw))
817
+ val_raw = concatenate_datasets(val_parts) if val_parts else None
818
+ logger.info("Raw paragraph counts -> train=%d val=%d test=%d",
819
+ len(train_raw),
820
+ sum(len(p) for p in val_parts),
821
+ sum(len(p) for p in test_parts))
822
+
823
+ out_dir = cfg.output_dir
824
+ if cfg.diagnose_tokenizer:
825
+ n = len(train_raw)
826
+ m = min(cfg.tokenizer_sample_size, n)
827
+ idx = np.random.default_rng(cfg.split_seed).choice(n, size=m, replace=False)
828
+ sample_texts = train_raw.select(idx)["text"]
829
+ diagnose_tokenizer(sample_texts, tokenizer, cfg, out_dir)
830
+ if val_raw is not None:
831
+ representativeness_report(train_raw, val_raw, cfg, out_dir)
832
+
833
+ if cfg.diagnose_only:
834
+ logger.info("--diagnose-only set: wrote diagnostics to %s, exiting before "
835
+ "packing/training.", out_dir)
836
+ sys.exit(0)
837
+
838
+ bundle: Dict[str, Dataset] = {}
839
+ bundle["train"] = pack_doc_sentences(tokenize_paragraphs(train_raw, tokenizer, cfg),
840
+ tokenizer, cfg, "train")
841
+ if val_raw is not None:
842
+ bundle["validation"] = pack_doc_sentences(tokenize_paragraphs(val_raw, tokenizer, cfg),
843
+ tokenizer, cfg, "validation")
844
+ if len(sources) > 1:
845
+ for tag in ("med", "canon"):
846
+ sub = val_raw.filter(lambda b: [s == tag for s in b["source"]], batched=True)
847
+ if len(sub):
848
+ bundle[f"validation_{tag}"] = pack_doc_sentences(
849
+ tokenize_paragraphs(sub, tokenizer, cfg), tokenizer, cfg, f"validation_{tag}")
850
+ if test_parts:
851
+ test_raw = concatenate_datasets(test_parts)
852
+ bundle["test"] = pack_doc_sentences(tokenize_paragraphs(test_raw, tokenizer, cfg),
853
+ tokenizer, cfg, "test")
854
+
855
+ os.makedirs(cache_root, exist_ok=True)
856
+ for name, d in bundle.items():
857
+ d.save_to_disk(os.path.join(cache_root, name))
858
+ logger.info("Saved packed datasets to %s", cache_root)
859
+ return bundle
860
+
861
+
862
+ # --------------------------------------------------------------------------- #
863
+ # Metrics
864
+ # --------------------------------------------------------------------------- #
865
+ def preprocess_logits_for_metrics(logits, labels):
866
+ if isinstance(logits, tuple):
867
+ logits = logits[0]
868
+ return logits.argmax(dim=-1)
869
+
870
+
871
+ def compute_metrics(eval_pred):
872
+ preds, labels = eval_pred
873
+ labels = labels.reshape(-1)
874
+ preds = preds.reshape(-1)
875
+ mask = labels != -100
876
+ if mask.sum() == 0:
877
+ return {"masked_accuracy": 0.0}
878
+ correct = (preds[mask] == labels[mask]).sum()
879
+ return {"masked_accuracy": float(correct) / float(mask.sum())}
880
+
881
+
882
+ class PerplexityCallback(TrainerCallback):
883
+ def on_evaluate(self, args, state, control, metrics=None, **kwargs):
884
+ if not metrics:
885
+ return
886
+ for k in list(metrics.keys()):
887
+ if k.endswith("loss") and ("eval" in k):
888
+ try:
889
+ metrics[k.replace("loss", "perplexity")] = math.exp(min(metrics[k], 20))
890
+ except OverflowError:
891
+ metrics[k.replace("loss", "perplexity")] = float("inf")
892
+
893
+
894
+ class JsonlLoggingCallback(TrainerCallback):
895
+ def __init__(self, path: str):
896
+ self.path = path
897
+ os.makedirs(os.path.dirname(path), exist_ok=True)
898
+
899
+ def on_log(self, args, state, control, logs=None, **kwargs):
900
+ if not logs:
901
+ return
902
+ record = dict(logs)
903
+ record["step"] = state.global_step
904
+ record["epoch"] = state.epoch
905
+ record["wall_time"] = datetime.now().isoformat(timespec="seconds")
906
+ with open(self.path, "a", encoding="utf-8") as fh:
907
+ fh.write(json.dumps(record, ensure_ascii=False) + "\n")
908
+ fh.flush()
909
+ os.fsync(fh.fileno())
910
+
911
+
912
+ # --------------------------------------------------------------------------- #
913
+ # Environment / run metadata
914
+ # --------------------------------------------------------------------------- #
915
+ def capture_environment(cfg: Config) -> dict:
916
+ import transformers
917
+ import datasets as ds_lib
918
+ info = {
919
+ "timestamp": datetime.now().isoformat(timespec="seconds"),
920
+ "python": platform.python_version(),
921
+ "platform": platform.platform(),
922
+ "torch": torch.__version__,
923
+ "transformers": transformers.__version__,
924
+ "datasets": ds_lib.__version__,
925
+ "cuda_available": torch.cuda.is_available(),
926
+ }
927
+ if torch.cuda.is_available():
928
+ info["cuda"] = torch.version.cuda
929
+ info["gpu_name"] = torch.cuda.get_device_name(0)
930
+ props = torch.cuda.get_device_properties(0)
931
+ info["gpu_total_memory_gb"] = round(props.total_memory / 1024**3, 1)
932
+ cap = f"{props.major}.{props.minor}"
933
+ info["gpu_capability"] = cap
934
+ info["bf16_supported"] = torch.cuda.is_bf16_supported()
935
+ try:
936
+ arch_list = torch.cuda.get_arch_list()
937
+ except Exception:
938
+ arch_list = []
939
+ info["torch_arch_list"] = arch_list
940
+ sm_tag = f"sm_{props.major}{props.minor}"
941
+ info["gpu_arch_supported_by_torch"] = any(sm_tag == a for a in arch_list)
942
+ return info
943
+
944
+
945
+ # --------------------------------------------------------------------------- #
946
+ # Argparse
947
+ # --------------------------------------------------------------------------- #
948
+ def parse_args() -> Config:
949
+ cfg = Config()
950
+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
951
+ for f in cfg.__dataclass_fields__.values():
952
+ name = "--" + f.name.replace("_", "-")
953
+ default = getattr(cfg, f.name)
954
+ if isinstance(default, bool):
955
+ if default:
956
+ p.add_argument("--no-" + f.name.replace("_", "-"), dest=f.name, action="store_false")
957
+ else:
958
+ p.add_argument(name, dest=f.name, action="store_true")
959
+ elif default is None:
960
+ p.add_argument(name, dest=f.name, default=None, type=str)
961
+ else:
962
+ p.add_argument(name, dest=f.name, default=default, type=type(default))
963
+ args = p.parse_args()
964
+ return Config(**vars(args))
965
+
966
+
967
+ # --------------------------------------------------------------------------- #
968
+ # Main
969
+ # --------------------------------------------------------------------------- #
970
+ def main():
971
+ cfg = parse_args()
972
+ set_seed(cfg.seed)
973
+
974
+ torch.backends.cuda.matmul.allow_tf32 = True
975
+ torch.backends.cudnn.allow_tf32 = True
976
+ torch.set_float32_matmul_precision("high")
977
+
978
+ os.makedirs(cfg.output_dir, exist_ok=True)
979
+
980
+ file_handler = logging.FileHandler(os.path.join(cfg.output_dir, "train.log"))
981
+ file_handler.setFormatter(logging.Formatter(
982
+ "%(asctime)s | %(levelname)-7s | %(name)s | %(message)s", "%Y-%m-%d %H:%M:%S"))
983
+ logging.getLogger().addHandler(file_handler)
984
+
985
+ env = capture_environment(cfg)
986
+ logger.info("Environment:\n%s", json.dumps(env, indent=2))
987
+ with open(os.path.join(cfg.output_dir, "run_metadata.json"), "w") as fh:
988
+ json.dump({"config": cfg.__dict__, "environment": env}, fh, indent=2)
989
+
990
+ if not env["cuda_available"]:
991
+ logger.warning("CUDA not available — this script is intended to run on a GPU server.")
992
+ else:
993
+ if not env.get("bf16_supported", False):
994
+ logger.warning("bf16 not reported as supported on this GPU; consider fp16.")
995
+ if env.get("torch_arch_list") and not env.get("gpu_arch_supported_by_torch", True):
996
+ logger.error(
997
+ "This torch build (%s, arch_list=%s) has NO kernels for your GPU "
998
+ "(capability %s). On Blackwell (RTX PRO 6000) install a cu128+ build, e.g.:\n"
999
+ " pip install --upgrade torch --index-url https://download.pytorch.org/whl/cu128\n"
1000
+ "CUDA ops will fail until this matches.",
1001
+ env["torch"], env.get("torch_arch_list"), env.get("gpu_capability"))
1002
+
1003
+ if cfg.save_steps % cfg.eval_steps != 0:
1004
+ new_save = max(cfg.eval_steps, (cfg.save_steps // cfg.eval_steps) * cfg.eval_steps)
1005
+ logger.warning("save_steps (%d) not a multiple of eval_steps (%d); adjusting to %d.",
1006
+ cfg.save_steps, cfg.eval_steps, new_save)
1007
+ cfg.save_steps = new_save
1008
+
1009
+ tokenizer = AutoTokenizer.from_pretrained(cfg.model_name, use_fast=True)
1010
+ tokenizer.model_max_length = cfg.max_seq_length
1011
+
1012
+ bundle = build_or_load_packed(cfg, tokenizer)
1013
+ train_ds = bundle["train"]
1014
+ eval_ds = bundle.get("validation")
1015
+
1016
+ def tok_count(d):
1017
+ lengths = d.map(lambda b: {"len": [len(x) for x in b["input_ids"]]},
1018
+ batched=True, remove_columns=d.column_names, desc="count tokens")["len"]
1019
+ return int(sum(lengths)), float(np.mean(lengths))
1020
+
1021
+ stats = {}
1022
+ for name, d in bundle.items():
1023
+ n_tok, mean_len = tok_count(d)
1024
+ stats[name] = {"sequences": len(d), "tokens": n_tok, "mean_seq_len": round(mean_len, 1)}
1025
+ logger.info("Packed dataset statistics:\n%s", json.dumps(stats, indent=2))
1026
+ with open(os.path.join(cfg.output_dir, "dataset_stats.json"), "w") as fh:
1027
+ json.dump(stats, fh, indent=2)
1028
+
1029
+ if cfg.dry_run:
1030
+ logger.info("--dry-run set: datasets built and cached, exiting before training.")
1031
+ return
1032
+
1033
+ train_ds = train_ds.remove_columns([c for c in train_ds.column_names if c != "input_ids"])
1034
+ if eval_ds is not None:
1035
+ eval_ds = eval_ds.remove_columns([c for c in eval_ds.column_names if c != "input_ids"])
1036
+
1037
+ model = AutoModelForMaskedLM.from_pretrained(
1038
+ cfg.model_name,
1039
+ attn_implementation=cfg.attn_implementation,
1040
+ )
1041
+ if cfg.gradient_checkpointing:
1042
+ model.config.use_cache = False
1043
+ logger.info("Model parameters: %.1fM", sum(p.numel() for p in model.parameters()) / 1e6)
1044
+
1045
+ collator = DataCollatorForLanguageModeling(
1046
+ tokenizer=tokenizer,
1047
+ mlm=True,
1048
+ mlm_probability=cfg.mlm_probability,
1049
+ pad_to_multiple_of=8,
1050
+ )
1051
+
1052
+ report_to = ["tensorboard"]
1053
+ if cfg.report_to_wandb:
1054
+ report_to.append("wandb")
1055
+
1056
+ do_eval = eval_ds is not None
1057
+ args = TrainingArguments(
1058
+ output_dir=cfg.output_dir,
1059
+ seed=cfg.seed,
1060
+ data_seed=cfg.seed,
1061
+ num_train_epochs=cfg.num_train_epochs,
1062
+ max_steps=cfg.max_steps,
1063
+ learning_rate=cfg.learning_rate,
1064
+ weight_decay=cfg.weight_decay,
1065
+ adam_beta1=cfg.adam_beta1,
1066
+ adam_beta2=cfg.adam_beta2,
1067
+ adam_epsilon=cfg.adam_epsilon,
1068
+ max_grad_norm=cfg.max_grad_norm,
1069
+ warmup_ratio=cfg.warmup_ratio,
1070
+ lr_scheduler_type=cfg.lr_scheduler_type,
1071
+ optim=cfg.optim,
1072
+ bf16=bool(env.get("bf16_supported")),
1073
+ per_device_train_batch_size=cfg.per_device_train_batch_size,
1074
+ per_device_eval_batch_size=cfg.per_device_eval_batch_size,
1075
+ gradient_accumulation_steps=cfg.gradient_accumulation_steps,
1076
+ gradient_checkpointing=cfg.gradient_checkpointing,
1077
+ auto_find_batch_size=cfg.auto_find_batch_size,
1078
+ torch_compile=cfg.torch_compile,
1079
+ dataloader_num_workers=cfg.dataloader_num_workers,
1080
+ dataloader_pin_memory=True,
1081
+ eval_accumulation_steps=cfg.eval_accumulation_steps,
1082
+ eval_strategy="steps" if do_eval else "no",
1083
+ eval_steps=cfg.eval_steps if do_eval else None,
1084
+ logging_strategy="steps",
1085
+ logging_steps=cfg.logging_steps,
1086
+ logging_first_step=True,
1087
+ save_strategy="steps",
1088
+ save_steps=cfg.save_steps,
1089
+ save_total_limit=cfg.save_total_limit,
1090
+ load_best_model_at_end=do_eval,
1091
+ metric_for_best_model="eval_loss" if do_eval else None,
1092
+ greater_is_better=False,
1093
+ report_to=report_to,
1094
+ run_name=os.path.basename(cfg.output_dir.rstrip("/")),
1095
+ include_num_input_tokens_seen=True,
1096
+ logging_dir=os.path.join(cfg.output_dir, "tb"),
1097
+ )
1098
+
1099
+ callbacks = [PerplexityCallback(),
1100
+ JsonlLoggingCallback(os.path.join(cfg.output_dir, "training_log.jsonl"))]
1101
+ if do_eval and cfg.early_stopping_patience > 0:
1102
+ callbacks.append(EarlyStoppingCallback(
1103
+ early_stopping_patience=cfg.early_stopping_patience,
1104
+ early_stopping_threshold=cfg.early_stopping_threshold))
1105
+
1106
+ tok_kwarg = {"processing_class": tokenizer} if _USE_PROCESSING_CLASS else {"tokenizer": tokenizer}
1107
+ trainer = Trainer(
1108
+ model=model,
1109
+ args=args,
1110
+ train_dataset=train_ds,
1111
+ eval_dataset=eval_ds,
1112
+ data_collator=collator,
1113
+ compute_metrics=compute_metrics if do_eval else None,
1114
+ preprocess_logits_for_metrics=preprocess_logits_for_metrics if do_eval else None,
1115
+ callbacks=callbacks,
1116
+ **tok_kwarg,
1117
+ )
1118
+
1119
+ resume_from = None
1120
+ if cfg.resume:
1121
+ last = get_last_checkpoint(cfg.output_dir)
1122
+ if last:
1123
+ logger.info("Resuming from checkpoint %s", last)
1124
+ resume_from = last
1125
+
1126
+ try:
1127
+ train_result = trainer.train(resume_from_checkpoint=resume_from)
1128
+ except BaseException as exc:
1129
+ tb_path = os.path.join(cfg.output_dir, "crash_traceback.txt")
1130
+ with open(tb_path, "w", encoding="utf-8") as fh:
1131
+ fh.write(f"Crashed at {datetime.now().isoformat()}\n\n")
1132
+ traceback.print_exc(file=fh)
1133
+ logger.error("Training crashed (%s: %s). Traceback written to %s",
1134
+ type(exc).__name__, exc, tb_path)
1135
+ try:
1136
+ emergency = os.path.join(cfg.output_dir, "emergency_checkpoint")
1137
+ trainer.save_model(emergency)
1138
+ trainer.save_state()
1139
+ logger.error("Emergency model state saved to %s", emergency)
1140
+ except Exception as save_exc:
1141
+ logger.error("Emergency save also failed: %s", save_exc)
1142
+ raise
1143
+
1144
+ trainer.save_model()
1145
+ tokenizer.save_pretrained(cfg.output_dir)
1146
+ trainer.save_state()
1147
+
1148
+ metrics = train_result.metrics
1149
+ trainer.log_metrics("train", metrics)
1150
+ trainer.save_metrics("train", metrics)
1151
+
1152
+ final = {}
1153
+ if eval_ds is not None:
1154
+ m = trainer.evaluate(eval_dataset=eval_ds, metric_key_prefix="final_val")
1155
+ m["final_val_perplexity"] = math.exp(min(m["final_val_loss"], 20))
1156
+ final.update(m)
1157
+ for tag in ("med", "canon"):
1158
+ sub = bundle.get(f"validation_{tag}")
1159
+ if sub is not None:
1160
+ sub = sub.remove_columns([c for c in sub.column_names if c != "input_ids"])
1161
+ m = trainer.evaluate(eval_dataset=sub, metric_key_prefix=f"final_val_{tag}")
1162
+ m[f"final_val_{tag}_perplexity"] = math.exp(min(m[f"final_val_{tag}_loss"], 20))
1163
+ final.update(m)
1164
+ if "test" in bundle:
1165
+ test_ds = bundle["test"].remove_columns(
1166
+ [c for c in bundle["test"].column_names if c != "input_ids"])
1167
+ m = trainer.evaluate(eval_dataset=test_ds, metric_key_prefix="final_test")
1168
+ m["final_test_perplexity"] = math.exp(min(m["final_test_loss"], 20))
1169
+ final.update(m)
1170
+
1171
+ if final:
1172
+ logger.info("Final evaluation:\n%s", json.dumps(final, indent=2))
1173
+ trainer.save_metrics("final_eval", final)
1174
+ logger.info("Done. Best model + tokenizer saved to %s", cfg.output_dir)
1175
+
1176
+
1177
+ if __name__ == "__main__":
1178
+ main()
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "num_input_tokens_seen": 1464637440,
4
+ "total_flos": 4.917799558028206e+18,
5
+ "train_loss": 21.80173199032777,
6
+ "train_runtime": 34212.9191,
7
+ "train_samples_per_second": 83.612,
8
+ "train_steps_per_second": 0.131
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,1043 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 4470,
3
+ "best_metric": 1.0481648445129395,
4
+ "best_model_checkpoint": "./MediBerta/checkpoint-4470",
5
+ "epoch": 10.0,
6
+ "eval_steps": 500,
7
+ "global_step": 4470,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.0022371364653243847,
14
+ "grad_norm": 32.67197036743164,
15
+ "learning_rate": 0.0,
16
+ "loss": 31.76995086669922,
17
+ "num_input_tokens_seen": 327680,
18
+ "step": 1,
19
+ "train_runtime": 7.8573,
20
+ "train_tokens_per_second": 41704.008
21
+ },
22
+ {
23
+ "epoch": 0.11185682326621924,
24
+ "grad_norm": 31.965707778930664,
25
+ "learning_rate": 1.821561338289963e-05,
26
+ "loss": 32.44386509486607,
27
+ "num_input_tokens_seen": 16384000,
28
+ "step": 50,
29
+ "train_runtime": 380.1137,
30
+ "train_tokens_per_second": 43102.889
31
+ },
32
+ {
33
+ "epoch": 0.22371364653243847,
34
+ "grad_norm": 34.0458984375,
35
+ "learning_rate": 3.680297397769517e-05,
36
+ "loss": 31.56456787109375,
37
+ "num_input_tokens_seen": 32768000,
38
+ "step": 100,
39
+ "train_runtime": 759.9744,
40
+ "train_tokens_per_second": 43117.243
41
+ },
42
+ {
43
+ "epoch": 0.33557046979865773,
44
+ "grad_norm": 36.13215255737305,
45
+ "learning_rate": 5.539033457249071e-05,
46
+ "loss": 30.703720703125,
47
+ "num_input_tokens_seen": 49152000,
48
+ "step": 150,
49
+ "train_runtime": 1140.0254,
50
+ "train_tokens_per_second": 43114.831
51
+ },
52
+ {
53
+ "epoch": 0.44742729306487694,
54
+ "grad_norm": 75.77301788330078,
55
+ "learning_rate": 7.397769516728626e-05,
56
+ "loss": 30.0342919921875,
57
+ "num_input_tokens_seen": 65536000,
58
+ "step": 200,
59
+ "train_runtime": 1519.7868,
60
+ "train_tokens_per_second": 43121.837
61
+ },
62
+ {
63
+ "epoch": 0.5592841163310962,
64
+ "grad_norm": 31.4502010345459,
65
+ "learning_rate": 9.256505576208179e-05,
66
+ "loss": 29.2935986328125,
67
+ "num_input_tokens_seen": 81920000,
68
+ "step": 250,
69
+ "train_runtime": 1899.4384,
70
+ "train_tokens_per_second": 43128.538
71
+ },
72
+ {
73
+ "epoch": 0.6711409395973155,
74
+ "grad_norm": 35.78462600708008,
75
+ "learning_rate": 9.928588431325875e-05,
76
+ "loss": 28.63412353515625,
77
+ "num_input_tokens_seen": 98304000,
78
+ "step": 300,
79
+ "train_runtime": 2279.106,
80
+ "train_tokens_per_second": 43132.703
81
+ },
82
+ {
83
+ "epoch": 0.7829977628635347,
84
+ "grad_norm": 32.92461395263672,
85
+ "learning_rate": 9.809569150202333e-05,
86
+ "loss": 27.89616455078125,
87
+ "num_input_tokens_seen": 114688000,
88
+ "step": 350,
89
+ "train_runtime": 2659.6638,
90
+ "train_tokens_per_second": 43121.24
91
+ },
92
+ {
93
+ "epoch": 0.8948545861297539,
94
+ "grad_norm": 31.881704330444336,
95
+ "learning_rate": 9.69054986907879e-05,
96
+ "loss": 27.38276123046875,
97
+ "num_input_tokens_seen": 131072000,
98
+ "step": 400,
99
+ "train_runtime": 3039.907,
100
+ "train_tokens_per_second": 43117.108
101
+ },
102
+ {
103
+ "epoch": 1.0067114093959733,
104
+ "grad_norm": 28.32501220703125,
105
+ "learning_rate": 9.57153058795525e-05,
106
+ "loss": 26.72684814453125,
107
+ "num_input_tokens_seen": 147446784,
108
+ "step": 450,
109
+ "train_runtime": 3419.3158,
110
+ "train_tokens_per_second": 43121.721
111
+ },
112
+ {
113
+ "epoch": 1.1185682326621924,
114
+ "grad_norm": 27.903852462768555,
115
+ "learning_rate": 9.452511306831707e-05,
116
+ "loss": 26.2695166015625,
117
+ "num_input_tokens_seen": 163830784,
118
+ "step": 500,
119
+ "train_runtime": 3799.2571,
120
+ "train_tokens_per_second": 43121.794
121
+ },
122
+ {
123
+ "epoch": 1.1185682326621924,
124
+ "eval_loss": 1.3941264152526855,
125
+ "eval_masked_accuracy": 0.7140736610865701,
126
+ "eval_runtime": 25.352,
127
+ "eval_samples_per_second": 219.627,
128
+ "eval_steps_per_second": 3.432,
129
+ "num_input_tokens_seen": 163830784,
130
+ "step": 500
131
+ },
132
+ {
133
+ "epoch": 1.2304250559284116,
134
+ "grad_norm": 25.51409339904785,
135
+ "learning_rate": 9.333492025708165e-05,
136
+ "loss": 25.86959228515625,
137
+ "num_input_tokens_seen": 180214784,
138
+ "step": 550,
139
+ "train_runtime": 4207.1627,
140
+ "train_tokens_per_second": 42835.231
141
+ },
142
+ {
143
+ "epoch": 1.342281879194631,
144
+ "grad_norm": 31.309749603271484,
145
+ "learning_rate": 9.214472744584624e-05,
146
+ "loss": 25.3655517578125,
147
+ "num_input_tokens_seen": 196598784,
148
+ "step": 600,
149
+ "train_runtime": 4586.3773,
150
+ "train_tokens_per_second": 42865.812
151
+ },
152
+ {
153
+ "epoch": 1.45413870246085,
154
+ "grad_norm": 28.270841598510742,
155
+ "learning_rate": 9.095453463461082e-05,
156
+ "loss": 25.1278271484375,
157
+ "num_input_tokens_seen": 212982784,
158
+ "step": 650,
159
+ "train_runtime": 4966.2335,
160
+ "train_tokens_per_second": 42886.18
161
+ },
162
+ {
163
+ "epoch": 1.5659955257270695,
164
+ "grad_norm": 27.522308349609375,
165
+ "learning_rate": 8.976434182337538e-05,
166
+ "loss": 24.80788818359375,
167
+ "num_input_tokens_seen": 229366784,
168
+ "step": 700,
169
+ "train_runtime": 5346.6697,
170
+ "train_tokens_per_second": 42899.0
171
+ },
172
+ {
173
+ "epoch": 1.6778523489932886,
174
+ "grad_norm": 27.093711853027344,
175
+ "learning_rate": 8.857414901213997e-05,
176
+ "loss": 24.6148046875,
177
+ "num_input_tokens_seen": 245750784,
178
+ "step": 750,
179
+ "train_runtime": 5726.4531,
180
+ "train_tokens_per_second": 42915.008
181
+ },
182
+ {
183
+ "epoch": 1.7897091722595078,
184
+ "grad_norm": 25.49102210998535,
185
+ "learning_rate": 8.738395620090455e-05,
186
+ "loss": 24.40305908203125,
187
+ "num_input_tokens_seen": 262134784,
188
+ "step": 800,
189
+ "train_runtime": 6106.8482,
190
+ "train_tokens_per_second": 42924.726
191
+ },
192
+ {
193
+ "epoch": 1.901565995525727,
194
+ "grad_norm": 25.048629760742188,
195
+ "learning_rate": 8.619376338966914e-05,
196
+ "loss": 24.0977783203125,
197
+ "num_input_tokens_seen": 278518784,
198
+ "step": 850,
199
+ "train_runtime": 6486.6563,
200
+ "train_tokens_per_second": 42937.189
201
+ },
202
+ {
203
+ "epoch": 2.0134228187919465,
204
+ "grad_norm": 25.157657623291016,
205
+ "learning_rate": 8.500357057843372e-05,
206
+ "loss": 23.89361572265625,
207
+ "num_input_tokens_seen": 294893568,
208
+ "step": 900,
209
+ "train_runtime": 6867.068,
210
+ "train_tokens_per_second": 42943.155
211
+ },
212
+ {
213
+ "epoch": 2.1252796420581657,
214
+ "grad_norm": 25.902904510498047,
215
+ "learning_rate": 8.381337776719829e-05,
216
+ "loss": 23.57096435546875,
217
+ "num_input_tokens_seen": 311277568,
218
+ "step": 950,
219
+ "train_runtime": 7246.8643,
220
+ "train_tokens_per_second": 42953.415
221
+ },
222
+ {
223
+ "epoch": 2.237136465324385,
224
+ "grad_norm": 24.553022384643555,
225
+ "learning_rate": 8.262318495596287e-05,
226
+ "loss": 23.42177490234375,
227
+ "num_input_tokens_seen": 327661568,
228
+ "step": 1000,
229
+ "train_runtime": 7626.2318,
230
+ "train_tokens_per_second": 42965.068
231
+ },
232
+ {
233
+ "epoch": 2.237136465324385,
234
+ "eval_loss": 1.2587531805038452,
235
+ "eval_masked_accuracy": 0.7357594334234474,
236
+ "eval_runtime": 25.3576,
237
+ "eval_samples_per_second": 219.579,
238
+ "eval_steps_per_second": 3.431,
239
+ "num_input_tokens_seen": 327661568,
240
+ "step": 1000
241
+ },
242
+ {
243
+ "epoch": 2.348993288590604,
244
+ "grad_norm": 24.940990447998047,
245
+ "learning_rate": 8.143299214472745e-05,
246
+ "loss": 23.259130859375,
247
+ "num_input_tokens_seen": 344045568,
248
+ "step": 1050,
249
+ "train_runtime": 8033.7212,
250
+ "train_tokens_per_second": 42825.182
251
+ },
252
+ {
253
+ "epoch": 2.460850111856823,
254
+ "grad_norm": 24.946720123291016,
255
+ "learning_rate": 8.024279933349202e-05,
256
+ "loss": 23.12722900390625,
257
+ "num_input_tokens_seen": 360429568,
258
+ "step": 1100,
259
+ "train_runtime": 8413.8773,
260
+ "train_tokens_per_second": 42837.512
261
+ },
262
+ {
263
+ "epoch": 2.5727069351230423,
264
+ "grad_norm": 24.135683059692383,
265
+ "learning_rate": 7.905260652225661e-05,
266
+ "loss": 22.9796826171875,
267
+ "num_input_tokens_seen": 376813568,
268
+ "step": 1150,
269
+ "train_runtime": 8793.1262,
270
+ "train_tokens_per_second": 42853.197
271
+ },
272
+ {
273
+ "epoch": 2.684563758389262,
274
+ "grad_norm": 25.850025177001953,
275
+ "learning_rate": 7.786241371102119e-05,
276
+ "loss": 22.77516845703125,
277
+ "num_input_tokens_seen": 393197568,
278
+ "step": 1200,
279
+ "train_runtime": 9172.793,
280
+ "train_tokens_per_second": 42865.632
281
+ },
282
+ {
283
+ "epoch": 2.796420581655481,
284
+ "grad_norm": 22.62376594543457,
285
+ "learning_rate": 7.667222089978577e-05,
286
+ "loss": 22.6738720703125,
287
+ "num_input_tokens_seen": 409581568,
288
+ "step": 1250,
289
+ "train_runtime": 9553.7727,
290
+ "train_tokens_per_second": 42871.186
291
+ },
292
+ {
293
+ "epoch": 2.9082774049217,
294
+ "grad_norm": 22.361886978149414,
295
+ "learning_rate": 7.548202808855035e-05,
296
+ "loss": 22.53384033203125,
297
+ "num_input_tokens_seen": 425965568,
298
+ "step": 1300,
299
+ "train_runtime": 9932.9496,
300
+ "train_tokens_per_second": 42884.096
301
+ },
302
+ {
303
+ "epoch": 3.0201342281879193,
304
+ "grad_norm": 24.093069076538086,
305
+ "learning_rate": 7.429183527731492e-05,
306
+ "loss": 22.38953125,
307
+ "num_input_tokens_seen": 442340352,
308
+ "step": 1350,
309
+ "train_runtime": 10313.033,
310
+ "train_tokens_per_second": 42891.393
311
+ },
312
+ {
313
+ "epoch": 3.131991051454139,
314
+ "grad_norm": 22.419540405273438,
315
+ "learning_rate": 7.310164246607951e-05,
316
+ "loss": 22.18701416015625,
317
+ "num_input_tokens_seen": 458724352,
318
+ "step": 1400,
319
+ "train_runtime": 10693.1337,
320
+ "train_tokens_per_second": 42898.964
321
+ },
322
+ {
323
+ "epoch": 3.243847874720358,
324
+ "grad_norm": 24.061424255371094,
325
+ "learning_rate": 7.191144965484409e-05,
326
+ "loss": 22.1206396484375,
327
+ "num_input_tokens_seen": 475108352,
328
+ "step": 1450,
329
+ "train_runtime": 11073.944,
330
+ "train_tokens_per_second": 42903.265
331
+ },
332
+ {
333
+ "epoch": 3.3557046979865772,
334
+ "grad_norm": 21.660381317138672,
335
+ "learning_rate": 7.072125684360867e-05,
336
+ "loss": 21.99459228515625,
337
+ "num_input_tokens_seen": 491492352,
338
+ "step": 1500,
339
+ "train_runtime": 11453.1596,
340
+ "train_tokens_per_second": 42913.254
341
+ },
342
+ {
343
+ "epoch": 3.3557046979865772,
344
+ "eval_loss": 1.194376826286316,
345
+ "eval_masked_accuracy": 0.7462490242091618,
346
+ "eval_runtime": 25.3329,
347
+ "eval_samples_per_second": 219.794,
348
+ "eval_steps_per_second": 3.434,
349
+ "num_input_tokens_seen": 491492352,
350
+ "step": 1500
351
+ },
352
+ {
353
+ "epoch": 3.4675615212527964,
354
+ "grad_norm": 23.65436553955078,
355
+ "learning_rate": 6.953106403237324e-05,
356
+ "loss": 21.8618212890625,
357
+ "num_input_tokens_seen": 507876352,
358
+ "step": 1550,
359
+ "train_runtime": 11860.4601,
360
+ "train_tokens_per_second": 42820.965
361
+ },
362
+ {
363
+ "epoch": 3.5794183445190155,
364
+ "grad_norm": 23.492694854736328,
365
+ "learning_rate": 6.834087122113782e-05,
366
+ "loss": 21.84644287109375,
367
+ "num_input_tokens_seen": 524260352,
368
+ "step": 1600,
369
+ "train_runtime": 12240.4952,
370
+ "train_tokens_per_second": 42829.995
371
+ },
372
+ {
373
+ "epoch": 3.6912751677852347,
374
+ "grad_norm": 22.143146514892578,
375
+ "learning_rate": 6.715067840990241e-05,
376
+ "loss": 21.695458984375,
377
+ "num_input_tokens_seen": 540644352,
378
+ "step": 1650,
379
+ "train_runtime": 12620.3553,
380
+ "train_tokens_per_second": 42839.075
381
+ },
382
+ {
383
+ "epoch": 3.8031319910514543,
384
+ "grad_norm": 22.582658767700195,
385
+ "learning_rate": 6.596048559866699e-05,
386
+ "loss": 21.49891357421875,
387
+ "num_input_tokens_seen": 557028352,
388
+ "step": 1700,
389
+ "train_runtime": 13000.3344,
390
+ "train_tokens_per_second": 42847.233
391
+ },
392
+ {
393
+ "epoch": 3.9149888143176734,
394
+ "grad_norm": 22.46770477294922,
395
+ "learning_rate": 6.477029278743157e-05,
396
+ "loss": 21.4741015625,
397
+ "num_input_tokens_seen": 573412352,
398
+ "step": 1750,
399
+ "train_runtime": 13380.7098,
400
+ "train_tokens_per_second": 42853.657
401
+ },
402
+ {
403
+ "epoch": 4.026845637583893,
404
+ "grad_norm": 22.605365753173828,
405
+ "learning_rate": 6.358009997619616e-05,
406
+ "loss": 21.39329833984375,
407
+ "num_input_tokens_seen": 589787136,
408
+ "step": 1800,
409
+ "train_runtime": 13759.8334,
410
+ "train_tokens_per_second": 42862.956
411
+ },
412
+ {
413
+ "epoch": 4.138702460850112,
414
+ "grad_norm": 21.95088005065918,
415
+ "learning_rate": 6.238990716496072e-05,
416
+ "loss": 21.18496826171875,
417
+ "num_input_tokens_seen": 606171136,
418
+ "step": 1850,
419
+ "train_runtime": 14139.2097,
420
+ "train_tokens_per_second": 42871.642
421
+ },
422
+ {
423
+ "epoch": 4.250559284116331,
424
+ "grad_norm": 22.34777069091797,
425
+ "learning_rate": 6.11997143537253e-05,
426
+ "loss": 21.2357177734375,
427
+ "num_input_tokens_seen": 622555136,
428
+ "step": 1900,
429
+ "train_runtime": 14518.7366,
430
+ "train_tokens_per_second": 42879.429
431
+ },
432
+ {
433
+ "epoch": 4.3624161073825505,
434
+ "grad_norm": 22.553071975708008,
435
+ "learning_rate": 6.000952154248989e-05,
436
+ "loss": 21.16220458984375,
437
+ "num_input_tokens_seen": 638939136,
438
+ "step": 1950,
439
+ "train_runtime": 14898.759,
440
+ "train_tokens_per_second": 42885.393
441
+ },
442
+ {
443
+ "epoch": 4.47427293064877,
444
+ "grad_norm": 22.87005615234375,
445
+ "learning_rate": 5.8819328731254464e-05,
446
+ "loss": 21.01521240234375,
447
+ "num_input_tokens_seen": 655323136,
448
+ "step": 2000,
449
+ "train_runtime": 15278.457,
450
+ "train_tokens_per_second": 42891.971
451
+ },
452
+ {
453
+ "epoch": 4.47427293064877,
454
+ "eval_loss": 1.14796781539917,
455
+ "eval_masked_accuracy": 0.7536936821619982,
456
+ "eval_runtime": 25.3387,
457
+ "eval_samples_per_second": 219.743,
458
+ "eval_steps_per_second": 3.433,
459
+ "num_input_tokens_seen": 655323136,
460
+ "step": 2000
461
+ },
462
+ {
463
+ "epoch": 4.586129753914989,
464
+ "grad_norm": 22.233734130859375,
465
+ "learning_rate": 5.762913592001905e-05,
466
+ "loss": 20.9041943359375,
467
+ "num_input_tokens_seen": 671707136,
468
+ "step": 2050,
469
+ "train_runtime": 15686.3972,
470
+ "train_tokens_per_second": 42820.995
471
+ },
472
+ {
473
+ "epoch": 4.697986577181208,
474
+ "grad_norm": 21.8343448638916,
475
+ "learning_rate": 5.6438943108783625e-05,
476
+ "loss": 20.91035888671875,
477
+ "num_input_tokens_seen": 688091136,
478
+ "step": 2100,
479
+ "train_runtime": 16066.5233,
480
+ "train_tokens_per_second": 42827.631
481
+ },
482
+ {
483
+ "epoch": 4.809843400447427,
484
+ "grad_norm": 21.778446197509766,
485
+ "learning_rate": 5.52487502975482e-05,
486
+ "loss": 20.8124365234375,
487
+ "num_input_tokens_seen": 704475136,
488
+ "step": 2150,
489
+ "train_runtime": 16446.1615,
490
+ "train_tokens_per_second": 42835.231
491
+ },
492
+ {
493
+ "epoch": 4.921700223713646,
494
+ "grad_norm": 21.686033248901367,
495
+ "learning_rate": 5.4058557486312786e-05,
496
+ "loss": 20.7349560546875,
497
+ "num_input_tokens_seen": 720859136,
498
+ "step": 2200,
499
+ "train_runtime": 16825.7207,
500
+ "train_tokens_per_second": 42842.69
501
+ },
502
+ {
503
+ "epoch": 5.033557046979865,
504
+ "grad_norm": 21.80782127380371,
505
+ "learning_rate": 5.286836467507736e-05,
506
+ "loss": 20.7165283203125,
507
+ "num_input_tokens_seen": 737233920,
508
+ "step": 2250,
509
+ "train_runtime": 17206.3274,
510
+ "train_tokens_per_second": 42846.675
511
+ },
512
+ {
513
+ "epoch": 5.145413870246085,
514
+ "grad_norm": 23.681589126586914,
515
+ "learning_rate": 5.167817186384194e-05,
516
+ "loss": 20.5538916015625,
517
+ "num_input_tokens_seen": 753617920,
518
+ "step": 2300,
519
+ "train_runtime": 17586.976,
520
+ "train_tokens_per_second": 42850.91
521
+ },
522
+ {
523
+ "epoch": 5.257270693512305,
524
+ "grad_norm": 23.062753677368164,
525
+ "learning_rate": 5.0487979052606524e-05,
526
+ "loss": 20.505771484375,
527
+ "num_input_tokens_seen": 770001920,
528
+ "step": 2350,
529
+ "train_runtime": 17966.3576,
530
+ "train_tokens_per_second": 42857.987
531
+ },
532
+ {
533
+ "epoch": 5.369127516778524,
534
+ "grad_norm": 22.336442947387695,
535
+ "learning_rate": 4.92977862413711e-05,
536
+ "loss": 20.479666748046874,
537
+ "num_input_tokens_seen": 786385920,
538
+ "step": 2400,
539
+ "train_runtime": 18346.1922,
540
+ "train_tokens_per_second": 42863.713
541
+ },
542
+ {
543
+ "epoch": 5.480984340044743,
544
+ "grad_norm": 22.454973220825195,
545
+ "learning_rate": 4.8107593430135685e-05,
546
+ "loss": 20.462833251953125,
547
+ "num_input_tokens_seen": 802769920,
548
+ "step": 2450,
549
+ "train_runtime": 18726.6308,
550
+ "train_tokens_per_second": 42867.824
551
+ },
552
+ {
553
+ "epoch": 5.592841163310962,
554
+ "grad_norm": 21.835525512695312,
555
+ "learning_rate": 4.691740061890026e-05,
556
+ "loss": 20.33416748046875,
557
+ "num_input_tokens_seen": 819153920,
558
+ "step": 2500,
559
+ "train_runtime": 19106.2918,
560
+ "train_tokens_per_second": 42873.517
561
+ },
562
+ {
563
+ "epoch": 5.592841163310962,
564
+ "eval_loss": 1.1132277250289917,
565
+ "eval_masked_accuracy": 0.7584756667111459,
566
+ "eval_runtime": 25.3527,
567
+ "eval_samples_per_second": 219.621,
568
+ "eval_steps_per_second": 3.432,
569
+ "num_input_tokens_seen": 819153920,
570
+ "step": 2500
571
+ },
572
+ {
573
+ "epoch": 5.704697986577181,
574
+ "grad_norm": 23.234500885009766,
575
+ "learning_rate": 4.5727207807664845e-05,
576
+ "loss": 20.25551513671875,
577
+ "num_input_tokens_seen": 835537920,
578
+ "step": 2550,
579
+ "train_runtime": 19513.5356,
580
+ "train_tokens_per_second": 42818.377
581
+ },
582
+ {
583
+ "epoch": 5.8165548098434,
584
+ "grad_norm": 21.292207717895508,
585
+ "learning_rate": 4.453701499642942e-05,
586
+ "loss": 20.156134033203124,
587
+ "num_input_tokens_seen": 851921920,
588
+ "step": 2600,
589
+ "train_runtime": 19893.1597,
590
+ "train_tokens_per_second": 42824.867
591
+ },
592
+ {
593
+ "epoch": 5.9284116331096195,
594
+ "grad_norm": 20.622861862182617,
595
+ "learning_rate": 4.3346822185194e-05,
596
+ "loss": 20.159989013671876,
597
+ "num_input_tokens_seen": 868305920,
598
+ "step": 2650,
599
+ "train_runtime": 20273.9368,
600
+ "train_tokens_per_second": 42828.679
601
+ },
602
+ {
603
+ "epoch": 6.040268456375839,
604
+ "grad_norm": 23.077646255493164,
605
+ "learning_rate": 4.215662937395858e-05,
606
+ "loss": 20.13439697265625,
607
+ "num_input_tokens_seen": 884680704,
608
+ "step": 2700,
609
+ "train_runtime": 20654.2855,
610
+ "train_tokens_per_second": 42832.791
611
+ },
612
+ {
613
+ "epoch": 6.152125279642058,
614
+ "grad_norm": 21.280527114868164,
615
+ "learning_rate": 4.096643656272317e-05,
616
+ "loss": 19.994425048828123,
617
+ "num_input_tokens_seen": 901064704,
618
+ "step": 2750,
619
+ "train_runtime": 21033.1909,
620
+ "train_tokens_per_second": 42840.133
621
+ },
622
+ {
623
+ "epoch": 6.263982102908278,
624
+ "grad_norm": 23.213178634643555,
625
+ "learning_rate": 3.977624375148774e-05,
626
+ "loss": 19.980313720703126,
627
+ "num_input_tokens_seen": 917448704,
628
+ "step": 2800,
629
+ "train_runtime": 21412.6494,
630
+ "train_tokens_per_second": 42846.109
631
+ },
632
+ {
633
+ "epoch": 6.375838926174497,
634
+ "grad_norm": 22.994335174560547,
635
+ "learning_rate": 3.858605094025232e-05,
636
+ "loss": 19.9128857421875,
637
+ "num_input_tokens_seen": 933832704,
638
+ "step": 2850,
639
+ "train_runtime": 21792.7904,
640
+ "train_tokens_per_second": 42850.534
641
+ },
642
+ {
643
+ "epoch": 6.487695749440716,
644
+ "grad_norm": 25.134178161621094,
645
+ "learning_rate": 3.7395858129016905e-05,
646
+ "loss": 19.874437255859377,
647
+ "num_input_tokens_seen": 950216704,
648
+ "step": 2900,
649
+ "train_runtime": 22173.1567,
650
+ "train_tokens_per_second": 42854.372
651
+ },
652
+ {
653
+ "epoch": 6.599552572706935,
654
+ "grad_norm": 20.69732093811035,
655
+ "learning_rate": 3.620566531778148e-05,
656
+ "loss": 19.910538330078126,
657
+ "num_input_tokens_seen": 966600704,
658
+ "step": 2950,
659
+ "train_runtime": 22553.2542,
660
+ "train_tokens_per_second": 42858.591
661
+ },
662
+ {
663
+ "epoch": 6.7114093959731544,
664
+ "grad_norm": 21.227550506591797,
665
+ "learning_rate": 3.501547250654606e-05,
666
+ "loss": 19.87051025390625,
667
+ "num_input_tokens_seen": 982984704,
668
+ "step": 3000,
669
+ "train_runtime": 22933.4022,
670
+ "train_tokens_per_second": 42862.576
671
+ },
672
+ {
673
+ "epoch": 6.7114093959731544,
674
+ "eval_loss": 1.0944103002548218,
675
+ "eval_masked_accuracy": 0.7619272139314145,
676
+ "eval_runtime": 25.3399,
677
+ "eval_samples_per_second": 219.732,
678
+ "eval_steps_per_second": 3.433,
679
+ "num_input_tokens_seen": 982984704,
680
+ "step": 3000
681
+ },
682
+ {
683
+ "epoch": 6.823266219239374,
684
+ "grad_norm": 21.8638858795166,
685
+ "learning_rate": 3.382527969531064e-05,
686
+ "loss": 19.65084228515625,
687
+ "num_input_tokens_seen": 999368704,
688
+ "step": 3050,
689
+ "train_runtime": 23340.7393,
690
+ "train_tokens_per_second": 42816.497
691
+ },
692
+ {
693
+ "epoch": 6.935123042505593,
694
+ "grad_norm": 25.967254638671875,
695
+ "learning_rate": 3.263508688407522e-05,
696
+ "loss": 19.7602978515625,
697
+ "num_input_tokens_seen": 1015752704,
698
+ "step": 3100,
699
+ "train_runtime": 23721.0833,
700
+ "train_tokens_per_second": 42820.671
701
+ },
702
+ {
703
+ "epoch": 7.046979865771812,
704
+ "grad_norm": 29.288837432861328,
705
+ "learning_rate": 3.1444894072839803e-05,
706
+ "loss": 19.64591064453125,
707
+ "num_input_tokens_seen": 1032127488,
708
+ "step": 3150,
709
+ "train_runtime": 24100.8116,
710
+ "train_tokens_per_second": 42825.425
711
+ },
712
+ {
713
+ "epoch": 7.158836689038031,
714
+ "grad_norm": 20.559537887573242,
715
+ "learning_rate": 3.025470126160438e-05,
716
+ "loss": 19.643092041015624,
717
+ "num_input_tokens_seen": 1048511488,
718
+ "step": 3200,
719
+ "train_runtime": 24479.6774,
720
+ "train_tokens_per_second": 42831.916
721
+ },
722
+ {
723
+ "epoch": 7.27069351230425,
724
+ "grad_norm": 21.216949462890625,
725
+ "learning_rate": 2.906450845036896e-05,
726
+ "loss": 19.5406787109375,
727
+ "num_input_tokens_seen": 1064895488,
728
+ "step": 3250,
729
+ "train_runtime": 24859.9356,
730
+ "train_tokens_per_second": 42835.81
731
+ },
732
+ {
733
+ "epoch": 7.382550335570469,
734
+ "grad_norm": 19.948974609375,
735
+ "learning_rate": 2.787431563913354e-05,
736
+ "loss": 19.52265625,
737
+ "num_input_tokens_seen": 1081279488,
738
+ "step": 3300,
739
+ "train_runtime": 25239.4624,
740
+ "train_tokens_per_second": 42840.829
741
+ },
742
+ {
743
+ "epoch": 7.494407158836689,
744
+ "grad_norm": 24.806529998779297,
745
+ "learning_rate": 2.668412282789812e-05,
746
+ "loss": 19.483994140625,
747
+ "num_input_tokens_seen": 1097663488,
748
+ "step": 3350,
749
+ "train_runtime": 25619.1507,
750
+ "train_tokens_per_second": 42845.428
751
+ },
752
+ {
753
+ "epoch": 7.6062639821029085,
754
+ "grad_norm": 22.306764602661133,
755
+ "learning_rate": 2.54939300166627e-05,
756
+ "loss": 19.500614013671875,
757
+ "num_input_tokens_seen": 1114047488,
758
+ "step": 3400,
759
+ "train_runtime": 25998.4643,
760
+ "train_tokens_per_second": 42850.511
761
+ },
762
+ {
763
+ "epoch": 7.718120805369128,
764
+ "grad_norm": 20.259811401367188,
765
+ "learning_rate": 2.430373720542728e-05,
766
+ "loss": 19.44253173828125,
767
+ "num_input_tokens_seen": 1130431488,
768
+ "step": 3450,
769
+ "train_runtime": 26378.2281,
770
+ "train_tokens_per_second": 42854.716
771
+ },
772
+ {
773
+ "epoch": 7.829977628635347,
774
+ "grad_norm": 31.171615600585938,
775
+ "learning_rate": 2.311354439419186e-05,
776
+ "loss": 19.37197021484375,
777
+ "num_input_tokens_seen": 1146815488,
778
+ "step": 3500,
779
+ "train_runtime": 26757.8966,
780
+ "train_tokens_per_second": 42858.955
781
+ },
782
+ {
783
+ "epoch": 7.829977628635347,
784
+ "eval_loss": 1.0732760429382324,
785
+ "eval_masked_accuracy": 0.7661423014370904,
786
+ "eval_runtime": 25.3349,
787
+ "eval_samples_per_second": 219.775,
788
+ "eval_steps_per_second": 3.434,
789
+ "num_input_tokens_seen": 1146815488,
790
+ "step": 3500
791
+ },
792
+ {
793
+ "epoch": 7.941834451901566,
794
+ "grad_norm": 19.443883895874023,
795
+ "learning_rate": 2.192335158295644e-05,
796
+ "loss": 19.302330322265625,
797
+ "num_input_tokens_seen": 1163199488,
798
+ "step": 3550,
799
+ "train_runtime": 27165.3477,
800
+ "train_tokens_per_second": 42819.238
801
+ },
802
+ {
803
+ "epoch": 8.053691275167786,
804
+ "grad_norm": 19.65326690673828,
805
+ "learning_rate": 2.073315877172102e-05,
806
+ "loss": 19.334464111328124,
807
+ "num_input_tokens_seen": 1179574272,
808
+ "step": 3600,
809
+ "train_runtime": 27545.6424,
810
+ "train_tokens_per_second": 42822.536
811
+ },
812
+ {
813
+ "epoch": 8.165548098434005,
814
+ "grad_norm": 20.760732650756836,
815
+ "learning_rate": 1.9542965960485597e-05,
816
+ "loss": 19.248583984375,
817
+ "num_input_tokens_seen": 1195958272,
818
+ "step": 3650,
819
+ "train_runtime": 27925.9284,
820
+ "train_tokens_per_second": 42826.088
821
+ },
822
+ {
823
+ "epoch": 8.277404921700224,
824
+ "grad_norm": 19.925457000732422,
825
+ "learning_rate": 1.835277314925018e-05,
826
+ "loss": 19.22830322265625,
827
+ "num_input_tokens_seen": 1212342272,
828
+ "step": 3700,
829
+ "train_runtime": 28305.1518,
830
+ "train_tokens_per_second": 42831.152
831
+ },
832
+ {
833
+ "epoch": 8.389261744966444,
834
+ "grad_norm": 20.903831481933594,
835
+ "learning_rate": 1.7162580338014758e-05,
836
+ "loss": 19.20845458984375,
837
+ "num_input_tokens_seen": 1228726272,
838
+ "step": 3750,
839
+ "train_runtime": 28684.7612,
840
+ "train_tokens_per_second": 42835.506
841
+ },
842
+ {
843
+ "epoch": 8.501118568232663,
844
+ "grad_norm": 25.746238708496094,
845
+ "learning_rate": 1.597238752677934e-05,
846
+ "loss": 19.181019287109375,
847
+ "num_input_tokens_seen": 1245110272,
848
+ "step": 3800,
849
+ "train_runtime": 29065.5625,
850
+ "train_tokens_per_second": 42837.99
851
+ },
852
+ {
853
+ "epoch": 8.612975391498882,
854
+ "grad_norm": 19.735681533813477,
855
+ "learning_rate": 1.4782194715543917e-05,
856
+ "loss": 19.200714111328125,
857
+ "num_input_tokens_seen": 1261494272,
858
+ "step": 3850,
859
+ "train_runtime": 29444.9518,
860
+ "train_tokens_per_second": 42842.463
861
+ },
862
+ {
863
+ "epoch": 8.724832214765101,
864
+ "grad_norm": 20.348718643188477,
865
+ "learning_rate": 1.35920019043085e-05,
866
+ "loss": 19.14758056640625,
867
+ "num_input_tokens_seen": 1277878272,
868
+ "step": 3900,
869
+ "train_runtime": 29825.2862,
870
+ "train_tokens_per_second": 42845.466
871
+ },
872
+ {
873
+ "epoch": 8.83668903803132,
874
+ "grad_norm": 19.208051681518555,
875
+ "learning_rate": 1.2401809093073078e-05,
876
+ "loss": 19.07127685546875,
877
+ "num_input_tokens_seen": 1294262272,
878
+ "step": 3950,
879
+ "train_runtime": 30205.5851,
880
+ "train_tokens_per_second": 42848.442
881
+ },
882
+ {
883
+ "epoch": 8.94854586129754,
884
+ "grad_norm": 20.35933494567871,
885
+ "learning_rate": 1.1211616281837658e-05,
886
+ "loss": 19.101888427734377,
887
+ "num_input_tokens_seen": 1310646272,
888
+ "step": 4000,
889
+ "train_runtime": 30585.6001,
890
+ "train_tokens_per_second": 42851.743
891
+ },
892
+ {
893
+ "epoch": 8.94854586129754,
894
+ "eval_loss": 1.0551483631134033,
895
+ "eval_masked_accuracy": 0.7687605175839743,
896
+ "eval_runtime": 25.3745,
897
+ "eval_samples_per_second": 219.433,
898
+ "eval_steps_per_second": 3.429,
899
+ "num_input_tokens_seen": 1310646272,
900
+ "step": 4000
901
+ },
902
+ {
903
+ "epoch": 9.060402684563758,
904
+ "grad_norm": 19.61882972717285,
905
+ "learning_rate": 1.0021423470602239e-05,
906
+ "loss": 19.01905517578125,
907
+ "num_input_tokens_seen": 1327021056,
908
+ "step": 4050,
909
+ "train_runtime": 30993.7338,
910
+ "train_tokens_per_second": 42815.785
911
+ },
912
+ {
913
+ "epoch": 9.172259507829978,
914
+ "grad_norm": 20.218429565429688,
915
+ "learning_rate": 8.831230659366818e-06,
916
+ "loss": 19.05180419921875,
917
+ "num_input_tokens_seen": 1343405056,
918
+ "step": 4100,
919
+ "train_runtime": 31373.034,
920
+ "train_tokens_per_second": 42820.374
921
+ },
922
+ {
923
+ "epoch": 9.284116331096197,
924
+ "grad_norm": 19.39629554748535,
925
+ "learning_rate": 7.641037848131398e-06,
926
+ "loss": 19.03988525390625,
927
+ "num_input_tokens_seen": 1359789056,
928
+ "step": 4150,
929
+ "train_runtime": 31753.5802,
930
+ "train_tokens_per_second": 42823.173
931
+ },
932
+ {
933
+ "epoch": 9.395973154362416,
934
+ "grad_norm": 20.272720336914062,
935
+ "learning_rate": 6.4508450368959775e-06,
936
+ "loss": 18.968330078125,
937
+ "num_input_tokens_seen": 1376173056,
938
+ "step": 4200,
939
+ "train_runtime": 32133.9665,
940
+ "train_tokens_per_second": 42826.119
941
+ },
942
+ {
943
+ "epoch": 9.507829977628635,
944
+ "grad_norm": 19.632165908813477,
945
+ "learning_rate": 5.260652225660558e-06,
946
+ "loss": 18.9827783203125,
947
+ "num_input_tokens_seen": 1392557056,
948
+ "step": 4250,
949
+ "train_runtime": 32514.0065,
950
+ "train_tokens_per_second": 42829.451
951
+ },
952
+ {
953
+ "epoch": 9.619686800894854,
954
+ "grad_norm": 19.80462646484375,
955
+ "learning_rate": 4.0704594144251375e-06,
956
+ "loss": 18.956285400390627,
957
+ "num_input_tokens_seen": 1408941056,
958
+ "step": 4300,
959
+ "train_runtime": 32894.5758,
960
+ "train_tokens_per_second": 42832.018
961
+ },
962
+ {
963
+ "epoch": 9.731543624161073,
964
+ "grad_norm": 20.74568748474121,
965
+ "learning_rate": 2.8802666031897166e-06,
966
+ "loss": 18.866341552734376,
967
+ "num_input_tokens_seen": 1425325056,
968
+ "step": 4350,
969
+ "train_runtime": 33275.028,
970
+ "train_tokens_per_second": 42834.676
971
+ },
972
+ {
973
+ "epoch": 9.843400447427292,
974
+ "grad_norm": 19.217836380004883,
975
+ "learning_rate": 1.6900737919542966e-06,
976
+ "loss": 18.920970458984375,
977
+ "num_input_tokens_seen": 1441709056,
978
+ "step": 4400,
979
+ "train_runtime": 33654.5878,
980
+ "train_tokens_per_second": 42838.411
981
+ },
982
+ {
983
+ "epoch": 9.955257270693512,
984
+ "grad_norm": 19.824554443359375,
985
+ "learning_rate": 4.998809807188764e-07,
986
+ "loss": 18.913228759765627,
987
+ "num_input_tokens_seen": 1458093056,
988
+ "step": 4450,
989
+ "train_runtime": 34033.2944,
990
+ "train_tokens_per_second": 42843.136
991
+ },
992
+ {
993
+ "epoch": 10.0,
994
+ "eval_loss": 1.0481648445129395,
995
+ "eval_masked_accuracy": 0.7704025727385737,
996
+ "eval_runtime": 25.3462,
997
+ "eval_samples_per_second": 219.678,
998
+ "eval_steps_per_second": 3.432,
999
+ "num_input_tokens_seen": 1464637440,
1000
+ "step": 4470
1001
+ },
1002
+ {
1003
+ "epoch": 10.0,
1004
+ "num_input_tokens_seen": 1464637440,
1005
+ "step": 4470,
1006
+ "total_flos": 4.917799558028206e+18,
1007
+ "train_loss": 21.80173199032777,
1008
+ "train_runtime": 34212.9191,
1009
+ "train_samples_per_second": 83.612,
1010
+ "train_steps_per_second": 0.131
1011
+ }
1012
+ ],
1013
+ "logging_steps": 50,
1014
+ "max_steps": 4470,
1015
+ "num_input_tokens_seen": 1464637440,
1016
+ "num_train_epochs": 10,
1017
+ "save_steps": 500,
1018
+ "stateful_callbacks": {
1019
+ "EarlyStoppingCallback": {
1020
+ "args": {
1021
+ "early_stopping_patience": 5,
1022
+ "early_stopping_threshold": 0.0001
1023
+ },
1024
+ "attributes": {
1025
+ "early_stopping_patience_counter": 0
1026
+ }
1027
+ },
1028
+ "TrainerControl": {
1029
+ "args": {
1030
+ "should_epoch_stop": false,
1031
+ "should_evaluate": false,
1032
+ "should_log": false,
1033
+ "should_save": true,
1034
+ "should_training_stop": true
1035
+ },
1036
+ "attributes": {}
1037
+ }
1038
+ },
1039
+ "total_flos": 4.917799558028206e+18,
1040
+ "train_batch_size": 40,
1041
+ "trial_name": null,
1042
+ "trial_params": null
1043
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6df1f2af59cfef522294063f953ccb9171365cb47145a67c1538a269dde99f1b
3
+ size 5265
training_log.jsonl ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"loss": 32.84220504760742, "grad_norm": 30.264930725097656, "learning_rate": 0.0, "epoch": 0.002516910492370615, "num_input_tokens_seen": 368640, "train_runtime": 8.8205, "train_tokens_per_second": 41793.643, "step": 1, "wall_time": "2026-06-21T18:29:45"}
2
+ {"loss": 31.76995086669922, "grad_norm": 32.67197036743164, "learning_rate": 0.0, "epoch": 0.0022371364653243847, "num_input_tokens_seen": 327680, "train_runtime": 7.8573, "train_tokens_per_second": 41704.008, "step": 1, "wall_time": "2026-06-21T18:29:54"}
3
+ {"loss": 32.44386509486607, "grad_norm": 31.965707778930664, "learning_rate": 1.821561338289963e-05, "epoch": 0.11185682326621924, "num_input_tokens_seen": 16384000, "train_runtime": 380.1137, "train_tokens_per_second": 43102.889, "step": 50, "wall_time": "2026-06-21T18:36:06"}
4
+ {"loss": 31.56456787109375, "grad_norm": 34.0458984375, "learning_rate": 3.680297397769517e-05, "epoch": 0.22371364653243847, "num_input_tokens_seen": 32768000, "train_runtime": 759.9744, "train_tokens_per_second": 43117.243, "step": 100, "wall_time": "2026-06-21T18:42:26"}
5
+ {"loss": 30.703720703125, "grad_norm": 36.13215255737305, "learning_rate": 5.539033457249071e-05, "epoch": 0.33557046979865773, "num_input_tokens_seen": 49152000, "train_runtime": 1140.0254, "train_tokens_per_second": 43114.831, "step": 150, "wall_time": "2026-06-21T18:48:46"}
6
+ {"loss": 30.0342919921875, "grad_norm": 75.77301788330078, "learning_rate": 7.397769516728626e-05, "epoch": 0.44742729306487694, "num_input_tokens_seen": 65536000, "train_runtime": 1519.7868, "train_tokens_per_second": 43121.837, "step": 200, "wall_time": "2026-06-21T18:55:06"}
7
+ {"loss": 29.2935986328125, "grad_norm": 31.4502010345459, "learning_rate": 9.256505576208179e-05, "epoch": 0.5592841163310962, "num_input_tokens_seen": 81920000, "train_runtime": 1899.4384, "train_tokens_per_second": 43128.538, "step": 250, "wall_time": "2026-06-21T19:01:26"}
8
+ {"loss": 28.63412353515625, "grad_norm": 35.78462600708008, "learning_rate": 9.928588431325875e-05, "epoch": 0.6711409395973155, "num_input_tokens_seen": 98304000, "train_runtime": 2279.106, "train_tokens_per_second": 43132.703, "step": 300, "wall_time": "2026-06-21T19:07:45"}
9
+ {"loss": 27.89616455078125, "grad_norm": 32.92461395263672, "learning_rate": 9.809569150202333e-05, "epoch": 0.7829977628635347, "num_input_tokens_seen": 114688000, "train_runtime": 2659.6638, "train_tokens_per_second": 43121.24, "step": 350, "wall_time": "2026-06-21T19:14:06"}
10
+ {"loss": 27.38276123046875, "grad_norm": 31.881704330444336, "learning_rate": 9.69054986907879e-05, "epoch": 0.8948545861297539, "num_input_tokens_seen": 131072000, "train_runtime": 3039.907, "train_tokens_per_second": 43117.108, "step": 400, "wall_time": "2026-06-21T19:20:26"}
11
+ {"loss": 26.72684814453125, "grad_norm": 28.32501220703125, "learning_rate": 9.57153058795525e-05, "epoch": 1.0067114093959733, "num_input_tokens_seen": 147446784, "train_runtime": 3419.3158, "train_tokens_per_second": 43121.721, "step": 450, "wall_time": "2026-06-21T19:26:45"}
12
+ {"loss": 26.2695166015625, "grad_norm": 27.903852462768555, "learning_rate": 9.452511306831707e-05, "epoch": 1.1185682326621924, "num_input_tokens_seen": 163830784, "train_runtime": 3799.2571, "train_tokens_per_second": 43121.794, "step": 500, "wall_time": "2026-06-21T19:33:05"}
13
+ {"eval_loss": 1.3941264152526855, "eval_masked_accuracy": 0.7140736610865701, "eval_runtime": 25.352, "eval_samples_per_second": 219.627, "eval_steps_per_second": 3.432, "epoch": 1.1185682326621924, "num_input_tokens_seen": 163830784, "step": 500, "wall_time": "2026-06-21T19:33:31"}
14
+ {"loss": 25.86959228515625, "grad_norm": 25.51409339904785, "learning_rate": 9.333492025708165e-05, "epoch": 1.2304250559284116, "num_input_tokens_seen": 180214784, "train_runtime": 4207.1627, "train_tokens_per_second": 42835.231, "step": 550, "wall_time": "2026-06-21T19:39:53"}
15
+ {"loss": 25.3655517578125, "grad_norm": 31.309749603271484, "learning_rate": 9.214472744584624e-05, "epoch": 1.342281879194631, "num_input_tokens_seen": 196598784, "train_runtime": 4586.3773, "train_tokens_per_second": 42865.812, "step": 600, "wall_time": "2026-06-21T19:46:12"}
16
+ {"loss": 25.1278271484375, "grad_norm": 28.270841598510742, "learning_rate": 9.095453463461082e-05, "epoch": 1.45413870246085, "num_input_tokens_seen": 212982784, "train_runtime": 4966.2335, "train_tokens_per_second": 42886.18, "step": 650, "wall_time": "2026-06-21T19:52:32"}
17
+ {"loss": 24.80788818359375, "grad_norm": 27.522308349609375, "learning_rate": 8.976434182337538e-05, "epoch": 1.5659955257270695, "num_input_tokens_seen": 229366784, "train_runtime": 5346.6697, "train_tokens_per_second": 42899.0, "step": 700, "wall_time": "2026-06-21T19:58:53"}
18
+ {"loss": 24.6148046875, "grad_norm": 27.093711853027344, "learning_rate": 8.857414901213997e-05, "epoch": 1.6778523489932886, "num_input_tokens_seen": 245750784, "train_runtime": 5726.4531, "train_tokens_per_second": 42915.008, "step": 750, "wall_time": "2026-06-21T20:05:13"}
19
+ {"loss": 24.40305908203125, "grad_norm": 25.49102210998535, "learning_rate": 8.738395620090455e-05, "epoch": 1.7897091722595078, "num_input_tokens_seen": 262134784, "train_runtime": 6106.8482, "train_tokens_per_second": 42924.726, "step": 800, "wall_time": "2026-06-21T20:11:33"}
20
+ {"loss": 24.0977783203125, "grad_norm": 25.048629760742188, "learning_rate": 8.619376338966914e-05, "epoch": 1.901565995525727, "num_input_tokens_seen": 278518784, "train_runtime": 6486.6563, "train_tokens_per_second": 42937.189, "step": 850, "wall_time": "2026-06-21T20:17:53"}
21
+ {"loss": 23.89361572265625, "grad_norm": 25.157657623291016, "learning_rate": 8.500357057843372e-05, "epoch": 2.0134228187919465, "num_input_tokens_seen": 294893568, "train_runtime": 6867.068, "train_tokens_per_second": 42943.155, "step": 900, "wall_time": "2026-06-21T20:24:13"}
22
+ {"loss": 23.57096435546875, "grad_norm": 25.902904510498047, "learning_rate": 8.381337776719829e-05, "epoch": 2.1252796420581657, "num_input_tokens_seen": 311277568, "train_runtime": 7246.8643, "train_tokens_per_second": 42953.415, "step": 950, "wall_time": "2026-06-21T20:30:33"}
23
+ {"loss": 23.42177490234375, "grad_norm": 24.553022384643555, "learning_rate": 8.262318495596287e-05, "epoch": 2.237136465324385, "num_input_tokens_seen": 327661568, "train_runtime": 7626.2318, "train_tokens_per_second": 42965.068, "step": 1000, "wall_time": "2026-06-21T20:36:52"}
24
+ {"eval_loss": 1.2587531805038452, "eval_masked_accuracy": 0.7357594334234474, "eval_runtime": 25.3576, "eval_samples_per_second": 219.579, "eval_steps_per_second": 3.431, "epoch": 2.237136465324385, "num_input_tokens_seen": 327661568, "step": 1000, "wall_time": "2026-06-21T20:37:18"}
25
+ {"loss": 23.259130859375, "grad_norm": 24.940990447998047, "learning_rate": 8.143299214472745e-05, "epoch": 2.348993288590604, "num_input_tokens_seen": 344045568, "train_runtime": 8033.7212, "train_tokens_per_second": 42825.182, "step": 1050, "wall_time": "2026-06-21T20:43:40"}
26
+ {"loss": 23.12722900390625, "grad_norm": 24.946720123291016, "learning_rate": 8.024279933349202e-05, "epoch": 2.460850111856823, "num_input_tokens_seen": 360429568, "train_runtime": 8413.8773, "train_tokens_per_second": 42837.512, "step": 1100, "wall_time": "2026-06-21T20:50:00"}
27
+ {"loss": 22.9796826171875, "grad_norm": 24.135683059692383, "learning_rate": 7.905260652225661e-05, "epoch": 2.5727069351230423, "num_input_tokens_seen": 376813568, "train_runtime": 8793.1262, "train_tokens_per_second": 42853.197, "step": 1150, "wall_time": "2026-06-21T20:56:19"}
28
+ {"loss": 22.77516845703125, "grad_norm": 25.850025177001953, "learning_rate": 7.786241371102119e-05, "epoch": 2.684563758389262, "num_input_tokens_seen": 393197568, "train_runtime": 9172.793, "train_tokens_per_second": 42865.632, "step": 1200, "wall_time": "2026-06-21T21:02:39"}
29
+ {"loss": 22.6738720703125, "grad_norm": 22.62376594543457, "learning_rate": 7.667222089978577e-05, "epoch": 2.796420581655481, "num_input_tokens_seen": 409581568, "train_runtime": 9553.7727, "train_tokens_per_second": 42871.186, "step": 1250, "wall_time": "2026-06-21T21:09:00"}
30
+ {"loss": 22.53384033203125, "grad_norm": 22.361886978149414, "learning_rate": 7.548202808855035e-05, "epoch": 2.9082774049217, "num_input_tokens_seen": 425965568, "train_runtime": 9932.9496, "train_tokens_per_second": 42884.096, "step": 1300, "wall_time": "2026-06-21T21:15:19"}
31
+ {"loss": 22.38953125, "grad_norm": 24.093069076538086, "learning_rate": 7.429183527731492e-05, "epoch": 3.0201342281879193, "num_input_tokens_seen": 442340352, "train_runtime": 10313.033, "train_tokens_per_second": 42891.393, "step": 1350, "wall_time": "2026-06-21T21:21:39"}
32
+ {"loss": 22.18701416015625, "grad_norm": 22.419540405273438, "learning_rate": 7.310164246607951e-05, "epoch": 3.131991051454139, "num_input_tokens_seen": 458724352, "train_runtime": 10693.1337, "train_tokens_per_second": 42898.964, "step": 1400, "wall_time": "2026-06-21T21:27:59"}
33
+ {"loss": 22.1206396484375, "grad_norm": 24.061424255371094, "learning_rate": 7.191144965484409e-05, "epoch": 3.243847874720358, "num_input_tokens_seen": 475108352, "train_runtime": 11073.944, "train_tokens_per_second": 42903.265, "step": 1450, "wall_time": "2026-06-21T21:34:20"}
34
+ {"loss": 21.99459228515625, "grad_norm": 21.660381317138672, "learning_rate": 7.072125684360867e-05, "epoch": 3.3557046979865772, "num_input_tokens_seen": 491492352, "train_runtime": 11453.1596, "train_tokens_per_second": 42913.254, "step": 1500, "wall_time": "2026-06-21T21:40:39"}
35
+ {"eval_loss": 1.194376826286316, "eval_masked_accuracy": 0.7462490242091618, "eval_runtime": 25.3329, "eval_samples_per_second": 219.794, "eval_steps_per_second": 3.434, "epoch": 3.3557046979865772, "num_input_tokens_seen": 491492352, "step": 1500, "wall_time": "2026-06-21T21:41:05"}
36
+ {"loss": 21.8618212890625, "grad_norm": 23.65436553955078, "learning_rate": 6.953106403237324e-05, "epoch": 3.4675615212527964, "num_input_tokens_seen": 507876352, "train_runtime": 11860.4601, "train_tokens_per_second": 42820.965, "step": 1550, "wall_time": "2026-06-21T21:47:27"}
37
+ {"loss": 21.84644287109375, "grad_norm": 23.492694854736328, "learning_rate": 6.834087122113782e-05, "epoch": 3.5794183445190155, "num_input_tokens_seen": 524260352, "train_runtime": 12240.4952, "train_tokens_per_second": 42829.995, "step": 1600, "wall_time": "2026-06-21T21:53:47"}
38
+ {"loss": 21.695458984375, "grad_norm": 22.143146514892578, "learning_rate": 6.715067840990241e-05, "epoch": 3.6912751677852347, "num_input_tokens_seen": 540644352, "train_runtime": 12620.3553, "train_tokens_per_second": 42839.075, "step": 1650, "wall_time": "2026-06-21T22:00:06"}
39
+ {"loss": 21.49891357421875, "grad_norm": 22.582658767700195, "learning_rate": 6.596048559866699e-05, "epoch": 3.8031319910514543, "num_input_tokens_seen": 557028352, "train_runtime": 13000.3344, "train_tokens_per_second": 42847.233, "step": 1700, "wall_time": "2026-06-21T22:06:26"}
40
+ {"loss": 21.4741015625, "grad_norm": 22.46770477294922, "learning_rate": 6.477029278743157e-05, "epoch": 3.9149888143176734, "num_input_tokens_seen": 573412352, "train_runtime": 13380.7098, "train_tokens_per_second": 42853.657, "step": 1750, "wall_time": "2026-06-21T22:12:47"}
41
+ {"loss": 21.39329833984375, "grad_norm": 22.605365753173828, "learning_rate": 6.358009997619616e-05, "epoch": 4.026845637583893, "num_input_tokens_seen": 589787136, "train_runtime": 13759.8334, "train_tokens_per_second": 42862.956, "step": 1800, "wall_time": "2026-06-21T22:19:06"}
42
+ {"loss": 21.18496826171875, "grad_norm": 21.95088005065918, "learning_rate": 6.238990716496072e-05, "epoch": 4.138702460850112, "num_input_tokens_seen": 606171136, "train_runtime": 14139.2097, "train_tokens_per_second": 42871.642, "step": 1850, "wall_time": "2026-06-21T22:25:25"}
43
+ {"loss": 21.2357177734375, "grad_norm": 22.34777069091797, "learning_rate": 6.11997143537253e-05, "epoch": 4.250559284116331, "num_input_tokens_seen": 622555136, "train_runtime": 14518.7366, "train_tokens_per_second": 42879.429, "step": 1900, "wall_time": "2026-06-21T22:31:45"}
44
+ {"loss": 21.16220458984375, "grad_norm": 22.553071975708008, "learning_rate": 6.000952154248989e-05, "epoch": 4.3624161073825505, "num_input_tokens_seen": 638939136, "train_runtime": 14898.759, "train_tokens_per_second": 42885.393, "step": 1950, "wall_time": "2026-06-21T22:38:05"}
45
+ {"loss": 21.01521240234375, "grad_norm": 22.87005615234375, "learning_rate": 5.8819328731254464e-05, "epoch": 4.47427293064877, "num_input_tokens_seen": 655323136, "train_runtime": 15278.457, "train_tokens_per_second": 42891.971, "step": 2000, "wall_time": "2026-06-21T22:44:25"}
46
+ {"eval_loss": 1.14796781539917, "eval_masked_accuracy": 0.7536936821619982, "eval_runtime": 25.3387, "eval_samples_per_second": 219.743, "eval_steps_per_second": 3.433, "epoch": 4.47427293064877, "num_input_tokens_seen": 655323136, "step": 2000, "wall_time": "2026-06-21T22:44:50"}
47
+ {"loss": 20.9041943359375, "grad_norm": 22.233734130859375, "learning_rate": 5.762913592001905e-05, "epoch": 4.586129753914989, "num_input_tokens_seen": 671707136, "train_runtime": 15686.3972, "train_tokens_per_second": 42820.995, "step": 2050, "wall_time": "2026-06-21T22:51:12"}
48
+ {"loss": 20.91035888671875, "grad_norm": 21.8343448638916, "learning_rate": 5.6438943108783625e-05, "epoch": 4.697986577181208, "num_input_tokens_seen": 688091136, "train_runtime": 16066.5233, "train_tokens_per_second": 42827.631, "step": 2100, "wall_time": "2026-06-21T22:57:33"}
49
+ {"loss": 20.8124365234375, "grad_norm": 21.778446197509766, "learning_rate": 5.52487502975482e-05, "epoch": 4.809843400447427, "num_input_tokens_seen": 704475136, "train_runtime": 16446.1615, "train_tokens_per_second": 42835.231, "step": 2150, "wall_time": "2026-06-21T23:03:52"}
50
+ {"loss": 20.7349560546875, "grad_norm": 21.686033248901367, "learning_rate": 5.4058557486312786e-05, "epoch": 4.921700223713646, "num_input_tokens_seen": 720859136, "train_runtime": 16825.7207, "train_tokens_per_second": 42842.69, "step": 2200, "wall_time": "2026-06-21T23:10:12"}
51
+ {"loss": 20.7165283203125, "grad_norm": 21.80782127380371, "learning_rate": 5.286836467507736e-05, "epoch": 5.033557046979865, "num_input_tokens_seen": 737233920, "train_runtime": 17206.3274, "train_tokens_per_second": 42846.675, "step": 2250, "wall_time": "2026-06-21T23:16:32"}
52
+ {"loss": 20.5538916015625, "grad_norm": 23.681589126586914, "learning_rate": 5.167817186384194e-05, "epoch": 5.145413870246085, "num_input_tokens_seen": 753617920, "train_runtime": 17586.976, "train_tokens_per_second": 42850.91, "step": 2300, "wall_time": "2026-06-21T23:22:53"}
53
+ {"loss": 20.505771484375, "grad_norm": 23.062753677368164, "learning_rate": 5.0487979052606524e-05, "epoch": 5.257270693512305, "num_input_tokens_seen": 770001920, "train_runtime": 17966.3576, "train_tokens_per_second": 42857.987, "step": 2350, "wall_time": "2026-06-21T23:29:12"}
54
+ {"loss": 20.479666748046874, "grad_norm": 22.336442947387695, "learning_rate": 4.92977862413711e-05, "epoch": 5.369127516778524, "num_input_tokens_seen": 786385920, "train_runtime": 18346.1922, "train_tokens_per_second": 42863.713, "step": 2400, "wall_time": "2026-06-21T23:35:32"}
55
+ {"loss": 20.462833251953125, "grad_norm": 22.454973220825195, "learning_rate": 4.8107593430135685e-05, "epoch": 5.480984340044743, "num_input_tokens_seen": 802769920, "train_runtime": 18726.6308, "train_tokens_per_second": 42867.824, "step": 2450, "wall_time": "2026-06-21T23:41:53"}
56
+ {"loss": 20.33416748046875, "grad_norm": 21.835525512695312, "learning_rate": 4.691740061890026e-05, "epoch": 5.592841163310962, "num_input_tokens_seen": 819153920, "train_runtime": 19106.2918, "train_tokens_per_second": 42873.517, "step": 2500, "wall_time": "2026-06-21T23:48:12"}
57
+ {"eval_loss": 1.1132277250289917, "eval_masked_accuracy": 0.7584756667111459, "eval_runtime": 25.3527, "eval_samples_per_second": 219.621, "eval_steps_per_second": 3.432, "epoch": 5.592841163310962, "num_input_tokens_seen": 819153920, "step": 2500, "wall_time": "2026-06-21T23:48:38"}
58
+ {"loss": 20.25551513671875, "grad_norm": 23.234500885009766, "learning_rate": 4.5727207807664845e-05, "epoch": 5.704697986577181, "num_input_tokens_seen": 835537920, "train_runtime": 19513.5356, "train_tokens_per_second": 42818.377, "step": 2550, "wall_time": "2026-06-21T23:55:00"}
59
+ {"loss": 20.156134033203124, "grad_norm": 21.292207717895508, "learning_rate": 4.453701499642942e-05, "epoch": 5.8165548098434, "num_input_tokens_seen": 851921920, "train_runtime": 19893.1597, "train_tokens_per_second": 42824.867, "step": 2600, "wall_time": "2026-06-22T00:01:19"}
60
+ {"loss": 20.159989013671876, "grad_norm": 20.622861862182617, "learning_rate": 4.3346822185194e-05, "epoch": 5.9284116331096195, "num_input_tokens_seen": 868305920, "train_runtime": 20273.9368, "train_tokens_per_second": 42828.679, "step": 2650, "wall_time": "2026-06-22T00:07:40"}
61
+ {"loss": 20.13439697265625, "grad_norm": 23.077646255493164, "learning_rate": 4.215662937395858e-05, "epoch": 6.040268456375839, "num_input_tokens_seen": 884680704, "train_runtime": 20654.2855, "train_tokens_per_second": 42832.791, "step": 2700, "wall_time": "2026-06-22T00:14:00"}
62
+ {"loss": 19.994425048828123, "grad_norm": 21.280527114868164, "learning_rate": 4.096643656272317e-05, "epoch": 6.152125279642058, "num_input_tokens_seen": 901064704, "train_runtime": 21033.1909, "train_tokens_per_second": 42840.133, "step": 2750, "wall_time": "2026-06-22T00:20:19"}
63
+ {"loss": 19.980313720703126, "grad_norm": 23.213178634643555, "learning_rate": 3.977624375148774e-05, "epoch": 6.263982102908278, "num_input_tokens_seen": 917448704, "train_runtime": 21412.6494, "train_tokens_per_second": 42846.109, "step": 2800, "wall_time": "2026-06-22T00:26:39"}
64
+ {"loss": 19.9128857421875, "grad_norm": 22.994335174560547, "learning_rate": 3.858605094025232e-05, "epoch": 6.375838926174497, "num_input_tokens_seen": 933832704, "train_runtime": 21792.7904, "train_tokens_per_second": 42850.534, "step": 2850, "wall_time": "2026-06-22T00:32:59"}
65
+ {"loss": 19.874437255859377, "grad_norm": 25.134178161621094, "learning_rate": 3.7395858129016905e-05, "epoch": 6.487695749440716, "num_input_tokens_seen": 950216704, "train_runtime": 22173.1567, "train_tokens_per_second": 42854.372, "step": 2900, "wall_time": "2026-06-22T00:39:19"}
66
+ {"loss": 19.910538330078126, "grad_norm": 20.69732093811035, "learning_rate": 3.620566531778148e-05, "epoch": 6.599552572706935, "num_input_tokens_seen": 966600704, "train_runtime": 22553.2542, "train_tokens_per_second": 42858.591, "step": 2950, "wall_time": "2026-06-22T00:45:39"}
67
+ {"loss": 19.87051025390625, "grad_norm": 21.227550506591797, "learning_rate": 3.501547250654606e-05, "epoch": 6.7114093959731544, "num_input_tokens_seen": 982984704, "train_runtime": 22933.4022, "train_tokens_per_second": 42862.576, "step": 3000, "wall_time": "2026-06-22T00:51:59"}
68
+ {"eval_loss": 1.0944103002548218, "eval_masked_accuracy": 0.7619272139314145, "eval_runtime": 25.3399, "eval_samples_per_second": 219.732, "eval_steps_per_second": 3.433, "epoch": 6.7114093959731544, "num_input_tokens_seen": 982984704, "step": 3000, "wall_time": "2026-06-22T00:52:25"}
69
+ {"loss": 19.65084228515625, "grad_norm": 21.8638858795166, "learning_rate": 3.382527969531064e-05, "epoch": 6.823266219239374, "num_input_tokens_seen": 999368704, "train_runtime": 23340.7393, "train_tokens_per_second": 42816.497, "step": 3050, "wall_time": "2026-06-22T00:58:47"}
70
+ {"loss": 19.7602978515625, "grad_norm": 25.967254638671875, "learning_rate": 3.263508688407522e-05, "epoch": 6.935123042505593, "num_input_tokens_seen": 1015752704, "train_runtime": 23721.0833, "train_tokens_per_second": 42820.671, "step": 3100, "wall_time": "2026-06-22T01:05:07"}
71
+ {"loss": 19.64591064453125, "grad_norm": 29.288837432861328, "learning_rate": 3.1444894072839803e-05, "epoch": 7.046979865771812, "num_input_tokens_seen": 1032127488, "train_runtime": 24100.8116, "train_tokens_per_second": 42825.425, "step": 3150, "wall_time": "2026-06-22T01:11:27"}
72
+ {"loss": 19.643092041015624, "grad_norm": 20.559537887573242, "learning_rate": 3.025470126160438e-05, "epoch": 7.158836689038031, "num_input_tokens_seen": 1048511488, "train_runtime": 24479.6774, "train_tokens_per_second": 42831.916, "step": 3200, "wall_time": "2026-06-22T01:17:46"}
73
+ {"loss": 19.5406787109375, "grad_norm": 21.216949462890625, "learning_rate": 2.906450845036896e-05, "epoch": 7.27069351230425, "num_input_tokens_seen": 1064895488, "train_runtime": 24859.9356, "train_tokens_per_second": 42835.81, "step": 3250, "wall_time": "2026-06-22T01:24:06"}
74
+ {"loss": 19.52265625, "grad_norm": 19.948974609375, "learning_rate": 2.787431563913354e-05, "epoch": 7.382550335570469, "num_input_tokens_seen": 1081279488, "train_runtime": 25239.4624, "train_tokens_per_second": 42840.829, "step": 3300, "wall_time": "2026-06-22T01:30:26"}
75
+ {"loss": 19.483994140625, "grad_norm": 24.806529998779297, "learning_rate": 2.668412282789812e-05, "epoch": 7.494407158836689, "num_input_tokens_seen": 1097663488, "train_runtime": 25619.1507, "train_tokens_per_second": 42845.428, "step": 3350, "wall_time": "2026-06-22T01:36:45"}
76
+ {"loss": 19.500614013671875, "grad_norm": 22.306764602661133, "learning_rate": 2.54939300166627e-05, "epoch": 7.6062639821029085, "num_input_tokens_seen": 1114047488, "train_runtime": 25998.4643, "train_tokens_per_second": 42850.511, "step": 3400, "wall_time": "2026-06-22T01:43:05"}
77
+ {"loss": 19.44253173828125, "grad_norm": 20.259811401367188, "learning_rate": 2.430373720542728e-05, "epoch": 7.718120805369128, "num_input_tokens_seen": 1130431488, "train_runtime": 26378.2281, "train_tokens_per_second": 42854.716, "step": 3450, "wall_time": "2026-06-22T01:49:24"}
78
+ {"loss": 19.37197021484375, "grad_norm": 31.171615600585938, "learning_rate": 2.311354439419186e-05, "epoch": 7.829977628635347, "num_input_tokens_seen": 1146815488, "train_runtime": 26757.8966, "train_tokens_per_second": 42858.955, "step": 3500, "wall_time": "2026-06-22T01:55:44"}
79
+ {"eval_loss": 1.0732760429382324, "eval_masked_accuracy": 0.7661423014370904, "eval_runtime": 25.3349, "eval_samples_per_second": 219.775, "eval_steps_per_second": 3.434, "epoch": 7.829977628635347, "num_input_tokens_seen": 1146815488, "step": 3500, "wall_time": "2026-06-22T01:56:09"}
80
+ {"loss": 19.302330322265625, "grad_norm": 19.443883895874023, "learning_rate": 2.192335158295644e-05, "epoch": 7.941834451901566, "num_input_tokens_seen": 1163199488, "train_runtime": 27165.3477, "train_tokens_per_second": 42819.238, "step": 3550, "wall_time": "2026-06-22T02:02:31"}
81
+ {"loss": 19.334464111328124, "grad_norm": 19.65326690673828, "learning_rate": 2.073315877172102e-05, "epoch": 8.053691275167786, "num_input_tokens_seen": 1179574272, "train_runtime": 27545.6424, "train_tokens_per_second": 42822.536, "step": 3600, "wall_time": "2026-06-22T02:08:52"}
82
+ {"loss": 19.248583984375, "grad_norm": 20.760732650756836, "learning_rate": 1.9542965960485597e-05, "epoch": 8.165548098434005, "num_input_tokens_seen": 1195958272, "train_runtime": 27925.9284, "train_tokens_per_second": 42826.088, "step": 3650, "wall_time": "2026-06-22T02:15:12"}
83
+ {"loss": 19.22830322265625, "grad_norm": 19.925457000732422, "learning_rate": 1.835277314925018e-05, "epoch": 8.277404921700224, "num_input_tokens_seen": 1212342272, "train_runtime": 28305.1518, "train_tokens_per_second": 42831.152, "step": 3700, "wall_time": "2026-06-22T02:21:31"}
84
+ {"loss": 19.20845458984375, "grad_norm": 20.903831481933594, "learning_rate": 1.7162580338014758e-05, "epoch": 8.389261744966444, "num_input_tokens_seen": 1228726272, "train_runtime": 28684.7612, "train_tokens_per_second": 42835.506, "step": 3750, "wall_time": "2026-06-22T02:27:51"}
85
+ {"loss": 19.181019287109375, "grad_norm": 25.746238708496094, "learning_rate": 1.597238752677934e-05, "epoch": 8.501118568232663, "num_input_tokens_seen": 1245110272, "train_runtime": 29065.5625, "train_tokens_per_second": 42837.99, "step": 3800, "wall_time": "2026-06-22T02:34:12"}
86
+ {"loss": 19.200714111328125, "grad_norm": 19.735681533813477, "learning_rate": 1.4782194715543917e-05, "epoch": 8.612975391498882, "num_input_tokens_seen": 1261494272, "train_runtime": 29444.9518, "train_tokens_per_second": 42842.463, "step": 3850, "wall_time": "2026-06-22T02:40:31"}
87
+ {"loss": 19.14758056640625, "grad_norm": 20.348718643188477, "learning_rate": 1.35920019043085e-05, "epoch": 8.724832214765101, "num_input_tokens_seen": 1277878272, "train_runtime": 29825.2862, "train_tokens_per_second": 42845.466, "step": 3900, "wall_time": "2026-06-22T02:46:51"}
88
+ {"loss": 19.07127685546875, "grad_norm": 19.208051681518555, "learning_rate": 1.2401809093073078e-05, "epoch": 8.83668903803132, "num_input_tokens_seen": 1294262272, "train_runtime": 30205.5851, "train_tokens_per_second": 42848.442, "step": 3950, "wall_time": "2026-06-22T02:53:12"}
89
+ {"loss": 19.101888427734377, "grad_norm": 20.35933494567871, "learning_rate": 1.1211616281837658e-05, "epoch": 8.94854586129754, "num_input_tokens_seen": 1310646272, "train_runtime": 30585.6001, "train_tokens_per_second": 42851.743, "step": 4000, "wall_time": "2026-06-22T02:59:32"}
90
+ {"eval_loss": 1.0551483631134033, "eval_masked_accuracy": 0.7687605175839743, "eval_runtime": 25.3745, "eval_samples_per_second": 219.433, "eval_steps_per_second": 3.429, "epoch": 8.94854586129754, "num_input_tokens_seen": 1310646272, "step": 4000, "wall_time": "2026-06-22T02:59:57"}
91
+ {"loss": 19.01905517578125, "grad_norm": 19.61882972717285, "learning_rate": 1.0021423470602239e-05, "epoch": 9.060402684563758, "num_input_tokens_seen": 1327021056, "train_runtime": 30993.7338, "train_tokens_per_second": 42815.785, "step": 4050, "wall_time": "2026-06-22T03:06:20"}
92
+ {"loss": 19.05180419921875, "grad_norm": 20.218429565429688, "learning_rate": 8.831230659366818e-06, "epoch": 9.172259507829978, "num_input_tokens_seen": 1343405056, "train_runtime": 31373.034, "train_tokens_per_second": 42820.374, "step": 4100, "wall_time": "2026-06-22T03:12:39"}
93
+ {"loss": 19.03988525390625, "grad_norm": 19.39629554748535, "learning_rate": 7.641037848131398e-06, "epoch": 9.284116331096197, "num_input_tokens_seen": 1359789056, "train_runtime": 31753.5802, "train_tokens_per_second": 42823.173, "step": 4150, "wall_time": "2026-06-22T03:19:00"}
94
+ {"loss": 18.968330078125, "grad_norm": 20.272720336914062, "learning_rate": 6.4508450368959775e-06, "epoch": 9.395973154362416, "num_input_tokens_seen": 1376173056, "train_runtime": 32133.9665, "train_tokens_per_second": 42826.119, "step": 4200, "wall_time": "2026-06-22T03:25:20"}
95
+ {"loss": 18.9827783203125, "grad_norm": 19.632165908813477, "learning_rate": 5.260652225660558e-06, "epoch": 9.507829977628635, "num_input_tokens_seen": 1392557056, "train_runtime": 32514.0065, "train_tokens_per_second": 42829.451, "step": 4250, "wall_time": "2026-06-22T03:31:40"}
96
+ {"loss": 18.956285400390627, "grad_norm": 19.80462646484375, "learning_rate": 4.0704594144251375e-06, "epoch": 9.619686800894854, "num_input_tokens_seen": 1408941056, "train_runtime": 32894.5758, "train_tokens_per_second": 42832.018, "step": 4300, "wall_time": "2026-06-22T03:38:01"}
97
+ {"loss": 18.866341552734376, "grad_norm": 20.74568748474121, "learning_rate": 2.8802666031897166e-06, "epoch": 9.731543624161073, "num_input_tokens_seen": 1425325056, "train_runtime": 33275.028, "train_tokens_per_second": 42834.676, "step": 4350, "wall_time": "2026-06-22T03:44:21"}
98
+ {"loss": 18.920970458984375, "grad_norm": 19.217836380004883, "learning_rate": 1.6900737919542966e-06, "epoch": 9.843400447427292, "num_input_tokens_seen": 1441709056, "train_runtime": 33654.5878, "train_tokens_per_second": 42838.411, "step": 4400, "wall_time": "2026-06-22T03:50:41"}
99
+ {"loss": 18.913228759765627, "grad_norm": 19.824554443359375, "learning_rate": 4.998809807188764e-07, "epoch": 9.955257270693512, "num_input_tokens_seen": 1458093056, "train_runtime": 34033.2944, "train_tokens_per_second": 42843.136, "step": 4450, "wall_time": "2026-06-22T03:56:59"}
100
+ {"eval_loss": 1.0481648445129395, "eval_masked_accuracy": 0.7704025727385737, "eval_runtime": 25.3462, "eval_samples_per_second": 219.678, "eval_steps_per_second": 3.432, "epoch": 10.0, "num_input_tokens_seen": 1464637440, "step": 4470, "wall_time": "2026-06-22T03:59:57"}
101
+ {"train_runtime": 34212.9191, "train_samples_per_second": 83.612, "train_steps_per_second": 0.131, "total_flos": 4.917799558028206e+18, "train_loss": 21.80173199032777, "epoch": 10.0, "num_input_tokens_seen": 1464637440, "step": 4470, "wall_time": "2026-06-22T03:59:59"}
102
+ {"final_val_loss": 1.0483721494674683, "final_val_masked_accuracy": 0.7699716443872566, "final_val_runtime": 25.2687, "final_val_samples_per_second": 220.351, "final_val_steps_per_second": 3.443, "epoch": 10.0, "num_input_tokens_seen": 1464637440, "step": 4470, "wall_time": "2026-06-22T04:00:28"}