| { | |
| "data": "cosmopedia_v2_3B_8k_digits.uint16.bin", | |
| "base": "MiniBananaMind-v3-9M", | |
| "revision": "", | |
| "tokenizer": "", | |
| "out": "checkpoints/MiniBananaMind-v3-9M-CosmopediaV2-3B-1024", | |
| "seq_len": 1024, | |
| "batch_size": 72, | |
| "grad_accum": 16, | |
| "max_steps": 2613, | |
| "lr": 0.0001, | |
| "min_lr": 1e-05, | |
| "warmup_steps": 200, | |
| "weight_decay": 0.1, | |
| "grad_clip": 1.0, | |
| "val_tokens": 5000000, | |
| "eval_every": 250, | |
| "eval_batches": 50, | |
| "save_every": 250, | |
| "device": "cuda", | |
| "dtype": "auto", | |
| "compile": true, | |
| "compile_mode": "default", | |
| "seed": 1337, | |
| "resume": "" | |
| } |