RWKV7-1.5B-SMI-20260822 / tokenizer_config.json
aabbdev's picture
Publish RWKV7-1.5B-SMI-20260822
1bd7982 verified
Raw History Blame Contribute Delete
3 kB
{
"added_tokens_decoder": {
"0": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65536": {
"content": "<|ctrl|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65537": {
"content": "<|sys|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65538": {
"content": "<|dev|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65539": {
"content": "<|caps|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65540": {
"content": "<|usr|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65541": {
"content": "<|obs|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65542": {
"content": "<|think|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65543": {
"content": "<|out|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65544": {
"content": "<|act|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"65545": {
"content": "<|eot|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"<|ctrl|>",
"<|sys|>",
"<|dev|>",
"<|caps|>",
"<|usr|>",
"<|obs|>",
"<|think|>",
"<|out|>",
"<|act|>",
"<|eot|>"
],
"backend": "tokenizers",
"eos_token": "<|endoftext|>",
"pad_token": "<|endoftext|>",
"padding_side": "left",
"response_template": {
"defaults": {
"role": "assistant"
},
"fields": {
"content": {
"close": "<|eot|>",
"content": "text",
"open": "<|out|>\n"
},
"thinking": {
"close_pattern": "(?=<\\|out\\|>|<\\|act\\|>|<\\|eot\\|>)",
"content": "text",
"open": "<|think|>\n"
},
"tool_calls": {
"close_pattern": "(?=<\\|act\\|>|<\\|eot\\|>)",
"content": "json",
"open": "<|act|>\n",
"repeats": true,
"transform": {
"function": "{content}",
"type": "function"
}
}
},
"start_anchor": "<|eot|>\n"
},
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<|endoftext|>"
}