Update tokenizer_config.json
Browse files- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
"bos_token": "<|im_start|>",
|
| 4 |
"eos_token": "<|endoftext|>",
|
| 5 |
"pad_token": "<|pad|>",
|
| 6 |
-
"model_max_length":
|
| 7 |
"chat_template": "{% for message in messages %}\n{% if message['role'] == 'system' %}<|im_start|>system\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'user' %}<|im_start|>user\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'assistant' %}\n{% if message.get('thinking') %}<|thinking|>\n{{ message['thinking'] }}<|/thinking|>{% endif %}\n{% if message.get('tool_calls') %}<|tool_calls|>\n{{ message['tool_calls'] }}<|/tool_calls|>{% endif %}\n<|im_start|>assistant\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'tool' %}<|im_start|>tool\n{{ message['content'] }}<|im_end|>\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
|
| 8 |
"add_bos_token": false,
|
| 9 |
"add_eos_token": false,
|
|
|
|
| 3 |
"bos_token": "<|im_start|>",
|
| 4 |
"eos_token": "<|endoftext|>",
|
| 5 |
"pad_token": "<|pad|>",
|
| 6 |
+
"model_max_length": 100000,
|
| 7 |
"chat_template": "{% for message in messages %}\n{% if message['role'] == 'system' %}<|im_start|>system\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'user' %}<|im_start|>user\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'assistant' %}\n{% if message.get('thinking') %}<|thinking|>\n{{ message['thinking'] }}<|/thinking|>{% endif %}\n{% if message.get('tool_calls') %}<|tool_calls|>\n{{ message['tool_calls'] }}<|/tool_calls|>{% endif %}\n<|im_start|>assistant\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'tool' %}<|im_start|>tool\n{{ message['content'] }}<|im_end|>\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
|
| 8 |
"add_bos_token": false,
|
| 9 |
"add_eos_token": false,
|