ThingsAI commited on
Commit
5a2e01b
·
verified ·
1 Parent(s): f862712

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -3,7 +3,7 @@
3
  "bos_token": "<|im_start|>",
4
  "eos_token": "<|endoftext|>",
5
  "pad_token": "<|pad|>",
6
- "model_max_length": 32768,
7
  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'system' %}<|im_start|>system\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'user' %}<|im_start|>user\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'assistant' %}\n{% if message.get('thinking') %}<|thinking|>\n{{ message['thinking'] }}<|/thinking|>{% endif %}\n{% if message.get('tool_calls') %}<|tool_calls|>\n{{ message['tool_calls'] }}<|/tool_calls|>{% endif %}\n<|im_start|>assistant\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'tool' %}<|im_start|>tool\n{{ message['content'] }}<|im_end|>\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
8
  "add_bos_token": false,
9
  "add_eos_token": false,
 
3
  "bos_token": "<|im_start|>",
4
  "eos_token": "<|endoftext|>",
5
  "pad_token": "<|pad|>",
6
+ "model_max_length": 100000,
7
  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'system' %}<|im_start|>system\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'user' %}<|im_start|>user\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'assistant' %}\n{% if message.get('thinking') %}<|thinking|>\n{{ message['thinking'] }}<|/thinking|>{% endif %}\n{% if message.get('tool_calls') %}<|tool_calls|>\n{{ message['tool_calls'] }}<|/tool_calls|>{% endif %}\n<|im_start|>assistant\n{{ message['content'] }}<|im_end|>\n{% elif message['role'] == 'tool' %}<|im_start|>tool\n{{ message['content'] }}<|im_end|>\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
8
  "add_bos_token": false,
9
  "add_eos_token": false,