Hugging Face's logo

hf-internal-testing
/
tiny-processor-qwen2_audio

qwen2_audio
Model card Files
xet
 Community
hf-transformers-bot commited on
Commit
d41ca4c
·
verified ·
1 Parent(s): f461480

Add tiny processor fixture for testing

Browse files
Files changed (5) hide show
  1. config.json +34 -0
  2. generation_config.json +11 -0
  3. preprocessor_config.json +14 -0
  4. tokenizer.json +825 -0
  5. tokenizer_config.json +0 -0
config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2AudioForConditionalGeneration"
4
+ ],
5
+ "audio_config": {
6
+ "model_type": "qwen2_audio_encoder",
7
+ "num_mel_bins": 128,
8
+ "encoder_layers": 32,
9
+ "encoder_attention_heads": 20,
10
+ "encoder_ffn_dim": 5120,
11
+ "d_model": 1280,
12
+ "activation_function": "gelu",
13
+ "scale_embedding": false,
14
+ "max_source_positions": 1500
15
+ },
16
+ "audio_token_index": 151646,
17
+ "ignore_index": -100,
18
+ "model_type": "qwen2_audio",
19
+ "text_config": {
20
+ "bos_token_id": 151643,
21
+ "eos_token_id": 151645,
22
+ "intermediate_size": 11008,
23
+ "max_position_embeddings": 8192,
24
+ "model_type": "qwen2",
25
+ "rope_theta": 10000,
26
+ "rms_norm_eps": 1e-05,
27
+ "sliding_window": 32768,
28
+ "torch_dtype": "bfloat16",
29
+ "use_mrope": false,
30
+ "vocab_size": 156032
31
+ },
32
+ "transformers_version": "4.38.1",
33
+ "vocab_size": 500
34
+ }
generation_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "chat_format": "chatml",
3
+ "eos_token_id": [151643,151645],
4
+ "pad_token_id": 151643,
5
+ "do_sample": true,
6
+ "top_k": 20,
7
+ "top_p": 0.5,
8
+ "temperature": 0.7,
9
+ "repetition_penalty": 1.1,
10
+ "transformers_version": "4.38.1"
11
+ }
preprocessor_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "chunk_length": 30,
3
+ "feature_extractor_type": "WhisperFeatureExtractor",
4
+ "feature_size": 128,
5
+ "hop_length": 160,
6
+ "n_fft": 400,
7
+ "n_samples": 480000,
8
+ "nb_max_frames": 3000,
9
+ "padding_side": "right",
10
+ "padding_value": 0.0,
11
+ "processor_class": "Qwen2AudioProcessor",
12
+ "return_attention_mask": true,
13
+ "sampling_rate": 16000
14
+ }
tokenizer.json ADDED
@@ -0,0 +1,825 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 500,
8
+ "content": "<|endoftext|>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 501,
17
+ "content": "<|im_start|>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 502,
26
+ "content": "<|im_end|>",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ }
33
+ ],
34
+ "normalizer": {
35
+ "type": "NFC"
36
+ },
37
+ "pre_tokenizer": {
38
+ "type": "Sequence",
39
+ "pretokenizers": [
40
+ {
41
+ "type": "Split",
42
+ "pattern": {
43
+ "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
44
+ },
45
+ "behavior": "Isolated",
46
+ "invert": false
47
+ },
48
+ {
49
+ "type": "ByteLevel",
50
+ "add_prefix_space": false,
51
+ "trim_offsets": false,
52
+ "use_regex": false
53
+ }
54
+ ]
55
+ },
56
+ "post_processor": {
57
+ "type": "ByteLevel",
58
+ "add_prefix_space": false,
59
+ "trim_offsets": false,
60
+ "use_regex": false
61
+ },
62
+ "decoder": {
63
+ "type": "ByteLevel",
64
+ "add_prefix_space": false,
65
+ "trim_offsets": false,
66
+ "use_regex": false
67
+ },
68
+ "model": {
69
+ "type": "BPE",
70
+ "dropout": null,
71
+ "unk_token": null,
72
+ "continuing_subword_prefix": "",
73
+ "end_of_word_suffix": "",
74
+ "fuse_unk": false,
75
+ "byte_fallback": false,
76
+ "vocab": {
77
+ "!": 0,
78
+ "\"": 1,
79
+ "#": 2,
80
+ "$": 3,
81
+ "%": 4,
82
+ "&": 5,
83
+ "'": 6,
84
+ "(": 7,
85
+ ")": 8,
86
+ "*": 9,
87
+ "+": 10,
88
+ ",": 11,
89
+ "-": 12,
90
+ ".": 13,
91
+ "/": 14,
92
+ "0": 15,
93
+ "1": 16,
94
+ "2": 17,
95
+ "3": 18,
96
+ "4": 19,
97
+ "5": 20,
98
+ "6": 21,
99
+ "7": 22,
100
+ "8": 23,
101
+ "9": 24,
102
+ ":": 25,
103
+ ";": 26,
104
+ "<": 27,
105
+ "=": 28,
106
+ ">": 29,
107
+ "?": 30,
108
+ "@": 31,
109
+ "A": 32,
110
+ "B": 33,
111
+ "C": 34,
112
+ "D": 35,
113
+ "E": 36,
114
+ "F": 37,
115
+ "G": 38,
116
+ "H": 39,
117
+ "I": 40,
118
+ "J": 41,
119
+ "K": 42,
120
+ "L": 43,
121
+ "M": 44,
122
+ "N": 45,
123
+ "O": 46,
124
+ "P": 47,
125
+ "Q": 48,
126
+ "R": 49,
127
+ "S": 50,
128
+ "T": 51,
129
+ "U": 52,
130
+ "V": 53,
131
+ "W": 54,
132
+ "X": 55,
133
+ "Y": 56,
134
+ "Z": 57,
135
+ "[": 58,
136
+ "\\": 59,
137
+ "]": 60,
138
+ "^": 61,
139
+ "_": 62,
140
+ "`": 63,
141
+ "a": 64,
142
+ "b": 65,
143
+ "c": 66,
144
+ "d": 67,
145
+ "e": 68,
146
+ "f": 69,
147
+ "g": 70,
148
+ "h": 71,
149
+ "i": 72,
150
+ "j": 73,
151
+ "k": 74,
152
+ "l": 75,
153
+ "m": 76,
154
+ "n": 77,
155
+ "o": 78,
156
+ "p": 79,
157
+ "q": 80,
158
+ "r": 81,
159
+ "s": 82,
160
+ "t": 83,
161
+ "u": 84,
162
+ "v": 85,
163
+ "w": 86,
164
+ "x": 87,
165
+ "y": 88,
166
+ "z": 89,
167
+ "{": 90,
168
+ "|": 91,
169
+ "}": 92,
170
+ "~": 93,
171
+ "¡": 94,
172
+ "¢": 95,
173
+ "£": 96,
174
+ "¤": 97,
175
+ "¥": 98,
176
+ "¦": 99,
177
+ "§": 100,
178
+ "¨": 101,
179
+ "©": 102,
180
+ "ª": 103,
181
+ "«": 104,
182
+ "¬": 105,
183
+ "®": 106,
184
+ "¯": 107,
185
+ "°": 108,
186
+ "±": 109,
187
+ "²": 110,
188
+ "³": 111,
189
+ "´": 112,
190
+ "µ": 113,
191
+ "¶": 114,
192
+ "·": 115,
193
+ "¸": 116,
194
+ "¹": 117,
195
+ "º": 118,
196
+ "»": 119,
197
+ "¼": 120,
198
+ "½": 121,
199
+ "¾": 122,
200
+ "¿": 123,
201
+ "À": 124,
202
+ "Á": 125,
203
+ "Â": 126,
204
+ "Ã": 127,
205
+ "Ä": 128,
206
+ "Å": 129,
207
+ "Æ": 130,
208
+ "Ç": 131,
209
+ "È": 132,
210
+ "É": 133,
211
+ "Ê": 134,
212
+ "Ë": 135,
213
+ "Ì": 136,
214
+ "Í": 137,
215
+ "Î": 138,
216
+ "Ï": 139,
217
+ "Ð": 140,
218
+ "Ñ": 141,
219
+ "Ò": 142,
220
+ "Ó": 143,
221
+ "Ô": 144,
222
+ "Õ": 145,
223
+ "Ö": 146,
224
+ "×": 147,
225
+ "Ø": 148,
226
+ "Ù": 149,
227
+ "Ú": 150,
228
+ "Û": 151,
229
+ "Ü": 152,
230
+ "Ý": 153,
231
+ "Þ": 154,
232
+ "ß": 155,
233
+ "à": 156,
234
+ "á": 157,
235
+ "â": 158,
236
+ "ã": 159,
237
+ "ä": 160,
238
+ "å": 161,
239
+ "æ": 162,
240
+ "ç": 163,
241
+ "è": 164,
242
+ "é": 165,
243
+ "ê": 166,
244
+ "ë": 167,
245
+ "ì": 168,
246
+ "í": 169,
247
+ "î": 170,
248
+ "ï": 171,
249
+ "ð": 172,
250
+ "ñ": 173,
251
+ "ò": 174,
252
+ "ó": 175,
253
+ "ô": 176,
254
+ "õ": 177,
255
+ "ö": 178,
256
+ "÷": 179,
257
+ "ø": 180,
258
+ "ù": 181,
259
+ "ú": 182,
260
+ "û": 183,
261
+ "ü": 184,
262
+ "ý": 185,
263
+ "þ": 186,
264
+ "ÿ": 187,
265
+ "Ā": 188,
266
+ "ā": 189,
267
+ "Ă": 190,
268
+ "ă": 191,
269
+ "Ą": 192,
270
+ "ą": 193,
271
+ "Ć": 194,
272
+ "ć": 195,
273
+ "Ĉ": 196,
274
+ "ĉ": 197,
275
+ "Ċ": 198,
276
+ "ċ": 199,
277
+ "Č": 200,
278
+ "č": 201,
279
+ "Ď": 202,
280
+ "ď": 203,
281
+ "Đ": 204,
282
+ "đ": 205,
283
+ "Ē": 206,
284
+ "ē": 207,
285
+ "Ĕ": 208,
286
+ "ĕ": 209,
287
+ "Ė": 210,
288
+ "ė": 211,
289
+ "Ę": 212,
290
+ "ę": 213,
291
+ "Ě": 214,
292
+ "ě": 215,
293
+ "Ĝ": 216,
294
+ "ĝ": 217,
295
+ "Ğ": 218,
296
+ "ğ": 219,
297
+ "Ġ": 220,
298
+ "ġ": 221,
299
+ "Ģ": 222,
300
+ "ģ": 223,
301
+ "Ĥ": 224,
302
+ "ĥ": 225,
303
+ "Ħ": 226,
304
+ "ħ": 227,
305
+ "Ĩ": 228,
306
+ "ĩ": 229,
307
+ "Ī": 230,
308
+ "ī": 231,
309
+ "Ĭ": 232,
310
+ "ĭ": 233,
311
+ "Į": 234,
312
+ "į": 235,
313
+ "İ": 236,
314
+ "ı": 237,
315
+ "IJ": 238,
316
+ "ij": 239,
317
+ "Ĵ": 240,
318
+ "ĵ": 241,
319
+ "Ķ": 242,
320
+ "ķ": 243,
321
+ "ĸ": 244,
322
+ "Ĺ": 245,
323
+ "ĺ": 246,
324
+ "Ļ": 247,
325
+ "ļ": 248,
326
+ "Ľ": 249,
327
+ "ľ": 250,
328
+ "Ŀ": 251,
329
+ "ŀ": 252,
330
+ "Ł": 253,
331
+ "ł": 254,
332
+ "Ń": 255,
333
+ "ĠĠ": 256,
334
+ "ĠĠĠĠ": 257,
335
+ "in": 258,
336
+ "Ġt": 259,
337
+ "ĠĠĠĠĠĠĠĠ": 260,
338
+ "er": 261,
339
+ "ĠĠĠ": 262,
340
+ "on": 263,
341
+ "Ġa": 264,
342
+ "re": 265,
343
+ "at": 266,
344
+ "st": 267,
345
+ "en": 268,
346
+ "or": 269,
347
+ "Ġth": 270,
348
+ "ĊĊ": 271,
349
+ "Ġc": 272,
350
+ "le": 273,
351
+ "Ġs": 274,
352
+ "it": 275,
353
+ "an": 276,
354
+ "ar": 277,
355
+ "al": 278,
356
+ "Ġthe": 279,
357
+ ";Ċ": 280,
358
+ "Ġp": 281,
359
+ "Ġf": 282,
360
+ "ou": 283,
361
+ "Ġ=": 284,
362
+ "is": 285,
363
+ "ĠĠĠĠĠĠĠ": 286,
364
+ "ing": 287,
365
+ "es": 288,
366
+ "Ġw": 289,
367
+ "ion": 290,
368
+ "ed": 291,
369
+ "ic": 292,
370
+ "Ġb": 293,
371
+ "Ġd": 294,
372
+ "et": 295,
373
+ "Ġm": 296,
374
+ "Ġo": 297,
375
+ "ĉĉ": 298,
376
+ "ro": 299,
377
+ "as": 300,
378
+ "el": 301,
379
+ "ct": 302,
380
+ "nd": 303,
381
+ "Ġin": 304,
382
+ "Ġh": 305,
383
+ "ent": 306,
384
+ "id": 307,
385
+ "Ġn": 308,
386
+ "am": 309,
387
+ "ĠĠĠĠĠĠĠĠĠĠĠ": 310,
388
+ "Ġto": 311,
389
+ "Ġre": 312,
390
+ "--": 313,
391
+ "Ġ{": 314,
392
+ "Ġof": 315,
393
+ "om": 316,
394
+ ");Ċ": 317,
395
+ "im": 318,
396
+ "čĊ": 319,
397
+ "Ġ(": 320,
398
+ "il": 321,
399
+ "//": 322,
400
+ "Ġand": 323,
401
+ "ur": 324,
402
+ "se": 325,
403
+ "Ġl": 326,
404
+ "ex": 327,
405
+ "ĠS": 328,
406
+ "ad": 329,
407
+ "Ġ\"": 330,
408
+ "ch": 331,
409
+ "ut": 332,
410
+ "if": 333,
411
+ "**": 334,
412
+ "Ġ}": 335,
413
+ "em": 336,
414
+ "ol": 337,
415
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 338,
416
+ "th": 339,
417
+ ")Ċ": 340,
418
+ "Ġ{Ċ": 341,
419
+ "Ġg": 342,
420
+ "ig": 343,
421
+ "iv": 344,
422
+ ",Ċ": 345,
423
+ "ce": 346,
424
+ "od": 347,
425
+ "Ġv": 348,
426
+ "ate": 349,
427
+ "ĠT": 350,
428
+ "ag": 351,
429
+ "ay": 352,
430
+ "Ġ*": 353,
431
+ "ot": 354,
432
+ "us": 355,
433
+ "ĠC": 356,
434
+ "Ġst": 357,
435
+ "ĠI": 358,
436
+ "un": 359,
437
+ "ul": 360,
438
+ "ue": 361,
439
+ "ĠA": 362,
440
+ "ow": 363,
441
+ "Ġ'": 364,
442
+ "ew": 365,
443
+ "Ġ<": 366,
444
+ "ation": 367,
445
+ "()": 368,
446
+ "Ġfor": 369,
447
+ "ab": 370,
448
+ "ort": 371,
449
+ "um": 372,
450
+ "ame": 373,
451
+ "Ġis": 374,
452
+ "pe": 375,
453
+ "tr": 376,
454
+ "ck": 377,
455
+ "âĢ": 378,
456
+ "Ġy": 379,
457
+ "ist": 380,
458
+ "----": 381,
459
+ ".ĊĊ": 382,
460
+ "he": 383,
461
+ "Ġe": 384,
462
+ "lo": 385,
463
+ "ĠM": 386,
464
+ "Ġbe": 387,
465
+ "ers": 388,
466
+ "Ġon": 389,
467
+ "Ġcon": 390,
468
+ "ap": 391,
469
+ "ub": 392,
470
+ "ĠP": 393,
471
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 394,
472
+ "ass": 395,
473
+ "int": 396,
474
+ ">Ċ": 397,
475
+ "ly": 398,
476
+ "urn": 399,
477
+ "Ġ$": 400,
478
+ ";ĊĊ": 401,
479
+ "av": 402,
480
+ "port": 403,
481
+ "ir": 404,
482
+ "->": 405,
483
+ "nt": 406,
484
+ "ction": 407,
485
+ "end": 408,
486
+ "Ġde": 409,
487
+ "ith": 410,
488
+ "out": 411,
489
+ "turn": 412,
490
+ "our": 413,
491
+ "ĠĠĠĠĠ": 414,
492
+ "lic": 415,
493
+ "res": 416,
494
+ "pt": 417,
495
+ "==": 418,
496
+ "Ġthis": 419,
497
+ "Ġwh": 420,
498
+ "Ġif": 421,
499
+ "ĠD": 422,
500
+ "ver": 423,
501
+ "age": 424,
502
+ "ĠB": 425,
503
+ "ht": 426,
504
+ "ext": 427,
505
+ "=\"": 428,
506
+ "Ġthat": 429,
507
+ "****": 430,
508
+ "ĠR": 431,
509
+ "Ġit": 432,
510
+ "ess": 433,
511
+ "ĠF": 434,
512
+ "Ġr": 435,
513
+ "os": 436,
514
+ "and": 437,
515
+ "Ġas": 438,
516
+ "ect": 439,
517
+ "ke": 440,
518
+ "rom": 441,
519
+ "Ġ//": 442,
520
+ "con": 443,
521
+ "ĠL": 444,
522
+ "(\"": 445,
523
+ "qu": 446,
524
+ "lass": 447,
525
+ "Ġwith": 448,
526
+ "iz": 449,
527
+ "de": 450,
528
+ "ĠN": 451,
529
+ "Ġal": 452,
530
+ "op": 453,
531
+ "up": 454,
532
+ "get": 455,
533
+ "Ġ}Ċ": 456,
534
+ "ile": 457,
535
+ "Ġan": 458,
536
+ "ata": 459,
537
+ "ore": 460,
538
+ "ri": 461,
539
+ "Ġpro": 462,
540
+ ";čĊ": 463,
541
+ "ĉĉĉĉ": 464,
542
+ "ter": 465,
543
+ "ain": 466,
544
+ "ĠW": 467,
545
+ "ĠE": 468,
546
+ "Ġcom": 469,
547
+ "Ġreturn": 470,
548
+ "art": 471,
549
+ "ĠH": 472,
550
+ "ack": 473,
551
+ "import": 474,
552
+ "ublic": 475,
553
+ "Ġor": 476,
554
+ "est": 477,
555
+ "ment": 478,
556
+ "ĠG": 479,
557
+ "able": 480,
558
+ "Ġ-": 481,
559
+ "ine": 482,
560
+ "ill": 483,
561
+ "ind": 484,
562
+ "ere": 485,
563
+ "::": 486,
564
+ "ity": 487,
565
+ "Ġ+": 488,
566
+ "Ġtr": 489,
567
+ "elf": 490,
568
+ "ight": 491,
569
+ "('": 492,
570
+ "orm": 493,
571
+ "ult": 494,
572
+ "str": 495,
573
+ "..": 496,
574
+ "\",": 497,
575
+ "Ġyou": 498,
576
+ "ype": 499
577
+ },
578
+ "merges": [
579
+ "Ġ Ġ",
580
+ "ĠĠ ĠĠ",
581
+ "i n",
582
+ "Ġ t",
583
+ "ĠĠĠĠ ĠĠĠĠ",
584
+ "e r",
585
+ "ĠĠ Ġ",
586
+ "o n",
587
+ "Ġ a",
588
+ "r e",
589
+ "a t",
590
+ "s t",
591
+ "e n",
592
+ "o r",
593
+ "Ġt h",
594
+ "Ċ Ċ",
595
+ "Ġ c",
596
+ "l e",
597
+ "Ġ s",
598
+ "i t",
599
+ "a n",
600
+ "a r",
601
+ "a l",
602
+ "Ġth e",
603
+ "; Ċ",
604
+ "Ġ p",
605
+ "Ġ f",
606
+ "o u",
607
+ "Ġ =",
608
+ "i s",
609
+ "ĠĠĠĠ ĠĠĠ",
610
+ "in g",
611
+ "e s",
612
+ "Ġ w",
613
+ "i on",
614
+ "e d",
615
+ "i c",
616
+ "Ġ b",
617
+ "Ġ d",
618
+ "e t",
619
+ "Ġ m",
620
+ "Ġ o",
621
+ "ĉ ĉ",
622
+ "r o",
623
+ "a s",
624
+ "e l",
625
+ "c t",
626
+ "n d",
627
+ "Ġ in",
628
+ "Ġ h",
629
+ "en t",
630
+ "i d",
631
+ "Ġ n",
632
+ "a m",
633
+ "ĠĠĠĠĠĠĠĠ ĠĠĠ",
634
+ "Ġt o",
635
+ "Ġ re",
636
+ "- -",
637
+ "Ġ {",
638
+ "Ġo f",
639
+ "o m",
640
+ ") ;Ċ",
641
+ "i m",
642
+ "č Ċ",
643
+ "Ġ (",
644
+ "i l",
645
+ "/ /",
646
+ "Ġa nd",
647
+ "u r",
648
+ "s e",
649
+ "Ġ l",
650
+ "e x",
651
+ "Ġ S",
652
+ "a d",
653
+ "Ġ \"",
654
+ "c h",
655
+ "u t",
656
+ "i f",
657
+ "* *",
658
+ "Ġ }",
659
+ "e m",
660
+ "o l",
661
+ "ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ",
662
+ "t h",
663
+ ") Ċ",
664
+ "Ġ{ Ċ",
665
+ "Ġ g",
666
+ "i g",
667
+ "i v",
668
+ ", Ċ",
669
+ "c e",
670
+ "o d",
671
+ "Ġ v",
672
+ "at e",
673
+ "Ġ T",
674
+ "a g",
675
+ "a y",
676
+ "Ġ *",
677
+ "o t",
678
+ "u s",
679
+ "Ġ C",
680
+ "Ġ st",
681
+ "Ġ I",
682
+ "u n",
683
+ "u l",
684
+ "u e",
685
+ "Ġ A",
686
+ "o w",
687
+ "Ġ '",
688
+ "e w",
689
+ "Ġ <",
690
+ "at ion",
691
+ "( )",
692
+ "Ġf or",
693
+ "a b",
694
+ "or t",
695
+ "u m",
696
+ "am e",
697
+ "Ġ is",
698
+ "p e",
699
+ "t r",
700
+ "c k",
701
+ "â Ģ",
702
+ "Ġ y",
703
+ "i st",
704
+ "-- --",
705
+ ". ĊĊ",
706
+ "h e",
707
+ "Ġ e",
708
+ "l o",
709
+ "Ġ M",
710
+ "Ġb e",
711
+ "er s",
712
+ "Ġ on",
713
+ "Ġc on",
714
+ "a p",
715
+ "u b",
716
+ "Ġ P",
717
+ "ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠ",
718
+ "as s",
719
+ "in t",
720
+ "> Ċ",
721
+ "l y",
722
+ "ur n",
723
+ "Ġ $",
724
+ "; ĊĊ",
725
+ "a v",
726
+ "p ort",
727
+ "i r",
728
+ "- >",
729
+ "n t",
730
+ "ct ion",
731
+ "en d",
732
+ "Ġd e",
733
+ "it h",
734
+ "ou t",
735
+ "t urn",
736
+ "ou r",
737
+ "ĠĠĠĠ Ġ",
738
+ "l ic",
739
+ "re s",
740
+ "p t",
741
+ "= =",
742
+ "Ġth is",
743
+ "Ġw h",
744
+ "Ġ if",
745
+ "Ġ D",
746
+ "v er",
747
+ "ag e",
748
+ "Ġ B",
749
+ "h t",
750
+ "ex t",
751
+ "= \"",
752
+ "Ġth at",
753
+ "** **",
754
+ "Ġ R",
755
+ "Ġ it",
756
+ "es s",
757
+ "Ġ F",
758
+ "Ġ r",
759
+ "o s",
760
+ "an d",
761
+ "Ġa s",
762
+ "e ct",
763
+ "k e",
764
+ "ro m",
765
+ "Ġ //",
766
+ "c on",
767
+ "Ġ L",
768
+ "( \"",
769
+ "q u",
770
+ "l ass",
771
+ "Ġw ith",
772
+ "i z",
773
+ "d e",
774
+ "Ġ N",
775
+ "Ġa l",
776
+ "o p",
777
+ "u p",
778
+ "g et",
779
+ "Ġ} Ċ",
780
+ "i le",
781
+ "Ġa n",
782
+ "at a",
783
+ "o re",
784
+ "r i",
785
+ "Ġp ro",
786
+ "; čĊ",
787
+ "ĉĉ ĉĉ",
788
+ "t er",
789
+ "a in",
790
+ "Ġ W",
791
+ "Ġ E",
792
+ "Ġc om",
793
+ "Ġre turn",
794
+ "ar t",
795
+ "Ġ H",
796
+ "a ck",
797
+ "im port",
798
+ "ub lic",
799
+ "Ġ or",
800
+ "e st",
801
+ "m ent",
802
+ "Ġ G",
803
+ "ab le",
804
+ "Ġ -",
805
+ "in e",
806
+ "il l",
807
+ "in d",
808
+ "er e",
809
+ ": :",
810
+ "it y",
811
+ "Ġ +",
812
+ "Ġt r",
813
+ "el f",
814
+ "ig ht",
815
+ "( '",
816
+ "or m",
817
+ "ul t",
818
+ "st r",
819
+ ". .",
820
+ "\" ,",
821
+ "Ġy ou",
822
+ "y pe"
823
+ ]
824
+ }
825
+ }
tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff