tiny-processor-glmasr / tokenizer.json
hf-transformers-bot's picture
Add tiny processor fixture for testing
a26cd09 verified
Raw
History Blame Contribute Delete
26.6 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 500,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 501,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 502,
"content": "[gMASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 503,
"content": "[sMASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 504,
"content": "<sop>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 505,
"content": "<eop>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 506,
"content": "<|system|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 507,
"content": "<|user|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 508,
"content": "<|assistant|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 509,
"content": "<|observation|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 510,
"content": "<|begin_of_image|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 511,
"content": "<|end_of_image|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 512,
"content": "<|begin_of_video|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 513,
"content": "<|end_of_video|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 514,
"content": "<|pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 515,
"content": "<|begin_of_audio|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 516,
"content": "<|end_of_audio|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Sequence",
"pretokenizers": [
{
"type": "Split",
"pattern": {
"Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
},
"behavior": "Isolated",
"invert": false
},
{
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": false
}
]
},
"post_processor": {
"type": "Sequence",
"processors": [
{
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
}
]
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": true,
"vocab": {
"Ā": 0,
"ā": 1,
"Ă": 2,
"ă": 3,
"Ą": 4,
"ą": 5,
"Ć": 6,
"ć": 7,
"Ĉ": 8,
"ĉ": 9,
"Ċ": 10,
"ċ": 11,
"Č": 12,
"č": 13,
"Ď": 14,
"ď": 15,
"Đ": 16,
"đ": 17,
"Ē": 18,
"ē": 19,
"Ĕ": 20,
"ĕ": 21,
"Ė": 22,
"ė": 23,
"Ę": 24,
"ę": 25,
"Ě": 26,
"ě": 27,
"Ĝ": 28,
"ĝ": 29,
"Ğ": 30,
"ğ": 31,
"Ġ": 32,
"!": 33,
"\"": 34,
"#": 35,
"$": 36,
"%": 37,
"&": 38,
"'": 39,
"(": 40,
")": 41,
"*": 42,
"+": 43,
",": 44,
"-": 45,
".": 46,
"/": 47,
"0": 48,
"1": 49,
"2": 50,
"3": 51,
"4": 52,
"5": 53,
"6": 54,
"7": 55,
"8": 56,
"9": 57,
":": 58,
";": 59,
"<": 60,
"=": 61,
">": 62,
"?": 63,
"@": 64,
"A": 65,
"B": 66,
"C": 67,
"D": 68,
"E": 69,
"F": 70,
"G": 71,
"H": 72,
"I": 73,
"J": 74,
"K": 75,
"L": 76,
"M": 77,
"N": 78,
"O": 79,
"P": 80,
"Q": 81,
"R": 82,
"S": 83,
"T": 84,
"U": 85,
"V": 86,
"W": 87,
"X": 88,
"Y": 89,
"Z": 90,
"[": 91,
"\\": 92,
"]": 93,
"^": 94,
"_": 95,
"`": 96,
"a": 97,
"b": 98,
"c": 99,
"d": 100,
"e": 101,
"f": 102,
"g": 103,
"h": 104,
"i": 105,
"j": 106,
"k": 107,
"l": 108,
"m": 109,
"n": 110,
"o": 111,
"p": 112,
"q": 113,
"r": 114,
"s": 115,
"t": 116,
"u": 117,
"v": 118,
"w": 119,
"x": 120,
"y": 121,
"z": 122,
"{": 123,
"|": 124,
"}": 125,
"~": 126,
"ġ": 127,
"Ģ": 128,
"ģ": 129,
"Ĥ": 130,
"ĥ": 131,
"Ħ": 132,
"ħ": 133,
"Ĩ": 134,
"ĩ": 135,
"Ī": 136,
"ī": 137,
"Ĭ": 138,
"ĭ": 139,
"Į": 140,
"į": 141,
"İ": 142,
"ı": 143,
"IJ": 144,
"ij": 145,
"Ĵ": 146,
"ĵ": 147,
"Ķ": 148,
"ķ": 149,
"ĸ": 150,
"Ĺ": 151,
"ĺ": 152,
"Ļ": 153,
"ļ": 154,
"Ľ": 155,
"ľ": 156,
"Ŀ": 157,
"ŀ": 158,
"Ł": 159,
"ł": 160,
"¡": 161,
"¢": 162,
"£": 163,
"¤": 164,
"¥": 165,
"¦": 166,
"§": 167,
"¨": 168,
"©": 169,
"ª": 170,
"«": 171,
"¬": 172,
"Ń": 173,
"®": 174,
"¯": 175,
"°": 176,
"±": 177,
"²": 178,
"³": 179,
"´": 180,
"µ": 181,
"¶": 182,
"·": 183,
"¸": 184,
"¹": 185,
"º": 186,
"»": 187,
"¼": 188,
"½": 189,
"¾": 190,
"¿": 191,
"À": 192,
"Á": 193,
"Â": 194,
"Ã": 195,
"Ä": 196,
"Å": 197,
"Æ": 198,
"Ç": 199,
"È": 200,
"É": 201,
"Ê": 202,
"Ë": 203,
"Ì": 204,
"Í": 205,
"Î": 206,
"Ï": 207,
"Ð": 208,
"Ñ": 209,
"Ò": 210,
"Ó": 211,
"Ô": 212,
"Õ": 213,
"Ö": 214,
"×": 215,
"Ø": 216,
"Ù": 217,
"Ú": 218,
"Û": 219,
"Ü": 220,
"Ý": 221,
"Þ": 222,
"ß": 223,
"à": 224,
"á": 225,
"â": 226,
"ã": 227,
"ä": 228,
"å": 229,
"æ": 230,
"ç": 231,
"è": 232,
"é": 233,
"ê": 234,
"ë": 235,
"ì": 236,
"í": 237,
"î": 238,
"ï": 239,
"ð": 240,
"ñ": 241,
"ò": 242,
"ó": 243,
"ô": 244,
"õ": 245,
"ö": 246,
"÷": 247,
"ø": 248,
"ù": 249,
"ú": 250,
"û": 251,
"ü": 252,
"ý": 253,
"þ": 254,
"ÿ": 255,
"ĠĠ": 256,
"ĠĠĠĠ": 257,
"Ġt": 258,
"in": 259,
"er": 260,
"Ġa": 261,
"on": 262,
"en": 263,
"ä¸": 264,
"re": 265,
"he": 266,
"Ġs": 267,
"at": 268,
"ĠÐ": 269,
"or": 270,
"es": 271,
"Ġc": 272,
"о": 273,
"Ġd": 274,
"an": 275,
"Ġp": 276,
"ï¼": 277,
"it": 278,
"çļ": 279,
"ĠĠĠĠĠĠĠĠ": 280,
"çļĦ": 281,
"is": 282,
"Ġthe": 283,
"al": 284,
"ar": 285,
"е": 286,
"а": 287,
"ãĢ": 288,
"Ġo": 289,
"le": 290,
"Ġw": 291,
"Ġf": 292,
"äº": 293,
"Ġm": 294,
"ĠĠĠ": 295,
"и": 296,
"ou": 297,
"Ġb": 298,
"ÑĤ": 299,
"ï¼Į": 300,
"ic": 301,
"ing": 302,
"ion": 303,
"ed": 304,
"as": 305,
"nd": 306,
"н": 307,
"ro": 308,
"Ġin": 309,
"ent": 310,
"åı": 311,
"Ñģ": 312,
"ÑĢ": 313,
"æľ": 314,
"ä»": 315,
"el": 316,
"оÐ": 317,
"et": 318,
"Ġn": 319,
"Ġto": 320,
"st": 321,
"Ġl": 322,
"åħ": 323,
"âĢ": 324,
"Ġof": 325,
"ãĢĤ": 326,
"Ġh": 327,
"ct": 328,
"om": 329,
"å¤": 330,
"il": 331,
"Ġand": 332,
"åĪ": 333,
".Ċ": 334,
"Ġth": 335,
"è¿": 336,
"å®": 337,
"ä½": 338,
"Ġre": 339,
"Ġe": 340,
"am": 341,
"id": 342,
"åIJ": 343,
"æĺ": 344,
"åľ": 345,
"ch": 346,
"çĶ": 347,
"im": 348,
"ut": 349,
"ur": 350,
";Ċ": 351,
"ad": 352,
"л": 353,
"ol": 354,
"Ġg": 355,
"Ġv": 356,
"ç»": 357,
"Ġu": 358,
"ä¹": 359,
"Ġde": 360,
"æĪ": 361,
"åĬ": 362,
"ĠS": 363,
"ig": 364,
"ä¸Ģ": 365,
"se": 366,
"iv": 367,
"ãģ": 368,
"ot": 369,
"åĽ": 370,
"è¯": 371,
"æĺ¯": 372,
"æĸ": 373,
"ĠĠĠĠĠĠĠ": 374,
"ĠT": 375,
"ce": 376,
"Ġ=": 377,
"ĊĊ": 378,
"Ñĥ": 379,
"к": 380,
"å°": 381,
"ĠA": 382,
"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 383,
"Ġy": 384,
"åŃ": 385,
"Ġ(": 386,
"ĠC": 387,
"æĹ": 388,
"ation": 389,
"qu": 390,
"ra": 391,
"ĠI": 392,
"ul": 393,
"--": 394,
"em": 395,
"ãĢģ": 396,
"os": 397,
"Ġis": 398,
"è®": 399,
"ow": 400,
"ä¼": 401,
"å¹": 402,
"od": 403,
"if": 404,
"est": 405,
"åį": 406,
"Ġst": 407,
"ag": 408,
"Ñı": 409,
"æĿ": 410,
"åľ¨": 411,
"Ġfor": 412,
"un": 413,
"Ġcon": 414,
"ä¸į": 415,
"ا": 416,
"æľī": 417,
"в": 418,
"ers": 419,
"ir": 420,
"м": 421,
"Ġ{": 422,
"Ñĭ": 423,
"Ġbe": 424,
"Ġ\"": 425,
"ay": 426,
"人": 427,
"æī": 428,
"å¼": 429,
"ab": 430,
"éĩ": 431,
"ter": 432,
"ĠM": 433,
"åº": 434,
"ĠP": 435,
"ate": 436,
"th": 437,
"äºĨ": 438,
"Ġon": 439,
"ly": 440,
"å·": 441,
"è¡": 442,
"us": 443,
"Ġп": 444,
"ac": 445,
"èĢ": 446,
"ĉĉ": 447,
"æķ": 448,
"åĨ": 449,
"ÑĮ": 450,
"åĩ": 451,
"ie": 452,
"å¯": 453,
"å¸": 454,
"Ġyou": 455,
"å¾": 456,
"Ġpro": 457,
"д": 458,
"pe": 459,
"ä¸Ń": 460,
"um": 461,
"åĮ": 462,
"ith": 463,
"ist": 464,
"éĢ": 465,
"为": 466,
"ÑģÑĤ": 467,
"æĢ": 468,
"Ġal": 469,
"oc": 470,
"é": 471,
"ĠB": 472,
"çĽ": 473,
"åĴ": 474,
"ĠD": 475,
"Ġthat": 476,
"ort": 477,
"ve": 478,
"res": 479,
"è§": 480,
"ĥ½": 481,
"Ġk": 482,
"ver": 483,
"ess": 484,
"te": 485,
"end": 486,
"Ġan": 487,
"Ġwh": 488,
"æł": 489,
"åĴĮ": 490,
"and": 491,
"Ġв": 492,
",Ċ": 493,
"Ġ{Ċ": 494,
"ht": 495,
"ç§": 496,
")Ċ": 497,
");Ċ": 498,
"//": 499
},
"merges": [
[
"Ġ",
"Ġ"
],
[
"Ġ",
"ĠĠĠ"
],
[
"ĠĠ",
"ĠĠ"
],
[
"ĠĠĠ",
"Ġ"
],
[
"Ġ",
"t"
],
[
"i",
"n"
],
[
"e",
"r"
],
[
"Ġ",
"a"
],
[
"o",
"n"
],
[
"e",
"n"
],
[
"ä",
"¸"
],
[
"r",
"e"
],
[
"h",
"e"
],
[
"Ġ",
"s"
],
[
"a",
"t"
],
[
"Ġ",
"Ð"
],
[
"o",
"r"
],
[
"e",
"s"
],
[
"Ġ",
"c"
],
[
"Ð",
"¾"
],
[
"Ġ",
"d"
],
[
"a",
"n"
],
[
"Ġ",
"p"
],
[
"ï",
"¼"
],
[
"i",
"t"
],
[
"ç",
"ļ"
],
[
"Ġ",
"ĠĠĠĠĠĠĠ"
],
[
"ĠĠĠĠ",
"ĠĠĠĠ"
],
[
"ĠĠĠĠĠĠĠ",
"Ġ"
],
[
"çļ",
"Ħ"
],
[
"i",
"s"
],
[
"Ġt",
"he"
],
[
"Ġth",
"e"
],
[
"a",
"l"
],
[
"a",
"r"
],
[
"Ð",
"µ"
],
[
"Ð",
"°"
],
[
"ã",
"Ģ"
],
[
"Ġ",
"o"
],
[
"l",
"e"
],
[
"Ġ",
"w"
],
[
"Ġ",
"f"
],
[
"ä",
"º"
],
[
"Ġ",
"m"
],
[
"Ġ",
"ĠĠ"
],
[
"ĠĠ",
"Ġ"
],
[
"Ð",
"¸"
],
[
"o",
"u"
],
[
"Ġ",
"b"
],
[
"Ñ",
"Ĥ"
],
[
"ï¼",
"Į"
],
[
"i",
"c"
],
[
"in",
"g"
],
[
"i",
"on"
],
[
"e",
"d"
],
[
"a",
"s"
],
[
"n",
"d"
],
[
"Ð",
"½"
],
[
"r",
"o"
],
[
"Ġ",
"in"
],
[
"en",
"t"
],
[
"å",
"ı"
],
[
"Ñ",
"ģ"
],
[
"Ñ",
"Ģ"
],
[
"æ",
"ľ"
],
[
"ä",
"»"
],
[
"e",
"l"
],
[
"о",
"Ð"
],
[
"e",
"t"
],
[
"Ġ",
"n"
],
[
"Ġt",
"o"
],
[
"s",
"t"
],
[
"Ġ",
"l"
],
[
"å",
"ħ"
],
[
"â",
"Ģ"
],
[
"Ġo",
"f"
],
[
"ãĢ",
"Ĥ"
],
[
"Ġ",
"h"
],
[
"c",
"t"
],
[
"o",
"m"
],
[
"å",
"¤"
],
[
"i",
"l"
],
[
"Ġ",
"and"
],
[
"Ġa",
"nd"
],
[
"Ġan",
"d"
],
[
"å",
"Ī"
],
[
".",
"Ċ"
],
[
"Ġ",
"th"
],
[
"Ġt",
"h"
],
[
"è",
"¿"
],
[
"å",
"®"
],
[
"ä",
"½"
],
[
"Ġ",
"re"
],
[
"Ġ",
"e"
],
[
"a",
"m"
],
[
"i",
"d"
],
[
"å",
"IJ"
],
[
"æ",
"ĺ"
],
[
"å",
"ľ"
],
[
"c",
"h"
],
[
"ç",
"Ķ"
],
[
"i",
"m"
],
[
"u",
"t"
],
[
"u",
"r"
],
[
";",
"Ċ"
],
[
"a",
"d"
],
[
"Ð",
"»"
],
[
"o",
"l"
],
[
"Ġ",
"g"
],
[
"Ġ",
"v"
],
[
"ç",
"»"
],
[
"Ġ",
"u"
],
[
"ä",
"¹"
],
[
"Ġd",
"e"
],
[
"æ",
"Ī"
],
[
"å",
"Ĭ"
],
[
"Ġ",
"S"
],
[
"i",
"g"
],
[
"ä¸",
"Ģ"
],
[
"s",
"e"
],
[
"i",
"v"
],
[
"ã",
"ģ"
],
[
"o",
"t"
],
[
"å",
"Ľ"
],
[
"è",
"¯"
],
[
"æĺ",
"¯"
],
[
"æ",
"ĸ"
],
[
"ĠĠĠĠ",
"ĠĠĠ"
],
[
"ĠĠĠ",
"ĠĠĠĠ"
],
[
"Ġ",
"T"
],
[
"c",
"e"
],
[
"Ġ",
"="
],
[
"Ċ",
"Ċ"
],
[
"Ñ",
"ĥ"
],
[
"Ð",
"º"
],
[
"å",
"°"
],
[
"Ġ",
"A"
],
[
"ĠĠĠĠĠĠĠĠ",
"ĠĠĠĠĠĠĠĠ"
],
[
"Ġ",
"y"
],
[
"å",
"Ń"
],
[
"Ġ",
"("
],
[
"Ġ",
"C"
],
[
"æ",
"Ĺ"
],
[
"at",
"ion"
],
[
"q",
"u"
],
[
"r",
"a"
],
[
"Ġ",
"I"
],
[
"u",
"l"
],
[
"-",
"-"
],
[
"e",
"m"
],
[
"ãĢ",
"ģ"
],
[
"o",
"s"
],
[
"Ġ",
"is"
],
[
"è",
"®"
],
[
"o",
"w"
],
[
"ä",
"¼"
],
[
"å",
"¹"
],
[
"o",
"d"
],
[
"i",
"f"
],
[
"e",
"st"
],
[
"es",
"t"
],
[
"å",
"į"
],
[
"Ġ",
"st"
],
[
"Ġs",
"t"
],
[
"a",
"g"
],
[
"Ñ",
"ı"
],
[
"æ",
"Ŀ"
],
[
"åľ",
"¨"
],
[
"Ġf",
"or"
],
[
"u",
"n"
],
[
"Ġc",
"on"
],
[
"ä¸",
"į"
],
[
"Ø",
"§"
],
[
"æľ",
"ī"
],
[
"Ð",
"²"
],
[
"er",
"s"
],
[
"i",
"r"
],
[
"Ð",
"¼"
],
[
"Ġ",
"{"
],
[
"Ñ",
"ĭ"
],
[
"Ġb",
"e"
],
[
"Ġ",
"\""
],
[
"a",
"y"
],
[
"äº",
"º"
],
[
"æ",
"ī"
],
[
"å",
"¼"
],
[
"a",
"b"
],
[
"é",
"ĩ"
],
[
"t",
"er"
],
[
"te",
"r"
],
[
"Ġ",
"M"
],
[
"å",
"º"
],
[
"Ġ",
"P"
],
[
"a",
"te"
],
[
"at",
"e"
],
[
"t",
"h"
],
[
"äº",
"Ĩ"
],
[
"Ġ",
"on"
],
[
"Ġo",
"n"
],
[
"l",
"y"
],
[
"å",
"·"
],
[
"è",
"¡"
],
[
"u",
"s"
],
[
"ĠÐ",
"¿"
],
[
"a",
"c"
],
[
"è",
"Ģ"
],
[
"ĉ",
"ĉ"
],
[
"æ",
"ķ"
],
[
"å",
"Ĩ"
],
[
"Ñ",
"Į"
],
[
"å",
"ĩ"
],
[
"i",
"e"
],
[
"å",
"¯"
],
[
"å",
"¸"
],
[
"Ġy",
"ou"
],
[
"å",
"¾"
],
[
"Ġp",
"ro"
],
[
"Ð",
"´"
],
[
"p",
"e"
],
[
"ä¸",
"Ń"
],
[
"u",
"m"
],
[
"å",
"Į"
],
[
"i",
"th"
],
[
"it",
"h"
],
[
"i",
"st"
],
[
"is",
"t"
],
[
"é",
"Ģ"
],
[
"ä¸",
"º"
],
[
"Ñģ",
"ÑĤ"
],
[
"æ",
"Ģ"
],
[
"Ġ",
"al"
],
[
"Ġa",
"l"
],
[
"o",
"c"
],
[
"Ã",
"©"
],
[
"Ġ",
"B"
],
[
"ç",
"Ľ"
],
[
"å",
"Ĵ"
],
[
"Ġ",
"D"
],
[
"Ġth",
"at"
],
[
"or",
"t"
],
[
"v",
"e"
],
[
"r",
"es"
],
[
"re",
"s"
],
[
"è",
"§"
],
[
"ĥ",
"½"
],
[
"Ġ",
"k"
],
[
"v",
"er"
],
[
"ve",
"r"
],
[
"es",
"s"
],
[
"t",
"e"
],
[
"e",
"nd"
],
[
"en",
"d"
],
[
"Ġ",
"an"
],
[
"Ġa",
"n"
],
[
"Ġw",
"h"
],
[
"æ",
"ł"
],
[
"åĴ",
"Į"
],
[
"a",
"nd"
],
[
"an",
"d"
],
[
"Ġ",
"в"
],
[
"ĠÐ",
"²"
],
[
",",
"Ċ"
],
[
"Ġ{",
"Ċ"
],
[
"h",
"t"
],
[
"ç",
"§"
],
[
")",
"Ċ"
],
[
")",
";Ċ"
],
[
"/",
"/"
]
]
}
}