{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 500, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 501, "content": "[MASK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 502, "content": "[gMASK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 503, "content": "[sMASK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 504, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 505, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 506, "content": "<|system|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 507, "content": "<|user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 508, "content": "<|assistant|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 509, "content": "<|observation|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 510, "content": "<|begin_of_image|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 511, "content": "<|end_of_image|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 512, "content": "<|begin_of_video|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 513, "content": "<|end_of_video|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 514, "content": "<|pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 515, "content": "<|begin_of_audio|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 516, "content": "<|end_of_audio|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "post_processor": { "type": "Sequence", "processors": [ { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": false, "use_regex": true } ] }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": true, "vocab": { "Ā": 0, "ā": 1, "Ă": 2, "ă": 3, "Ą": 4, "ą": 5, "Ć": 6, "ć": 7, "Ĉ": 8, "ĉ": 9, "Ċ": 10, "ċ": 11, "Č": 12, "č": 13, "Ď": 14, "ď": 15, "Đ": 16, "đ": 17, "Ē": 18, "ē": 19, "Ĕ": 20, "ĕ": 21, "Ė": 22, "ė": 23, "Ę": 24, "ę": 25, "Ě": 26, "ě": 27, "Ĝ": 28, "ĝ": 29, "Ğ": 30, "ğ": 31, "Ġ": 32, "!": 33, "\"": 34, "#": 35, "$": 36, "%": 37, "&": 38, "'": 39, "(": 40, ")": 41, "*": 42, "+": 43, ",": 44, "-": 45, ".": 46, "/": 47, "0": 48, "1": 49, "2": 50, "3": 51, "4": 52, "5": 53, "6": 54, "7": 55, "8": 56, "9": 57, ":": 58, ";": 59, "<": 60, "=": 61, ">": 62, "?": 63, "@": 64, "A": 65, "B": 66, "C": 67, "D": 68, "E": 69, "F": 70, "G": 71, "H": 72, "I": 73, "J": 74, "K": 75, "L": 76, "M": 77, "N": 78, "O": 79, "P": 80, "Q": 81, "R": 82, "S": 83, "T": 84, "U": 85, "V": 86, "W": 87, "X": 88, "Y": 89, "Z": 90, "[": 91, "\\": 92, "]": 93, "^": 94, "_": 95, "`": 96, "a": 97, "b": 98, "c": 99, "d": 100, "e": 101, "f": 102, "g": 103, "h": 104, "i": 105, "j": 106, "k": 107, "l": 108, "m": 109, "n": 110, "o": 111, "p": 112, "q": 113, "r": 114, "s": 115, "t": 116, "u": 117, "v": 118, "w": 119, "x": 120, "y": 121, "z": 122, "{": 123, "|": 124, "}": 125, "~": 126, "ġ": 127, "Ģ": 128, "ģ": 129, "Ĥ": 130, "ĥ": 131, "Ħ": 132, "ħ": 133, "Ĩ": 134, "ĩ": 135, "Ī": 136, "ī": 137, "Ĭ": 138, "ĭ": 139, "Į": 140, "į": 141, "İ": 142, "ı": 143, "IJ": 144, "ij": 145, "Ĵ": 146, "ĵ": 147, "Ķ": 148, "ķ": 149, "ĸ": 150, "Ĺ": 151, "ĺ": 152, "Ļ": 153, "ļ": 154, "Ľ": 155, "ľ": 156, "Ŀ": 157, "ŀ": 158, "Ł": 159, "ł": 160, "¡": 161, "¢": 162, "£": 163, "¤": 164, "¥": 165, "¦": 166, "§": 167, "¨": 168, "©": 169, "ª": 170, "«": 171, "¬": 172, "Ń": 173, "®": 174, "¯": 175, "°": 176, "±": 177, "²": 178, "³": 179, "´": 180, "µ": 181, "¶": 182, "·": 183, "¸": 184, "¹": 185, "º": 186, "»": 187, "¼": 188, "½": 189, "¾": 190, "¿": 191, "À": 192, "Á": 193, "Â": 194, "Ã": 195, "Ä": 196, "Å": 197, "Æ": 198, "Ç": 199, "È": 200, "É": 201, "Ê": 202, "Ë": 203, "Ì": 204, "Í": 205, "Î": 206, "Ï": 207, "Ð": 208, "Ñ": 209, "Ò": 210, "Ó": 211, "Ô": 212, "Õ": 213, "Ö": 214, "×": 215, "Ø": 216, "Ù": 217, "Ú": 218, "Û": 219, "Ü": 220, "Ý": 221, "Þ": 222, "ß": 223, "à": 224, "á": 225, "â": 226, "ã": 227, "ä": 228, "å": 229, "æ": 230, "ç": 231, "è": 232, "é": 233, "ê": 234, "ë": 235, "ì": 236, "í": 237, "î": 238, "ï": 239, "ð": 240, "ñ": 241, "ò": 242, "ó": 243, "ô": 244, "õ": 245, "ö": 246, "÷": 247, "ø": 248, "ù": 249, "ú": 250, "û": 251, "ü": 252, "ý": 253, "þ": 254, "ÿ": 255, "ĠĠ": 256, "ĠĠĠĠ": 257, "Ġt": 258, "in": 259, "er": 260, "Ġa": 261, "on": 262, "en": 263, "ä¸": 264, "re": 265, "he": 266, "Ġs": 267, "at": 268, "ĠÐ": 269, "or": 270, "es": 271, "Ġc": 272, "о": 273, "Ġd": 274, "an": 275, "Ġp": 276, "ï¼": 277, "it": 278, "çļ": 279, "ĠĠĠĠĠĠĠĠ": 280, "çļĦ": 281, "is": 282, "Ġthe": 283, "al": 284, "ar": 285, "е": 286, "а": 287, "ãĢ": 288, "Ġo": 289, "le": 290, "Ġw": 291, "Ġf": 292, "äº": 293, "Ġm": 294, "ĠĠĠ": 295, "и": 296, "ou": 297, "Ġb": 298, "ÑĤ": 299, "ï¼Į": 300, "ic": 301, "ing": 302, "ion": 303, "ed": 304, "as": 305, "nd": 306, "н": 307, "ro": 308, "Ġin": 309, "ent": 310, "åı": 311, "Ñģ": 312, "ÑĢ": 313, "æľ": 314, "ä»": 315, "el": 316, "оÐ": 317, "et": 318, "Ġn": 319, "Ġto": 320, "st": 321, "Ġl": 322, "åħ": 323, "âĢ": 324, "Ġof": 325, "ãĢĤ": 326, "Ġh": 327, "ct": 328, "om": 329, "å¤": 330, "il": 331, "Ġand": 332, "åĪ": 333, ".Ċ": 334, "Ġth": 335, "è¿": 336, "å®": 337, "ä½": 338, "Ġre": 339, "Ġe": 340, "am": 341, "id": 342, "åIJ": 343, "æĺ": 344, "åľ": 345, "ch": 346, "çĶ": 347, "im": 348, "ut": 349, "ur": 350, ";Ċ": 351, "ad": 352, "л": 353, "ol": 354, "Ġg": 355, "Ġv": 356, "ç»": 357, "Ġu": 358, "ä¹": 359, "Ġde": 360, "æĪ": 361, "åĬ": 362, "ĠS": 363, "ig": 364, "ä¸Ģ": 365, "se": 366, "iv": 367, "ãģ": 368, "ot": 369, "åĽ": 370, "è¯": 371, "æĺ¯": 372, "æĸ": 373, "ĠĠĠĠĠĠĠ": 374, "ĠT": 375, "ce": 376, "Ġ=": 377, "ĊĊ": 378, "Ñĥ": 379, "к": 380, "å°": 381, "ĠA": 382, "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 383, "Ġy": 384, "åŃ": 385, "Ġ(": 386, "ĠC": 387, "æĹ": 388, "ation": 389, "qu": 390, "ra": 391, "ĠI": 392, "ul": 393, "--": 394, "em": 395, "ãĢģ": 396, "os": 397, "Ġis": 398, "è®": 399, "ow": 400, "ä¼": 401, "å¹": 402, "od": 403, "if": 404, "est": 405, "åį": 406, "Ġst": 407, "ag": 408, "Ñı": 409, "æĿ": 410, "åľ¨": 411, "Ġfor": 412, "un": 413, "Ġcon": 414, "ä¸į": 415, "ا": 416, "æľī": 417, "в": 418, "ers": 419, "ir": 420, "м": 421, "Ġ{": 422, "Ñĭ": 423, "Ġbe": 424, "Ġ\"": 425, "ay": 426, "人": 427, "æī": 428, "å¼": 429, "ab": 430, "éĩ": 431, "ter": 432, "ĠM": 433, "åº": 434, "ĠP": 435, "ate": 436, "th": 437, "äºĨ": 438, "Ġon": 439, "ly": 440, "å·": 441, "è¡": 442, "us": 443, "Ġп": 444, "ac": 445, "èĢ": 446, "ĉĉ": 447, "æķ": 448, "åĨ": 449, "ÑĮ": 450, "åĩ": 451, "ie": 452, "å¯": 453, "å¸": 454, "Ġyou": 455, "å¾": 456, "Ġpro": 457, "д": 458, "pe": 459, "ä¸Ń": 460, "um": 461, "åĮ": 462, "ith": 463, "ist": 464, "éĢ": 465, "为": 466, "ÑģÑĤ": 467, "æĢ": 468, "Ġal": 469, "oc": 470, "é": 471, "ĠB": 472, "çĽ": 473, "åĴ": 474, "ĠD": 475, "Ġthat": 476, "ort": 477, "ve": 478, "res": 479, "è§": 480, "ĥ½": 481, "Ġk": 482, "ver": 483, "ess": 484, "te": 485, "end": 486, "Ġan": 487, "Ġwh": 488, "æł": 489, "åĴĮ": 490, "and": 491, "Ġв": 492, ",Ċ": 493, "Ġ{Ċ": 494, "ht": 495, "ç§": 496, ")Ċ": 497, ");Ċ": 498, "//": 499 }, "merges": [ [ "Ġ", "Ġ" ], [ "Ġ", "ĠĠĠ" ], [ "ĠĠ", "ĠĠ" ], [ "ĠĠĠ", "Ġ" ], [ "Ġ", "t" ], [ "i", "n" ], [ "e", "r" ], [ "Ġ", "a" ], [ "o", "n" ], [ "e", "n" ], [ "ä", "¸" ], [ "r", "e" ], [ "h", "e" ], [ "Ġ", "s" ], [ "a", "t" ], [ "Ġ", "Ð" ], [ "o", "r" ], [ "e", "s" ], [ "Ġ", "c" ], [ "Ð", "¾" ], [ "Ġ", "d" ], [ "a", "n" ], [ "Ġ", "p" ], [ "ï", "¼" ], [ "i", "t" ], [ "ç", "ļ" ], [ "Ġ", "ĠĠĠĠĠĠĠ" ], [ "ĠĠĠĠ", "ĠĠĠĠ" ], [ "ĠĠĠĠĠĠĠ", "Ġ" ], [ "çļ", "Ħ" ], [ "i", "s" ], [ "Ġt", "he" ], [ "Ġth", "e" ], [ "a", "l" ], [ "a", "r" ], [ "Ð", "µ" ], [ "Ð", "°" ], [ "ã", "Ģ" ], [ "Ġ", "o" ], [ "l", "e" ], [ "Ġ", "w" ], [ "Ġ", "f" ], [ "ä", "º" ], [ "Ġ", "m" ], [ "Ġ", "ĠĠ" ], [ "ĠĠ", "Ġ" ], [ "Ð", "¸" ], [ "o", "u" ], [ "Ġ", "b" ], [ "Ñ", "Ĥ" ], [ "ï¼", "Į" ], [ "i", "c" ], [ "in", "g" ], [ "i", "on" ], [ "e", "d" ], [ "a", "s" ], [ "n", "d" ], [ "Ð", "½" ], [ "r", "o" ], [ "Ġ", "in" ], [ "en", "t" ], [ "å", "ı" ], [ "Ñ", "ģ" ], [ "Ñ", "Ģ" ], [ "æ", "ľ" ], [ "ä", "»" ], [ "e", "l" ], [ "о", "Ð" ], [ "e", "t" ], [ "Ġ", "n" ], [ "Ġt", "o" ], [ "s", "t" ], [ "Ġ", "l" ], [ "å", "ħ" ], [ "â", "Ģ" ], [ "Ġo", "f" ], [ "ãĢ", "Ĥ" ], [ "Ġ", "h" ], [ "c", "t" ], [ "o", "m" ], [ "å", "¤" ], [ "i", "l" ], [ "Ġ", "and" ], [ "Ġa", "nd" ], [ "Ġan", "d" ], [ "å", "Ī" ], [ ".", "Ċ" ], [ "Ġ", "th" ], [ "Ġt", "h" ], [ "è", "¿" ], [ "å", "®" ], [ "ä", "½" ], [ "Ġ", "re" ], [ "Ġ", "e" ], [ "a", "m" ], [ "i", "d" ], [ "å", "IJ" ], [ "æ", "ĺ" ], [ "å", "ľ" ], [ "c", "h" ], [ "ç", "Ķ" ], [ "i", "m" ], [ "u", "t" ], [ "u", "r" ], [ ";", "Ċ" ], [ "a", "d" ], [ "Ð", "»" ], [ "o", "l" ], [ "Ġ", "g" ], [ "Ġ", "v" ], [ "ç", "»" ], [ "Ġ", "u" ], [ "ä", "¹" ], [ "Ġd", "e" ], [ "æ", "Ī" ], [ "å", "Ĭ" ], [ "Ġ", "S" ], [ "i", "g" ], [ "ä¸", "Ģ" ], [ "s", "e" ], [ "i", "v" ], [ "ã", "ģ" ], [ "o", "t" ], [ "å", "Ľ" ], [ "è", "¯" ], [ "æĺ", "¯" ], [ "æ", "ĸ" ], [ "ĠĠĠĠ", "ĠĠĠ" ], [ "ĠĠĠ", "ĠĠĠĠ" ], [ "Ġ", "T" ], [ "c", "e" ], [ "Ġ", "=" ], [ "Ċ", "Ċ" ], [ "Ñ", "ĥ" ], [ "Ð", "º" ], [ "å", "°" ], [ "Ġ", "A" ], [ "ĠĠĠĠĠĠĠĠ", "ĠĠĠĠĠĠĠĠ" ], [ "Ġ", "y" ], [ "å", "Ń" ], [ "Ġ", "(" ], [ "Ġ", "C" ], [ "æ", "Ĺ" ], [ "at", "ion" ], [ "q", "u" ], [ "r", "a" ], [ "Ġ", "I" ], [ "u", "l" ], [ "-", "-" ], [ "e", "m" ], [ "ãĢ", "ģ" ], [ "o", "s" ], [ "Ġ", "is" ], [ "è", "®" ], [ "o", "w" ], [ "ä", "¼" ], [ "å", "¹" ], [ "o", "d" ], [ "i", "f" ], [ "e", "st" ], [ "es", "t" ], [ "å", "į" ], [ "Ġ", "st" ], [ "Ġs", "t" ], [ "a", "g" ], [ "Ñ", "ı" ], [ "æ", "Ŀ" ], [ "åľ", "¨" ], [ "Ġf", "or" ], [ "u", "n" ], [ "Ġc", "on" ], [ "ä¸", "į" ], [ "Ø", "§" ], [ "æľ", "ī" ], [ "Ð", "²" ], [ "er", "s" ], [ "i", "r" ], [ "Ð", "¼" ], [ "Ġ", "{" ], [ "Ñ", "ĭ" ], [ "Ġb", "e" ], [ "Ġ", "\"" ], [ "a", "y" ], [ "äº", "º" ], [ "æ", "ī" ], [ "å", "¼" ], [ "a", "b" ], [ "é", "ĩ" ], [ "t", "er" ], [ "te", "r" ], [ "Ġ", "M" ], [ "å", "º" ], [ "Ġ", "P" ], [ "a", "te" ], [ "at", "e" ], [ "t", "h" ], [ "äº", "Ĩ" ], [ "Ġ", "on" ], [ "Ġo", "n" ], [ "l", "y" ], [ "å", "·" ], [ "è", "¡" ], [ "u", "s" ], [ "ĠÐ", "¿" ], [ "a", "c" ], [ "è", "Ģ" ], [ "ĉ", "ĉ" ], [ "æ", "ķ" ], [ "å", "Ĩ" ], [ "Ñ", "Į" ], [ "å", "ĩ" ], [ "i", "e" ], [ "å", "¯" ], [ "å", "¸" ], [ "Ġy", "ou" ], [ "å", "¾" ], [ "Ġp", "ro" ], [ "Ð", "´" ], [ "p", "e" ], [ "ä¸", "Ń" ], [ "u", "m" ], [ "å", "Į" ], [ "i", "th" ], [ "it", "h" ], [ "i", "st" ], [ "is", "t" ], [ "é", "Ģ" ], [ "ä¸", "º" ], [ "Ñģ", "ÑĤ" ], [ "æ", "Ģ" ], [ "Ġ", "al" ], [ "Ġa", "l" ], [ "o", "c" ], [ "Ã", "©" ], [ "Ġ", "B" ], [ "ç", "Ľ" ], [ "å", "Ĵ" ], [ "Ġ", "D" ], [ "Ġth", "at" ], [ "or", "t" ], [ "v", "e" ], [ "r", "es" ], [ "re", "s" ], [ "è", "§" ], [ "ĥ", "½" ], [ "Ġ", "k" ], [ "v", "er" ], [ "ve", "r" ], [ "es", "s" ], [ "t", "e" ], [ "e", "nd" ], [ "en", "d" ], [ "Ġ", "an" ], [ "Ġa", "n" ], [ "Ġw", "h" ], [ "æ", "ł" ], [ "åĴ", "Į" ], [ "a", "nd" ], [ "an", "d" ], [ "Ġ", "в" ], [ "ĠÐ", "²" ], [ ",", "Ċ" ], [ "Ġ{", "Ċ" ], [ "h", "t" ], [ "ç", "§" ], [ ")", "Ċ" ], [ ")", ";Ċ" ], [ "/", "/" ] ] } }