{
    "version": "1.0",
    "truncation": null,
    "padding": null,
    "added_tokens": [
        {
            "id": 0,
            "content": "<|endoftext|>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        },
        {
            "id": 1,
            "content": "<|padding|>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        },
        {
            "id": 50254,
            "content": "                        ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50255,
            "content": "                       ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50256,
            "content": "                      ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50257,
            "content": "                     ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50258,
            "content": "                    ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50259,
            "content": "                   ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50260,
            "content": "                  ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50261,
            "content": "                 ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50262,
            "content": "                ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50263,
            "content": "               ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50264,
            "content": "              ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50265,
            "content": "             ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50266,
            "content": "            ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50267,
            "content": "           ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50268,
            "content": "          ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50269,
            "content": "         ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50270,
            "content": "        ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50271,
            "content": "       ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50272,
            "content": "      ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50273,
            "content": "     ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50274,
            "content": "    ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50275,
            "content": "   ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50276,
            "content": "  ",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": true,
            "special": false
        },
        {
            "id": 50277,
            "content": "<|im_start|>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        },
        {
            "id": 50278,
            "content": "<|im_end|>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        }
    ],
    "normalizer": {
        "type": "NFC"
    },
    "pre_tokenizer": {
        "type": "ByteLevel",
        "add_prefix_space": false,
        "trim_offsets": true,
        "use_regex": true
    },
    "post_processor": {
        "type": "ByteLevel",
        "add_prefix_space": false,
        "trim_offsets": true,
        "use_regex": true
    },
    "decoder": {
        "type": "ByteLevel",
        "add_prefix_space": false,
        "trim_offsets": true,
        "use_regex": true
    },
    "model": {
        "type": "BPE",
        "dropout": null,
        "unk_token": null,
        "continuing_subword_prefix": null,
        "end_of_word_suffix": null,
        "fuse_unk": false,
        "byte_fallback": false,
        "vocab": null,
        "merges": null
    }
}