Tokenizer_Test.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyN3kQKzIXK68Q6sL6h/umus",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bebechien/gemma/blob/main/Tokenizer_Test.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Setup & Install dependencies"
      ],
      "metadata": {
        "id": "2SNOwNf6OV28"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from google.colab import userdata\n",
        "\n",
        "# Note: `userdata.get` is a Colab API. If you're not using Colab, set the env\n",
        "# vars as appropriate for your system.\n",
        "os.environ[\"KAGGLE_USERNAME\"] = userdata.get(\"KAGGLE_USERNAME\")\n",
        "os.environ[\"KAGGLE_KEY\"] = userdata.get(\"KAGGLE_KEY\")\n",
        "\n",
        "!pip install -q -U keras-nlp\n",
        "!pip install -q -U keras\n",
        "\n",
        "# Set the backbend before importing Keras\n",
        "os.environ[\"KERAS_BACKEND\"] = \"jax\"\n",
        "# Avoid memory fragmentation on JAX backend.\n",
        "os.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"] = \"1.00\"\n",
        "\n",
        "import keras_nlp"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CcMRCmR4OVdx",
        "outputId": "034e6083-9b55-4838-f6f7-eff5354413b4"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/548.4 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K   \u001b[91m━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.4/548.4 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K   \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m542.7/548.4 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m548.4/548.4 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.2/5.2 MB\u001b[0m \u001b[31m82.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### How to check the Tokenizer already have the name in vocabulary"
      ],
      "metadata": {
        "id": "0DSjJE4aOKc7"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wp2aaKJ8OE-y",
        "outputId": "307f425b-dff7-4c37-f5e1-869882431f93"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[   108  56985  32012 235465  78638   4758 235394  79900   4758 235394\n",
            " 122224   4758 235394  98647   4758 235394 119177   4758 235394 169944\n",
            "   4758 235394 113698   4758 235394 119392   4758 235394 117774   4758\n",
            " 235394 140762   4758    108 235308 235276 235274 235893 235465 235585\n",
            " 240763   4758 235394 239713 235493   4758 235394 235469 236872   4758\n",
            " 235394 241122 235722   4758 235394 238303 235841   4758 235394 157967\n",
            "   4758 235394 238803 238313   4758 235394 235861 235875   4758 235394\n",
            " 236447 236063   4758 235394 240052 236872   4758    108 235304 235324\n",
            " 235276 235276 235274 235893 235465 235661 236316   4758 235394 238881\n",
            " 235842   4758 235394 235469 239319   4758 235394 237338 235502   4758\n",
            " 235394 237550 236228   4758 235394 235648 235858   4758 235394 243141\n",
            " 238022   4758 235394 235771    450    389    391   4758 235394 236497\n",
            " 241335   4758 235394 235990 242446   4758    108]\n",
            "   108 -> \n",
            "\n",
            " 56985 -> トップ\n",
            " 32012 -> テン\n",
            "235465 -> ：\n",
            " 78638 -> 佐藤\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            " 79900 -> 鈴木\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "122224 -> 高橋\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            " 98647 -> 田中\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "119177 -> 伊藤\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "169944 -> 渡辺\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "113698 -> 山本\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "119392 -> 中村\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "117774 -> 小林\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "140762 -> 加藤\n",
            "  4758 -> さん\n",
            "   108 -> \n",
            "\n",
            "235308 -> 5\n",
            "235276 -> 0\n",
            "235274 -> 1\n",
            "235893 -> 位\n",
            "235465 -> ：\n",
            "235585 -> 小\n",
            "240763 -> 嶋\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "239713 -> 畑\n",
            "235493 -> 中\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "235469 -> 大\n",
            "236872 -> 井\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "241122 -> 磯\n",
            "235722 -> 部\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "238303 -> 浅\n",
            "235841 -> 見\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "157967 -> 秋田\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "238803 -> 芳\n",
            "238313 -> 賀\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "235861 -> 相\n",
            "235875 -> 原\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "236447 -> 細\n",
            "236063 -> 田\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "240052 -> 坪\n",
            "236872 -> 井\n",
            "  4758 -> さん\n",
            "   108 -> \n",
            "\n",
            "235304 -> 3\n",
            "235324 -> 7\n",
            "235276 -> 0\n",
            "235276 -> 0\n",
            "235274 -> 1\n",
            "235893 -> 位\n",
            "235465 -> ：\n",
            "235661 -> 法\n",
            "236316 -> 川\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "238881 -> 乙\n",
            "235842 -> 間\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "235469 -> 大\n",
            "239319 -> 舌\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "237338 -> 巻\n",
            "235502 -> 上\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "237550 -> 戸\n",
            "236228 -> 住\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "235648 -> 前\n",
            "235858 -> 更\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "243141 -> 梶\n",
            "238022 -> 浜\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "235771 -> 加\n",
            "   450 -> �\n",
            "   389 -> �\n",
            "   391 -> �\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "236497 -> 千\n",
            "241335 -> 艘\n",
            "  4758 -> さん\n",
            "235394 -> 、\n",
            "235990 -> 西\n",
            "242446 -> 胤\n",
            "  4758 -> さん\n",
            "   108 -> \n",
            "\n"
          ]
        }
      ],
      "source": [
        "tokenizer = keras_nlp.models.GemmaTokenizer.from_preset(\"gemma2_instruct_2b_en\")\n",
        "\n",
        "def detoken(tokens):\n",
        "  print(tokens)\n",
        "  for x in tokens:\n",
        "    word = tokenizer.detokenize([x])\n",
        "    print(f\"{x:6} -> {word}\")\n",
        "\n",
        "detoken(tokenizer(\"\"\"\n",
        "トップテン：佐藤さん、鈴木さん、高橋さん、田中さん、伊藤さん、渡辺さん、山本さん、中村さん、小林さん、加藤さん\n",
        "501位：小嶋さん、畑中さん、大井さん、磯部さん、浅見さん、秋田さん、芳賀さん、相原さん、細田さん、坪井さん\n",
        "37001位：法川さん、乙間さん、大舌さん、巻上さん、戸住さん、前更さん、梶浜さん、加鬮さん、千艘さん、西胤さん\n",
        "\"\"\"))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenizer.id_to_token(450))\n",
        "print(tokenizer.id_to_token(389))\n",
        "print(tokenizer.id_to_token(391))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pMFgoID3FAcJ",
        "outputId": "970ea086-f42c-4a19-9bf6-2b92b46d75af"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<0xE9>\n",
            "<0xAC>\n",
            "<0xAE>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.get_vocabulary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ggigihbgvkLc",
        "outputId": "63280078-658f-4520-c1da-2498bc91cfbc"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['<pad>',\n",
              " '<eos>',\n",
              " '<bos>',\n",
              " '<unk>',\n",
              " '<mask>',\n",
              " '<2mass>',\n",
              " '[@BOS@]',\n",
              " '<unused0>',\n",
              " '<unused1>',\n",
              " '<unused2>',\n",
              " '<unused3>',\n",
              " '<unused4>',\n",
              " '<unused5>',\n",
              " '<unused6>',\n",
              " '<unused7>',\n",
              " '<unused8>',\n",
              " '<unused9>',\n",
              " '<unused10>',\n",
              " '<unused11>',\n",
              " '<unused12>',\n",
              " '<unused13>',\n",
              " '<unused14>',\n",
              " '<unused15>',\n",
              " '<unused16>',\n",
              " '<unused17>',\n",
              " '<unused18>',\n",
              " '<unused19>',\n",
              " '<unused20>',\n",
              " '<unused21>',\n",
              " '<unused22>',\n",
              " '<unused23>',\n",
              " '<unused24>',\n",
              " '<unused25>',\n",
              " '<unused26>',\n",
              " '<unused27>',\n",
              " '<unused28>',\n",
              " '<unused29>',\n",
              " '<unused30>',\n",
              " '<unused31>',\n",
              " '<unused32>',\n",
              " '<unused33>',\n",
              " '<unused34>',\n",
              " '<unused35>',\n",
              " '<unused36>',\n",
              " '<unused37>',\n",
              " '<unused38>',\n",
              " '<unused39>',\n",
              " '<unused40>',\n",
              " '<unused41>',\n",
              " '<unused42>',\n",
              " '<unused43>',\n",
              " '<unused44>',\n",
              " '<unused45>',\n",
              " '<unused46>',\n",
              " '<unused47>',\n",
              " '<unused48>',\n",
              " '<unused49>',\n",
              " '<unused50>',\n",
              " '<unused51>',\n",
              " '<unused52>',\n",
              " '<unused53>',\n",
              " '<unused54>',\n",
              " '<unused55>',\n",
              " '<unused56>',\n",
              " '<unused57>',\n",
              " '<unused58>',\n",
              " '<unused59>',\n",
              " '<unused60>',\n",
              " '<unused61>',\n",
              " '<unused62>',\n",
              " '<unused63>',\n",
              " '<unused64>',\n",
              " '<unused65>',\n",
              " '<unused66>',\n",
              " '<unused67>',\n",
              " '<unused68>',\n",
              " '<unused69>',\n",
              " '<unused70>',\n",
              " '<unused71>',\n",
              " '<unused72>',\n",
              " '<unused73>',\n",
              " '<unused74>',\n",
              " '<unused75>',\n",
              " '<unused76>',\n",
              " '<unused77>',\n",
              " '<unused78>',\n",
              " '<unused79>',\n",
              " '<unused80>',\n",
              " '<unused81>',\n",
              " '<unused82>',\n",
              " '<unused83>',\n",
              " '<unused84>',\n",
              " '<unused85>',\n",
              " '<unused86>',\n",
              " '<unused87>',\n",
              " '<unused88>',\n",
              " '<unused89>',\n",
              " '<unused90>',\n",
              " '<unused91>',\n",
              " '<unused92>',\n",
              " '<unused93>',\n",
              " '<unused94>',\n",
              " '<unused95>',\n",
              " '<unused96>',\n",
              " '<unused97>',\n",
              " '<unused98>',\n",
              " '<start_of_turn>',\n",
              " '<end_of_turn>',\n",
              " '\\n',\n",
              " '\\n\\n',\n",
              " '\\n\\n\\n',\n",
              " '\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n',\n",
              " '▁▁',\n",
              " '▁▁▁',\n",
              " '▁▁▁▁',\n",
              " '▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁',\n",
              " '<table>',\n",
              " '<caption>',\n",
              " '<thead>',\n",
              " '<tbody>',\n",
              " '<tfoot>',\n",
              " '<tr>',\n",
              " '<th>',\n",
              " '<td>',\n",
              " '</table>',\n",
              " '</caption>',\n",
              " '</thead>',\n",
              " '</tbody>',\n",
              " '</tfoot>',\n",
              " '</tr>',\n",
              " '</th>',\n",
              " '</td>',\n",
              " '<h1>',\n",
              " '<h2>',\n",
              " '<h3>',\n",
              " '<h4>',\n",
              " '<h5>',\n",
              " '<h6>',\n",
              " '<blockquote>',\n",
              " '</h1>',\n",
              " '</h2>',\n",
              " '</h3>',\n",
              " '</h4>',\n",
              " '</h5>',\n",
              " '</h6>',\n",
              " '</blockquote>',\n",
              " '<strong>',\n",
              " '<em>',\n",
              " '<b>',\n",
              " '<i>',\n",
              " '<u>',\n",
              " '<s>',\n",
              " '<sub>',\n",
              " '<sup>',\n",
              " '<code>',\n",
              " '</strong>',\n",
              " '</em>',\n",
              " '</b>',\n",
              " '</i>',\n",
              " '</u>',\n",
              " '</s>',\n",
              " '</sub>',\n",
              " '</sup>',\n",
              " '</code>',\n",
              " '<0x00>',\n",
              " '<0x01>',\n",
              " '<0x02>',\n",
              " '<0x03>',\n",
              " '<0x04>',\n",
              " '<0x05>',\n",
              " '<0x06>',\n",
              " '<0x07>',\n",
              " '<0x08>',\n",
              " '<0x09>',\n",
              " '<0x0A>',\n",
              " '<0x0B>',\n",
              " '<0x0C>',\n",
              " '<0x0D>',\n",
              " '<0x0E>',\n",
              " '<0x0F>',\n",
              " '<0x10>',\n",
              " '<0x11>',\n",
              " '<0x12>',\n",
              " '<0x13>',\n",
              " '<0x14>',\n",
              " '<0x15>',\n",
              " '<0x16>',\n",
              " '<0x17>',\n",
              " '<0x18>',\n",
              " '<0x19>',\n",
              " '<0x1A>',\n",
              " '<0x1B>',\n",
              " '<0x1C>',\n",
              " '<0x1D>',\n",
              " '<0x1E>',\n",
              " '<0x1F>',\n",
              " '<0x20>',\n",
              " '<0x21>',\n",
              " '<0x22>',\n",
              " '<0x23>',\n",
              " '<0x24>',\n",
              " '<0x25>',\n",
              " '<0x26>',\n",
              " '<0x27>',\n",
              " '<0x28>',\n",
              " '<0x29>',\n",
              " '<0x2A>',\n",
              " '<0x2B>',\n",
              " '<0x2C>',\n",
              " '<0x2D>',\n",
              " '<0x2E>',\n",
              " '<0x2F>',\n",
              " '<0x30>',\n",
              " '<0x31>',\n",
              " '<0x32>',\n",
              " '<0x33>',\n",
              " '<0x34>',\n",
              " '<0x35>',\n",
              " '<0x36>',\n",
              " '<0x37>',\n",
              " '<0x38>',\n",
              " '<0x39>',\n",
              " '<0x3A>',\n",
              " '<0x3B>',\n",
              " '<0x3C>',\n",
              " '<0x3D>',\n",
              " '<0x3E>',\n",
              " '<0x3F>',\n",
              " '<0x40>',\n",
              " '<0x41>',\n",
              " '<0x42>',\n",
              " '<0x43>',\n",
              " '<0x44>',\n",
              " '<0x45>',\n",
              " '<0x46>',\n",
              " '<0x47>',\n",
              " '<0x48>',\n",
              " '<0x49>',\n",
              " '<0x4A>',\n",
              " '<0x4B>',\n",
              " '<0x4C>',\n",
              " '<0x4D>',\n",
              " '<0x4E>',\n",
              " '<0x4F>',\n",
              " '<0x50>',\n",
              " '<0x51>',\n",
              " '<0x52>',\n",
              " '<0x53>',\n",
              " '<0x54>',\n",
              " '<0x55>',\n",
              " '<0x56>',\n",
              " '<0x57>',\n",
              " '<0x58>',\n",
              " '<0x59>',\n",
              " '<0x5A>',\n",
              " '<0x5B>',\n",
              " '<0x5C>',\n",
              " '<0x5D>',\n",
              " '<0x5E>',\n",
              " '<0x5F>',\n",
              " '<0x60>',\n",
              " '<0x61>',\n",
              " '<0x62>',\n",
              " '<0x63>',\n",
              " '<0x64>',\n",
              " '<0x65>',\n",
              " '<0x66>',\n",
              " '<0x67>',\n",
              " '<0x68>',\n",
              " '<0x69>',\n",
              " '<0x6A>',\n",
              " '<0x6B>',\n",
              " '<0x6C>',\n",
              " '<0x6D>',\n",
              " '<0x6E>',\n",
              " '<0x6F>',\n",
              " '<0x70>',\n",
              " '<0x71>',\n",
              " '<0x72>',\n",
              " '<0x73>',\n",
              " '<0x74>',\n",
              " '<0x75>',\n",
              " '<0x76>',\n",
              " '<0x77>',\n",
              " '<0x78>',\n",
              " '<0x79>',\n",
              " '<0x7A>',\n",
              " '<0x7B>',\n",
              " '<0x7C>',\n",
              " '<0x7D>',\n",
              " '<0x7E>',\n",
              " '<0x7F>',\n",
              " '<0x80>',\n",
              " '<0x81>',\n",
              " '<0x82>',\n",
              " '<0x83>',\n",
              " '<0x84>',\n",
              " '<0x85>',\n",
              " '<0x86>',\n",
              " '<0x87>',\n",
              " '<0x88>',\n",
              " '<0x89>',\n",
              " '<0x8A>',\n",
              " '<0x8B>',\n",
              " '<0x8C>',\n",
              " '<0x8D>',\n",
              " '<0x8E>',\n",
              " '<0x8F>',\n",
              " '<0x90>',\n",
              " '<0x91>',\n",
              " '<0x92>',\n",
              " '<0x93>',\n",
              " '<0x94>',\n",
              " '<0x95>',\n",
              " '<0x96>',\n",
              " '<0x97>',\n",
              " '<0x98>',\n",
              " '<0x99>',\n",
              " '<0x9A>',\n",
              " '<0x9B>',\n",
              " '<0x9C>',\n",
              " '<0x9D>',\n",
              " '<0x9E>',\n",
              " '<0x9F>',\n",
              " '<0xA0>',\n",
              " '<0xA1>',\n",
              " '<0xA2>',\n",
              " '<0xA3>',\n",
              " '<0xA4>',\n",
              " '<0xA5>',\n",
              " '<0xA6>',\n",
              " '<0xA7>',\n",
              " '<0xA8>',\n",
              " '<0xA9>',\n",
              " '<0xAA>',\n",
              " '<0xAB>',\n",
              " '<0xAC>',\n",
              " '<0xAD>',\n",
              " '<0xAE>',\n",
              " '<0xAF>',\n",
              " '<0xB0>',\n",
              " '<0xB1>',\n",
              " '<0xB2>',\n",
              " '<0xB3>',\n",
              " '<0xB4>',\n",
              " '<0xB5>',\n",
              " '<0xB6>',\n",
              " '<0xB7>',\n",
              " '<0xB8>',\n",
              " '<0xB9>',\n",
              " '<0xBA>',\n",
              " '<0xBB>',\n",
              " '<0xBC>',\n",
              " '<0xBD>',\n",
              " '<0xBE>',\n",
              " '<0xBF>',\n",
              " '<0xC0>',\n",
              " '<0xC1>',\n",
              " '<0xC2>',\n",
              " '<0xC3>',\n",
              " '<0xC4>',\n",
              " '<0xC5>',\n",
              " '<0xC6>',\n",
              " '<0xC7>',\n",
              " '<0xC8>',\n",
              " '<0xC9>',\n",
              " '<0xCA>',\n",
              " '<0xCB>',\n",
              " '<0xCC>',\n",
              " '<0xCD>',\n",
              " '<0xCE>',\n",
              " '<0xCF>',\n",
              " '<0xD0>',\n",
              " '<0xD1>',\n",
              " '<0xD2>',\n",
              " '<0xD3>',\n",
              " '<0xD4>',\n",
              " '<0xD5>',\n",
              " '<0xD6>',\n",
              " '<0xD7>',\n",
              " '<0xD8>',\n",
              " '<0xD9>',\n",
              " '<0xDA>',\n",
              " '<0xDB>',\n",
              " '<0xDC>',\n",
              " '<0xDD>',\n",
              " '<0xDE>',\n",
              " '<0xDF>',\n",
              " '<0xE0>',\n",
              " '<0xE1>',\n",
              " '<0xE2>',\n",
              " '<0xE3>',\n",
              " '<0xE4>',\n",
              " '<0xE5>',\n",
              " '<0xE6>',\n",
              " '<0xE7>',\n",
              " '<0xE8>',\n",
              " '<0xE9>',\n",
              " '<0xEA>',\n",
              " '<0xEB>',\n",
              " '<0xEC>',\n",
              " '<0xED>',\n",
              " '<0xEE>',\n",
              " '<0xEF>',\n",
              " '<0xF0>',\n",
              " '<0xF1>',\n",
              " '<0xF2>',\n",
              " '<0xF3>',\n",
              " '<0xF4>',\n",
              " '<0xF5>',\n",
              " '<0xF6>',\n",
              " '<0xF7>',\n",
              " '<0xF8>',\n",
              " '<0xF9>',\n",
              " '<0xFA>',\n",
              " '<0xFB>',\n",
              " '<0xFC>',\n",
              " '<0xFD>',\n",
              " '<0xFE>',\n",
              " '<0xFF>',\n",
              " 'in',\n",
              " '▁t',\n",
              " 'er',\n",
              " '▁a',\n",
              " 'on',\n",
              " 're',\n",
              " 'en',\n",
              " 'he',\n",
              " 'an',\n",
              " 'at',\n",
              " 'or',\n",
              " 'es',\n",
              " '▁s',\n",
              " 'ar',\n",
              " 'ti',\n",
              " 'te',\n",
              " 'th',\n",
              " 'st',\n",
              " 'nd',\n",
              " 'al',\n",
              " '▁o',\n",
              " 'le',\n",
              " 'de',\n",
              " '▁i',\n",
              " 'se',\n",
              " '▁c',\n",
              " '▁d',\n",
              " 'it',\n",
              " 'nt',\n",
              " 'is',\n",
              " '▁p',\n",
              " 'me',\n",
              " 'ri',\n",
              " 'ra',\n",
              " 'ou',\n",
              " 'as',\n",
              " 'ed',\n",
              " 'ne',\n",
              " 'to',\n",
              " 'ng',\n",
              " '▁w',\n",
              " 'ro',\n",
              " 'li',\n",
              " 'ta',\n",
              " '▁f',\n",
              " '▁b',\n",
              " '▁m',\n",
              " 'ic',\n",
              " 'el',\n",
              " 'la',\n",
              " 'et',\n",
              " 've',\n",
              " 'ur',\n",
              " '▁e',\n",
              " 'ha',\n",
              " 'co',\n",
              " 'll',\n",
              " 'ch',\n",
              " '▁h',\n",
              " 'ce',\n",
              " '▁l',\n",
              " 'ma',\n",
              " 'ea',\n",
              " 'io',\n",
              " 'om',\n",
              " 'tr',\n",
              " 'id',\n",
              " 'il',\n",
              " 'ge',\n",
              " 'si',\n",
              " 'di',\n",
              " 'hi',\n",
              " 'lo',\n",
              " 'ec',\n",
              " 'ie',\n",
              " '▁r',\n",
              " 'un',\n",
              " 'ac',\n",
              " 'ct',\n",
              " '▁n',\n",
              " 'us',\n",
              " 'pe',\n",
              " 'be',\n",
              " 'na',\n",
              " 'ca',\n",
              " 'ns',\n",
              " 'of',\n",
              " 'ut',\n",
              " 'ol',\n",
              " 'ot',\n",
              " 'am',\n",
              " 'ss',\n",
              " 'em',\n",
              " 'ad',\n",
              " 'po',\n",
              " 'os',\n",
              " 'pa',\n",
              " '▁S',\n",
              " 'im',\n",
              " 'ho',\n",
              " '▁the',\n",
              " 'ing',\n",
              " '▁in',\n",
              " '▁of',\n",
              " '▁to',\n",
              " '▁and',\n",
              " 'ent',\n",
              " '▁th',\n",
              " '▁de',\n",
              " '▁re',\n",
              " '▁g',\n",
              " '▁T',\n",
              " '▁C',\n",
              " '▁A',\n",
              " 'ion',\n",
              " 'tion',\n",
              " '▁=',\n",
              " '▁I',\n",
              " '▁(',\n",
              " 'qu',\n",
              " '▁v',\n",
              " ');',\n",
              " '▁M',\n",
              " '▁P',\n",
              " '▁y',\n",
              " 'ver',\n",
              " '▁B',\n",
              " 'um',\n",
              " 'ul',\n",
              " 'ig',\n",
              " '▁is',\n",
              " '▁for',\n",
              " 'ly',\n",
              " '▁u',\n",
              " 'ate',\n",
              " '▁D',\n",
              " 'ow',\n",
              " 'ter',\n",
              " '▁on',\n",
              " '▁{',\n",
              " '▁wi',\n",
              " '▁be',\n",
              " 'end',\n",
              " 'ir',\n",
              " 'ts',\n",
              " 'ers',\n",
              " 'ue',\n",
              " 'hat',\n",
              " 'ati',\n",
              " 'ine',\n",
              " 'ck',\n",
              " 'res',\n",
              " '▁R',\n",
              " 'ay',\n",
              " '=\"',\n",
              " 'gh',\n",
              " '▁L',\n",
              " 'ons',\n",
              " 'men',\n",
              " '▁con',\n",
              " '▁F',\n",
              " '//',\n",
              " 'int',\n",
              " 'cti',\n",
              " '▁E',\n",
              " '▁k',\n",
              " 'and',\n",
              " '▁H',\n",
              " 'ort',\n",
              " 'our',\n",
              " 'od',\n",
              " 'est',\n",
              " '()',\n",
              " '▁N',\n",
              " '▁W',\n",
              " 'if',\n",
              " '▁*',\n",
              " 'ht',\n",
              " 'The',\n",
              " 'ith',\n",
              " '▁G',\n",
              " 'ere',\n",
              " 'ke',\n",
              " '▁pro',\n",
              " 'ub',\n",
              " 'pp',\n",
              " '▁en',\n",
              " 'ble',\n",
              " '▁ha',\n",
              " '\\\\[',\n",
              " '\\\\]',\n",
              " '▁\"',\n",
              " '▁it',\n",
              " 'oo',\n",
              " 'ci',\n",
              " 'all',\n",
              " 'ess',\n",
              " 'ab',\n",
              " '▁an',\n",
              " 'ret',\n",
              " 'ation',\n",
              " '▁that',\n",
              " '▁with',\n",
              " 'ction',\n",
              " 'ment',\n",
              " '▁j',\n",
              " '▁ne',\n",
              " '▁st',\n",
              " '▁com',\n",
              " '▁me',\n",
              " '▁la',\n",
              " 'ity',\n",
              " '▁as',\n",
              " 'tem',\n",
              " '▁O',\n",
              " '**',\n",
              " '▁or',\n",
              " 'urn',\n",
              " 'ted',\n",
              " '▁you',\n",
              " '▁he',\n",
              " 'ist',\n",
              " 'rom',\n",
              " '▁at',\n",
              " '▁$',\n",
              " '▁ex',\n",
              " '▁se',\n",
              " '▁qu',\n",
              " 'mp',\n",
              " '▁li',\n",
              " 'line',\n",
              " 'per',\n",
              " '▁mo',\n",
              " 'ld',\n",
              " 'port',\n",
              " '▁are',\n",
              " '▁le',\n",
              " '}\\r',\n",
              " 'ure',\n",
              " '▁so',\n",
              " '▁J',\n",
              " '▁The',\n",
              " 'op',\n",
              " 'iz',\n",
              " '▁al',\n",
              " '▁ma',\n",
              " 'ass',\n",
              " 'lic',\n",
              " 'xt',\n",
              " 'ore',\n",
              " '..',\n",
              " '▁&',\n",
              " 'ite',\n",
              " '/*',\n",
              " '</',\n",
              " '▁-',\n",
              " '▁was',\n",
              " '▁\\\\',\n",
              " '▁by',\n",
              " 'age',\n",
              " '▁wh',\n",
              " 'ame',\n",
              " 'ard',\n",
              " '▁this',\n",
              " 'ag',\n",
              " 'ak',\n",
              " 'ust',\n",
              " 'ia',\n",
              " 'ant',\n",
              " '▁su',\n",
              " 'hl',\n",
              " '▁V',\n",
              " 'out',\n",
              " 'for',\n",
              " '▁K',\n",
              " '▁un',\n",
              " '▁do',\n",
              " 'ap',\n",
              " '▁di',\n",
              " '▁U',\n",
              " 'ath',\n",
              " 'ang',\n",
              " '▁par',\n",
              " 'ty',\n",
              " 'ord',\n",
              " 'ber',\n",
              " 'con',\n",
              " 'art',\n",
              " 'ff',\n",
              " 'ys',\n",
              " 'du',\n",
              " 'ome',\n",
              " 'tu',\n",
              " 'ze',\n",
              " 'ind',\n",
              " '\">',\n",
              " '▁po',\n",
              " 'ex',\n",
              " 'mat',\n",
              " 'las',\n",
              " 'return',\n",
              " '▁from',\n",
              " '/**',\n",
              " 'hline',\n",
              " \"▁'\",\n",
              " 'gin',\n",
              " 'ans',\n",
              " '▁not',\n",
              " 'rou',\n",
              " 'av',\n",
              " '▁we',\n",
              " 'ata',\n",
              " 'one',\n",
              " 'cl',\n",
              " 'ack',\n",
              " '▁ch',\n",
              " '},',\n",
              " 'ther',\n",
              " '▁have',\n",
              " 'au',\n",
              " '▁no',\n",
              " '--',\n",
              " 'ase',\n",
              " 'ould',\n",
              " 'ich',\n",
              " '▁can',\n",
              " '▁tr',\n",
              " 'sed',\n",
              " 'sp',\n",
              " 'tic',\n",
              " 'ud',\n",
              " 'ote',\n",
              " 'ublic',\n",
              " 'ell',\n",
              " '▁id',\n",
              " 'ong',\n",
              " 'import',\n",
              " 'ep',\n",
              " 'ect',\n",
              " 'ks',\n",
              " '▁ar',\n",
              " '};',\n",
              " 'ght',\n",
              " 'oc',\n",
              " 'sion',\n",
              " 'get',\n",
              " 'ous',\n",
              " 'ru',\n",
              " '();',\n",
              " '▁el',\n",
              " '),',\n",
              " '\",',\n",
              " 'mo',\n",
              " '▁lo',\n",
              " 'ran',\n",
              " 'ary',\n",
              " '▁*/',\n",
              " 'cc',\n",
              " 'ry',\n",
              " '▁all',\n",
              " 'ps',\n",
              " 'ount',\n",
              " '▁ad',\n",
              " 'ph',\n",
              " 'ork',\n",
              " 'lass',\n",
              " 'ound',\n",
              " 'ated',\n",
              " '▁ab',\n",
              " '▁per',\n",
              " 'lock',\n",
              " 'ire',\n",
              " 'ear',\n",
              " ').',\n",
              " '▁sp',\n",
              " '▁des',\n",
              " 'ER',\n",
              " 'ven',\n",
              " 'ance',\n",
              " 'va',\n",
              " 'lu',\n",
              " 'red',\n",
              " 'ult',\n",
              " 'iew',\n",
              " 'erv',\n",
              " 'ally',\n",
              " '});',\n",
              " '*/',\n",
              " '▁your',\n",
              " 'ile',\n",
              " 'ice',\n",
              " 'che',\n",
              " '▁:',\n",
              " 'der',\n",
              " '▁sa',\n",
              " '▁z',\n",
              " 'og',\n",
              " 'fer',\n",
              " '▁go',\n",
              " 'com',\n",
              " 'ON',\n",
              " 'row',\n",
              " '▁es',\n",
              " 'ence',\n",
              " '▁will',\n",
              " '▁In',\n",
              " 'public',\n",
              " 'ign',\n",
              " 'set',\n",
              " 'ime',\n",
              " 'this',\n",
              " 'item',\n",
              " '(\"',\n",
              " 'In',\n",
              " 'able',\n",
              " '▁new',\n",
              " 'ize',\n",
              " '▁Y',\n",
              " 'ull',\n",
              " '▁[',\n",
              " 'vi',\n",
              " 'vel',\n",
              " 'are',\n",
              " 'ont',\n",
              " 'ast',\n",
              " '▁п',\n",
              " '▁ro',\n",
              " 'ple',\n",
              " '▁but',\n",
              " 'begin',\n",
              " 'ations',\n",
              " '▁Th',\n",
              " '__',\n",
              " '▁uid',\n",
              " '▁que',\n",
              " '▁up',\n",
              " 'tring',\n",
              " '▁co',\n",
              " 'ight',\n",
              " 'ст',\n",
              " 'div',\n",
              " 'ide',\n",
              " 'ces',\n",
              " '▁man',\n",
              " '▁us',\n",
              " 'ial',\n",
              " '▁has',\n",
              " \"',\",\n",
              " '▁out',\n",
              " '▁ا',\n",
              " 'block',\n",
              " 'ies',\n",
              " 'lay',\n",
              " '▁his',\n",
              " 'section',\n",
              " 'IN',\n",
              " 'ory',\n",
              " '▁und',\n",
              " 'so',\n",
              " 'sel',\n",
              " 'ook',\n",
              " 'val',\n",
              " 'ft',\n",
              " 'ill',\n",
              " 'log',\n",
              " 'ption',\n",
              " 'ded',\n",
              " 'ni',\n",
              " '▁с',\n",
              " '▁comp',\n",
              " '▁sh',\n",
              " 'ree',\n",
              " 'quote',\n",
              " 'Th',\n",
              " '▁get',\n",
              " '▁which',\n",
              " '->',\n",
              " 'ater',\n",
              " '▁pid',\n",
              " 'ens',\n",
              " '▁pre',\n",
              " '▁.',\n",
              " '...',\n",
              " 'no',\n",
              " 'iv',\n",
              " 'Con',\n",
              " '▁ti',\n",
              " 'pri',\n",
              " 'form',\n",
              " '{\\r',\n",
              " '▁+',\n",
              " 'par',\n",
              " 'ten',\n",
              " 'we',\n",
              " 'ach',\n",
              " '▁<',\n",
              " 'wer',\n",
              " '▁my',\n",
              " 'ond',\n",
              " '▁ver',\n",
              " 'blockquote',\n",
              " '▁one',\n",
              " '▁в',\n",
              " 'ors',\n",
              " 'mer',\n",
              " '▁more',\n",
              " 'ink',\n",
              " 'EN',\n",
              " 'на',\n",
              " 'les',\n",
              " ';\\r',\n",
              " '▁they',\n",
              " 'ain',\n",
              " 'ystem',\n",
              " 'Re',\n",
              " 'ES',\n",
              " '▁dis',\n",
              " 'ents',\n",
              " 'ild',\n",
              " '▁au',\n",
              " 'RE',\n",
              " '_{',\n",
              " 'mb',\n",
              " 'ни',\n",
              " '▁St',\n",
              " 'ра',\n",
              " 'ms',\n",
              " ...]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    }
  ]
}