Lao_Word2Vec.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lao-Word2Vec.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO59K021FYQIVM5F5pZwNy/",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PyThaiNLP/lao-language/blob/master/Lao_Word2Vec.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LwwOptiWYvYF",
        "colab_type": "text"
      },
      "source": [
        "# Lao Word Embedding using Word2Vec\n",
        "\n",
        "Data from https://traces1.inria.fr/oscar/\n",
        "\n",
        "Python | Word Embedding using Word2Vec https://www.geeksforgeeks.org/python-word-embedding-using-word2vec/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MmrKXZ9_WYgh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://traces1.inria.fr/oscar/files/compressed-orig/lo.txt.gz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Qi6GJIPWaGN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!gunzip lo.txt.gz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j9bT4L4hWh8h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jiylUsmjWiuc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ls -l --block-size=M lo.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SR6k8BpkW-wn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install pythainlp"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hCeCf69hXGrI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget /~https://github.com/google/language-resources/raw/master/lo/data_sets/lo_spellcheck_dict.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "02IeHfwwWuq0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gensim \n",
        "from gensim.models import Word2Vec "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0A4Hhhs4W-C_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pythainlp.tokenize import Tokenizer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PsE70ZbmWnhN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lao_ws = Tokenizer(\"lo_spellcheck_dict.txt\", engine = \"mm\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZeCQ933AXLym",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(\"lo.txt\", \"r\",encoding=\"utf-8-sig\") as f:\n",
        "  s = [i.strip() for i in f.read().split('.')]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "08-hb0aiXmex",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d8806213-e4ba-4050-891e-b90484c07a3a"
      },
      "source": [
        "len(s)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "367841"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9QBaT78XXcjF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tqdm import tqdm\n",
        "data = []"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PL7FTncdXe2C",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "233ace44-fb23-4ab2-9f87-fa881741e4c6"
      },
      "source": [
        "# iterate through each sentence in the file \n",
        "for i in tqdm(s): \n",
        "    temp = [] \n",
        "      \n",
        "    # tokenize the sentence into words \n",
        "    for j in lao_ws.word_tokenize(i): \n",
        "        temp.append(j.lower()) \n",
        "  \n",
        "    data.append(temp) "
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 367841/367841 [03:43<00:00, 1644.93it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PwdT43doXUZu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#CBOW model\n",
        "model = gensim.models.Word2Vec(data, min_count = 1,  \n",
        "                              size = 300, window = 5) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8D0pYkSFbDHT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "b1c78aaf-2bbf-48ba-d34b-fce79e3e7314"
      },
      "source": [
        "print(\"Cosine similarity between 'ວຽງຈັນ' \" +\n",
        "          \"and 'ເມືອງ' - Skip Gram : \", \n",
        "    model.wv.similarity('ວຽງຈັນ', 'ເມືອງ')) "
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cosine similarity between 'ວຽງຈັນ' and 'ເມືອງ' - Skip Gram :  0.18443461\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KybdqK1BYS0F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Skip Gram model \n",
        "model2 = gensim.models.Word2Vec(data, min_count = 1, size = 300, \n",
        "                                             window = 5, sg = 1) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pojeG4PBaQSg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "69392489-2104-4fb1-b030-62128c2e992f"
      },
      "source": [
        "print(\"Cosine similarity between 'ວຽງຈັນ' \" +\n",
        "          \"and 'ເມືອງ' - Skip Gram : \", \n",
        "    model2.wv.similarity('ວຽງຈັນ', 'ເມືອງ')) "
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cosine similarity between 'ວຽງຈັນ' and 'ເມືອງ' - Skip Gram :  0.35181516\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xMTFG9aOdVt_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}