Modern_NLP_S5.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Modern_NLP_S5.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a466d087c7b94d8eb3b7fd92bc307823": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1273a44497624d3db05c869c6c59a536",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_52e5e36437f845ca9208bd5d2821b0d1",
              "IPY_MODEL_e042c5d7c8b0463c8917e31ce584ea92"
            ]
          }
        },
        "1273a44497624d3db05c869c6c59a536": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "52e5e36437f845ca9208bd5d2821b0d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_427aaa6141e14126a7b522a75d259f21",
            "_dom_classes": [],
            "description": "Create embeddings: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 80,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 80,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_07018a2e91604a6bbabc66aec6fbd043"
          }
        },
        "e042c5d7c8b0463c8917e31ce584ea92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c3ad653417954babb7b99c2d63541090",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 80/80 [00:02&lt;00:00, 37.14 Docs/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7b28b1a0adbd4b72a554d8aae42532ee"
          }
        },
        "427aaa6141e14126a7b522a75d259f21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "07018a2e91604a6bbabc66aec6fbd043": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c3ad653417954babb7b99c2d63541090": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7b28b1a0adbd4b72a554d8aae42532ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_CSXtHd6F_9k"
      },
      "source": [
        "# ModernNLP: #5 \n",
        "\n",
        "## Retrieval-Augmented Generation (RAG)\n",
        "\n",
        "* \"Retrieval-augmented generation for knowledge-intensive NLP tasks\", [Lewis et al., 2020](https://arxiv.org/abs/2005.11401).\n",
        "* Use a pre-trained RAG model for question answering.\n",
        "* Finetune RAG for binary classification.\n",
        "\n",
        "\n",
        "> Authored by John Pavlopoulos & Vasiliki Kougia\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LBEATUHtrKGs"
      },
      "source": [
        "## Using a pre-trained RAG via Haystack \n",
        "* Taking a detour via [haystack](https://haystack.deepset.ai/docs/intromd), an open-source framework for building search systems."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5hk88KU-jQic"
      },
      "source": [
        "# Install\n",
        "%%capture\n",
        "#!pip install torch==1.8.1\n",
        "!pip install git+/~https://github.com/deepset-ai/haystack.git\n",
        "!pip install urllib3==1.25.4"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EB91mKMbG7kW"
      },
      "source": [
        "> Restart the notebook before you move on."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NMWtN0Z4jsD-"
      },
      "source": [
        "%%capture\n",
        "import requests\n",
        "import pandas as pd\n",
        "from typing import List\n",
        "from haystack import Document\n",
        "from haystack.document_store.faiss import FAISSDocumentStore\n",
        "from haystack.generator.transformers import RAGenerator\n",
        "from haystack.retriever.dense import DensePassageRetriever"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YPJac5A8l6Bb",
        "outputId": "9b315456-aba6-4c4f-8110-b5a667e11508"
      },
      "source": [
        "# Download sample dataset\n",
        "temp = requests.get(\"https://raw.githubusercontent.com/deepset-ai/haystack/master/tutorials/small_generator_dataset.csv\")\n",
        "open('small_generator_dataset.csv', 'wb').write(temp.content)\n",
        "\n",
        "# Create dataframe with columns \"title\" and \"text\"\n",
        "df = pd.read_csv(\"small_generator_dataset.csv\", sep=',')\n",
        "# Minimal cleaning\n",
        "df.fillna(value=\"\", inplace=True)\n",
        "\n",
        "print(df.head())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               title                                                                             text\n",
            "0  \"Albert Einstein\"  to Einstein in 1922. Footnotes Citations Albert Einstein Albert Einstein (; ...\n",
            "1  \"Albert Einstein\"  Albert Einstein Albert Einstein (; ; 14 March 1879 – 18 April 1955) was a Ge...\n",
            "2  \"Albert Einstein\"  observations were published in the international media, making Einstein worl...\n",
            "3  \"Albert Einstein\"  model for depictions of mad scientists and absent-minded professors; his exp...\n",
            "4     \"Alfred Nobel\"  was adopted as the standard technology for mining in the \"Age of Engineering...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g4pxp5ful-be"
      },
      "source": [
        "# Use the dataset to initialize Document objects\n",
        "titles = list(df[\"title\"].values)\n",
        "texts = list(df[\"text\"].values)\n",
        "documents: List[Document] = []\n",
        "for title, text in zip(titles, texts):\n",
        "    documents.append(\n",
        "        Document(\n",
        "            text=text,\n",
        "            meta={\n",
        "                \"name\": title or \"\"\n",
        "            }\n",
        "        )\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EIP1v6-kmArJ"
      },
      "source": [
        "%%capture\n",
        "# Initialize FAISS document store.\n",
        "# Set `return_embedding` to `True`, so generator doesn't have to perform re-embedding\n",
        "document_store = FAISSDocumentStore(\n",
        "    faiss_index_factory_str=\"Flat\",\n",
        "    return_embedding=True\n",
        ")\n",
        "\n",
        "# Initialize DPR Retriever to encode documents, encode question and query documents\n",
        "retriever = DensePassageRetriever(\n",
        "    document_store=document_store,\n",
        "    query_embedding_model=\"facebook/dpr-question_encoder-single-nq-base\",\n",
        "    passage_embedding_model=\"facebook/dpr-ctx_encoder-single-nq-base\",\n",
        "    use_gpu=True,\n",
        "    embed_title=True,\n",
        ")\n",
        "\n",
        "# Initialize RAG Generator\n",
        "generator = RAGenerator(\n",
        "    model_name_or_path=\"facebook/rag-token-nq\",\n",
        "    use_gpu=True,\n",
        "    top_k=1,\n",
        "    max_length=200,\n",
        "    min_length=2,\n",
        "    embed_title=True,\n",
        "    num_beams=2,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138,
          "referenced_widgets": [
            "a466d087c7b94d8eb3b7fd92bc307823",
            "1273a44497624d3db05c869c6c59a536",
            "52e5e36437f845ca9208bd5d2821b0d1",
            "e042c5d7c8b0463c8917e31ce584ea92",
            "427aaa6141e14126a7b522a75d259f21",
            "07018a2e91604a6bbabc66aec6fbd043",
            "c3ad653417954babb7b99c2d63541090",
            "7b28b1a0adbd4b72a554d8aae42532ee"
          ]
        },
        "id": "BoReLSqvmIQu",
        "outputId": "651963c5-317f-48fc-af20-a8f8e3d9c2b1"
      },
      "source": [
        "# Delete existing documents in documents store\n",
        "document_store.delete_all_documents()\n",
        "\n",
        "# Write documents to document store\n",
        "document_store.write_documents(documents)\n",
        "\n",
        "# Add documents embeddings to index\n",
        "document_store.update_embeddings(\n",
        "    retriever=retriever\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/haystack/document_store/sql.py:487: SAWarning: relationship 'DocumentORM.Meta' will copy column document.id to column meta.document_id, which conflicts with relationship(s): 'MetaORM.Document' (copies document.id to meta.document_id), 'DocumentORM.meta' (copies document.id to meta.document_id). If this is not the intention, consider if these relationships should be linked with back_populates, or if viewonly=True should be applied to one or more if they are read-only. For the less common case that foreign key constraints are partially overlapping, the orm.foreign() annotation can be used to isolate the columns that should be written towards.   The 'overlaps' parameter may be used to remove this warning.\n",
            "  self.session.query(DocumentORM).filter_by(index=index).delete(synchronize_session=False)\n",
            "/usr/local/lib/python3.7/dist-packages/haystack/document_store/sql.py:487: SAWarning: relationship 'MetaORM.documents' will copy column document.id to column meta.document_id, which conflicts with relationship(s): 'MetaORM.Document' (copies document.id to meta.document_id), 'DocumentORM.meta' (copies document.id to meta.document_id). If this is not the intention, consider if these relationships should be linked with back_populates, or if viewonly=True should be applied to one or more if they are read-only. For the less common case that foreign key constraints are partially overlapping, the orm.foreign() annotation can be used to isolate the columns that should be written towards.   The 'overlaps' parameter may be used to remove this warning.\n",
            "  self.session.query(DocumentORM).filter_by(index=index).delete(synchronize_session=False)\n",
            "Updating Embedding:   0%|          | 0/68 [00:00<?, ? docs/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a466d087c7b94d8eb3b7fd92bc307823",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Create embeddings', max=80.0, style=ProgressStyle(descrip…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Documents Processed: 10000 docs [00:02, 4181.71 docs/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ifft8UOmKBp"
      },
      "source": [
        "QUESTIONS = [\n",
        "    \"who got the first nobel prize in physics\",\n",
        "    \"when is the next deadpool movie being released\",\n",
        "    \"which mode is used for short wave broadcast service\",\n",
        "    \"who is the owner of reading football club\",\n",
        "    \"when is the next scandal episode coming out\",\n",
        "    \"when is the last time the philadelphia won the superbowl\",\n",
        "    \"what is the most current adobe flash player version\",\n",
        "    \"how many episodes are there in dragon ball z\",\n",
        "    \"what is the first step in the evolution of the eye\",\n",
        "    \"where is gall bladder situated in human body\",\n",
        "    \"what is the main mineral in lithium batteries\",\n",
        "    \"who is the president of usa right now\",\n",
        "    \"where do the greasers live in the outsiders\",\n",
        "    \"panda is a national animal of which country\",\n",
        "    \"what is the name of manchester united stadium\",\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "chrHhwUEmMgN",
        "outputId": "a0826060-10d7-4286-f559-7965408602a0"
      },
      "source": [
        "# Now generate an answer for each question\n",
        "for question in QUESTIONS:\n",
        "    # Retrieve related documents from retriever\n",
        "    retriever_results = retriever.retrieve(\n",
        "        query=question\n",
        "    )\n",
        "\n",
        "    # Now generate answer from question and retrieved documents\n",
        "    predicted_result = generator.predict(\n",
        "        query=question,\n",
        "        documents=retriever_results,\n",
        "        top_k=1\n",
        "    )\n",
        "\n",
        "    # Print you answer\n",
        "    answers = predicted_result[\"answers\"]\n",
        "    print(f'Generated answer is \\'{answers[0][\"answer\"]}\\' for the question = \\'{question}\\'')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated answer is ' albert einstein' for the question = 'who got the first nobel prize in physics'\n",
            "Generated answer is ' september 22, 2017' for the question = 'when is the next deadpool movie being released'\n",
            "Generated answer is ' amplitude modulation' for the question = 'which mode is used for short wave broadcast service'\n",
            "Generated answer is ' stefan persson' for the question = 'who is the owner of reading football club'\n",
            "Generated answer is ' april 20, 2018' for the question = 'when is the next scandal episode coming out'\n",
            "Generated answer is ' the 1970s' for the question = 'when is the last time the philadelphia won the superbowl'\n",
            "Generated answer is ' 7.1. 2' for the question = 'what is the most current adobe flash player version'\n",
            "Generated answer is ' 13' for the question = 'how many episodes are there in dragon ball z'\n",
            "Generated answer is ' step by step' for the question = 'what is the first step in the evolution of the eye'\n",
            "Generated answer is ' stomach' for the question = 'where is gall bladder situated in human body'\n",
            "Generated answer is ' spodumene' for the question = 'what is the main mineral in lithium batteries'\n",
            "Generated answer is ' ronald reagan' for the question = 'who is the president of usa right now'\n",
            "Generated answer is ' northern new jersey' for the question = 'where do the greasers live in the outsiders'\n",
            "Generated answer is ' india' for the question = 'panda is a national animal of which country'\n",
            "Generated answer is ', england' for the question = 'what is the name of manchester united stadium'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z5cF4qcRmOCS",
        "outputId": "a0ca13d7-65c6-4863-ddd2-6242a596dfac"
      },
      "source": [
        "# Try a new question\n",
        "from haystack.pipeline import GenerativeQAPipeline\n",
        "\n",
        "NEW_QUESTIONS = [\n",
        "    \"who is the best president USA\",\n",
        "]\n",
        "\n",
        "pipe = GenerativeQAPipeline(generator=generator, retriever=retriever)\n",
        "for question in NEW_QUESTIONS:\n",
        "    res = pipe.run(query=question, top_k_generator=1, top_k_retriever=50)\n",
        "    print(res)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'query': 'who is the best president USA', 'answers': [{'query': 'who is the best president USA', 'answer': ' trump', 'meta': {'doc_ids': ['5e30f241d1c389b17b3e0d894589bb6d', '677b399a549c301443097e99e579b504', '4bc744a58c35759e701f569c3afbec12', '756703c5a38a72a1445026e10187e6d1', '2aedb0c40e3a20c805d5b404a39543a', 'd34ecd3aeacc8cbd33555e07f9a98c73', 'ad7b335b2aa0618a83a484712609f21f', 'a9f5a2bc2b302574c4813daa71b52b09', '69c8b13a944a0878822e417f028aae49', '7fe7251032cc4b8b249a06fe8c975213', '179c700282b1784abd793bb989ba453c', 'e86ca2c67ea5220c1bad5de0868fad3f', '9dfc548a82f643420f5a3fa219088773', '326bf35becf4b0fbb1983d0939b192e', '6e4109839970876be7e585365a5d8c02', '4a347e0fc7c1b7d06fa7d7a2aa580555', '87d904c0a6eb166df1e2d7d2414ff8f5', '23c07d7b51ea44a1db9af1265ac6c276', '1ecd0e520a520197fa2f74549ccad50', '792312bbd1b1e8c8c4365b5966dee573', '7b6800d75331a3619db6cd07bfde9f4e', '549e4d922dd841db8244599f1d9ac30c', 'bd3e02e3146be9bc4081cc806bbee7af', '2df48a633cba33fa64736240303513cf', '9d152c99bcd246e1c83eddd6cb795293', '7165b780af3df9b242d6e364754e0396', 'ce27ab4c90c16d47bf3577fac072ce12', 'd2e527ae5f2c2098b156dbd589525a52', 'c38368e3e17050542f6f999d57296ffb', '52e07cab880d630926fd490560f94f9a', 'ab6983f94425886f00fd550b2ac0f037', '7d43847de1e042f71c892e1be05c274e', 'b85487f3f7fd1b9d5d4f4ed0aea4ef09', '18380b136b3b8a2180cfbf4c6a83d6a5', 'be1495b4311eb772ea3604d5c955b147', 'edd15c743ed7f5f430d64a9e46ac98c2', '2e0b3942b6eb6f0b2f921a2199869f29', 'c8650ec4df2bc410a2c2fe17e8a36731', '431aed1e948f4cad51e20cb1be573a7a', '82fe278b9f77e6e24998b5c17dc3dbf8', '42614e437e45c1d4a359b285b6cbb87d', '931941e863ae8d029da932c0be99119c', '7e03befae842ed80030881d500a961d6', '997369f2a55c7b9385f7655508e07834', 'b32ed755e70609744f2b8c85733352f3', 'd40c6b861499986feba50af45e970ca6', '3c527839680f7f5ff3cb758454fa5079', '26b6cd0ca87c357c7fb305de992d7155', '7ad3f74425e06ec7306cb35c8b2ac5b', 'f2094dc4797e7998fe69b65ca48d1f43'], 'doc_scores': [62.372536, 61.74641, 61.084663, 60.869537, 59.221924, 58.607178, 57.951942, 57.763813, 57.315228, 57.073277, 56.316273, 56.30035, 56.276676, 55.895508, 55.642876, 55.208935, 54.586555, 54.411022, 53.620083, 53.54629, 53.311058, 53.166843, 53.019936, 52.998352, 52.989677, 52.65008, 51.27361, 51.214336, 50.994926, 50.922623, 50.39163, 50.3737, 49.765076, 49.00429, 48.682755, 48.661804, 48.267754, 48.069225, 47.70882, 47.543976, 47.516464, 47.203434, 46.79912, 46.783535, 46.382507, 45.95205, 45.673927, 45.496597, 45.482845, 44.315422], 'doc_probabilities': [0.6510653480112909, 0.6496415790705735, 0.6481339065232966, 0.6476431417382152, 0.6438741798425061, 0.6424633222896239, 0.640956818409401, 0.640523759240922, 0.6394902275446966, 0.6389322396958976, 0.6371840180766493, 0.6371472074431214, 0.6370924735673603, 0.6362107310665432, 0.6356258215092473, 0.634620199688149, 0.6331758372366212, 0.6327680410153929, 0.6309281918923826, 0.6307563460185781, 0.6302083127994146, 0.6298721636040814, 0.6295296074074592, 0.6294792683792738, 0.6294590358880636, 0.6286666138238562, 0.6254476588609319, 0.6253087944477049, 0.624794580813735, 0.6246250664591322, 0.623379229817899, 0.6233371354359795, 0.621907091719437, 0.6201165420493762, 0.619358798811144, 0.6193094064745144, 0.6183799362575937, 0.6179113269315054, 0.6170600610745459, 0.6166704634399509, 0.6166054272388615, 0.6158651442015514, 0.6149081947000853, 0.6148712849779459, 0.613921196634564, 0.6129004174821592, 0.6122403566033592, 0.6118192877172309, 0.6117866267425138, 0.6090103611887389], 'texts': ['broadcast of the Tiangong-1 launch. The song is often included in songbooks in a wide variety of religious congregations in the United States. Bing Crosby included the song in a medley on his album \"101 Gang Songs\" (1961). In 1976, while the United States celebrated its bicentennial, a soulful version popularized by Ray Charles peaked at number 98 on the US R&B Charts. Ray Charles did this again in 1984 to re-elect Ronald Reagan. Ray Charles did this yet again in Miami, Florida in 1999. Three different renditions of the song have entered the Hot Country Songs charts. The first', \"A mid-October 2013 announcement revealed that Burberry executive Angela Ahrendts will commence as a senior vice president at Apple in mid-2014. Ahrendts oversaw Burberry's digital strategy for almost eight years and, during her tenure, sales increased to about US$3.2 billion and shares gained more than threefold. Alongside Google vice-president Vint Cerf and AT&T CEO Randall Stephenson, Cook attended a closed-door summit held by President Obama on August 8, 2013, in regard to government surveillance and the Internet in the wake of the Edward Snowden NSA incident. On February 4, 2014, Cook met with Abdullah Gül, the President of Turkey, in\", \"CEO. Two months later, Jobs died, marking the end of an era for the company. Apple is well known for its size and revenues. Its worldwide annual revenue totaled $265billion for the 2018 fiscal year. Apple is the world's largest information technology company by revenue and the world's third-largest mobile phone manufacturer after Samsung and Huawei. In August 2018, Apple became the first public U.S. company to be valued at over US$1 trillion. The company employs 123,000 full-time employees and maintains 504 retail stores in 24 countries . It operates the iTunes Store, which is the world's largest music retailer.\", 'of $250 billion, officially confirmed by Apple as specifically $256.8 billion a few days later. , Apple is the largest publicly traded corporation in the world by market capitalization. On August 2, 2018, Apple became the first publicly traded U.S. company to reach a $1 trillion market value. Apple is currently ranked #4 on the Fortune 500 rankings of the largest United States corporations by total revenue. Apple has created subsidiaries in low-tax places such as Ireland, the Netherlands, Luxembourg and the British Virgin Islands to cut the taxes it pays around the world. According to \"The New York Times,\"', 'status as: \"I am not one of those hyphenated Americans who claim allegiance to two countries.\" Despite this declaration, Bell has been proudly claimed as a \"native son\" by all three countries he resided in: the United States, Canada, and the United Kingdom. By 1885, a new summer retreat was contemplated. That summer, the Bells had a vacation on Cape Breton Island in Nova Scotia, spending time at the small village of Baddeck. Returning in 1886, Bell started building an estate on a point across from Baddeck, overlooking Bras d\\'Or Lake. By 1889, a large house, christened \"The Lodge\" was', 'model for depictions of mad scientists and absent-minded professors; his expressive face and distinctive hairstyle have been widely copied and exaggerated. \"Time\" magazine\\'s Frederic Golden wrote that Einstein was \"a cartoonist\\'s dream come true\". Many popular quotations are often misattributed to him. Einstein received numerous awards and honors and in 1922 he was awarded the 1921 Nobel Prize in Physics \"for his services to Theoretical Physics, and especially for his discovery of the law of the photoelectric effect\". None of the nominations in 1921 met the criteria set by Alfred Nobel, so the 1921 prize was carried forward and awarded', '\"the United States of North America\" in the first sentence, then \"the said United States\" afterwards; \"the United States of America\" and \"the United States of North America\" derive from \"the United Colonies of America\" and \"the United Colonies of North America\". The Treaty of Peace and Amity of September 5, 1795, between the United States and the Barbary States contains the usages \"the United States of North America\", \"citizens of the United States\", and \"American Citizens\". U.S. President George Washington, in his 1796 \"Farewell Address\", declaimed that \"The name of American, which belongs to you in your national capacity,', 'ranging from the reenactment of battles, to statues and memorial halls erected, to films being produced, to stamps and coins with Civil War themes being issued, all of which helped to shape public memory. This varied advent occurred in greater proportions on the 100th and 150th anniversary. Hollywood\\'s take on the war has been especially influential in shaping public memory, as seen in such film classics as \"Birth of a Nation\" (1915), \"Gone with the Wind\" (1939), and more recently \"Lincoln\" (2012). Ken Burns produced a notable PBS series on television titled \"The Civil War\" (1990). It was digitally remastered', 'an app that allows iPad and iPhone users to make and edit videos. The app provides a way to produce short videos to share with other users on the Messages app, Instagram, Facebook and other social networks. Apple also introduced Live Titles for Clips that allows users to add live animated captions and titles using their voice. In May 2017, Apple refreshed two of its website designs. Their public relations \"Apple Press Info\" website was changed to an \"Apple Newsroom\" site, featuring a greater emphasis on imagery and therefore lower information density, and combines press releases, news items, and photos.', 'been achieved and that Reconstruction should end. They ran a presidential ticket in 1872 but were decisively defeated. In 1874, Democrats, primarily Southern, took control of Congress and opposed any more reconstruction. The Compromise of 1877 closed with a national consensus that the Civil War had finally ended. With the withdrawal of federal troops, however, whites retook control of every Southern legislature; the Jim Crow period of disenfranchisement and legal segregation was about to begin. The Civil War is one of the central events in American collective memory. There are innumerable statues, commemorations, books and archival collections. The memory includes', 'a letter to a local Aspen newspaper informing them that, on Christmas Eve, he was going to use napalm to burn a number of dogs and hopefully any humans they find. The letter protests against the burning of Vietnamese people occurring overseas. The 2012 film \"Butcher Boys,\" written by Kim Henkel, is said to be loosely based on Jonathan Swift\\'s \"A Modest Proposal.\" The film\\'s opening scene takes place in a restaurant named \"J. Swift\\'s\". On November 30, 2017, Jonathan Swift\\'s 350th birthday, The Washington Post published a column entitled \\'Why Alabamians should consider eating Democrats\\' babies\", by the humorous', 'for 2020. Patrick Frater writing for \"Variety\" magazine in May 2017 stated that another two unfinished films by Kurosawa were planned, with \"Silvering Spear\" to start filming in 2018. In September 2011, it was reported that remake rights to most of Kurosawa\\'s movies and unproduced screenplays were assigned by the Akira Kurosawa 100 Project to the L.A.-based company Splendent. Splendent\\'s chief Sakiko Yamada, stated that he aimed to \"help contemporary film-makers introduce a new generation of moviegoers to these unforgettable stories\". Kurosawa Production Co., established in 1959, continues to oversee many of the aspects of Kurosawa\\'s legacy. The director\\'s son,', '2018 after being delayed from its initial December 2017 release. It also features 7 tweeters in the base, a four-inch woofer in the top, and six microphones for voice control and acoustic optimization On September 12, 2018, Apple announced that HomePod is adding new features—search by lyrics, set multiple timers, make and receive phone calls, Find My iPhone, Siri Shortcuts—and Siri languages. Apple develops its own operating systems to run on its devices, including macOS for Mac personal computers, iOS for its iPhone, iPad and iPod Touch smartphones and tablets, watchOS for its Apple Watch smartwatches, and tvOS for its', '\"Friendship 7\". \"Columbia\" was moved in 2017 to the NASM Mary Baker Engen Restoration Hangar at the Steven F. Udvar-Hazy Center in Chantilly, Virginia, to be readied for a four-city tour titled \"Destination Moon: The Apollo 11 Mission\". This included Space Center Houston from October 14, 2017 to March 18, 2018, the Saint Louis Science Center from April 14 to September 3, 2018, the Senator John Heinz History Center in Pittsburgh from September 29, 2018 to February 18, 2019, and the Seattle Museum of Flight from March 16 to September 2, 2019. For 40 years Armstrong\\'s and Aldrin\\'s space suits', '(\"Ashoka Chakra\") from its base was placed onto the center of the National Flag of India. The capital contains four lions (Indian / Asiatic Lions), standing back to back, mounted on a short cylindrical abacus, with a frieze carrying sculptures in high relief of an elephant, a galloping horse, a bull, and a lion, separated by intervening spoked chariot-wheels over a bell-shaped lotus. Carved out of a single block of polished sandstone, the capital was believed to be crowned by a \\'Wheel of Dharma\\' (Dharmachakra popularly known in India as the \"Ashoka Chakra\"). The Sarnath pillar bears one of the', 'observations were published in the international media, making Einstein world-famous. On 7 November 1919, the leading British newspaper \"The Times\" printed a banner headline that read: \"Revolution in Science – New Theory of the Universe – Newtonian Ideas Overthrown\". In 1920, he became a Foreign Member of the Royal Netherlands Academy of Arts and Sciences. In 1922, he was awarded the 1921 Nobel Prize in Physics \"for his services to Theoretical Physics, and especially for his discovery of the law of the photoelectric effect\". While the general theory of relativity was still considered somewhat controversial, the citation also does not', 'Aquarii among its 12 stars. 88, 89, and 98 Aquarii represent \"Fou-youe\", the axes used as weapons and for hostage executions. Also in Aquarius is \"Loui-pi-tchin\", the ramparts that stretch from 29 and 27 Piscium and 33 and 30 Aquarii through Phi, Lambda, Sigma, and Iota Aquarii to Delta, Gamma, Kappa, and Epsilon Capricorni. Near the border with Cetus, the axe \"Fuyue\" was represented by three stars; its position is disputed and may have instead been located in Sculptor. \"Tienliecheng\" also has a disputed position; the 13-star castle replete with ramparts may have possessed Nu and Xi Aquarii but may', '2016, Apple introduced the iPhone 7 and the iPhone 7 Plus, which feature improved system and graphics performance, add water resistance, a new rear dual-camera system on the 7 Plus model, and, controversially, remove the 3.5 mm headphone jack. On September 12, 2017, Apple introduced the iPhone 8 and iPhone 8 Plus, standing as evolutionary updates to its previous phones with a faster processor, improved display technology, upgraded camera systems and wireless charging. The company also announced iPhone X, which radically changes the hardware of the iPhone lineup, removing the home button in favor of facial recognition technology and featuring', 'the quickest - in terms of balls faced - a team has been bowled out in the first innings of a Test match. With victory by an innings and 78 runs on the morning of the third day of the Fourth Test, England regained the Ashes. During the buildup, the 2017–18 Ashes series was regarded as a turning point for both sides. Australia were criticised for being too reliant on captain Steve Smith and vice-captain David Warner, while England was said to have a shoddy middle to lower order. Off the field, England all-rounder Ben Stokes was ruled out of', \"Flash to store games, programs, and to cache the current media playing. The release also coincided with the opening of a separate Apple TV App Store and a new Siri Remote with a glass touchpad, gyroscope, and microphone. At the September 12, 2017 event, Apple released a new 4K Apple TV with the same form factor as the 4th Generation model. The 4K model is powered by the A10X SoC designed in-house that also powers their second-generation iPad Pro. The 4K model also has support for high dynamic range. Apple's first smart speaker, the HomePod was released on February 9,\", 'therefore will not host an Ashes Test until at least 2027. Trent Bridge is also not due to host an Ashes Test in 2019 or 2023. The popularity and reputation of the cricket series has led to other sports or games, and/or their followers, using the name \"Ashes\" for contests between England and Australia. The best-known and longest-running of these events is the rugby league rivalry between Great Britain and Australia (see rugby league \"Ashes\"). Use of the name \"Ashes\" was suggested by the Australian team when rugby league matches between the two countries commenced in 1908. Other examples included', 'was adopted as the standard technology for mining in the \"Age of Engineering\" bringing Nobel a great amount of financial success, though at a significant cost to his health. An offshoot of this research resulted in Nobel\\'s invention of ballistite, the precursor of many modern smokeless powder explosives and still used as a rocket propellant. In 1888 Alfred\\'s brother Ludvig died while visiting Cannes and a French newspaper erroneously published Alfred\\'s obituary. It condemned him for his invention of dynamite and is said to have brought about his decision to leave a better legacy after his death. The obituary stated,', \"a faster processor, and brighter display. On September 12, 2017, Apple introduced the Apple Watch Series 3 featuring LTE cellular connectivity, giving the wearable independence from an iPhone except for the setup process. On September 12, 2018, Apple introduced the Apple Watch Series 4, featuring new display, electrocardiogram and fall detection. At the 2007 Macworld conference, Jobs demonstrated the Apple TV (previously known as the iTV), a set-top video device intended to bridge the sale of content from iTunes with high-definition televisions. The device links up to a user's TV and syncs, either via Wi-Fi or a wired network, with\", \"The national animal of Azerbaijan is the Karabakh horse, a mountain-steppe racing and riding horse endemic to Azerbaijan. The Karabakh horse has a reputation for its good temper, speed, elegance and intelligence. It is one of the oldest breeds, with ancestry dating to the ancient world. However, today the horse is an endangered species. Azerbaijan's flora consists of more than 4,500 species of higher plants. Due to the unique climate in Azerbaijan, the flora is much richer in the number of species than the flora of the other republics of the South Caucasus. About 67 percent of the species growing\", 'pre-independence versions of the flag. The Ashoka Chakra can also been seen on the base of the Lion Capital of Ashoka which has been adopted as the National Emblem of India. The Ashoka Chakra was created by Ashoka during his reign. Chakra is a Sanskrit word which also means \"cycle\" or \"self-repeating process\". The process it signifies is the cycle of time—as in how the world changes with time. A few days before India became independent in August 1947, the specially-formed Constituent Assembly decided that the flag of India must be acceptable to all parties and communities. A flag with', 'The Ashes The Ashes is a Test cricket series played between England and Australia. The Ashes are regarded as being held by the team that most recently won the Test series. If the test series is drawn, the team that currently holds the Ashes retains the trophy. The term originated in a satirical obituary published in a British newspaper, \"The Sporting Times\", immediately after Australia\\'s 1882 victory at The Oval, its first Test win on English soil. The obituary stated that English cricket had died, and \"the body will be cremated and the ashes taken to Australia\". The mythical ashes', 'include access to the iTunes Store to rent movies and TV shows (purchasing has been discontinued), streaming from internet video sources, including YouTube and Netflix, and media streaming from an iTunes library. Apple also reduced the price of the device to $99. A third generation of the device was introduced at an Apple event on March 7, 2012, with new features such as higher resolution (1080p) and a new user interface. At the September 9, 2015, event, Apple unveiled an overhauled Apple TV, which now runs a variant of macOS, called tvOS, and contains 32GB or 64 GB of NAND', 'Wheel of Dharma). The wheel has 24 spokes which represent the 12 Laws of Dependent Origination and the 12 Laws of Dependent Termination. The Ashoka Chakra has been widely inscribed on many relics of the Mauryan Emperor, most prominent among which is the Lion Capital of Sarnath and The Ashoka Pillar. The most visible use of the Ashoka Chakra today is at the centre of the National flag of the Republic of India (adopted on 22 July 1947), where it is rendered in a Navy-blue color on a White background, by replacing the symbol of Charkha (Spinning wheel) of the', 'a near bezel-less design along with wireless charging. On September 12, 2018, Apple introduced the iPhone XS, iPhone XS Max and iPhone XR. The iPhone XS and iPhone XS Max features Super Retina displays, a faster and improved dual camera system that offers breakthrough photo and video features, the first 7-nanometer chip in a smartphone — the A12 Bionic chip with next-generation Neural Engine — faster Face ID, wider stereo sound and introduces Dual SIM to iPhone. The iPhone XR comes in an all-screen glass and aluminium design with the most advanced LCD in a smartphone featuring a 6.1-inch Liquid', 'Jack London\\'s novel and starring Ethan Hawke, was filmed in and around Haines. Steven Seagal\\'s 1994 \"On Deadly Ground\", starring Michael Caine, was filmed in part at the Worthington Glacier near Valdez. The 1999 John Sayles film \"Limbo\", starring David Strathairn, Mary Elizabeth Mastrantonio, and Kris Kristofferson, was filmed in Juneau. The psychological thriller \"Insomnia\", starring Al Pacino and Robin Williams, was shot in Canada, but was set in Alaska. The 2007 film directed by Sean Penn, \"Into The Wild\", was partially filmed and set in Alaska. The film, which is based on the novel of the same name, follows', 'to Einstein in 1922. Footnotes Citations Albert Einstein Albert Einstein (; ; 14 March 1879 – 18 April 1955) was a German-born theoretical physicist who developed the theory of relativity, one of the two pillars of modern physics (alongside quantum mechanics). His work is also known for its influence on the philosophy of science. He is best known to the general public for his mass–energy equivalence formula , which has been dubbed \"the world\\'s most famous equation\". He received the 1921 Nobel Prize in Physics \"for his services to theoretical physics, and especially for his discovery of the law of', 'D1b is found throughout the Japanese Archipelago, but with very high frequencies among the Ainu of Hokkaido in the far north, and to a lesser extent among the Ryukyuans in the Ryukyu Islands of the far south. The only places outside Japan in which Y-haplogroup D is common are Tibet in China and the Andaman Islands in the Indian Ocean. A study by Tajima \"et al.\" (2004) found two out of a sample of sixteen (or 12.5%) Ainu men to belong to Haplogroup C-M217, which is the most common Y-chromosome haplogroup among the indigenous populations of Siberia and Mongolia. Hammer', 'Albert Einstein Albert Einstein (; ; 14 March 1879 – 18 April 1955) was a German-born theoretical physicist who developed the theory of relativity, one of the two pillars of modern physics (alongside quantum mechanics). His work is also known for its influence on the philosophy of science. He is best known to the general public for his mass–energy equivalence formula , which has been dubbed \"the world\\'s most famous equation\". He received the 1921 Nobel Prize in Physics \"for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect\", a pivotal step', 'captain Alastair Cook being bowled out for 0 (his first duck in 26 innings as captain), Australia looked to be in with a significant chance of a win, keeping its series hopes alive. By lunch England were 37–3, but on resumption of play only 3 balls were bowled before rain stopped play. This rain persisted and, at 16:40, the captains shook hands and the match was declared a draw. With England 2–0 up with two Tests to play, England retained the Ashes on 5 August 2013. In the Fourth Test, England won the toss and batted first, putting on 238', \"desire to support the Americans the following year, hoping to weaken Britain's empire. The Portuguese threat was neutralized in the Spanish–Portuguese War (1776–77). On 12 April 1779, Spain signed the Treaty of Aranjuez with France and went to war against Britain. Spain sought to recover Gibraltar and Menorca in Europe, as well as Mobile and Pensacola in Florida, and also to expel the British from Central America. Meanwhile, George III had given up on subduing America while Britain had a European war to fight. He did not welcome war with France, but he believed that Britain had made all necessary\", 'signed the Treaty of Paris in which Great Britain agreed to recognize the sovereignty of the United States and formally end the war. French involvement had proven decisive, but France made few gains and incurred crippling debts. Spain made some territorial gains but failed in its primary aim of recovering Gibraltar. The Dutch were defeated on all counts and were compelled to cede territory to Great Britain. In India, the war against Mysore and its allies concluded in 1784 without any territorial changes. Parliament passed the Stamp Act in 1765. Colonists condemned the tax because their rights as Englishmen protected', 'pouch where it latches on to a nipple and completes its development. Humans have the overall body plan of a mammal. Humans have a head, neck, trunk (which includes the thorax and abdomen), two arms and hands, and two legs and feet. Generally, students of certain biological sciences, paramedics, prosthetists and orthotists, physiotherapists, occupational therapists, nurses, podiatrists, and medical students learn gross anatomy and microscopic anatomy from anatomical models, skeletons, textbooks, diagrams, photographs, lectures and tutorials, and in addition, medical students generally also learn gross anatomy through practical experience of dissection and inspection of cadavers. The study of microscopic anatomy', '14, 1784. Copies were sent back to Europe for ratification by the other parties involved, the first reaching France in March 1784. British ratification occurred on April 9, 1784, and the ratified versions were exchanged in Paris on May 12, 1784. The war formally concluded on September 3, 1783. The last British troops departed New York City on November 25, 1783, marking the end of British rule in the new United States. The total loss of life throughout the conflict is largely unknown. As was typical in wars of the era, diseases such as smallpox claimed more lives than battle.', 'mind, but are also aimed at the general music market, and therefore often allude only vaguely or not at all to the themes or plot of the series. Pop and rock songs are also sometimes used as incidental music (\"insert songs\") in an episode, often to highlight particularly important scenes. The animation industry consists of more than 430 production companies with some of the major studios including Toei Animation, Gainax, Madhouse, Gonzo, Sunrise, Bones, TMS Entertainment, Nippon Animation, P.A.Works, Studio Pierrot and Studio Ghibli. Many of the studios are organized into a trade association, The Association of Japanese Animations. There', 'Akira Kurosawa Akira Kurosawa (, \"Kurosawa Akira\"; March 23, 1910 – September 6, 1998) was a Japanese film director and screenwriter, who directed 30 films in a career spanning 57 years. He is regarded as one of the most important and influential filmmakers in the history of cinema. Kurosawa entered the Japanese film industry in 1936, following a brief stint as a painter. After years of working on numerous films as an assistant director and scriptwriter, he made his debut as a director during World War II with the popular action film \"Sanshiro Sugata\" (a.k.a. \"Judo Saga\"). After the war,', '1978–79; 1981; 1985; 1989; 1993 and 1997). Australians have made 264 centuries in Ashes Tests, of which 23 have been scores over 200, while Englishmen have scored 212 centuries, of which 10 have been over 200. Australians have taken 10 wickets in a match on 41 occasions, Englishmen 38 times. The series alternates between the United Kingdom and Australia, and within each country each of the usually five matches is held at different grounds. In Australia, the grounds currently used are the Gabba in Brisbane (first staged an England–Australia Test in the 1932–33 season), Adelaide Oval (1884–85), the Melbourne Cricket', 'tail which continues the spinal cord and vertebrae but not the gut. The mouth is found at the anterior end of the animal, and the anus at the base of the tail. The defining characteristic of a vertebrate is the vertebral column, formed in the development of the segmented series of vertebrae. In most vertebrates the notochord becomes the nucleus pulposus of the intervertebral discs. However, a few vertebrates, such as the sturgeon and the coelacanth retain the notochord into adulthood. Jawed vertebrates are typified by paired appendages, fins or legs, which may be secondarily lost. The limbs of vertebrates', 'both names. On June 6, 2008, the Japanese Diet passed a bipartisan, non-binding resolution calling upon the government to recognize the Ainu people as indigenous to Japan, and urging an end to discrimination against the group. The resolution recognised the Ainu people as \"an indigenous people with a distinct language, religion and culture\". The government immediately followed with a statement acknowledging its recognition, stating, \"The government would like to solemnly accept the historical fact that many Ainu were discriminated against and forced into poverty with the advancement of modernization, despite being legally equal to (Japanese) people.\" As a result of', 'Ground (MCG) (1876–77), and the Sydney Cricket Ground (SCG) (1881–82). A single Test was held at the Brisbane Exhibition Ground in 1928–29. Traditionally, Melbourne hosts the Boxing Day Test and Sydney hosts the New Year Test. Additionally the WACA in Perth (1970–71) hosted its final Ashes Test in 2017–18 and is due to be replaced by Perth Stadium for the 2021–22 series. Cricket Australia proposed that the 2010–11 series consist of six Tests, with the additional game to be played at Bellerive Oval in Hobart. The England and Wales Cricket Board declined and the series was played over five Tests.', \"In England and Wales, the grounds used are: Old Trafford in Manchester (1884), The Oval in Kennington, South London (1884); Lord's in St John's Wood, North London (1884); Trent Bridge at West Bridgford, Nottinghamshire (1899), Headingley in Leeds (1899); Edgbaston in Birmingham (1902); Sophia Gardens in Cardiff, Wales (2009); and the Riverside Ground in Chester-le-Street, County Durham (2013). One Test was also held at Bramall Lane in Sheffield in 1902. Traditionally the final Test of the series is played at the Oval. Sophia Gardens and the Riverside were excluded as Test grounds between the years of 2020 and 2024 and\", 'through the Second World War and beyond. The narrative centers on yearly birthday celebrations with his former students, during which the protagonist declares his unwillingness to die just yet—a theme that was becoming increasingly relevant for the film\\'s 81-year-old creator. Filming began in February 1992 and wrapped by the end of September. Its release on April 17, 1993, was greeted by an even more disappointed reaction than had been the case with his two preceding works. Kurosawa nevertheless continued to work. He wrote the original screenplays \"The Sea is Watching\" in 1993 and \"After the Rain\" in 1995. While putting', 'theory and stellar aberration. He was informed that he was trying to break down open doors (\"\"il enfonçait des portes ouvertes\"\"), and directed to classical works on optics. On 12 July 1815, as Fresnel was about to leave Paris, Arago left him a note on a new topic: Fresnel would not have ready access to these works outside Paris, and could not read English. But, in Mathieu — with a point-source of light made by focusing sunlight with a drop of honey, a crude micrometer of his own construction, and supporting apparatus made by a local locksmith — he began', \"for outdoor adventure activities, like surfing, kite-surfing and sea kayaking. Fishing and watersports are also popular. Sailing regattas featuring a local vessel type, the Achill Yawl, have been popular since the 19th century, though most present-day yawls, unlike their traditional working boat ancestors, have been structurally modified to promote greater speed under sail. The island's waters and underwater sites are occasionally visited by scuba divers, though Achill's unpredictable weather generally has precluded a commercially successful recreational diving industry. In 2011, the population was 2,569. The island's population has declined from around 6,000 before the Great Hunger. The table below reports\", 'organs, including the stomach. All vertebrates have a similar basic body plan and at some point in their lives, mostly in the embryonic stage, share the major chordate characteristics; a stiffening rod, the notochord; a dorsal hollow tube of nervous material, the neural tube; pharyngeal arches; and a tail posterior to the anus. The spinal cord is protected by the vertebral column and is above the notochord and the gastrointestinal tract is below it. Nervous tissue is derived from the ectoderm, connective tissues are derived from mesoderm, and gut is derived from the endoderm. At the posterior end is a', 'seen with the naked eye, and also includes superficial anatomy or surface anatomy, the study by sight of the external body features. Microscopic anatomy is the study of structures on a microscopic scale, along with histology (the study of tissues), and embryology (the study of an organism in its immature condition). Anatomy can be studied using both invasive and non-invasive methods with the goal of obtaining information about the structure and organization of organs and systems. Methods used include dissection, in which a body is opened and its organs studied, and endoscopy, in which a video camera-equipped instrument is inserted'], 'titles': ['\"America the Beautiful\"', '\"Apple Inc.\"', '\"Apple Inc.\"', '\"Apple Inc.\"', '\"Alexander Graham Bell\"', '\"Albert Einstein\"', '\"American (word)\"', '\"American Civil War\"', '\"Apple Inc.\"', '\"American Civil War\"', '\"A Modest Proposal\"', '\"Akira Kurosawa\"', '\"Apple Inc.\"', '\"Apollo 11\"', 'Ashoka', '\"Albert Einstein\"', '\"Aquarius (constellation)\"', '\"Apple Inc.\"', '\"The Ashes\"', '\"Apple Inc.\"', '\"The Ashes\"', '\"Alfred Nobel\"', '\"Apple Inc.\"', 'Azerbaijan', 'Ashoka', '\"The Ashes\"', '\"Apple Inc.\"', 'Ashoka', '\"Apple Inc.\"', 'Alaska', '\"Albert Einstein\"', '\"Ainu people\"', '\"Albert Einstein\"', '\"The Ashes\"', '\"American Revolutionary War\"', '\"American Revolutionary War\"', 'Anatomy', '\"American Revolutionary War\"', 'Anime', '\"Akira Kurosawa\"', '\"The Ashes\"', 'Anatomy', '\"Ainu people\"', '\"The Ashes\"', '\"The Ashes\"', '\"Akira Kurosawa\"', '\"Augustin-Jean Fresnel\"', '\"Achill Island\"', 'Anatomy', 'Anatomy']}}]}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "38wxiM2oMnFQ"
      },
      "source": [
        "## Finetune RAG via Hugging face\n",
        "\n",
        "* The following code does not run on Colab due to dependecy issues. The faiss library need to be installed, either with conda or building it from source. For more details: /~https://github.com/facebookresearch/faiss/blob/master/INSTALL.md.\n",
        "* Finetuning RAG is resource intensive and needs more than one GPU to run. This code can be ran on a GPU server and uses many GPUs in parallel. \n",
        "* We will finetune RAG on the humor detection dataset for binary text classification, which means the generator will take only one step to generate the label.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h6OPF81joCMQ"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install datasets\n",
        "!conda install -c pytorch faiss-gpu"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DyTDaGvjSph1"
      },
      "source": [
        "# Imports\n",
        "import torch\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "from transformers import AdamW\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "from sklearn.model_selection import train_test_split as tts\n",
        "from transformers import RagTokenizer, RagRetriever, RagSequenceForGeneration"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SdEizF5_Nekj"
      },
      "source": [
        "# Add the numbers of available GPUs in the d_ids array\n",
        "d_ids = [0,1]\n",
        "device = f\"cuda:{d_ids[0]}\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cbET6y_kUjlP"
      },
      "source": [
        "#### Set up the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 243
        },
        "id": "L3isfbBaSgic",
        "outputId": "0bfe8f17-318a-4cd4-df7e-faab4644a3db"
      },
      "source": [
        "# Download humor detection data\n",
        "# Paper: https://arxiv.org/abs/2004.12765\n",
        "data = pd.read_csv(\"https://raw.githubusercontent.com/Moradnejad/ColBERT-Using-BERT-Sentence-Embedding-for-Humor-Detection/master/Data/dataset.csv\")\n",
        "print(\"\\nThere are\", len(data), \"sentences\")\n",
        "data.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "There are 200000 sentences\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>humor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Joe biden rules out 2020 bid: 'guys, i'm not r...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Watch: darvish gave hitter whiplash with slow ...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>What do you call a turtle without its shell? d...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5 reasons the 2016 election feels so personal</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Pasco police shot mexican migrant from behind,...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  humor\n",
              "0  Joe biden rules out 2020 bid: 'guys, i'm not r...  False\n",
              "1  Watch: darvish gave hitter whiplash with slow ...  False\n",
              "2  What do you call a turtle without its shell? d...   True\n",
              "3      5 reasons the 2016 election feels so personal  False\n",
              "4  Pasco police shot mexican migrant from behind,...  False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "61fayTg_Tvhu"
      },
      "source": [
        "# Split to train, val and test\n",
        "train, test = tts(data[[\"text\", \"humor\"]], random_state=42, test_size=0.1)\n",
        "train, val = tts(train, random_state=42, test_size=test.shape[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0tWSmn1oUpn2"
      },
      "source": [
        "#### Set up the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w6MWLpXtQnES"
      },
      "source": [
        "# Use the pre-trained RAG sequence\n",
        "# Define the tokenizer\n",
        "tokenizer = RagTokenizer.from_pretrained(\"facebook/rag-sequence-nq\")\n",
        "# Define the retriever\n",
        "# index_name is the dataset to be used, here we use the \"wiki_dpr\"\n",
        "retriever = RagRetriever.from_pretrained(\"facebook/rag-sequence-nq\", index_name=\"exact\")\n",
        "# Initialize the RAG model\n",
        "model = RagSequenceForGeneration.from_pretrained(\"facebook/rag-sequence-nq\", retriever=retriever, n_docs=1)\n",
        "# Run the model in parallel\n",
        "model = nn.DataParallel(model, device_ids=d_ids)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RQoWegdHUs82"
      },
      "source": [
        "#### Encode the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9OGBIo9MUoHP"
      },
      "source": [
        "# Find the maximum length\n",
        "max_len = max([len(tokenizer(s)[\"input_ids\"]) for s in train.text.to_list()])\n",
        "print(\"The maximum sentence length in training based on BERT BPEs is\", max_len)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9rqjKM2oVEjM"
      },
      "source": [
        "# Encode input sequences\n",
        "x_train = tokenizer.__call__(\n",
        "        list(train.text.to_list()),\n",
        "        max_length=max_len,\n",
        "        padding=True,\n",
        "        truncation=True,\n",
        "        return_attention_mask=True,\n",
        "        return_tensors=\"pt\"\n",
        ")\n",
        "\n",
        "x_val = tokenizer.__call__(\n",
        "        list(val.text.to_list()),\n",
        "        max_length=max_len,\n",
        "        padding=True,\n",
        "        truncation=True,\n",
        "        return_attention_mask=True,\n",
        "        return_tensors=\"pt\"\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ggjq5Ob0VQly"
      },
      "source": [
        "# Encode target sequences\n",
        "with tokenizer.as_target_tokenizer():\n",
        "    y_train = tokenizer.__call__(\n",
        "        list(train.humor.to_list()),\n",
        "        max_length=1,\n",
        "        padding=True,\n",
        "        truncation=True,\n",
        "        return_attention_mask=True,\n",
        "        return_tensors=\"pt\"\n",
        "    )\n",
        "    y_val = tokenizer.__call__(\n",
        "        list(val.humor.to_list()),\n",
        "        max_length=1,\n",
        "        padding=True,\n",
        "        truncation=True,\n",
        "        return_attention_mask=True,\n",
        "        return_tensors=\"pt\"\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "14nzz8wJVkjV"
      },
      "source": [
        "#### Set up the dataloaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Imt29f1OVm74"
      },
      "source": [
        "batch_size = 4\n",
        "\n",
        "# Create a dataloader for each set\n",
        "print(\"Creating dataloaders...\")\n",
        "\n",
        "train_dataset = TensorDataset(x_train['input_ids'], x_train['attention_mask'], y_train[\"input_ids\"])\n",
        "train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=batch_size)\n",
        "\n",
        "val_dataset = TensorDataset(x_val['input_ids'], x_val['attention_mask'], y_val[\"input_ids\"])\n",
        "val_dataloader = DataLoader(val_dataset, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v42BA-BhVCLN"
      },
      "source": [
        "#### Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "parmBtG5QYx5"
      },
      "source": [
        "# Training method\n",
        "def training():\n",
        "    # Set to train mode\n",
        "    model.train()\n",
        "    total_loss = 0\n",
        "    # Iterate through the training batches\n",
        "    for batch in tqdm(train_dataloader, desc=\"Iteration\"):\n",
        "        # Push the batch to gpu\n",
        "        batch = [r.to(device) for r in batch]\n",
        "        sent_id, mask, labels = batch\n",
        "        # Clear gradients\n",
        "        model.zero_grad()\n",
        "        # Get model outputs\n",
        "        outputs = model(sent_id, attention_mask=mask, labels=labels)\n",
        "        # Get loss\n",
        "        loss = torch.mean(outputs.loss)\n",
        "        # Add to the total loss\n",
        "        total_loss = total_loss + loss\n",
        "        # Backward pass to calculate the gradients\n",
        "        loss.backward()\n",
        "        # Update parameters\n",
        "        optimizer.step()\n",
        "    # Compute the training loss of the epoch\n",
        "    epoch_loss = total_loss / len(train_dataloader)\n",
        "\n",
        "    return epoch_loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zCyiQfsoQd0f"
      },
      "source": [
        "# Evaluation method\n",
        "def evaluate():\n",
        "    print(\"\\nEvaluating...\")\n",
        "    # Set to eval mode\n",
        "    model.eval()\n",
        "    total_loss, total_accuracy = 0, 0\n",
        "    # Iterate through the validation batches\n",
        "    for batch in val_dataloader:\n",
        "        # Push the batch to gpu\n",
        "        batch = [t.to(device) for t in batch]\n",
        "        sent_id, mask, labels = batch\n",
        "        # Deactivate autograd\n",
        "        with torch.no_grad():\n",
        "            # Get model outputs\n",
        "            outputs = model(sent_id, attention_mask=mask, labels=labels)\n",
        "            # Get loss\n",
        "            loss = torch.mean(outputs.loss)\n",
        "            total_loss = total_loss + loss\n",
        "\n",
        "    # Compute the validation loss of the epoch\n",
        "    epoch_loss = total_loss / len(val_dataloader)\n",
        "\n",
        "    return epoch_loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X15mu3JxVz3O"
      },
      "source": [
        "# Define the optimizer and the learning rate\n",
        "optimizer = AdamW(model.parameters(), lr = 2e-5)\n",
        "\n",
        "best_val_loss = float('inf')\n",
        "best_epoch = -1\n",
        "train_losses=[]\n",
        "val_losses=[]\n",
        "epochs = 100\n",
        "# Define the number of epochs to wait for early stopping\n",
        "patience = 3\n",
        "\n",
        "print(\"Starting training...\")\n",
        "# Train the model\n",
        "for epoch in range(epochs):\n",
        "    print('\\n Epoch {:} / {:}'.format(epoch + 1, epochs))\n",
        "    train_loss = training()\n",
        "    val_loss = evaluate()\n",
        "\n",
        "    train_losses.append(train_loss)\n",
        "    val_losses.append(val_loss)\n",
        "\n",
        "    print(\"\\nTraining Loss:\", train_loss)\n",
        "    print(\"Validation Loss:\", val_loss)\n",
        "\n",
        "    # Save the model with the best validation loss\n",
        "    if val_loss < best_val_loss:\n",
        "        best_val_loss = val_loss\n",
        "        best_epoch = epoch\n",
        "\n",
        "        # Save the model\n",
        "        model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self\n",
        "        torch.save(model_to_save.state_dict(), \"rag.bin\")\n",
        "\n",
        "    # Early stopping\n",
        "    if ((epoch - best_epoch) >= patience):\n",
        "        print(\"No improvement in\", patience, \"epochs. Stopped training.\")\n",
        "        break"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RX4dSAleQg75"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}