runner.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "vxiBomdXUqLP"
      },
      "source": [
        "#### Import Libraries\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vRirsUF4RRWa",
        "outputId": "1181da80-e214-4e55-b7f5-97556f06309f"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 1,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "tfEUucgPfO5-"
      },
      "outputs": [],
      "source": [
        "from nltk.tokenize import MWETokenizer, word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import SnowballStemmer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "U_6T9xvwus8b"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import pathlib\n",
        "import os\n",
        "import re\n",
        "import string\n",
        "import time\n",
        "from collections import defaultdict\n",
        "from typing import List, Union\n",
        "import pprint\n",
        "import json"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "t3EJ087UXZCJ"
      },
      "source": [
        "#### Mount to Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PVYSLV7ARvQ9",
        "outputId": "e37871f4-6d90-4128-967e-eab772d63926"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "simpsons_dir = 'Simpsons2022'\n",
        "drive_path = \"/content/drive\" + \"/My Drive/\" + simpsons_dir\n",
        "files = os.listdir(drive_path)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "S8Fnpg9RSiId"
      },
      "source": [
        "#### Implementation of Inverted Index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "4Mnq6xnaONyC"
      },
      "outputs": [],
      "source": [
        "from IPython.utils import process\n",
        "\n",
        "from inverted_index import InvertedIndex"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "dyk1msW9SxI1"
      },
      "source": [
        "#### Demonstration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_Oh0C1LXONyG",
        "outputId": "99c707eb-5d0c-4ba6-ef3f-a89af804ec13"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Size of index is: 11100\n",
            "Total time to create the inverted index: 16.5426 sec\n",
            "--------------- PART I ---------------\n",
            "Enter the way you would like to run Part I:\n",
            "Type q - (query) or p - (path):q\n",
            "Enter your query: river\n",
            "\"river\" is processed as \"river\"\n",
            "Document frequency: 3\n",
            "Document IDs: ['5.11', '5.20', '7.5']\n",
            "\n",
            "['5.11', '5.20', '7.5']\n",
            "\n",
            "--------------- PART II ---------------\n",
            "You will be asked to enter two terms and a window size.\n",
            "Each of these must be provided one per line.\n",
            "Enter term 1: episode\n",
            "Enter term 2: 11\n",
            "Enter proximity window size: 1\n",
            "In document 4.11 pre-processed terms: \"episod\" & \"11\" are within a window of 1 at positions: [[16, 17]]\n",
            "In document 5.11 pre-processed terms: \"episod\" & \"11\" are within a window of 1 at positions: [[14, 15]]\n",
            "In document 6.11 pre-processed terms: \"episod\" & \"11\" are within a window of 1 at positions: [[15, 16]]\n",
            "In document 7.11 pre-processed terms: \"episod\" & \"11\" are within a window of 1 at positions: [[16, 17]]\n",
            "\"episode\" and \"11\" appear within a pre-defined window of 1 in 4 documents. These are:\n",
            "['4.11', '5.11', '6.11', '7.11']\n"
          ]
        }
      ],
      "source": [
        "def main():\n",
        "    index = InvertedIndex() # initilaise the index\n",
        "    drive_path = \"/content/drive/My Drive/Simpsons2022\"\n",
        "    corpus = index.read_data(drive_path) # specify the directory path in which files are located\n",
        "    \n",
        "    index.index_corpus(corpus) # index documents/corpus\n",
        "    \n",
        "    # write a demo to check entered search terms against the inverted index\n",
        "    print('--------------- PART I ---------------')\n",
        "    print('Enter the way you would like to run Part I:')\n",
        "    option = input(\"Type q - (query) or p - (path):\")\n",
        "    if option not in ['q', 'p']:\n",
        "        print('Wrong option. Try again.')\n",
        "        return 0\n",
        "    if option == 'q':  # Run a single term query\n",
        "        search_term = input(\"Enter your query: \") # insert a query\n",
        "        docs = index.dump(examples=[search_term])  # includes preprocessing of query\n",
        "    else:  # Run on a file containing a term per line\n",
        "        file_path = input(\"Specify the relative path to the file: \") # specify a path\n",
        "        drive_path = \"/content/drive/My Drive/\" + file_path  # development-examples.txt\n",
        "        with open(drive_path) as f_open:\n",
        "            search_terms = f_open.readlines()\n",
        "        search_term_merged = []\n",
        "        for search_term in search_terms:\n",
        "            # Assume terms are per line\n",
        "            search_term_merged.append(search_term.replace('\\n', ''))\n",
        "        docs = index.dump(examples=search_term_merged)\n",
        "    print(docs)\n",
        "    print()\n",
        "    \n",
        "    print('--------------- PART II ---------------')\n",
        "    print('You will be asked to enter two terms and a window size.')\n",
        "    print('Each of these must be provided one per line.')\n",
        "    search_term1 = input(\"Enter term 1: \") \n",
        "    search_term2 = input(\"Enter term 2: \") \n",
        "    window_size = int(input(\"Enter proximity window size: \"))\n",
        "    assert window_size > 0\n",
        "    coexistance_dict = index.proximity_search(term1=search_term1, term2=search_term2, window=window_size)\n",
        "    print(f'\"{search_term1}\" and \"{search_term2}\" appear within a pre-defined window of {window_size} in {len(coexistance_dict.keys())} documents. These are:')\n",
        "    print(list(coexistance_dict.keys()))\n",
        "\n",
        "    return index\n",
        "    \n",
        "index = main()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "QvhvxBn1KrFb"
      },
      "source": [
        "#### Testing Input"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "ZO1RF5ZKKtbs"
      },
      "outputs": [],
      "source": [
        "prepared_test_samples_task1 = \"\"\"Bart\n",
        "first\n",
        "image\n",
        "montage\n",
        "well\n",
        "top\n",
        "arguably\n",
        "best\n",
        "number\n",
        "humor\n",
        "dollarydoos\n",
        "Bart Simpson\n",
        "Gordie Howe\n",
        "recalled\n",
        "Bart the Lover\n",
        "cents\n",
        "won\n",
        "voice-overs\n",
        "Simpsonovi\"\"\".split('\\n')\n",
        "\n",
        "my_test_samples_task1_SINGLE_TERM = [\n",
        "    'United States', \n",
        "    'Springfield Elementary School',\n",
        "    'Krusty the Clown',\n",
        "    'Supreme Court',\n",
        "    'Saint Sebastian',\n",
        "    'Steven Spielberg',\n",
        "    'John Swartzwelder',\n",
        "    'Episode 10',\n",
        "]\n",
        "\n",
        "my_test_samples_task1_MULTI_TERM = [\n",
        "    ['United', 'States'], \n",
        "    ['Bart', 'Simpson'],\n",
        "    ['Bart', 'Lover'],\n",
        "    ['Springfield', 'Elementary', 'School'],\n",
        "    ['Krusty', 'the', 'Clown'],\n",
        "    ['Supreme', 'Court'],\n",
        "    ['Saint', 'Sebastian'],\n",
        "    ['Steven', 'Spielberg'],\n",
        "    ['John', 'Swartzwelder'],\n",
        "    ['Episode', '10'],\n",
        "]\n",
        "\n",
        "my_test_samples_task2 = [\n",
        "    ['United', 'States', 1],\n",
        "    ['Bart', 'Simpson', 1],\n",
        "    ['Bart', 'Lover', 2], \n",
        "    ['Springfield', 'Elementary', 1],\n",
        "    ['Krusty', 'Clown', 2],\n",
        "    ['Supreme', 'Court', 1],\n",
        "    ['Saint', 'Sebastian', 1],\n",
        "    ['Steven', 'Spielberg', 1],\n",
        "    ['John', 'Swartzwelder', 1],\n",
        "    ['Episode', '10', 1],\n",
        "]"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "FTtQRJBLSnep"
      },
      "source": [
        "#### Testing methods"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "uiyfyNW8UnLp"
      },
      "outputs": [],
      "source": [
        "def run_tests_with_data_task1(index, data: List[str], is_prepared: bool = True):\n",
        "    test_results = {}\n",
        "    for search_term in data:\n",
        "        docs = index.dump(examples=[search_term])  # includes preprocessing of query\n",
        "        test_results[search_term] = len([d for d in docs if d])  # None is returned when nothing is found\n",
        "    output = json.dumps(dict(sorted(test_results.items(), key=lambda item: item[0])), indent=4)\n",
        "    if is_prepared:\n",
        "        print('--------------- Prepared SINGLE-TERM test data ---------------')\n",
        "    else:\n",
        "        print('--------------- Personal SINGLE-TERM test data ---------------')\n",
        "    print(output)\n",
        "\n",
        "\n",
        "def run_tests_with_MULTI_data_task1(index, data, is_prepared: bool = True):\n",
        "    test_results = {}\n",
        "    for search_term in data:\n",
        "        docs = index.dump(examples=search_term)\n",
        "        search_term_str = re.sub(r\"\\[|]|\\'|,\", \"\", str(search_term))  # remove list-like chars\n",
        "        test_results[search_term_str] = len([d for d in docs if d])  # None is returned when nothing is found\n",
        "    output = json.dumps(dict(sorted(test_results.items(), key=lambda item: item[0])), indent=4)\n",
        "    if is_prepared:\n",
        "        print('--------------- Prepared SET-OF-TERMS test data ---------------')\n",
        "    else:\n",
        "        print('--------------- Personal SET-OF-TERMS test data ---------------')\n",
        "    print(output)\n",
        "\n",
        "\n",
        "def run_tests_with_data_task2(index, data, is_prepared: bool = True):\n",
        "    test_results = {}\n",
        "    for (search_term1, search_term2, window) in data:\n",
        "        docs = index.proximity_search(term1=search_term1, term2=search_term2, window=window)\n",
        "        test_results[search_term1 + ' ' + search_term2 + f\" {window}\"] = len(\n",
        "            [d for d in docs if d])  # None is returned when nothing is found\n",
        "    output = json.dumps(dict(sorted(test_results.items(), key=lambda item: item[0])), indent=4)\n",
        "    if is_prepared:\n",
        "        print('--------------- Prepared TWO-TERMS test data ---------------')\n",
        "    else:\n",
        "        print('--------------- Personal TWO-TERMS test data ---------------')\n",
        "    print(output)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6L6RVUY9J1Dy",
        "outputId": "f2dde826-a9da-4ac8-a65c-da4637faa873"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Size of index is: 11100\n",
            "Total time to create the inverted index: 7.3152 sec\n",
            "--------------- PART I ---------------\n",
            "\"Bart\" is processed as \"bart\"\n",
            "Document frequency: 113\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"first\" is processed as \"first\"\n",
            "Document frequency: 102\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '4.1', '4.2', '4.3', '4.4', '4.6', '4.8', '4.9', '4.10', '4.11', '4.12', '4.14', '4.15', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.7', '5.8', '5.9', '5.10', '5.12', '5.13', '5.15', '5.17', '5.18', '5.19', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.11', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20', '7.21', '7.23']\n",
            "\n",
            "\"image\" is processed as \"imag\"\n",
            "Document frequency: 15\n",
            "Document IDs: ['3.7', '3.9', '3.16', '3.18', '3.24', '4.15', '5.4', '5.12', '5.13', '7.5', '7.6', '7.18', '7.21', '7.25', '7.2']\n",
            "\n",
            "\"montage\" is processed as \"montag\"\n",
            "Document frequency: 7\n",
            "Document IDs: ['3.16', '3.21', '4.4', '4.18', '6.3', '7.3', '7.10']\n",
            "\n",
            "\"well\" is processed as \"well\"\n",
            "Document frequency: 72\n",
            "Document IDs: ['3.1', '3.2', '3.7', '3.10', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.21', '3.22', '3.23', '4.1', '4.2', '4.3', '4.4', '4.9', '4.10', '4.12', '4.13', '4.15', '4.17', '4.18', '4.21', '5.1', '5.2', '5.5', '5.8', '5.9', '5.10', '5.13', '5.14', '5.15', '5.16', '5.17', '5.19', '5.20', '5.21', '5.22', '6.1', '6.4', '6.5', '6.6', '6.9', '6.10', '6.11', '6.13', '6.15', '6.16', '6.18', '6.19', '6.21', '6.22', '6.23', '6.25', '5.3', '7.1', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.13', '7.14', '7.19', '7.20', '7.21', '7.24']\n",
            "\n",
            "\"top\" is processed as \"top\"\n",
            "Document frequency: 51\n",
            "Document IDs: ['3.1', '3.4', '3.5', '3.6', '3.7', '3.8', '3.10', '3.16', '3.17', '3.23', '4.1', '4.2', '4.3', '4.6', '4.8', '4.9', '4.12', '4.15', '4.16', '4.17', '4.22', '5.1', '5.4', '5.9', '5.13', '5.15', '5.18', '5.22', '6.1', '6.2', '6.4', '6.6', '6.9', '6.14', '6.15', '6.16', '6.19', '6.20', '6.21', '6.23', '6.24', '6.25', '5.3', '7.1', '7.4', '7.6', '7.7', '7.18', '7.19', '7.21', '7.24']\n",
            "\n",
            "\"arguably\" is processed as \"arguabl\"\n",
            "Document frequency: 3\n",
            "Document IDs: ['3.16', '4.12', '5.9']\n",
            "\n",
            "\"best\" is processed as \"best\"\n",
            "Document frequency: 98\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.2', '4.3', '4.4', '4.5', '4.6', '4.7', '4.9', '4.10', '4.11', '4.12', '4.14', '4.15', '4.17', '4.20', '4.22', '5.1', '5.2', '5.4', '5.5', '5.7', '5.8', '5.9', '5.10', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.21', '5.22', '6.2', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.11', '6.12', '6.15', '6.16', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20', '7.21', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"number\" is processed as \"number\"\n",
            "Document frequency: 47\n",
            "Document IDs: ['3.1', '3.5', '3.7', '3.9', '3.16', '3.17', '3.18', '3.19', '3.21', '4.1', '4.2', '4.7', '4.9', '4.11', '4.12', '4.15', '4.16', '4.17', '4.21', '4.22', '5.1', '5.2', '5.8', '5.10', '5.11', '5.13', '5.14', '5.21', '5.22', '6.2', '6.4', '6.6', '6.12', '6.20', '6.22', '6.24', '6.25', '5.3', '7.1', '7.6', '7.7', '7.8', '7.10', '7.13', '7.19', '7.21', '7.23']\n",
            "\n",
            "\"humor\" is processed as \"humor\"\n",
            "Document frequency: 32\n",
            "Document IDs: ['3.1', '3.3', '3.5', '3.7', '3.10', '3.13', '3.15', '3.16', '3.20', '3.24', '4.11', '4.15', '4.19', '5.1', '5.7', '5.10', '5.11', '5.12', '5.13', '5.14', '6.6', '6.10', '6.14', '6.16', '6.20', '6.22', '7.4', '7.5', '7.10', '7.14', '7.17', '7.18']\n",
            "\n",
            "\"dollarydoos\" is processed as \"dollarydoo\"\n",
            "Document frequency: 1\n",
            "Document IDs: ['6.16']\n",
            "\n",
            "\"Bart Simpson\" is processed as \"bart_simpson\"\n",
            "Document frequency: 18\n",
            "Document IDs: ['3.4', '3.13', '4.1', '4.2', '4.7', '5.1', '5.5', '5.15', '6.5', '6.7', '6.14', '6.18', '6.21', '6.23', '6.25', '5.3', '7.1', '7.4']\n",
            "\n",
            "\"Gordie Howe\" is processed as \"gordi_howe\"\n",
            "\"gordi_howe\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['gordi_howe']\n",
            "Input is not in the inverted index\n",
            "\"recalled\" is processed as \"recal\"\n",
            "Document frequency: 21\n",
            "Document IDs: ['3.1', '3.7', '3.11', '3.14', '3.16', '3.17', '3.19', '5.1', '5.8', '5.16', '5.19', '6.2', '6.3', '6.4', '6.11', '7.4', '7.18', '7.21', '7.22', '7.25', '7.2']\n",
            "\n",
            "\"Bart the Lover\" is processed as \"bart_the_lover\"\n",
            "\"bart_the_lover\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['bart_the_lover']\n",
            "Input is not in the inverted index\n",
            "\"cents\" is processed as \"cent\"\n",
            "Document frequency: 4\n",
            "Document IDs: ['3.11', '3.16', '4.15', '5.1']\n",
            "\n",
            "\"won\" is processed as \"won\"\n",
            "Document frequency: 22\n",
            "Document IDs: ['3.6', '3.8', '3.14', '3.16', '3.17', '3.18', '3.24', '4.2', '4.6', '4.9', '4.10', '4.17', '4.20', '4.22', '5.8', '5.15', '5.17', '6.19', '7.5', '7.6', '7.8', '7.12']\n",
            "\n",
            "\"voice-overs\" is processed as \"voice-ov\"\n",
            "Document frequency: 9\n",
            "Document IDs: ['3.1', '3.6', '3.8', '3.12', '3.14', '3.16', '3.18', '4.9', '5.13']\n",
            "\n",
            "\"Simpsonovi\" is processed as \"simpsonovi\"\n",
            "\"simpsonovi\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['simpsonovi']\n",
            "Input is not in the inverted index\n",
            "--------------- Prepared SINGLE-TERM test data ---------------\n",
            "{\n",
            "    \"Bart\": 113,\n",
            "    \"Bart Simpson\": 18,\n",
            "    \"Bart the Lover\": 0,\n",
            "    \"Gordie Howe\": 0,\n",
            "    \"Simpsonovi\": 0,\n",
            "    \"arguably\": 3,\n",
            "    \"best\": 98,\n",
            "    \"cents\": 4,\n",
            "    \"dollarydoos\": 1,\n",
            "    \"first\": 102,\n",
            "    \"humor\": 32,\n",
            "    \"image\": 15,\n",
            "    \"montage\": 7,\n",
            "    \"number\": 47,\n",
            "    \"recalled\": 21,\n",
            "    \"top\": 51,\n",
            "    \"voice-overs\": 9,\n",
            "    \"well\": 72,\n",
            "    \"won\": 22\n",
            "}\n",
            "\"United States\" is processed as \"unit_state\"\n",
            "\"unit_state\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['unit_state']\n",
            "Input is not in the inverted index\n",
            "\"Springfield Elementary School\" is processed as \"springfield_elementari_school\"\n",
            "Document frequency: 19\n",
            "Document IDs: ['3.16', '3.18', '3.23', '4.4', '4.6', '4.7', '4.15', '5.1', '5.19', '6.1', '6.2', '6.8', '6.22', '6.25', '5.3', '7.1', '7.6', '7.18', '7.22']\n",
            "\n",
            "\"Krusty the Clown\" is processed as \"krusti_the_clown\"\n",
            "Document frequency: 21\n",
            "Document IDs: ['3.5', '3.6', '3.21', '4.1', '4.3', '4.4', '4.5', '4.7', '4.10', '4.15', '4.22', '5.12', '5.17', '5.18', '6.15', '6.18', '6.22', '7.9', '7.10', '7.15', '7.21']\n",
            "\n",
            "\"Supreme Court\" is processed as \"suprem_court\"\n",
            "\"suprem_court\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['suprem_court']\n",
            "Input is not in the inverted index\n",
            "\"Saint Sebastian\" is processed as \"saint_sebastian\"\n",
            "\"saint_sebastian\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['saint_sebastian']\n",
            "Input is not in the inverted index\n",
            "\"Steven Spielberg\" is processed as \"steven_spielberg\"\n",
            "\"steven_spielberg\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['steven_spielberg']\n",
            "Input is not in the inverted index\n",
            "\"John Swartzwelder\" is processed as \"john_swartzweld\"\n",
            "\"john_swartzweld\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['john_swartzweld']\n",
            "Input is not in the inverted index\n",
            "\"Episode 10\" is processed as \"episod_10\"\n",
            "\"episod_10\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['episod_10']\n",
            "Input is not in the inverted index\n",
            "--------------- Personal SINGLE-TERM test data ---------------\n",
            "{\n",
            "    \"Episode 10\": 0,\n",
            "    \"John Swartzwelder\": 0,\n",
            "    \"Krusty the Clown\": 21,\n",
            "    \"Saint Sebastian\": 0,\n",
            "    \"Springfield Elementary School\": 19,\n",
            "    \"Steven Spielberg\": 0,\n",
            "    \"Supreme Court\": 0,\n",
            "    \"United States\": 0\n",
            "}\n",
            "\"United\" is processed as \"unit\"\n",
            "Document frequency: 118\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.11', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"States\" is processed as \"state\"\n",
            "Document frequency: 118\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.11', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "['unit', 'state'] are in ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.11', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\"Bart\" is processed as \"bart\"\n",
            "Document frequency: 113\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"Simpson\" is processed as \"simpson\"\n",
            "Document frequency: 115\n",
            "Document IDs: ['3.1', '3.2', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.17', '4.18', '4.19', '4.21', '4.22', '5.1', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.11', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "['bart', 'simpson'] are in ['3.1', '3.2', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.18', '4.19', '4.21', '4.22', '5.1', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\"Bart\" is processed as \"bart\"\n",
            "Document frequency: 113\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"Lover\" is processed as \"lover\"\n",
            "Document frequency: 7\n",
            "Document IDs: ['3.15', '3.16', '3.17', '5.20', '5.21', '5.22', '6.3']\n",
            "\n",
            "['bart', 'lover'] are in ['3.15', '3.16', '3.17', '5.20', '5.21', '5.22', '6.3']\n",
            "\"Springfield\" is processed as \"springfield\"\n",
            "Document frequency: 81\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.10', '3.13', '3.16', '3.17', '3.19', '3.20', '3.22', '3.24', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.9', '4.12', '4.14', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.4', '5.6', '5.7', '5.8', '5.10', '5.11', '5.13', '5.15', '5.16', '5.17', '5.20', '5.22', '6.1', '6.2', '6.5', '6.7', '6.8', '6.11', '6.12', '6.13', '6.14', '6.15', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.6', '7.7', '7.8', '7.9', '7.13', '7.14', '7.15', '7.16', '7.17', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"Elementary\" is processed as \"elementari\"\n",
            "Document frequency: 6\n",
            "Document IDs: ['3.22', '4.5', '4.14', '6.2', '6.21', '7.20']\n",
            "\n",
            "\"School\" is processed as \"school\"\n",
            "Document frequency: 47\n",
            "Document IDs: ['3.3', '3.4', '3.6', '3.8', '3.10', '3.13', '3.16', '3.18', '3.19', '3.22', '3.23', '4.2', '4.4', '4.5', '4.7', '4.8', '4.15', '4.16', '4.17', '4.19', '4.20', '5.2', '5.5', '5.8', '5.9', '5.19', '5.20', '6.2', '6.8', '6.17', '6.19', '6.21', '6.22', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.11', '7.12', '7.18', '7.20', '7.24', '7.25', '7.2']\n",
            "\n",
            "['springfield', 'elementari', 'school'] are in ['3.22', '4.5', '6.2', '6.21', '7.20']\n",
            "\"Krusty\" is processed as \"krusti\"\n",
            "Document frequency: 32\n",
            "Document IDs: ['3.6', '3.10', '3.13', '3.20', '3.21', '4.1', '4.4', '4.10', '4.11', '4.14', '4.15', '4.21', '4.22', '5.1', '5.7', '5.10', '5.12', '6.1', '6.3', '6.11', '6.15', '6.18', '6.19', '6.22', '6.25', '5.3', '7.1', '7.6', '7.9', '7.10', '7.15', '7.18']\n",
            "\n",
            "\"the\" is processed as \"the\"\n",
            "Document frequency: 118\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.11', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"Clown\" is processed as \"clown\"\n",
            "Document frequency: 14\n",
            "Document IDs: ['3.5', '3.6', '3.7', '4.5', '4.10', '4.22', '6.14', '6.15', '6.16', '6.25', '5.3', '7.1', '7.4', '7.15']\n",
            "\n",
            "['krusti', 'the', 'clown'] are in ['3.6', '4.10', '4.22', '6.15', '6.25', '5.3', '7.1', '7.15']\n",
            "\"Supreme\" is processed as \"suprem\"\n",
            "Document frequency: 4\n",
            "Document IDs: ['4.2', '4.6', '4.21', '5.22']\n",
            "\n",
            "\"Court\" is processed as \"court\"\n",
            "Document frequency: 13\n",
            "Document IDs: ['3.4', '3.5', '3.15', '3.18', '4.2', '4.6', '4.8', '5.18', '5.20', '5.22', '6.5', '6.15', '7.18']\n",
            "\n",
            "['suprem', 'court'] are in ['4.2', '4.6', '5.22']\n",
            "\"Saint\" is processed as \"saint\"\n",
            "Document frequency: 2\n",
            "Document IDs: ['3.23', '6.7']\n",
            "\n",
            "\"Sebastian\" is processed as \"sebastian\"\n",
            "Document frequency: 2\n",
            "Document IDs: ['3.23', '4.12']\n",
            "\n",
            "['saint', 'sebastian'] are in ['3.23']\n",
            "\"Steven\" is processed as \"steven\"\n",
            "Document frequency: 7\n",
            "Document IDs: ['3.2', '3.23', '4.19', '6.18', '6.22', '7.11', '7.17']\n",
            "\n",
            "\"Spielberg\" is processed as \"spielberg\"\n",
            "Document frequency: 3\n",
            "Document IDs: ['3.23', '4.19', '6.18']\n",
            "\n",
            "['steven', 'spielberg'] are in ['3.23', '4.19', '6.18']\n",
            "\"John\" is processed as \"john\"\n",
            "Document frequency: 55\n",
            "Document IDs: ['3.1', '3.3', '3.4', '3.7', '3.8', '3.10', '3.14', '3.17', '3.18', '3.19', '3.20', '3.24', '3.23', '4.6', '4.10', '4.12', '4.14', '4.15', '4.18', '4.19', '4.20', '4.22', '5.1', '5.4', '5.5', '5.7', '5.8', '5.11', '5.12', '5.17', '5.20', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.11', '6.12', '6.14', '6.15', '6.23', '6.25', '5.3', '7.1', '7.3', '7.5', '7.6', '7.13', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20']\n",
            "\n",
            "\"Swartzwelder\" is processed as \"swartzweld\"\n",
            "Document frequency: 29\n",
            "Document IDs: ['3.3', '3.4', '3.7', '3.17', '3.18', '3.19', '3.24', '4.6', '4.14', '4.18', '4.19', '4.20', '4.22', '5.4', '5.11', '5.12', '5.17', '5.20', '6.3', '6.4', '6.12', '6.14', '6.15', '7.3', '7.5', '7.6', '7.15', '7.17', '7.18']\n",
            "\n",
            "['john', 'swartzweld'] are in ['3.3', '3.4', '3.7', '3.17', '3.18', '3.19', '3.24', '4.6', '4.14', '4.18', '4.19', '4.20', '4.22', '5.4', '5.11', '5.12', '5.17', '5.20', '6.3', '6.4', '6.12', '6.14', '6.15', '7.3', '7.5', '7.6', '7.15', '7.17', '7.18']\n",
            "\"Episode\" is processed as \"episod\"\n",
            "Document frequency: 118\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.11', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"10\" is processed as \"10\"\n",
            "Document frequency: 33\n",
            "Document IDs: ['3.4', '3.8', '3.10', '3.11', '3.17', '4.9', '4.10', '4.12', '4.17', '4.20', '4.22', '5.4', '5.9', '5.10', '5.13', '5.15', '5.17', '5.20', '6.6', '6.9', '6.10', '6.19', '6.20', '6.24', '6.25', '5.3', '7.1', '7.6', '7.10', '7.20', '7.21', '7.25', '7.2']\n",
            "\n",
            "['episod', '10'] are in ['3.4', '3.8', '3.10', '3.11', '3.17', '4.9', '4.10', '4.12', '4.17', '4.20', '4.22', '5.4', '5.9', '5.10', '5.13', '5.15', '5.17', '5.20', '6.6', '6.9', '6.10', '6.19', '6.20', '6.24', '6.25', '5.3', '7.1', '7.6', '7.10', '7.20', '7.21', '7.25', '7.2']\n",
            "--------------- Personal SET-OF-TERMS test data ---------------\n",
            "{\n",
            "    \"Bart Lover\": 7,\n",
            "    \"Bart Simpson\": 110,\n",
            "    \"Episode 10\": 33,\n",
            "    \"John Swartzwelder\": 29,\n",
            "    \"Krusty the Clown\": 8,\n",
            "    \"Saint Sebastian\": 1,\n",
            "    \"Springfield Elementary School\": 5,\n",
            "    \"Steven Spielberg\": 3,\n",
            "    \"Supreme Court\": 3,\n",
            "    \"United States\": 118\n",
            "}\n",
            "--------------- PART II ---------------\n",
            "In document 3.1 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[146, 147], [1653, 1654]]\n",
            "In document 3.2 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[120, 121], [137, 138], [1591, 1592], [1957, 1958]]\n",
            "In document 3.3 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[124, 125], [779, 780]]\n",
            "In document 3.4 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[135, 136]]\n",
            "In document 3.5 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[139, 140]]\n",
            "In document 3.6 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[133, 134]]\n",
            "In document 3.7 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[106, 107]]\n",
            "In document 3.8 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[136, 137], [858, 859]]\n",
            "In document 3.9 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[120, 121], [1204, 1205]]\n",
            "In document 3.10 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[131, 132]]\n",
            "In document 3.11 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[155, 156]]\n",
            "In document 3.12 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[168, 169]]\n",
            "In document 3.13 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[114, 115]]\n",
            "In document 3.14 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[118, 119]]\n",
            "In document 3.15 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[124, 125]]\n",
            "In document 3.16 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[108, 109], [1002, 1003]]\n",
            "In document 3.17 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[166, 167], [1325, 1326]]\n",
            "In document 3.18 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[116, 117]]\n",
            "In document 3.19 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[129, 130], [1569, 1570]]\n",
            "In document 3.20 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[156, 157], [3058, 3059]]\n",
            "In document 3.21 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[151, 152]]\n",
            "In document 3.22 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[135, 136]]\n",
            "In document 3.24 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[127, 128]]\n",
            "In document 3.23 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[146, 147]]\n",
            "In document 4.1 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[112, 113]]\n",
            "In document 4.2 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[130, 131]]\n",
            "In document 4.3 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[145, 146]]\n",
            "In document 4.4 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[148, 149]]\n",
            "In document 4.5 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[125, 126]]\n",
            "In document 4.6 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[116, 117], [500, 501]]\n",
            "In document 4.7 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[147, 148]]\n",
            "In document 4.8 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[167, 168]]\n",
            "In document 4.9 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[135, 136]]\n",
            "In document 4.10 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[133, 134], [2116, 2117]]\n",
            "In document 4.11 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[127, 128]]\n",
            "In document 4.12 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[162, 163]]\n",
            "In document 4.13 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[132, 133]]\n",
            "In document 4.14 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[135, 136], [542, 543]]\n",
            "In document 4.15 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[136, 137]]\n",
            "In document 4.16 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[132, 133]]\n",
            "In document 4.17 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[135, 136]]\n",
            "In document 4.18 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[133, 134]]\n",
            "In document 4.19 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[120, 121]]\n",
            "In document 4.20 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[121, 122]]\n",
            "In document 4.21 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[127, 128], [875, 876]]\n",
            "In document 4.22 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[162, 163], [998, 999]]\n",
            "In document 5.1 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[175, 176], [1697, 1698], [2186, 2187]]\n",
            "In document 5.2 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[114, 115]]\n",
            "In document 5.4 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[115, 116]]\n",
            "In document 5.5 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[183, 184]]\n",
            "In document 5.6 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[122, 123], [253, 254]]\n",
            "In document 5.7 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[121, 122]]\n",
            "In document 5.8 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[146, 147]]\n",
            "In document 5.9 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[151, 152], [424, 425]]\n",
            "In document 5.10 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[178, 179], [1702, 1703]]\n",
            "In document 5.11 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[130, 131], [818, 819]]\n",
            "In document 5.12 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[141, 142]]\n",
            "In document 5.13 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[136, 137]]\n",
            "In document 5.14 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[118, 119], [739, 740], [1049, 1050]]\n",
            "In document 5.15 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[174, 175], [1391, 1392]]\n",
            "In document 5.16 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[123, 124], [711, 712]]\n",
            "In document 5.17 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[129, 130], [1464, 1465]]\n",
            "In document 5.18 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[136, 137]]\n",
            "In document 5.19 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[157, 158], [524, 525], [1023, 1024]]\n",
            "In document 5.20 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[149, 150], [1365, 1366]]\n",
            "In document 5.21 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[149, 150]]\n",
            "In document 5.22 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[140, 141]]\n",
            "In document 6.1 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[125, 126]]\n",
            "In document 6.2 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[132, 133]]\n",
            "In document 6.3 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[147, 148]]\n",
            "In document 6.4 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[118, 119]]\n",
            "In document 6.5 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[125, 126], [266, 267], [966, 967]]\n",
            "In document 6.6 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[139, 140]]\n",
            "In document 6.7 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[134, 135], [1142, 1143]]\n",
            "In document 6.8 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[120, 121]]\n",
            "In document 6.9 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[125, 126]]\n",
            "In document 6.10 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[133, 134], [897, 898]]\n",
            "In document 6.11 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[151, 152]]\n",
            "In document 6.12 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[137, 138]]\n",
            "In document 6.13 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[115, 116]]\n",
            "In document 6.14 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[113, 114]]\n",
            "In document 6.15 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[143, 144]]\n",
            "In document 6.16 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[130, 131]]\n",
            "In document 6.17 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[132, 133]]\n",
            "In document 6.18 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[139, 140], [441, 442], [1631, 1632]]\n",
            "In document 6.19 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[129, 130]]\n",
            "In document 6.20 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[141, 142]]\n",
            "In document 6.21 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[138, 139]]\n",
            "In document 6.22 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[145, 146]]\n",
            "In document 6.23 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[123, 124]]\n",
            "In document 6.24 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[119, 120]]\n",
            "In document 6.25 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[249, 250], [3139, 3140], [3964, 3965]]\n",
            "In document 5.3 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[249, 250], [3139, 3140], [3964, 3965]]\n",
            "In document 7.1 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[249, 250], [3139, 3140], [3964, 3965]]\n",
            "In document 7.3 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[136, 137]]\n",
            "In document 7.4 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[126, 127]]\n",
            "In document 7.5 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[124, 125], [1878, 1879], [2088, 2089]]\n",
            "In document 7.6 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[156, 157]]\n",
            "In document 7.7 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[137, 138]]\n",
            "In document 7.8 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[150, 151]]\n",
            "In document 7.9 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[139, 140], [1296, 1297]]\n",
            "In document 7.10 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[162, 163]]\n",
            "In document 7.11 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[143, 144], [938, 939]]\n",
            "In document 7.12 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[133, 134]]\n",
            "In document 7.13 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[152, 153], [184, 185], [908, 909], [1323, 1324], [1519, 1520]]\n",
            "In document 7.14 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[126, 127]]\n",
            "In document 7.15 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[139, 140]]\n",
            "In document 7.16 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[116, 117]]\n",
            "In document 7.17 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[105, 106]]\n",
            "In document 7.18 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[172, 173], [1845, 1846]]\n",
            "In document 7.19 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[122, 123]]\n",
            "In document 7.20 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[131, 132]]\n",
            "In document 7.21 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[197, 198], [1837, 1838]]\n",
            "In document 7.22 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[154, 155]]\n",
            "In document 7.23 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[184, 185], [228, 229], [526, 527]]\n",
            "In document 7.24 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[126, 127]]\n",
            "In document 7.25 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[126, 127]]\n",
            "In document 7.2 pre-processed terms: \"unit\" & \"state\" are within a window of 1 at positions: [[126, 127]]\n",
            "In document 3.15 pre-processed terms: \"bart\" & \"lover\" are within a window of 2 at positions: [[81, 83]]\n",
            "In document 3.16 pre-processed terms: \"bart\" & \"lover\" are within a window of 2 at positions: [[0, 2], [4, 2], [4, 6], [80, 82], [777, 779], [1511, 1513], [1640, 1642], [1751, 1753]]\n",
            "In document 3.17 pre-processed terms: \"bart\" & \"lover\" are within a window of 2 at positions: [[121, 123]]\n",
            "In document 3.22 pre-processed terms: \"springfield\" & \"elementari\" are within a window of 1 at positions: [[183, 184], [666, 667]]\n",
            "In document 4.5 pre-processed terms: \"springfield\" & \"elementari\" are within a window of 1 at positions: [[725, 726], [868, 869]]\n",
            "In document 4.14 pre-processed terms: \"springfield\" & \"elementari\" are within a window of 1 at positions: [[487, 488]]\n",
            "In document 6.2 pre-processed terms: \"springfield\" & \"elementari\" are within a window of 1 at positions: [[296, 297], [411, 412]]\n",
            "In document 6.21 pre-processed terms: \"springfield\" & \"elementari\" are within a window of 1 at positions: [[156, 157], [860, 861]]\n",
            "In document 7.20 pre-processed terms: \"springfield\" & \"elementari\" are within a window of 1 at positions: [[314, 315]]\n",
            "In document 4.2 pre-processed terms: \"suprem\" & \"court\" are within a window of 1 at positions: [[970, 971]]\n",
            "In document 4.6 pre-processed terms: \"suprem\" & \"court\" are within a window of 1 at positions: [[241, 242]]\n",
            "In document 5.22 pre-processed terms: \"suprem\" & \"court\" are within a window of 1 at positions: [[858, 859]]\n",
            "In document 3.23 pre-processed terms: \"saint\" & \"sebastian\" are within a window of 1 at positions: [[469, 470], [531, 532], [965, 966]]\n",
            "In document 3.23 pre-processed terms: \"steven\" & \"spielberg\" are within a window of 1 at positions: [[794, 795]]\n",
            "In document 4.19 pre-processed terms: \"steven\" & \"spielberg\" are within a window of 1 at positions: [[593, 594], [669, 670]]\n",
            "In document 6.18 pre-processed terms: \"steven\" & \"spielberg\" are within a window of 1 at positions: [[570, 571], [1228, 1229]]\n",
            "In document 3.3 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[1016, 1017]]\n",
            "In document 3.4 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[22, 23], [192, 193], [710, 711]]\n",
            "In document 3.7 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[34, 35], [270, 271], [916, 917], [1461, 1462]]\n",
            "In document 3.17 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[24, 25], [221, 222], [840, 841]]\n",
            "In document 3.18 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[760, 761]]\n",
            "In document 3.19 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[22, 23], [224, 225], [615, 616], [632, 633], [1555, 1556]]\n",
            "In document 3.24 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[28, 29], [232, 233], [629, 630], [1780, 1781], [1830, 1831]]\n",
            "In document 4.6 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[24, 25], [178, 179], [563, 564], [646, 647]]\n",
            "In document 4.14 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[791, 792]]\n",
            "In document 4.18 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[633, 634]]\n",
            "In document 4.19 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[1149, 1150], [1249, 1250]]\n",
            "In document 4.20 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[20, 21], [193, 194], [760, 761]]\n",
            "In document 4.22 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[41, 42], [218, 219], [867, 868]]\n",
            "In document 5.4 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[30, 31], [175, 176], [865, 866]]\n",
            "In document 5.11 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[22, 23], [223, 224], [685, 686]]\n",
            "In document 5.12 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[35, 36], [195, 196], [711, 712]]\n",
            "In document 5.17 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[24, 25], [218, 219], [762, 763], [769, 770]]\n",
            "In document 5.20 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[42, 43], [225, 226], [897, 898]]\n",
            "In document 6.3 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[502, 503]]\n",
            "In document 6.4 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[18, 19], [138, 139], [415, 416]]\n",
            "In document 6.12 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[34, 35], [161, 162], [543, 544], [1163, 1164]]\n",
            "In document 6.14 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[20, 21], [186, 187], [414, 415], [424, 425]]\n",
            "In document 6.15 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[39, 40], [186, 187], [503, 504], [522, 523]]\n",
            "In document 7.3 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[643, 644]]\n",
            "In document 7.5 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[1019, 1020]]\n",
            "In document 7.6 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[34, 35], [260, 261], [1399, 1400]]\n",
            "In document 7.15 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[33, 34], [191, 192], [551, 552]]\n",
            "In document 7.17 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[23, 24], [133, 134], [528, 529], [535, 536]]\n",
            "In document 7.18 pre-processed terms: \"john\" & \"swartzweld\" are within a window of 1 at positions: [[66, 67], [182, 183], [891, 892], [2135, 2136]]\n",
            "In document 3.8 pre-processed terms: \"episod\" & \"10\" are within a window of 1 at positions: [[1886, 1885]]\n",
            "In document 3.10 pre-processed terms: \"episod\" & \"10\" are within a window of 1 at positions: [[26, 27]]\n",
            "In document 3.11 pre-processed terms: \"episod\" & \"10\" are within a window of 1 at positions: [[27, 28]]\n",
            "In document 4.10 pre-processed terms: \"episod\" & \"10\" are within a window of 1 at positions: [[30, 31]]\n",
            "In document 4.22 pre-processed terms: \"episod\" & \"10\" are within a window of 1 at positions: [[2015, 2014]]\n",
            "In document 5.10 pre-processed terms: \"episod\" & \"10\" are within a window of 1 at positions: [[32, 33]]\n",
            "In document 6.10 pre-processed terms: \"episod\" & \"10\" are within a window of 1 at positions: [[16, 17]]\n",
            "In document 7.10 pre-processed terms: \"episod\" & \"10\" are within a window of 1 at positions: [[16, 17]]\n",
            "--------------- Personal TWO-TERMS test data ---------------\n",
            "{\n",
            "    \"Bart Lover 2\": 3,\n",
            "    \"Bart Simpson 1\": 0,\n",
            "    \"Episode 10 1\": 8,\n",
            "    \"John Swartzwelder 1\": 29,\n",
            "    \"Krusty Clown 2\": 0,\n",
            "    \"Saint Sebastian 1\": 1,\n",
            "    \"Springfield Elementary 1\": 6,\n",
            "    \"Steven Spielberg 1\": 3,\n",
            "    \"Supreme Court 1\": 3,\n",
            "    \"United States 1\": 118\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "def test():\n",
        "    index = InvertedIndex() # initilaise the index\n",
        "    corpus = index.read_data(drive_path) # specify the directory path in which files are located\n",
        "    index.index_corpus(corpus) # index documents/corpus\n",
        "    \n",
        "    print('--------------- PART I ---------------')\n",
        "    run_tests_with_data_task1(index=index, data=prepared_test_samples_task1, is_prepared=True)\n",
        "    run_tests_with_data_task1(index=index, data=my_test_samples_task1_SINGLE_TERM, is_prepared=False)\n",
        "    run_tests_with_MULTI_data_task1(index=index, data=my_test_samples_task1_MULTI_TERM, is_prepared=False)\n",
        "    \n",
        "    print('--------------- PART II ---------------')\n",
        "    run_tests_with_data_task2(index=index, data=my_test_samples_task2, is_prepared=False)\n",
        "\n",
        "    return index\n",
        "    \n",
        "index = test()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "mdkAUqGUS1ZZ"
      },
      "source": [
        "#### Report Examples & Demo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_28_yYTjwwby",
        "outputId": "e4ccede5-0fc7-425c-f026-7d751b125288"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[['bart_simpson'], ['bart', 'the', 'lover'], ['simpsonovi']]"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "blackboard_set = ['Bart Simpson', 'Bart the Lover', 'Simpsonovi']\n",
        "blackboard_set_processed = [index.process_document(d) for d in blackboard_set]\n",
        "blackboard_set_processed"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QVPjeA1cxfiE",
        "outputId": "5191ba55-e73f-4364-8c4d-a10f10ca38ab"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\"Bart the Lover\" is processed as \"bart_the_lover\"\n",
            "\"bart_the_lover\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['bart_the_lover']\n",
            "Input is not in the inverted index\n",
            "[None]\n"
          ]
        }
      ],
      "source": [
        "print(index.dump(['Bart the Lover']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zh2tUNu70uB3",
        "outputId": "7c926353-4024-4607-9db6-fa0a9d429860"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\"Bart\" is processed as \"bart\"\n",
            "Document frequency: 113\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"the\" is processed as \"the\"\n",
            "Document frequency: 118\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15', '3.16', '3.17', '3.18', '3.19', '3.20', '3.21', '3.22', '3.24', '3.23', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9', '4.10', '4.11', '4.12', '4.13', '4.14', '4.15', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.1', '5.2', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9', '5.10', '5.11', '5.12', '5.13', '5.14', '5.15', '5.16', '5.17', '5.18', '5.19', '5.20', '5.21', '5.22', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8', '6.9', '6.10', '6.11', '6.12', '6.13', '6.14', '6.15', '6.16', '6.17', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12', '7.13', '7.14', '7.15', '7.16', '7.17', '7.18', '7.19', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"Lover\" is processed as \"lover\"\n",
            "Document frequency: 7\n",
            "Document IDs: ['3.15', '3.16', '3.17', '5.20', '5.21', '5.22', '6.3']\n",
            "\n",
            "['bart', 'the', 'lover'] are in ['3.15', '3.16', '3.17', '5.20', '5.21', '5.22', '6.3']\n",
            "['3.15', '3.16', '3.17', '5.20', '5.21', '5.22', '6.3']\n"
          ]
        }
      ],
      "source": [
        "print(index.dump('Bart the Lover'.split(' ')))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l2bKC0Y94-i5",
        "outputId": "e4f1723b-1ea0-4aa6-8323-4db078ca5106"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\"Lover\" is processed as \"lover\"\n",
            "Document frequency: 7\n",
            "Document IDs: ['3.15', '3.16', '3.17', '5.20', '5.21', '5.22', '6.3']\n",
            "\n",
            "\"nonexistentfiller\" is processed as \"nonexistentfil\"\n",
            "\"nonexistentfil\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['nonexistentfil']\n",
            "['3.15', '3.16', '3.17', '5.20', '5.21', '5.22', '6.3']\n"
          ]
        }
      ],
      "source": [
        "print(index.dump('Lover nonexistentfiller'.split(' ')))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y_lOOoL2l_zB",
        "outputId": "1530b713-b07e-4ba8-ded0-5541eff0a751"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Searching for (\"across\" AND \"run\")\n",
            "In document 6.23 pre-processed terms: \"across\" & \"run\" are within a window of 2 at positions: [[470, 472]]\n",
            "In document 6.25 pre-processed terms: \"across\" & \"run\" are within a window of 2 at positions: [[77, 76]]\n",
            "In document 5.3 pre-processed terms: \"across\" & \"run\" are within a window of 2 at positions: [[77, 76]]\n",
            "In document 7.1 pre-processed terms: \"across\" & \"run\" are within a window of 2 at positions: [[77, 76]]\n",
            "\"across\" and \"run\" appear within a pre-defined window of 2 in 4 documents. These are:\n",
            "['6.23', '6.25', '5.3', '7.1']\n"
          ]
        }
      ],
      "source": [
        "search_term2 = 'run'\n",
        "search_term1 = 'across'\n",
        "window_size = 2\n",
        "print(f'Searching for (\"{search_term1}\" AND \"{search_term2}\")')\n",
        "coexistance_dict = index.proximity_search(term1=search_term1, term2=search_term2, window=window_size)\n",
        "print(f'\"{search_term1}\" and \"{search_term2}\" appear within a pre-defined window of {window_size} in {len(coexistance_dict.keys())} documents. These are:')\n",
        "print(list(coexistance_dict.keys()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E65UTZqPmUQ_",
        "outputId": "531f9c42-c363-40e9-e46a-c673c6ce09f3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{0: [1, [2509]],\n",
              " 5: [2, [339, 1414]],\n",
              " 7: [1, [1422]],\n",
              " 11: [1, [1458]],\n",
              " 13: [1, [1532]],\n",
              " 15: [2, [1416, 1623]],\n",
              " 17: [1, [1004]],\n",
              " 32: [2, [217, 1894]],\n",
              " 57: [1, [1146]]}"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "index.inverted_index[\"voice-ov\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AFHf-p7_adGv",
        "outputId": "48f86b96-266a-49c5-8449-cedcbea4aca1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['springfield_elementari_school']"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "index.process_document(\"Springfield Elementary School\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MCoKmm4vt-Nr",
        "outputId": "190a781d-30ce-41ee-84f2-71763a199747"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['oper', 'function']"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "index.process_document(\"operational functionality\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7mpoOz1iuMTd",
        "outputId": "a468cb15-a353-46c1-875d-b96c43abfd0e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['oper', 'function']"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "index.process_document(\"operative functioning\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8EtfwliMamCh",
        "outputId": "d3dceebc-61c1-4093-b2a3-7a5f45f2ad34"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['3.16', '3.18', '3.23', '4.4', '4.6', '4.7', '4.15', '5.1', '5.19', '6.1', '6.2', '6.8', '6.22', '6.25', '5.3', '7.1', '7.6', '7.18', '7.22']\n"
          ]
        }
      ],
      "source": [
        "print(index.get_doc_filenames_from_ids(index.inverted_index['springfield_elementari_school'].keys()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fn1qoaASfAE2",
        "outputId": "222497f5-64c7-425d-c18f-c8d3e3b05d8f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\"Springfield\" is processed as \"springfield\"\n",
            "Document frequency: 81\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.10', '3.13', '3.16', '3.17', '3.19', '3.20', '3.22', '3.24', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.9', '4.12', '4.14', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.4', '5.6', '5.7', '5.8', '5.10', '5.11', '5.13', '5.15', '5.16', '5.17', '5.20', '5.22', '6.1', '6.2', '6.5', '6.7', '6.8', '6.11', '6.12', '6.13', '6.14', '6.15', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.6', '7.7', '7.8', '7.9', '7.13', '7.14', '7.15', '7.16', '7.17', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"Elementary\" is processed as \"elementari\"\n",
            "Document frequency: 6\n",
            "Document IDs: ['3.22', '4.5', '4.14', '6.2', '6.21', '7.20']\n",
            "\n",
            "\"School\" is processed as \"school\"\n",
            "Document frequency: 47\n",
            "Document IDs: ['3.3', '3.4', '3.6', '3.8', '3.10', '3.13', '3.16', '3.18', '3.19', '3.22', '3.23', '4.2', '4.4', '4.5', '4.7', '4.8', '4.15', '4.16', '4.17', '4.19', '4.20', '5.2', '5.5', '5.8', '5.9', '5.19', '5.20', '6.2', '6.8', '6.17', '6.19', '6.21', '6.22', '6.25', '5.3', '7.1', '7.3', '7.4', '7.5', '7.6', '7.11', '7.12', '7.18', '7.20', '7.24', '7.25', '7.2']\n",
            "\n",
            "['springfield', 'elementari', 'school'] are in ['3.22', '4.5', '6.2', '6.21', '7.20']\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "['3.22', '4.5', '6.2', '6.21', '7.20']"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "index.dump(['Springfield', 'Elementary', 'School'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cc2PfNPfsCfA",
        "outputId": "2fd2a87b-f5c2-4cd6-810b-444c39e51f91"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\"Gordie Howe\" is processed as \"gordi_howe\"\n",
            "\"gordi_howe\" does not occur in the inverted index\n",
            "\n",
            "These terms are not in the inverted index: ['gordi_howe']\n",
            "Input is not in the inverted index\n",
            "[None]\n"
          ]
        }
      ],
      "source": [
        "print(index.dump([\"Gordie Howe\"]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5WrLrPpTfTpD",
        "outputId": "3852c50b-d478-45bf-dd6c-ade5c7041d4c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\"Springfield\" is processed as \"springfield\"\n",
            "Document frequency: 81\n",
            "Document IDs: ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.10', '3.13', '3.16', '3.17', '3.19', '3.20', '3.22', '3.24', '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.9', '4.12', '4.14', '4.16', '4.17', '4.18', '4.19', '4.20', '4.21', '4.22', '5.4', '5.6', '5.7', '5.8', '5.10', '5.11', '5.13', '5.15', '5.16', '5.17', '5.20', '5.22', '6.1', '6.2', '6.5', '6.7', '6.8', '6.11', '6.12', '6.13', '6.14', '6.15', '6.18', '6.19', '6.20', '6.21', '6.22', '6.23', '6.24', '6.25', '5.3', '7.1', '7.3', '7.4', '7.6', '7.7', '7.8', '7.9', '7.13', '7.14', '7.15', '7.16', '7.17', '7.20', '7.21', '7.22', '7.23', '7.24', '7.25', '7.2']\n",
            "\n",
            "\"Elementary\" is processed as \"elementari\"\n",
            "Document frequency: 6\n",
            "Document IDs: ['3.22', '4.5', '4.14', '6.2', '6.21', '7.20']\n",
            "\n",
            "['springfield', 'elementari'] are in ['3.22', '4.5', '4.14', '6.2', '6.21', '7.20']\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "['3.22', '4.5', '4.14', '6.2', '6.21', '7.20']"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "index.dump(['Springfield', 'Elementary',])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "IPuPylZ2V9MD"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}