det.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0b553959",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as p\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d20a8f5",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'india-news-headlines.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_7880\\1702198925.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mreal\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'india-news-headlines.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[0;32m    910\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    911\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 912\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    913\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    914\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    575\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    576\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 577\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    578\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    579\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m   1405\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1406\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandles\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[1;33m|\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1407\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1408\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1409\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, f, engine)\u001b[0m\n\u001b[0;32m   1659\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[1;34m\"b\"\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1660\u001b[0m                     \u001b[0mmode\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;34m\"b\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1661\u001b[1;33m             self.handles = get_handle(\n\u001b[0m\u001b[0;32m   1662\u001b[0m                 \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1663\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda\\lib\\site-packages\\pandas\\io\\common.py\u001b[0m in \u001b[0;36mget_handle\u001b[1;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[0;32m    857\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;34m\"b\"\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    858\u001b[0m             \u001b[1;31m# Encoding\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 859\u001b[1;33m             handle = open(\n\u001b[0m\u001b[0;32m    860\u001b[0m                 \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    861\u001b[0m                 \u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'india-news-headlines.csv'"
     ]
    }
   ],
   "source": [
    "real = p.read_csv('india-news-headlines.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7493a320",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01ec3f66",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "26fc64b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>publish_date</th>\n",
       "      <th>headline_category</th>\n",
       "      <th>headline_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20010102</td>\n",
       "      <td>unknown</td>\n",
       "      <td>Status quo will not be disturbed at Ayodhya; s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20010102</td>\n",
       "      <td>unknown</td>\n",
       "      <td>Fissures in Hurriyat over Pak visit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20010102</td>\n",
       "      <td>unknown</td>\n",
       "      <td>America's unwanted heading for India?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20010102</td>\n",
       "      <td>unknown</td>\n",
       "      <td>For bigwigs; it is destination Goa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20010102</td>\n",
       "      <td>unknown</td>\n",
       "      <td>Extra buses to clear tourist traffic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650965</th>\n",
       "      <td>20220331</td>\n",
       "      <td>city.srinagar</td>\n",
       "      <td>J&amp;K sacks 2 cops; 3 other employees over terro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650966</th>\n",
       "      <td>20220331</td>\n",
       "      <td>entertainment.hindi.bollywood</td>\n",
       "      <td>Ranbir Kapoor says 'Rishi Kapoor enjoyed his a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650967</th>\n",
       "      <td>20220331</td>\n",
       "      <td>city.trichy</td>\n",
       "      <td>As Covid-19 cases drop to nil in southern dist...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650968</th>\n",
       "      <td>20220331</td>\n",
       "      <td>city.erode</td>\n",
       "      <td>Tamil Nadu sees marginal rise of Covid cases w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650969</th>\n",
       "      <td>20220331</td>\n",
       "      <td>city.salem</td>\n",
       "      <td>Tamil Nadu sees marginal rise of Covid cases w...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3650970 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         publish_date              headline_category  \\\n",
       "0            20010102                        unknown   \n",
       "1            20010102                        unknown   \n",
       "2            20010102                        unknown   \n",
       "3            20010102                        unknown   \n",
       "4            20010102                        unknown   \n",
       "...               ...                            ...   \n",
       "3650965      20220331                  city.srinagar   \n",
       "3650966      20220331  entertainment.hindi.bollywood   \n",
       "3650967      20220331                    city.trichy   \n",
       "3650968      20220331                     city.erode   \n",
       "3650969      20220331                     city.salem   \n",
       "\n",
       "                                             headline_text  \n",
       "0        Status quo will not be disturbed at Ayodhya; s...  \n",
       "1                      Fissures in Hurriyat over Pak visit  \n",
       "2                    America's unwanted heading for India?  \n",
       "3                       For bigwigs; it is destination Goa  \n",
       "4                     Extra buses to clear tourist traffic  \n",
       "...                                                    ...  \n",
       "3650965  J&K sacks 2 cops; 3 other employees over terro...  \n",
       "3650966  Ranbir Kapoor says 'Rishi Kapoor enjoyed his a...  \n",
       "3650967  As Covid-19 cases drop to nil in southern dist...  \n",
       "3650968  Tamil Nadu sees marginal rise of Covid cases w...  \n",
       "3650969  Tamil Nadu sees marginal rise of Covid cases w...  \n",
       "\n",
       "[3650970 rows x 3 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ed2db680",
   "metadata": {},
   "outputs": [],
   "source": [
    "real = real.drop(labels=['publish_date','headline_category'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7c920e9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>headline_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Status quo will not be disturbed at Ayodhya; s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Fissures in Hurriyat over Pak visit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>America's unwanted heading for India?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>For bigwigs; it is destination Goa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Extra buses to clear tourist traffic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650965</th>\n",
       "      <td>J&amp;K sacks 2 cops; 3 other employees over terro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650966</th>\n",
       "      <td>Ranbir Kapoor says 'Rishi Kapoor enjoyed his a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650967</th>\n",
       "      <td>As Covid-19 cases drop to nil in southern dist...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650968</th>\n",
       "      <td>Tamil Nadu sees marginal rise of Covid cases w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650969</th>\n",
       "      <td>Tamil Nadu sees marginal rise of Covid cases w...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3650970 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             headline_text\n",
       "0        Status quo will not be disturbed at Ayodhya; s...\n",
       "1                      Fissures in Hurriyat over Pak visit\n",
       "2                    America's unwanted heading for India?\n",
       "3                       For bigwigs; it is destination Goa\n",
       "4                     Extra buses to clear tourist traffic\n",
       "...                                                    ...\n",
       "3650965  J&K sacks 2 cops; 3 other employees over terro...\n",
       "3650966  Ranbir Kapoor says 'Rishi Kapoor enjoyed his a...\n",
       "3650967  As Covid-19 cases drop to nil in southern dist...\n",
       "3650968  Tamil Nadu sees marginal rise of Covid cases w...\n",
       "3650969  Tamil Nadu sees marginal rise of Covid cases w...\n",
       "\n",
       "[3650970 rows x 1 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d7c2a5c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "real = real.rename(columns={'headline_text':'text'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d930be1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Status quo will not be disturbed at Ayodhya; s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Fissures in Hurriyat over Pak visit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>America's unwanted heading for India?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>For bigwigs; it is destination Goa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Extra buses to clear tourist traffic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650965</th>\n",
       "      <td>J&amp;K sacks 2 cops; 3 other employees over terro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650966</th>\n",
       "      <td>Ranbir Kapoor says 'Rishi Kapoor enjoyed his a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650967</th>\n",
       "      <td>As Covid-19 cases drop to nil in southern dist...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650968</th>\n",
       "      <td>Tamil Nadu sees marginal rise of Covid cases w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650969</th>\n",
       "      <td>Tamil Nadu sees marginal rise of Covid cases w...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3650970 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      text\n",
       "0        Status quo will not be disturbed at Ayodhya; s...\n",
       "1                      Fissures in Hurriyat over Pak visit\n",
       "2                    America's unwanted heading for India?\n",
       "3                       For bigwigs; it is destination Goa\n",
       "4                     Extra buses to clear tourist traffic\n",
       "...                                                    ...\n",
       "3650965  J&K sacks 2 cops; 3 other employees over terro...\n",
       "3650966  Ranbir Kapoor says 'Rishi Kapoor enjoyed his a...\n",
       "3650967  As Covid-19 cases drop to nil in southern dist...\n",
       "3650968  Tamil Nadu sees marginal rise of Covid cases w...\n",
       "3650969  Tamil Nadu sees marginal rise of Covid cases w...\n",
       "\n",
       "[3650970 rows x 1 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b76ab213",
   "metadata": {},
   "outputs": [],
   "source": [
    "real['class']=1 # indicating this is true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "21774cc2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Status quo will not be disturbed at Ayodhya; s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Fissures in Hurriyat over Pak visit</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>America's unwanted heading for India?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>For bigwigs; it is destination Goa</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Extra buses to clear tourist traffic</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650965</th>\n",
       "      <td>J&amp;K sacks 2 cops; 3 other employees over terro...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650966</th>\n",
       "      <td>Ranbir Kapoor says 'Rishi Kapoor enjoyed his a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650967</th>\n",
       "      <td>As Covid-19 cases drop to nil in southern dist...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650968</th>\n",
       "      <td>Tamil Nadu sees marginal rise of Covid cases w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650969</th>\n",
       "      <td>Tamil Nadu sees marginal rise of Covid cases w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3650970 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      text  class\n",
       "0        Status quo will not be disturbed at Ayodhya; s...      1\n",
       "1                      Fissures in Hurriyat over Pak visit      1\n",
       "2                    America's unwanted heading for India?      1\n",
       "3                       For bigwigs; it is destination Goa      1\n",
       "4                     Extra buses to clear tourist traffic      1\n",
       "...                                                    ...    ...\n",
       "3650965  J&K sacks 2 cops; 3 other employees over terro...      1\n",
       "3650966  Ranbir Kapoor says 'Rishi Kapoor enjoyed his a...      1\n",
       "3650967  As Covid-19 cases drop to nil in southern dist...      1\n",
       "3650968  Tamil Nadu sees marginal rise of Covid cases w...      1\n",
       "3650969  Tamil Nadu sees marginal rise of Covid cases w...      1\n",
       "\n",
       "[3650970 rows x 2 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "22a89fcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "fake = p.read_csv('fake/news_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "eb1ab748",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>REAL</td>\n",
       "      <td>Payal has accused filmmaker Anurag Kashyap of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FAKE</td>\n",
       "      <td>A four-minute-long video of a woman criticisin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FAKE</td>\n",
       "      <td>Republic Poll, a fake Twitter account imitatin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>REAL</td>\n",
       "      <td>Delhi teen finds place on UN green list, turns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>REAL</td>\n",
       "      <td>Delhi: A high-level meeting underway at reside...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>REAL</td>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>REAL</td>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>FAKE</td>\n",
       "      <td>The Bengaluru City Police’s official Twitter h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>REAL</td>\n",
       "      <td>Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3728</th>\n",
       "      <td>REAL</td>\n",
       "      <td>Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3729 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0     REAL  Payal has accused filmmaker Anurag Kashyap of ...\n",
       "1     FAKE  A four-minute-long video of a woman criticisin...\n",
       "2     FAKE  Republic Poll, a fake Twitter account imitatin...\n",
       "3     REAL  Delhi teen finds place on UN green list, turns...\n",
       "4     REAL  Delhi: A high-level meeting underway at reside...\n",
       "...    ...                                                ...\n",
       "3724  REAL  19:17 (IST) Sep 20\\n\\nThe second round of coun...\n",
       "3725  REAL  19:17 (IST) Sep 20\\n\\nThe second round of coun...\n",
       "3726  FAKE  The Bengaluru City Police’s official Twitter h...\n",
       "3727  REAL  Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...\n",
       "3728  REAL  Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...\n",
       "\n",
       "[3729 rows x 2 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "58a75747",
   "metadata": {},
   "outputs": [],
   "source": [
    "fake = fake.drop(labels='label',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "12150077",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Payal has accused filmmaker Anurag Kashyap of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A four-minute-long video of a woman criticisin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Republic Poll, a fake Twitter account imitatin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Delhi teen finds place on UN green list, turns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Delhi: A high-level meeting underway at reside...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>The Bengaluru City Police’s official Twitter h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3728</th>\n",
       "      <td>Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3729 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0     Payal has accused filmmaker Anurag Kashyap of ...\n",
       "1     A four-minute-long video of a woman criticisin...\n",
       "2     Republic Poll, a fake Twitter account imitatin...\n",
       "3     Delhi teen finds place on UN green list, turns...\n",
       "4     Delhi: A high-level meeting underway at reside...\n",
       "...                                                 ...\n",
       "3724  19:17 (IST) Sep 20\\n\\nThe second round of coun...\n",
       "3725  19:17 (IST) Sep 20\\n\\nThe second round of coun...\n",
       "3726  The Bengaluru City Police’s official Twitter h...\n",
       "3727  Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...\n",
       "3728  Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...\n",
       "\n",
       "[3729 rows x 1 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "2adc5bea",
   "metadata": {},
   "outputs": [],
   "source": [
    "fake['class']=0 #indicating false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "0496beac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Payal has accused filmmaker Anurag Kashyap of ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A four-minute-long video of a woman criticisin...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Republic Poll, a fake Twitter account imitatin...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Delhi teen finds place on UN green list, turns...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Delhi: A high-level meeting underway at reside...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>The Bengaluru City Police’s official Twitter h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3728</th>\n",
       "      <td>Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3729 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  class\n",
       "0     Payal has accused filmmaker Anurag Kashyap of ...      0\n",
       "1     A four-minute-long video of a woman criticisin...      0\n",
       "2     Republic Poll, a fake Twitter account imitatin...      0\n",
       "3     Delhi teen finds place on UN green list, turns...      0\n",
       "4     Delhi: A high-level meeting underway at reside...      0\n",
       "...                                                 ...    ...\n",
       "3724  19:17 (IST) Sep 20\\n\\nThe second round of coun...      0\n",
       "3725  19:17 (IST) Sep 20\\n\\nThe second round of coun...      0\n",
       "3726  The Bengaluru City Police’s official Twitter h...      0\n",
       "3727  Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...      0\n",
       "3728  Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...      0\n",
       "\n",
       "[3729 rows x 2 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "69c89265",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "text     8\n",
       "class    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c1ac102e",
   "metadata": {},
   "outputs": [],
   "source": [
    "mix= p.concat([real,fake],axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3857be16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Status quo will not be disturbed at Ayodhya; s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Fissures in Hurriyat over Pak visit</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>America's unwanted heading for India?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>For bigwigs; it is destination Goa</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Extra buses to clear tourist traffic</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>The Bengaluru City Police’s official Twitter h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3728</th>\n",
       "      <td>Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3654699 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  class\n",
       "0     Status quo will not be disturbed at Ayodhya; s...      1\n",
       "1                   Fissures in Hurriyat over Pak visit      1\n",
       "2                 America's unwanted heading for India?      1\n",
       "3                    For bigwigs; it is destination Goa      1\n",
       "4                  Extra buses to clear tourist traffic      1\n",
       "...                                                 ...    ...\n",
       "3724  19:17 (IST) Sep 20\\n\\nThe second round of coun...      0\n",
       "3725  19:17 (IST) Sep 20\\n\\nThe second round of coun...      0\n",
       "3726  The Bengaluru City Police’s official Twitter h...      0\n",
       "3727  Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...      0\n",
       "3728  Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...      0\n",
       "\n",
       "[3654699 rows x 2 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "88fbdb5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "text     8\n",
       "class    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mix.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7e966e89",
   "metadata": {},
   "outputs": [],
   "source": [
    "mix.dropna(subset='text',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "dc99b96d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Status quo will not be disturbed at Ayodhya; s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Fissures in Hurriyat over Pak visit</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>America's unwanted heading for India?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>For bigwigs; it is destination Goa</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Extra buses to clear tourist traffic</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>19:17 (IST) Sep 20\\n\\nThe second round of coun...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>The Bengaluru City Police’s official Twitter h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3728</th>\n",
       "      <td>Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3654691 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  class\n",
       "0     Status quo will not be disturbed at Ayodhya; s...      1\n",
       "1                   Fissures in Hurriyat over Pak visit      1\n",
       "2                 America's unwanted heading for India?      1\n",
       "3                    For bigwigs; it is destination Goa      1\n",
       "4                  Extra buses to clear tourist traffic      1\n",
       "...                                                 ...    ...\n",
       "3724  19:17 (IST) Sep 20\\n\\nThe second round of coun...      0\n",
       "3725  19:17 (IST) Sep 20\\n\\nThe second round of coun...      0\n",
       "3726  The Bengaluru City Police’s official Twitter h...      0\n",
       "3727  Sep 20, 2020, 08:00AM IST\\n\\nSource: TOI.in\\n\\...      0\n",
       "3728  Read Also\\n\\nRead Also\\n\\nAdvocate Ishkaran Bh...      0\n",
       "\n",
       "[3654691 rows x 2 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b1b63809",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "text     0\n",
       "class    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mix.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c4797800",
   "metadata": {},
   "outputs": [],
   "source": [
    "mix = mix.sample(frac=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "2c85995f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>86772</th>\n",
       "      <td>D'Souza flays Parrikar's snub to Khalap</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1621334</th>\n",
       "      <td>Adi wanted me to look hot &amp; not sweet in Auran...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3062982</th>\n",
       "      <td>4 of a family booked for assaulting neighbour</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>775703</th>\n",
       "      <td>GSPC's KG basin discovery 10.8 TCF: Saurabh Patel</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2380680</th>\n",
       "      <td>Serial killer held; 2nd in 5 days; for killing...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71998</th>\n",
       "      <td>Candidates forced to espouse Naga cause</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1108949</th>\n",
       "      <td>Cops pin hopes on call records to crack double...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3557040</th>\n",
       "      <td>Megan Fox and Kourtney Kardashian strip in a p...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3585914</th>\n",
       "      <td>BJP stages one-day hunger strike against gover...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453225</th>\n",
       "      <td>Consumers suffer as Trai; DoT fight over ombud...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3654691 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      text  class\n",
       "86772              D'Souza flays Parrikar's snub to Khalap      1\n",
       "1621334  Adi wanted me to look hot & not sweet in Auran...      1\n",
       "3062982      4 of a family booked for assaulting neighbour      1\n",
       "775703   GSPC's KG basin discovery 10.8 TCF: Saurabh Patel      1\n",
       "2380680  Serial killer held; 2nd in 5 days; for killing...      1\n",
       "...                                                    ...    ...\n",
       "71998              Candidates forced to espouse Naga cause      1\n",
       "1108949  Cops pin hopes on call records to crack double...      1\n",
       "3557040  Megan Fox and Kourtney Kardashian strip in a p...      1\n",
       "3585914  BJP stages one-day hunger strike against gover...      1\n",
       "453225   Consumers suffer as Trai; DoT fight over ombud...      1\n",
       "\n",
       "[3654691 rows x 2 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b18d5a40",
   "metadata": {},
   "outputs": [],
   "source": [
    "mix.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "ab7ef7fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>86772</td>\n",
       "      <td>D'Souza flays Parrikar's snub to Khalap</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1621334</td>\n",
       "      <td>Adi wanted me to look hot &amp; not sweet in Auran...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3062982</td>\n",
       "      <td>4 of a family booked for assaulting neighbour</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>775703</td>\n",
       "      <td>GSPC's KG basin discovery 10.8 TCF: Saurabh Patel</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2380680</td>\n",
       "      <td>Serial killer held; 2nd in 5 days; for killing...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3654686</th>\n",
       "      <td>71998</td>\n",
       "      <td>Candidates forced to espouse Naga cause</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3654687</th>\n",
       "      <td>1108949</td>\n",
       "      <td>Cops pin hopes on call records to crack double...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3654688</th>\n",
       "      <td>3557040</td>\n",
       "      <td>Megan Fox and Kourtney Kardashian strip in a p...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3654689</th>\n",
       "      <td>3585914</td>\n",
       "      <td>BJP stages one-day hunger strike against gover...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3654690</th>\n",
       "      <td>453225</td>\n",
       "      <td>Consumers suffer as Trai; DoT fight over ombud...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3654691 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           index                                               text  class\n",
       "0          86772            D'Souza flays Parrikar's snub to Khalap      1\n",
       "1        1621334  Adi wanted me to look hot & not sweet in Auran...      1\n",
       "2        3062982      4 of a family booked for assaulting neighbour      1\n",
       "3         775703  GSPC's KG basin discovery 10.8 TCF: Saurabh Patel      1\n",
       "4        2380680  Serial killer held; 2nd in 5 days; for killing...      1\n",
       "...          ...                                                ...    ...\n",
       "3654686    71998            Candidates forced to espouse Naga cause      1\n",
       "3654687  1108949  Cops pin hopes on call records to crack double...      1\n",
       "3654688  3557040  Megan Fox and Kourtney Kardashian strip in a p...      1\n",
       "3654689  3585914  BJP stages one-day hunger strike against gover...      1\n",
       "3654690   453225  Consumers suffer as Trai; DoT fight over ombud...      1\n",
       "\n",
       "[3654691 rows x 3 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "49121f16",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = mix['text']\n",
    "y = mix['class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "1f28e659",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                    D'Souza flays Parrikar's snub to Khalap\n",
       "1          Adi wanted me to look hot & not sweet in Auran...\n",
       "2              4 of a family booked for assaulting neighbour\n",
       "3          GSPC's KG basin discovery 10.8 TCF: Saurabh Patel\n",
       "4          Serial killer held; 2nd in 5 days; for killing...\n",
       "                                 ...                        \n",
       "3654686              Candidates forced to espouse Naga cause\n",
       "3654687    Cops pin hopes on call records to crack double...\n",
       "3654688    Megan Fox and Kourtney Kardashian strip in a p...\n",
       "3654689    BJP stages one-day hunger strike against gover...\n",
       "3654690    Consumers suffer as Trai; DoT fight over ombud...\n",
       "Name: text, Length: 3654691, dtype: object"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "f6bd2d41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          1\n",
       "1          1\n",
       "2          1\n",
       "3          1\n",
       "4          1\n",
       "          ..\n",
       "3654686    1\n",
       "3654687    1\n",
       "3654688    1\n",
       "3654689    1\n",
       "3654690    1\n",
       "Name: class, Length: 3654691, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "ec80c16f",
   "metadata": {},
   "outputs": [],
   "source": [
    "mix2  =mix[mix['class']==0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "edbeeca3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>2228</td>\n",
       "      <td>A disturbing video from Rajasthan of a man tie...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>271</td>\n",
       "      <td>An image of a porn star is being shared on Fac...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1009</th>\n",
       "      <td>864</td>\n",
       "      <td>A video from October last year, showing UP Chi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2909</th>\n",
       "      <td>3227</td>\n",
       "      <td>\\n\\n\\n\\nA video of a landslide that occurred i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4088</th>\n",
       "      <td>1544</td>\n",
       "      <td>Hindi news websites Dainik Bhaskar, Amar Ujala...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650618</th>\n",
       "      <td>3719</td>\n",
       "      <td>A set of images is being shared on Facebook wi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3652992</th>\n",
       "      <td>2856</td>\n",
       "      <td>AMARAVATI: Examination for jobs in Andhra Prad...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3653376</th>\n",
       "      <td>2502</td>\n",
       "      <td>Tripura Chief Minister Biplab Deb used a photo...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3654018</th>\n",
       "      <td>3183</td>\n",
       "      <td>A scene from a online show where a group of pe...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3654434</th>\n",
       "      <td>3382</td>\n",
       "      <td>Dramatic footage of a rocket crashing seconds ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3721 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         index                                               text  class\n",
       "102       2228  A disturbing video from Rajasthan of a man tie...      0\n",
       "857        271  An image of a porn star is being shared on Fac...      0\n",
       "1009       864  A video from October last year, showing UP Chi...      0\n",
       "2909      3227  \\n\\n\\n\\nA video of a landslide that occurred i...      0\n",
       "4088      1544  Hindi news websites Dainik Bhaskar, Amar Ujala...      0\n",
       "...        ...                                                ...    ...\n",
       "3650618   3719  A set of images is being shared on Facebook wi...      0\n",
       "3652992   2856  AMARAVATI: Examination for jobs in Andhra Prad...      0\n",
       "3653376   2502  Tripura Chief Minister Biplab Deb used a photo...      0\n",
       "3654018   3183  A scene from a online show where a group of pe...      0\n",
       "3654434   3382  Dramatic footage of a rocket crashing seconds ...      0\n",
       "\n",
       "[3721 rows x 3 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mix2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "ca6281b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "2df4553d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3491318         FDI norms liberalised for small scale sector\n",
       "2281084                             Entire world in a family\n",
       "2555377    A week after agreeing to it; Nepal has not mov...\n",
       "1011272     Pune: Proxy writer for police examination nabbed\n",
       "1324296      Minority investors rush to be part of cos' mgmt\n",
       "                                 ...                        \n",
       "157309                                              Your say\n",
       "2672497    Human rights groups protest Sharon's visit to ...\n",
       "1926996    Devas may lose international arbitration winni...\n",
       "2478006           ISL: Atletico de Kolkata thrash FC Goa 4-0\n",
       "3541255      Archbishop leads prayer services on Palm Sunday\n",
       "Name: text, Length: 2558283, dtype: object"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "a4cb8889",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3275901            Delivery man cuts cake with machete; held\n",
       "867054          BJP leaders take party chief matter to Delhi\n",
       "2705103                     RTR service lane encroached upon\n",
       "2753440                                  The Kamal conundrum\n",
       "3417979              Sharapova seeks answer to Serena voodoo\n",
       "                                 ...                        \n",
       "1723249          Sports trials in 10 disciplines from Friday\n",
       "2089633    VVIP chopper scam: Italy papers reveal dalals'...\n",
       "1919903       Weed posing threat to Pili River near Mankapur\n",
       "1392591    Boy survives fall from terrace while chasing k...\n",
       "1960563       Liquid nitrogen: Visual drama; medical trauma?\n",
       "Name: text, Length: 1096408, dtype: object"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "18761f94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3491318    1\n",
       "2281084    1\n",
       "2555377    1\n",
       "1011272    1\n",
       "1324296    1\n",
       "          ..\n",
       "157309     1\n",
       "2672497    1\n",
       "1926996    1\n",
       "2478006    1\n",
       "3541255    1\n",
       "Name: class, Length: 2558283, dtype: int64"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "5598c9f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3275901    1\n",
       "867054     1\n",
       "2705103    1\n",
       "2753440    1\n",
       "3417979    1\n",
       "          ..\n",
       "1723249    1\n",
       "2089633    1\n",
       "1919903    1\n",
       "1392591    1\n",
       "1960563    1\n",
       "Name: class, Length: 1096408, dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "bc904bff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# vec\n",
    "vec =TfidfVectorizer()\n",
    "X_train_vec = vec.fit_transform(X_train)\n",
    "X_test_vec = vec.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "26c2ccc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2558283x196314 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 19799718 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "aac069c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1096408x196314 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 8455675 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "121e3c78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier()"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RC = RandomForestClassifier()\n",
    "RC.fit(X_train_vec,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "35ba23cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9999480120539069"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RC.score(X_test_vec,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "90015ef6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC()"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LS = LinearSVC()\n",
    "LS.fit(X_train_vec,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "88a99281",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9999936155153921"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LS.score(X_test_vec,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "f9005f53",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = 'The Twitter Files Revealed One Thing: Elon Musk Is Trapped'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "0dd98690",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "def wordopt(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub('\\[.*?\\]', '', text)\n",
    "    text = re.sub(\"\\\\W\",\" \",text) \n",
    "    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n",
    "    text = re.sub('<.*?>+', '', text)\n",
    "    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\n",
    "    text = re.sub('\\n', '', text)\n",
    "    text = re.sub('\\w*\\d\\w*', '', text)    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "d770997d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the twitter files revealed one thing  elon musk is trapped'"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordopt(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "e9827691",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The Twitter Files Revealed One Thing: Elon Musk Is Trapped'"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "3af94a2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def manual_testing(news):\n",
    "    testing_news = {\"text\":[news]}\n",
    "    new_def_test = p.DataFrame(testing_news)\n",
    "    new_def_test[\"text\"] = new_def_test[\"text\"].apply(wordopt) \n",
    "    new_x_test = new_def_test[\"text\"]\n",
    "    new_xv_test = vec.transform(new_x_test)\n",
    "    p_lr = LS.predict(new_xv_test)\n",
    "    print(p_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "f016de54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\n"
     ]
    }
   ],
   "source": [
    "manual_testing(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74a6f967",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}