03 Trending Topics in Comments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import *\n",
    "import time\n",
    "import pyspark.sql.functions as F\n",
    "from pyspark.sql.types import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_file = sc.textFile(\"hdfs:///pp-data/Comments.xml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<?xml version=\"1.0\" encoding=\"utf-8\"?>',\n",
       " '<comments>',\n",
       " '  <row Id=\"13\" PostId=\"23\" Score=\"0\" Text=\"Using /opt helps me keep track of the applications I\\'ve installed myself.\" CreationDate=\"2010-07-28T19:36:59.773\" UserId=\"10\" ContentLicense=\"CC BY-SA 2.5\" />']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_file.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "filteredRDD = text_file.filter(lambda x: x.startswith(\"  <row \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['  <row Id=\"13\" PostId=\"23\" Score=\"0\" Text=\"Using /opt helps me keep track of the applications I\\'ve installed myself.\" CreationDate=\"2010-07-28T19:36:59.773\" UserId=\"10\" ContentLicense=\"CC BY-SA 2.5\" />']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filteredRDD.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleanedRDD = filteredRDD.map(lambda x: x.lstrip(\"  \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<row Id=\"13\" PostId=\"23\" Score=\"0\" Text=\"Using /opt helps me keep track of the applications I\\'ve installed myself.\" CreationDate=\"2010-07-28T19:36:59.773\" UserId=\"10\" ContentLicense=\"CC BY-SA 2.5\" />']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleanedRDD.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "def parse_xml(rdd):\n",
    "    \"\"\"\n",
    "    Read the xml string from rdd, parse and extract the elements,\n",
    "    then return a list of list.\n",
    "    \"\"\"\n",
    "    root = ET.fromstring(rdd)\n",
    "    rec = []\n",
    "    \n",
    "    if \"Text\" in root.attrib:\n",
    "        rec.append(root.attrib['Text'])\n",
    "    else:\n",
    "        rec.append(\"N/A\")\n",
    "    \n",
    "    return rec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "records_rdd = cleanedRDD.map(lambda x : parse_xml(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[\"Using /opt helps me keep track of the applications I've installed myself.\"],\n",
       " [\"but popping in a live cd I already have isn't going to work huh?\"],\n",
       " ['That will revert the splash screen as well as the login? I almost did that, but grew hesitant.']]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records_rdd.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words_text = sc.textFile(\"file:///home/aarora7/P4-ayush-adarsh/03 stopwords.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'about', 'above']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words_text.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words_rdd = stop_words_text.map(lambda x: x.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['a'], ['about'], ['above']]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words_rdd.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words_df = spark.createDataFrame(stop_words_rdd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+\n",
      "|      _1|\n",
      "+--------+\n",
      "|       a|\n",
      "|   about|\n",
      "|   above|\n",
      "|  across|\n",
      "|   after|\n",
      "|   again|\n",
      "| against|\n",
      "|     all|\n",
      "|  almost|\n",
      "|   alone|\n",
      "|   along|\n",
      "| already|\n",
      "|    also|\n",
      "|although|\n",
      "|  always|\n",
      "|   among|\n",
      "|      an|\n",
      "|     and|\n",
      "| another|\n",
      "|     any|\n",
      "+--------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stop_words_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a',\n",
       " 'about',\n",
       " 'above',\n",
       " 'across',\n",
       " 'after',\n",
       " 'again',\n",
       " 'against',\n",
       " 'all',\n",
       " 'almost',\n",
       " 'alone',\n",
       " 'along',\n",
       " 'already',\n",
       " 'also',\n",
       " 'although',\n",
       " 'always',\n",
       " 'among',\n",
       " 'an',\n",
       " 'and',\n",
       " 'another',\n",
       " 'any',\n",
       " 'anybody',\n",
       " 'anyone',\n",
       " 'anything',\n",
       " 'anywhere',\n",
       " 'are',\n",
       " 'area',\n",
       " 'areas',\n",
       " 'around',\n",
       " 'as',\n",
       " 'ask',\n",
       " 'asked',\n",
       " 'asking',\n",
       " 'asks',\n",
       " 'at',\n",
       " 'away',\n",
       " 'b',\n",
       " 'back',\n",
       " 'backed',\n",
       " 'backing',\n",
       " 'backs',\n",
       " 'be',\n",
       " 'became',\n",
       " 'because',\n",
       " 'become',\n",
       " 'becomes',\n",
       " 'been',\n",
       " 'before',\n",
       " 'began',\n",
       " 'behind',\n",
       " 'being',\n",
       " 'beings',\n",
       " 'best',\n",
       " 'better',\n",
       " 'between',\n",
       " 'big',\n",
       " 'both',\n",
       " 'but',\n",
       " 'by',\n",
       " 'c',\n",
       " 'came',\n",
       " 'can',\n",
       " 'cannot',\n",
       " 'case',\n",
       " 'cases',\n",
       " 'certain',\n",
       " 'certainly',\n",
       " 'clear',\n",
       " 'clearly',\n",
       " 'come',\n",
       " 'could',\n",
       " 'd',\n",
       " 'did',\n",
       " 'differ',\n",
       " 'different',\n",
       " 'differently',\n",
       " 'do',\n",
       " 'does',\n",
       " 'done',\n",
       " 'down',\n",
       " 'down',\n",
       " 'downed',\n",
       " 'downing',\n",
       " 'downs',\n",
       " 'during',\n",
       " 'e',\n",
       " 'each',\n",
       " 'early',\n",
       " 'either',\n",
       " 'end',\n",
       " 'ended',\n",
       " 'ending',\n",
       " 'ends',\n",
       " 'enough',\n",
       " 'even',\n",
       " 'evenly',\n",
       " 'ever',\n",
       " 'every',\n",
       " 'everybody',\n",
       " 'everyone',\n",
       " 'everything',\n",
       " 'everywhere',\n",
       " 'f',\n",
       " 'face',\n",
       " 'faces',\n",
       " 'fact',\n",
       " 'facts',\n",
       " 'far',\n",
       " 'felt',\n",
       " 'few',\n",
       " 'find',\n",
       " 'finds',\n",
       " 'first',\n",
       " 'for',\n",
       " 'four',\n",
       " 'from',\n",
       " 'full',\n",
       " 'fully',\n",
       " 'further',\n",
       " 'furthered',\n",
       " 'furthering',\n",
       " 'furthers',\n",
       " 'g',\n",
       " 'gave',\n",
       " 'general',\n",
       " 'generally',\n",
       " 'get',\n",
       " 'gets',\n",
       " 'give',\n",
       " 'given',\n",
       " 'gives',\n",
       " 'go',\n",
       " 'going',\n",
       " 'good',\n",
       " 'goods',\n",
       " 'got',\n",
       " 'great',\n",
       " 'greater',\n",
       " 'greatest',\n",
       " 'group',\n",
       " 'grouped',\n",
       " 'grouping',\n",
       " 'groups',\n",
       " 'h',\n",
       " 'had',\n",
       " 'has',\n",
       " 'have',\n",
       " 'having',\n",
       " 'he',\n",
       " 'her',\n",
       " 'here',\n",
       " 'herself',\n",
       " 'high',\n",
       " 'high',\n",
       " 'high',\n",
       " 'higher',\n",
       " 'highest',\n",
       " 'him',\n",
       " 'himself',\n",
       " 'his',\n",
       " 'how',\n",
       " 'however',\n",
       " 'i',\n",
       " 'if',\n",
       " 'important',\n",
       " 'in',\n",
       " 'interest',\n",
       " 'interested',\n",
       " 'interesting',\n",
       " 'interests',\n",
       " 'into',\n",
       " 'is',\n",
       " 'it',\n",
       " 'its',\n",
       " 'itself',\n",
       " 'j',\n",
       " 'just',\n",
       " 'k',\n",
       " 'keep',\n",
       " 'keeps',\n",
       " 'kind',\n",
       " 'knew',\n",
       " 'know',\n",
       " 'known',\n",
       " 'knows',\n",
       " 'l',\n",
       " 'large',\n",
       " 'largely',\n",
       " 'last',\n",
       " 'later',\n",
       " 'latest',\n",
       " 'least',\n",
       " 'less',\n",
       " 'let',\n",
       " 'lets',\n",
       " 'like',\n",
       " 'likely',\n",
       " 'long',\n",
       " 'longer',\n",
       " 'longest',\n",
       " 'm',\n",
       " 'made',\n",
       " 'make',\n",
       " 'making',\n",
       " 'man',\n",
       " 'many',\n",
       " 'may',\n",
       " 'me',\n",
       " 'member',\n",
       " 'members',\n",
       " 'men',\n",
       " 'might',\n",
       " 'more',\n",
       " 'most',\n",
       " 'mostly',\n",
       " 'mr',\n",
       " 'mrs',\n",
       " 'much',\n",
       " 'must',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'n',\n",
       " 'necessary',\n",
       " 'need',\n",
       " 'needed',\n",
       " 'needing',\n",
       " 'needs',\n",
       " 'never',\n",
       " 'new',\n",
       " 'new',\n",
       " 'newer',\n",
       " 'newest',\n",
       " 'next',\n",
       " 'no',\n",
       " 'nobody',\n",
       " 'non',\n",
       " 'noone',\n",
       " 'not',\n",
       " 'nothing',\n",
       " 'now',\n",
       " 'nowhere',\n",
       " 'number',\n",
       " 'numbers',\n",
       " 'o',\n",
       " 'of',\n",
       " 'off',\n",
       " 'often',\n",
       " 'old',\n",
       " 'older',\n",
       " 'oldest',\n",
       " 'on',\n",
       " 'once',\n",
       " 'one',\n",
       " 'only',\n",
       " 'open',\n",
       " 'opened',\n",
       " 'opening',\n",
       " 'opens',\n",
       " 'or',\n",
       " 'order',\n",
       " 'ordered',\n",
       " 'ordering',\n",
       " 'orders',\n",
       " 'other',\n",
       " 'others',\n",
       " 'our',\n",
       " 'out',\n",
       " 'over',\n",
       " 'p',\n",
       " 'part',\n",
       " 'parted',\n",
       " 'parting',\n",
       " 'parts',\n",
       " 'per',\n",
       " 'perhaps',\n",
       " 'place',\n",
       " 'places',\n",
       " 'point',\n",
       " 'pointed',\n",
       " 'pointing',\n",
       " 'points',\n",
       " 'possible',\n",
       " 'present',\n",
       " 'presented',\n",
       " 'presenting',\n",
       " 'presents',\n",
       " 'problem',\n",
       " 'problems',\n",
       " 'put',\n",
       " 'puts',\n",
       " 'q',\n",
       " 'quite',\n",
       " 'r',\n",
       " 'rather',\n",
       " 'really',\n",
       " 'right',\n",
       " 'right',\n",
       " 'room',\n",
       " 'rooms',\n",
       " 's',\n",
       " 'said',\n",
       " 'same',\n",
       " 'saw',\n",
       " 'say',\n",
       " 'says',\n",
       " 'second',\n",
       " 'seconds',\n",
       " 'see',\n",
       " 'seem',\n",
       " 'seemed',\n",
       " 'seeming',\n",
       " 'seems',\n",
       " 'sees',\n",
       " 'several',\n",
       " 'shall',\n",
       " 'she',\n",
       " 'should',\n",
       " 'show',\n",
       " 'showed',\n",
       " 'showing',\n",
       " 'shows',\n",
       " 'side',\n",
       " 'sides',\n",
       " 'since',\n",
       " 'small',\n",
       " 'smaller',\n",
       " 'smallest',\n",
       " 'so',\n",
       " 'some',\n",
       " 'somebody',\n",
       " 'someone',\n",
       " 'something',\n",
       " 'somewhere',\n",
       " 'state',\n",
       " 'states',\n",
       " 'still',\n",
       " 'still',\n",
       " 'such',\n",
       " 'sure',\n",
       " 't',\n",
       " 'take',\n",
       " 'taken',\n",
       " 'than',\n",
       " 'that',\n",
       " 'the',\n",
       " 'their',\n",
       " 'them',\n",
       " 'then',\n",
       " 'there',\n",
       " 'therefore',\n",
       " 'these',\n",
       " 'they',\n",
       " 'thing',\n",
       " 'things',\n",
       " 'think',\n",
       " 'thinks',\n",
       " 'this',\n",
       " 'those',\n",
       " 'though',\n",
       " 'thought',\n",
       " 'thoughts',\n",
       " 'three',\n",
       " 'through',\n",
       " 'thus',\n",
       " 'to',\n",
       " 'today',\n",
       " 'together',\n",
       " 'too',\n",
       " 'took',\n",
       " 'toward',\n",
       " 'turn',\n",
       " 'turned',\n",
       " 'turning',\n",
       " 'turns',\n",
       " 'two',\n",
       " 'u',\n",
       " 'under',\n",
       " 'until',\n",
       " 'up',\n",
       " 'upon',\n",
       " 'us',\n",
       " 'use',\n",
       " 'used',\n",
       " 'uses',\n",
       " 'v',\n",
       " 'very',\n",
       " 'w',\n",
       " 'want',\n",
       " 'wanted',\n",
       " 'wanting',\n",
       " 'wants',\n",
       " 'was',\n",
       " 'way',\n",
       " 'ways',\n",
       " 'we',\n",
       " 'well',\n",
       " 'wells',\n",
       " 'went',\n",
       " 'were',\n",
       " 'what',\n",
       " 'when',\n",
       " 'where',\n",
       " 'whether',\n",
       " 'which',\n",
       " 'while',\n",
       " 'who',\n",
       " 'whole',\n",
       " 'whose',\n",
       " 'why',\n",
       " 'will',\n",
       " 'with',\n",
       " 'within',\n",
       " 'without',\n",
       " 'work',\n",
       " 'worked',\n",
       " 'working',\n",
       " 'works',\n",
       " 'would',\n",
       " 'x',\n",
       " 'y',\n",
       " 'year',\n",
       " 'years',\n",
       " 'yet',\n",
       " 'you',\n",
       " 'young',\n",
       " 'younger',\n",
       " 'youngest',\n",
       " 'your',\n",
       " 'yours',\n",
       " 'i',\n",
       " 'install',\n",
       " 'answer',\n",
       " 'using',\n",
       " \"don't\",\n",
       " 'question',\n",
       " 'it',\n",
       " 'file',\n",
       " '-',\n",
       " \"i'm\",\n",
       " \"it's\",\n",
       " 'this',\n",
       " 'tried',\n",
       " 'run']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words_list = list(stop_words_df.select('_1').toPandas()['_1'])\n",
    "stop_words_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stop_word(x):\n",
    "    list = []\n",
    "    s_split = x[0].split(\" \")\n",
    "    for i in s_split:\n",
    "        i = i.lower()\n",
    "        if i not in stop_words_list:\n",
    "            if not i.startswith('.') and not i.endswith('.') and not i.startswith('?') and not i.endswith('?') and not i.startswith('&') and not i.endswith('&'):\n",
    "                list.append((i,1))\n",
    "    return list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_text = records_rdd.flatMap(lambda x: remove_stop_word(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('/opt', 1), ('helps', 1), ('track', 1)]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_text.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_count = cleaned_text.reduceByKey(lambda y,x: x+y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('manpage,', 44), ('python', 6727), (\"doesn't\", 45536)]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_count.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_count_col = [\"topic\",\"count\"]\n",
    "topic_count_data = topic_count.toDF(topic_count_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+-----+\n",
      "|       topic|count|\n",
      "+------------+-----+\n",
      "|    manpage,|   44|\n",
      "|      python| 6727|\n",
      "|     doesn't|45536|\n",
      "|      @muru,|  315|\n",
      "|     exposes|   52|\n",
      "|        sane|  289|\n",
      "|     package|26528|\n",
      "|      expose|  162|\n",
      "|    because,|  284|\n",
      "|        hope| 7065|\n",
      "|        mean|19178|\n",
      "|     manpage|  380|\n",
      "| pepperflash|   47|\n",
      "|      player| 1593|\n",
      "|adobe-flash,|    1|\n",
      "|      plugin| 2332|\n",
      "|         cat| 1564|\n",
      "|         pay|  753|\n",
      "|      follow| 6359|\n",
      "|     instead|18999|\n",
      "+------------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "topic_count_data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "final = topic_count_data.orderBy(col('count').desc())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final.repartition(1).write.csv(\"hdfs:///topic_count\", sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}