PySpark- WordCount.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.system('klist')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### We are using Kerberos for Hadoop Security"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Password for md78484@NAMUXDEV.DYN.NSROOT.NET: ']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%!\n",
    "echo \"My_pwd\" | kinit md78484@NAMUXDEV.DYN.NSROOT.NET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ticket cache: FILE:/tmp/krb5cc_1010878484_2De9tj',\n",
       " 'Default principal: md78484@NAMUXDEV.DYN.NSROOT.NET',\n",
       " '',\n",
       " 'Valid starting     Expires            Service principal',\n",
       " '11/06/17 11:23:26  11/06/17 19:23:26  krbtgt/NAMUXDEV.DYN.NSROOT.NET@NAMUXDEV.DYN.NSROOT.NET',\n",
       " '\\trenew until 11/07/17 11:23:26']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%!\n",
    "klist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Welcome to',\n",
       " '      ____              __',\n",
       " '     / __/__  ___ _____/ /__',\n",
       " \"    _\\\\ \\\\/ _ \\\\/ _ `/ __/  '_/\",\n",
       " '   /___/ .__/\\\\_,_/_/ /_/\\\\_\\\\   version 1.6.0',\n",
       " '      /_/',\n",
       " '                        ',\n",
       " 'Type --help for more information.']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%!\n",
    "spark-submit --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spark Context, created!\n",
      "CPU times: user 403 ms, sys: 97 ms, total: 500 ms\n",
      "Wall time: 10 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-7-oracle-cloudera/jre\"\n",
    "os.environ[\"JAVA_HOME\"] = \"/usr/java/default\"\n",
    "os.environ[\"SPARK_HOME\"] = \"/opt/cloudera/parcels/CDH/lib/spark\"\n",
    "os.environ[\"PYLIB\"] = os.environ[\"SPARK_HOME\"] + \"/python/lib\"\n",
    "os.environ[\"PYSPARK_PYTHON\"] = \"/opt/cloudera/parcels/Anaconda2/bin/python\"\n",
    "\n",
    "sys.path.insert(0, os.environ[\"PYLIB\"] +\"/py4j-0.9-src.zip\") \n",
    "sys.path.insert(0, os.environ[\"PYLIB\"] +\"/pyspark.zip\")\n",
    "from pyspark import SparkConf\n",
    "from pyspark import SparkContext\n",
    "\n",
    "SparkContext.setSystemProperty('spark.executor.memory', '2g')\n",
    "SparkContext.setSystemProperty('spark.executor.cores', '4')\n",
    "SparkContext.setSystemProperty('spark.app.name', 'pyspark-WordCount')\n",
    "\n",
    "conf = SparkConf()\n",
    "conf.setMaster('yarn-client')#jupyter notebook\n",
    "\n",
    "sc = SparkContext(conf=conf)\n",
    "print(\"Spark Context, created!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "myfile=sc.textFile('hdfs://devtoolsRnD/data/everyone/work/hive/WordCount.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "hdfs://devtoolsRnD/data/everyone/work/hive/WordCount.txt MapPartitionsRDD[1] at textFile at NativeMethodAccessorImpl.java:-2"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "count_reduceByKey=myfile.flatMap(lambda line: line.split(\" \")).map(lambda word: (word,1)).reduceByKey(lambda w1,w2: w1 + w2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PythonRDD[6] at RDD at PythonRDD.scala:43"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_reduceByKey"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Blue', 1),\n",
       " (u'and', 8),\n",
       " (u'Superman.', 1),\n",
       " (u'old', 1),\n",
       " (u'have', 1),\n",
       " (u'', 1),\n",
       " (u'question', 1),\n",
       " (u'is', 1),\n",
       " (u'Lane', 1),\n",
       " (u'one.', 1),\n",
       " (u'twist', 1),\n",
       " (u'Action', 1),\n",
       " (u'been', 1),\n",
       " (u'Kent', 1),\n",
       " (u'Reborn\\ufffd', 1),\n",
       " (u'happy,', 1),\n",
       " (u'comic', 2),\n",
       " (u'saw', 1),\n",
       " (u'The', 3),\n",
       " (u'story', 1),\n",
       " (u'finally', 1),\n",
       " (u'from', 1),\n",
       " (u'Universe,', 1),\n",
       " (u'every', 1),\n",
       " (u'year),', 1),\n",
       " (u'make', 1),\n",
       " (u'father', 1),\n",
       " (u'behind', 1),\n",
       " (u'But', 2),\n",
       " (u'him.', 1),\n",
       " (u'fans', 1),\n",
       " (u'those', 1),\n",
       " (u'start,', 1),\n",
       " (u'mysterious', 1),\n",
       " (u'was', 1),\n",
       " (u'into', 1),\n",
       " (u'lingering', 1),\n",
       " (u'Steel', 1),\n",
       " (u'a', 6),\n",
       " (u'resounding', 1),\n",
       " (u'hero', 1),\n",
       " (u'issue\\ufffds', 1),\n",
       " (u'fate', 1),\n",
       " (u'Comics', 1),\n",
       " (u'outright', 1),\n",
       " (u'but', 1),\n",
       " (u'folded', 1),\n",
       " (u'biggest', 1),\n",
       " (u'While', 1),\n",
       " (u'sees', 1),\n",
       " (u'fan', 2),\n",
       " (u'somewhat', 1),\n",
       " (u'not', 1),\n",
       " (u'mystery', 1),\n",
       " (u'bit', 1),\n",
       " (u'delivered.', 1),\n",
       " (u'died', 2),\n",
       " (u'seemed:', 1),\n",
       " (u'And', 1),\n",
       " (u'changed.', 1),\n",
       " (u'last', 1),\n",
       " (u'this', 1),\n",
       " (u'leaves', 1),\n",
       " (u'#976,', 1),\n",
       " (u'back,', 1),\n",
       " (u'circumstances,', 1),\n",
       " (u'the', 15),\n",
       " (u'\\ufffd', 2),\n",
       " (u'husband', 1),\n",
       " (u'months', 1),\n",
       " (u'version', 1),\n",
       " (u'be', 1),\n",
       " (u'(closer', 1),\n",
       " (u'fond', 1),\n",
       " (u'DC', 2),\n",
       " (u'DC\\ufffds', 1),\n",
       " (u'son', 1),\n",
       " (u'surrounding', 1),\n",
       " (u'himself', 1),\n",
       " (u'at', 1),\n",
       " (u'in', 1),\n",
       " (u'New', 4),\n",
       " (u'Jonathan', 1),\n",
       " (u'to', 4),\n",
       " (u'colleagues,', 1),\n",
       " (u'has', 1),\n",
       " (u'uncertainty', 1),\n",
       " (u'younger,', 1),\n",
       " (u'It\\ufffds', 1),\n",
       " (u'Big', 1),\n",
       " (u'there', 1),\n",
       " (u'launch', 1),\n",
       " (u'few', 1),\n",
       " (u'boost.', 1),\n",
       " (u'book', 2),\n",
       " (u'under', 1),\n",
       " (u'new', 1),\n",
       " (u'52', 4),\n",
       " (u'sure', 1),\n",
       " (u'Man', 1),\n",
       " (u'Boy', 1),\n",
       " (u'his', 1),\n",
       " (u'tease', 1),\n",
       " (u'returned', 1),\n",
       " (u'success,', 1),\n",
       " (u'after', 1),\n",
       " (u'it', 1),\n",
       " (u'were', 1),\n",
       " (u'scout', 1),\n",
       " (u'never', 1),\n",
       " (u'previous', 1),\n",
       " (u'needed', 1),\n",
       " (u'\\ufffdRebirth\\ufffd', 1),\n",
       " (u'reinvigorated,', 1),\n",
       " (u'like', 1),\n",
       " (u'wife', 1),\n",
       " (u'all.', 1),\n",
       " (u'of', 9),\n",
       " (u'older', 2),\n",
       " (u'Lois', 1),\n",
       " (u'Superman\\ufffds', 1),\n",
       " (u'good', 1),\n",
       " (u'replaced.', 1),\n",
       " (u'time', 1),\n",
       " (u'things', 1),\n",
       " (u'everything', 1),\n",
       " (u'\\ufffdSuperman', 1),\n",
       " (u'having', 1),\n",
       " (u'Superman', 4)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_reduceByKey.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Method 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from operator import add"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "count=myfile.flatMap(lambda line: line.split(\" \")).map(lambda word: (word,1)).reduceByKey(add)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Blue', 1),\n",
       " (u'and', 8),\n",
       " (u'Superman.', 1),\n",
       " (u'old', 1),\n",
       " (u'have', 1),\n",
       " (u'', 1),\n",
       " (u'question', 1),\n",
       " (u'is', 1),\n",
       " (u'Lane', 1),\n",
       " (u'one.', 1),\n",
       " (u'twist', 1),\n",
       " (u'Action', 1),\n",
       " (u'been', 1),\n",
       " (u'Kent', 1),\n",
       " (u'Reborn\\ufffd', 1),\n",
       " (u'happy,', 1),\n",
       " (u'comic', 2),\n",
       " (u'saw', 1),\n",
       " (u'The', 3),\n",
       " (u'story', 1),\n",
       " (u'finally', 1),\n",
       " (u'from', 1),\n",
       " (u'Universe,', 1),\n",
       " (u'every', 1),\n",
       " (u'year),', 1),\n",
       " (u'make', 1),\n",
       " (u'father', 1),\n",
       " (u'behind', 1),\n",
       " (u'But', 2),\n",
       " (u'him.', 1),\n",
       " (u'fans', 1),\n",
       " (u'those', 1),\n",
       " (u'start,', 1),\n",
       " (u'mysterious', 1),\n",
       " (u'was', 1),\n",
       " (u'into', 1),\n",
       " (u'lingering', 1),\n",
       " (u'Steel', 1),\n",
       " (u'a', 6),\n",
       " (u'resounding', 1),\n",
       " (u'hero', 1),\n",
       " (u'issue\\ufffds', 1),\n",
       " (u'fate', 1),\n",
       " (u'Comics', 1),\n",
       " (u'outright', 1),\n",
       " (u'but', 1),\n",
       " (u'folded', 1),\n",
       " (u'biggest', 1),\n",
       " (u'While', 1),\n",
       " (u'sees', 1),\n",
       " (u'fan', 2),\n",
       " (u'somewhat', 1),\n",
       " (u'not', 1),\n",
       " (u'mystery', 1),\n",
       " (u'bit', 1),\n",
       " (u'delivered.', 1),\n",
       " (u'died', 2),\n",
       " (u'seemed:', 1),\n",
       " (u'And', 1),\n",
       " (u'changed.', 1),\n",
       " (u'last', 1),\n",
       " (u'this', 1),\n",
       " (u'leaves', 1),\n",
       " (u'#976,', 1),\n",
       " (u'back,', 1),\n",
       " (u'circumstances,', 1),\n",
       " (u'the', 15),\n",
       " (u'\\ufffd', 2),\n",
       " (u'husband', 1),\n",
       " (u'months', 1),\n",
       " (u'version', 1),\n",
       " (u'be', 1),\n",
       " (u'(closer', 1),\n",
       " (u'fond', 1),\n",
       " (u'DC', 2),\n",
       " (u'DC\\ufffds', 1),\n",
       " (u'son', 1),\n",
       " (u'surrounding', 1),\n",
       " (u'himself', 1),\n",
       " (u'at', 1),\n",
       " (u'in', 1),\n",
       " (u'New', 4),\n",
       " (u'Jonathan', 1),\n",
       " (u'to', 4),\n",
       " (u'colleagues,', 1),\n",
       " (u'has', 1),\n",
       " (u'uncertainty', 1),\n",
       " (u'younger,', 1),\n",
       " (u'It\\ufffds', 1),\n",
       " (u'Big', 1),\n",
       " (u'there', 1),\n",
       " (u'launch', 1),\n",
       " (u'few', 1),\n",
       " (u'boost.', 1),\n",
       " (u'book', 2),\n",
       " (u'under', 1),\n",
       " (u'new', 1),\n",
       " (u'52', 4),\n",
       " (u'sure', 1),\n",
       " (u'Man', 1),\n",
       " (u'Boy', 1),\n",
       " (u'his', 1),\n",
       " (u'tease', 1),\n",
       " (u'returned', 1),\n",
       " (u'success,', 1),\n",
       " (u'after', 1),\n",
       " (u'it', 1),\n",
       " (u'were', 1),\n",
       " (u'scout', 1),\n",
       " (u'never', 1),\n",
       " (u'previous', 1),\n",
       " (u'needed', 1),\n",
       " (u'\\ufffdRebirth\\ufffd', 1),\n",
       " (u'reinvigorated,', 1),\n",
       " (u'like', 1),\n",
       " (u'wife', 1),\n",
       " (u'all.', 1),\n",
       " (u'of', 9),\n",
       " (u'older', 2),\n",
       " (u'Lois', 1),\n",
       " (u'Superman\\ufffds', 1),\n",
       " (u'good', 1),\n",
       " (u'replaced.', 1),\n",
       " (u'time', 1),\n",
       " (u'things', 1),\n",
       " (u'everything', 1),\n",
       " (u'\\ufffdSuperman', 1),\n",
       " (u'having', 1),\n",
       " (u'Superman', 4)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Method 3: Groupby"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "count_groupByKey=myfile.flatMap(lambda line: line.split(\" \")).map(lambda word: (word,1)).groupByKey().map(lambda (word,count): (word,sum(count)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Blue', 1),\n",
       " (u'and', 8),\n",
       " (u'Superman.', 1),\n",
       " (u'old', 1),\n",
       " (u'have', 1),\n",
       " (u'', 1),\n",
       " (u'question', 1),\n",
       " (u'is', 1),\n",
       " (u'Lane', 1),\n",
       " (u'one.', 1),\n",
       " (u'twist', 1),\n",
       " (u'Action', 1),\n",
       " (u'been', 1),\n",
       " (u'Kent', 1),\n",
       " (u'Reborn\\ufffd', 1),\n",
       " (u'happy,', 1),\n",
       " (u'comic', 2),\n",
       " (u'saw', 1),\n",
       " (u'The', 3),\n",
       " (u'story', 1),\n",
       " (u'finally', 1),\n",
       " (u'from', 1),\n",
       " (u'Universe,', 1),\n",
       " (u'every', 1),\n",
       " (u'year),', 1),\n",
       " (u'make', 1),\n",
       " (u'father', 1),\n",
       " (u'behind', 1),\n",
       " (u'But', 2),\n",
       " (u'him.', 1),\n",
       " (u'fans', 1),\n",
       " (u'those', 1),\n",
       " (u'start,', 1),\n",
       " (u'mysterious', 1),\n",
       " (u'was', 1),\n",
       " (u'into', 1),\n",
       " (u'lingering', 1),\n",
       " (u'Steel', 1),\n",
       " (u'a', 6),\n",
       " (u'resounding', 1),\n",
       " (u'hero', 1),\n",
       " (u'issue\\ufffds', 1),\n",
       " (u'fate', 1),\n",
       " (u'Comics', 1),\n",
       " (u'outright', 1),\n",
       " (u'but', 1),\n",
       " (u'folded', 1),\n",
       " (u'biggest', 1),\n",
       " (u'While', 1),\n",
       " (u'sees', 1),\n",
       " (u'fan', 2),\n",
       " (u'somewhat', 1),\n",
       " (u'not', 1),\n",
       " (u'mystery', 1),\n",
       " (u'bit', 1),\n",
       " (u'delivered.', 1),\n",
       " (u'died', 2),\n",
       " (u'seemed:', 1),\n",
       " (u'And', 1),\n",
       " (u'changed.', 1),\n",
       " (u'last', 1),\n",
       " (u'this', 1),\n",
       " (u'leaves', 1),\n",
       " (u'#976,', 1),\n",
       " (u'back,', 1),\n",
       " (u'circumstances,', 1),\n",
       " (u'the', 15),\n",
       " (u'\\ufffd', 2),\n",
       " (u'husband', 1),\n",
       " (u'months', 1),\n",
       " (u'version', 1),\n",
       " (u'be', 1),\n",
       " (u'(closer', 1),\n",
       " (u'fond', 1),\n",
       " (u'DC', 2),\n",
       " (u'DC\\ufffds', 1),\n",
       " (u'son', 1),\n",
       " (u'surrounding', 1),\n",
       " (u'himself', 1),\n",
       " (u'at', 1),\n",
       " (u'in', 1),\n",
       " (u'New', 4),\n",
       " (u'Jonathan', 1),\n",
       " (u'to', 4),\n",
       " (u'colleagues,', 1),\n",
       " (u'has', 1),\n",
       " (u'uncertainty', 1),\n",
       " (u'younger,', 1),\n",
       " (u'It\\ufffds', 1),\n",
       " (u'Big', 1),\n",
       " (u'there', 1),\n",
       " (u'launch', 1),\n",
       " (u'few', 1),\n",
       " (u'boost.', 1),\n",
       " (u'book', 2),\n",
       " (u'under', 1),\n",
       " (u'new', 1),\n",
       " (u'52', 4),\n",
       " (u'sure', 1),\n",
       " (u'Man', 1),\n",
       " (u'Boy', 1),\n",
       " (u'his', 1),\n",
       " (u'tease', 1),\n",
       " (u'returned', 1),\n",
       " (u'success,', 1),\n",
       " (u'after', 1),\n",
       " (u'it', 1),\n",
       " (u'were', 1),\n",
       " (u'scout', 1),\n",
       " (u'never', 1),\n",
       " (u'previous', 1),\n",
       " (u'needed', 1),\n",
       " (u'\\ufffdRebirth\\ufffd', 1),\n",
       " (u'reinvigorated,', 1),\n",
       " (u'like', 1),\n",
       " (u'wife', 1),\n",
       " (u'all.', 1),\n",
       " (u'of', 9),\n",
       " (u'older', 2),\n",
       " (u'Lois', 1),\n",
       " (u'Superman\\ufffds', 1),\n",
       " (u'good', 1),\n",
       " (u'replaced.', 1),\n",
       " (u'time', 1),\n",
       " (u'things', 1),\n",
       " (u'everything', 1),\n",
       " (u'\\ufffdSuperman', 1),\n",
       " (u'having', 1),\n",
       " (u'Superman', 4)]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_groupByKey.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}