beautiful_soup_scrape_table.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:ccfbc6e59ff033c9cebbcbd45224b27b280b9487226bc719772a84909f12f03c"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Scraping a HTML with Beauitful Soup\n",
      "\n",
      "This snippit was written by [Chris R. Albon](http://www.chrisralbon.com/) and is part of his collection of [well-documented Python snippits](/~https://github.com/chrisalbon/code_py). All code is written in Python 3 in iPython notebook and offered under the [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Import required modules\n",
      "import requests\n",
      "from bs4 import BeautifulSoup\n",
      "import pandas as pd"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 105
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Create a dataframe. We will scrape iPython's HTML table output"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Create a values as dictionary of lists\n",
      "raw_data = {'first_name': ['Jason', 'Molly', 'Tina', 'Jake', 'Amy'], \n",
      "        'last_name': ['Miller', 'Jacobson', 'Ali', 'Milner', 'Cooze'], \n",
      "        'age': [42, 52, 36, 24, 73], \n",
      "        'preTestScore': [4, 24, 31, 2, 3],\n",
      "        'postTestScore': [25, 94, 57, 62, 70]}\n",
      "\n",
      "# Create a dataframe\n",
      "raw_df = pd.DataFrame(raw_data, columns = ['first_name', 'last_name', 'age', 'preTestScore', 'postTestScore'])\n",
      "\n",
      "# View a dataframe\n",
      "raw_df"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>first_name</th>\n",
        "      <th>last_name</th>\n",
        "      <th>age</th>\n",
        "      <th>preTestScore</th>\n",
        "      <th>postTestScore</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> Jason</td>\n",
        "      <td>   Miller</td>\n",
        "      <td> 42</td>\n",
        "      <td>  4</td>\n",
        "      <td> 25</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> Molly</td>\n",
        "      <td> Jacobson</td>\n",
        "      <td> 52</td>\n",
        "      <td> 24</td>\n",
        "      <td> 94</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  Tina</td>\n",
        "      <td>      Ali</td>\n",
        "      <td> 36</td>\n",
        "      <td> 31</td>\n",
        "      <td> 57</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  Jake</td>\n",
        "      <td>   Milner</td>\n",
        "      <td> 24</td>\n",
        "      <td>  2</td>\n",
        "      <td> 62</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>   Amy</td>\n",
        "      <td>    Cooze</td>\n",
        "      <td> 73</td>\n",
        "      <td>  3</td>\n",
        "      <td> 70</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>5 rows \u00d7 5 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 106,
       "text": [
        "  first_name last_name  age  preTestScore  postTestScore\n",
        "0      Jason    Miller   42             4             25\n",
        "1      Molly  Jacobson   52            24             94\n",
        "2       Tina       Ali   36            31             57\n",
        "3       Jake    Milner   24             2             62\n",
        "4        Amy     Cooze   73             3             70\n",
        "\n",
        "[5 rows x 5 columns]"
       ]
      }
     ],
     "prompt_number": 106
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Download the HTML and create a Beautiful Soup object"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Create a variable with the URL to this tutorial\n",
      "url = 'http://nbviewer.ipython.org/github/chrisalbon/code_py/blob/master/beautiful_soup_scrape_table.ipynb'\n",
      "\n",
      "# Scrape the HTML at the url\n",
      "r = requests.get(url)\n",
      "\n",
      "# Turn the HTML into a Beautiful Soup object\n",
      "soup = BeautifulSoup(r.text)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 107
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Parse the Beautiful Soup object"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Create four variables to score the scraped data in\n",
      "first_name = []\n",
      "last_name = []\n",
      "age = []\n",
      "preTestScore = []\n",
      "postTestScore = []\n",
      "\n",
      "# Create an object of the first object that is class=dataframe\n",
      "table = soup.find(class_='dataframe')\n",
      "\n",
      "# Find all the <tr> tag pairs, skip the first one, then for each.\n",
      "for row in table.find_all('tr')[1:]:\n",
      "    # Create a variable of all the <td> tag pairs in each <tr> tag pair,\n",
      "    col = row.find_all('td')\n",
      "    \n",
      "    # Create a variable of the string inside 1st <td> tag pair,\n",
      "    column_1 = col[0].string.strip()\n",
      "    # and append it to first_name variable\n",
      "    first_name.append(column_1)\n",
      "    \n",
      "    # Create a variable of the string inside 2nd <td> tag pair,\n",
      "    column_2 = col[1].string.strip()\n",
      "    # and append it to last_name variable\n",
      "    last_name.append(column_2)\n",
      "    \n",
      "    # Create a variable of the string inside 3rd <td> tag pair,\n",
      "    column_3 = col[2].string.strip()\n",
      "    # and append it to age variable\n",
      "    age.append(column_3)\n",
      "    \n",
      "    # Create a variable of the string inside 4th <td> tag pair,\n",
      "    column_4 = col[3].string.strip()\n",
      "    # and append it to preTestScore variable\n",
      "    preTestScore.append(column_4)\n",
      "    \n",
      "    # Create a variable of the string inside 5th <td> tag pair,\n",
      "    column_5 = col[4].string.strip()\n",
      "    # and append it to postTestScore variable\n",
      "    postTestScore.append(column_5)\n",
      "\n",
      "# Create a variable of the value of the columns\n",
      "columns = {'first_name': first_name, 'last_name': last_name, 'age': age, 'preTestScore': preTestScore, 'postTestScore': postTestScore}\n",
      "\n",
      "# Create a dataframe from the columns variable\n",
      "df = pd.DataFrame(columns)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 108
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# View the dataframe\n",
      "df"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>age</th>\n",
        "      <th>first_name</th>\n",
        "      <th>last_name</th>\n",
        "      <th>postTestScore</th>\n",
        "      <th>preTestScore</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 42</td>\n",
        "      <td> Jason</td>\n",
        "      <td>   Miller</td>\n",
        "      <td> 25</td>\n",
        "      <td>  4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 52</td>\n",
        "      <td> Molly</td>\n",
        "      <td> Jacobson</td>\n",
        "      <td> 94</td>\n",
        "      <td> 24</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 36</td>\n",
        "      <td>  Tina</td>\n",
        "      <td>      Ali</td>\n",
        "      <td> 57</td>\n",
        "      <td> 31</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 24</td>\n",
        "      <td>  Jake</td>\n",
        "      <td>   Milner</td>\n",
        "      <td> 62</td>\n",
        "      <td>  2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 73</td>\n",
        "      <td>   Amy</td>\n",
        "      <td>    Cooze</td>\n",
        "      <td> 70</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>5 rows \u00d7 5 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 109,
       "text": [
        "  age first_name last_name postTestScore preTestScore\n",
        "0  42      Jason    Miller            25            4\n",
        "1  52      Molly  Jacobson            94           24\n",
        "2  36       Tina       Ali            57           31\n",
        "3  24       Jake    Milner            62            2\n",
        "4  73        Amy     Cooze            70            3\n",
        "\n",
        "[5 rows x 5 columns]"
       ]
      }
     ],
     "prompt_number": 109
    }
   ],
   "metadata": {}
  }
 ]
}