01_AutoTrader_scrape.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e289cfbd",
   "metadata": {},
   "source": [
    "# Notebook Context\n",
    "\n",
    "This is 1 of 6 Jupyter Notebooks associated with the used car project.\n",
    "\n",
    "This Notebook covers web scraping of the data to be used in the project from [autotrader.co.uk](https://www.autotrader.co.uk/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "830b6570",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-04T14:16:00.133076Z",
     "start_time": "2022-03-04T14:15:58.112549Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Import packages \n",
    "\n",
    "import pandas as pd\n",
    "import glob\n",
    "import numpy as np\n",
    "import re\n",
    "import regex\n",
    "import time\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import requests\n",
    "import cloudscraper\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib.parse\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e27de7fa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-09T21:15:45.954440Z",
     "start_time": "2022-03-09T21:15:45.941949Z"
    }
   },
   "source": [
    "# AutoTrader Scraping Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d07f6cd2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-04T14:16:00.169626Z",
     "start_time": "2022-03-04T14:16:00.134843Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# AutoTrader used car scraping function \n",
    "\n",
    "def get_used_cars(\n",
    "    postcode=\"KT12\",\n",
    "    radius=1500,\n",
    "    min_year=1995,\n",
    "    max_year=1995,\n",
    "    include_writeoff=\"include\",\n",
    "    max_attempts_per_page=5,\n",
    "    verbose=False):\n",
    "\n",
    "    # To bypass Cloudflare protection\n",
    "    scraper = cloudscraper.create_scraper()\n",
    "\n",
    "    # Basic variables\n",
    "    results = []\n",
    "    n_this_year_results = 0\n",
    "    # If a search returns > 1000 results then only the first 1000 are made available.\n",
    "    # These such searches are recorded in the extra_make_list for later attention \n",
    "    extra_make_list = []\n",
    "    url = \"https://www.autotrader.co.uk/results-car-search\"\n",
    "\n",
    "    # Keywords commonly used on Autotrader in each of the fields being scraped\n",
    "    keywords = {}\n",
    "    keywords[\"mileage\"] = [\"miles\"]\n",
    "    keywords[\"BHP\"] = [\"BHP\",\"PS\"]\n",
    "    keywords[\"transmission\"] = [\"Automatic\", \"Manual\"]\n",
    "    keywords[\"fuel\"] = [\"Petrol\", \"Diesel\", \"Electric\", \"Hybrid – Diesel/Electric Plug-in\", \n",
    "                        \"Hybrid – Petrol/Electric\", \"Hybrid – Petrol/Electric Plug-in\", \"Bi Fuel\",\n",
    "                        \"Diesel Hybrid\",\"Diesel Plug-in Hybrid\",\"Hydrogen\",\"Natural Gas\",\"Petrol Hybrid\"\n",
    "                        \"Petrol Plug-in Hybrid\"]\n",
    "    keywords[\"owners\"] = [\"owners\", \"owner\",\"own\"]\n",
    "    keywords[\"body\"] = [\"Coupe\", \"Convertible\", \"Estate\", \"Hatchback\", \"MPV\", \"Pickup\", \"SUV\", \"Saloon\"]\n",
    "    keywords[\"ULEZ\"] = [\"ULEZ\"]\n",
    "    keywords[\"year\"] = [\" reg)\",\"new\"]\n",
    "    keywords[\"engine\"] = [\"engine\"]\n",
    "\n",
    "    # Set up parameters for query to autotrader.co.uk\n",
    "    mpgall = ['OVER_60','OVER_50','OVER_40','OVER_30']\n",
    "    drivetrainall = ['Four Wheel Drive','Front Wheel Drive','Rear Wheel Drive']\n",
    "    makeall = ['AC','Abarth','Aixam','Alfa Romeo','Alpine','Ariel','Aston Martin','Audi','Austin',\n",
    "        'BAC','BMW','Beauford','Bentley','Bowler','Bugatti','Buick',\n",
    "        'CUPRA','Cadillac','Carbodies','Caterham','Chesil','Chevrolet','Chrysler','Citroen','Corvette',\n",
    "        'DAF','DFSK','DS Automobiles','Dacia','Daewoo','Daihatsu','Daimler','Datsun','Delorean','Dodge',\n",
    "        'Ferrari','Fiat','Ford','GMC','Great Wall',\n",
    "        'Hillman','Honda','Hummer','Hyundai','Infiniti','Isuzu','Iveco','Jaguar','Jeep','Jensen','KIA',\n",
    "        'LEVC','Lada','Lamborghini','Lancia','Land Rover','Lexus','Lincoln','London Taxis International','Lotus',\n",
    "        'MG','MINI','Mahindra','Maserati','Maybach','Mazda','McLaren','Mercedes-Benz','Microcar','Mitsubishi',\n",
    "        'Mitsuoka','Morgan','Morris','Nissan','Noble','Opel','Packard','Perodua','Peugeot','Pilgrim','Polestar',\n",
    "        'Pontiac','Porsche','Proton','REO','Radical','Rage','Raptor','Reliant','Renault','Replica','Reva','Riley',\n",
    "        'Rolls-Royce','Rover','SEAT','SKODA','Saab','Sebring','Singer','Smart','Spyker','Ssangyong','Subaru',\n",
    "        'Sunbeam','Suzuki','TVR','Tesla','Tiger','Toyota','Triumph','Ultima','Vauxhall','Venturi','Volkswagen',\n",
    "        'Volvo','Westfield','Wolseley','Yamaha','Zenos']\n",
    "    body = [\"Coupe\", \"Convertible\", \"Estate\", \"Hatchback\", \"MPV\", \"Pickup\", \"SUV\", \"Saloon\"]\n",
    "    trans = [\"Automatic\", \"Manual\"]\n",
    "    sort = [\"price-asc\",\"price-desc\"]\n",
    "    \n",
    "    # Iterate over combinations of the search key words. We make such specific searches in an aim\n",
    "    # to return <1000 results per search\n",
    "    for divide in tqdm(list(itertools.product(mpgall, drivetrainall, makeall, body, trans, sort))):  \n",
    "        params = {\n",
    "            \"sort\": divide[5],\n",
    "            \"postcode\": postcode,\n",
    "            \"radius\": radius,\n",
    "            \"make\": divide[2],\n",
    "            \"search-results-price-type\": \"total-price\",\n",
    "            \"search-results-year\": \"select-year\",\n",
    "            \"exclude-writeoff-categories\":\"on\",\n",
    "            \"fuel-consumption\":divide[0],\n",
    "            \"drivetrain\":divide[1],\n",
    "            \"body-type\":divide[3],\n",
    "            \"transmission\":divide[4]\n",
    "             }    \n",
    "    \n",
    "        # Set up writeoff parameters for query. Included by default args. \n",
    "        if (include_writeoff == \"include\"):\n",
    "            params[\"writeoff-categories\"] = \"on\"\n",
    "        elif (include_writeoff == \"exclude\"):\n",
    "            params[\"exclude-writeoff-categories\"] = \"on\"\n",
    "        elif (include_writeoff == \"writeoff-only\"):\n",
    "            params[\"only-writeoff-categories\"] = \"on\"\n",
    "\n",
    "        # Set up year parameters for query. Start at year_min and grow to year_max  \n",
    "        year = min_year\n",
    "        page = 1\n",
    "        attempt = 1\n",
    "\n",
    "        try:\n",
    "            while year <= max_year:\n",
    "                params[\"year-from\"] = year\n",
    "                params[\"year-to\"] = year\n",
    "                params[\"page\"] = page\n",
    "\n",
    "                #Sleep timer was not required with VPN cloudscraper combination.\n",
    "                #time.sleep(random.randint(1,9))\n",
    "                r = scraper.get(url, params=params)\n",
    "                \n",
    "                if verbose:\n",
    "                    print(\"Year:     \", year,\"\\t Page:     \", page,\"\\t Response: \", r)\n",
    "\n",
    "                try:\n",
    "                    if r.status_code != 200: # if not successful (e.g. due to bot protection), log as an attempt\n",
    "                        attempt = attempt + 1\n",
    "                        if attempt <= max_attempts_per_page: # break if max_attempts reached\n",
    "                            if verbose:\n",
    "                                print(\"Exception. Starting attempt #\", attempt, \"and keeping at page #\", page)\n",
    "                        if r.status_code == 500: # break if internal website error\n",
    "                            break\n",
    "                        else:\n",
    "                            page = page + 1\n",
    "                            attempt = 1\n",
    "                            if verbose:\n",
    "                                print(\"Exception. All attempts exhausted for this page. Skipping to next page #\", page)\n",
    "                            if page > 100: # Autotrader does not return sensible results beyond 100 pages\n",
    "                                break\n",
    "                    else:\n",
    "\n",
    "                        j = r.json()\n",
    "                        s = BeautifulSoup(j[\"html\"], features=\"html.parser\")\n",
    "\n",
    "                        # Use beautifulsoup to generate list of cars on each page - called articles \n",
    "                        articles = s.find_all(\"article\", attrs={\"data-standout-type\":\"\"})\n",
    "\n",
    "                        # if no results or reached end of results, report this then increment for next search\n",
    "                        if len(articles) == 0 or r.url[r.url.find(\"page=\")+5:] != str(page):\n",
    "                            if verbose:\n",
    "                                print(\"Found total\", n_this_year_results, \"results for year\", year, \"and brand\", divide, \"across\", page-1, \"pages\")\n",
    "                                # If search returnd > 1000 cars, add car band to the list.\n",
    "                                if n_this_year_results == 1000:\n",
    "                                    extra_make_list.append(params[\"make\"])\n",
    "                                if year+1 <= max_year:\n",
    "                                    print(\"Moving on to year\", year + 1)\n",
    "                                    print(\"---------------------------------\")\n",
    "\n",
    "                            # Increment year and reset relevant variables\n",
    "                            year = year + 1\n",
    "                            page = 1\n",
    "                            attempt = 1\n",
    "                            n_this_year_results = 0\n",
    "                        else:\n",
    "                            # For each car, build a dictionary. Some from seach parameters, some from produced article\n",
    "                            for article in articles:\n",
    "                                car = {}\n",
    "                                seller_href=[]\n",
    "                                car[\"name\"] = article.find(\"h3\", {\"class\": \"product-card-details__title\"}).text.strip()\n",
    "                                car[\"name_subtitle\"] = article.find(\"h3\", {\"class\": \"product-card-details__title\"}).findNext('p').text.strip()                       \n",
    "                                car[\"link\"] = \"https://www.autotrader.co.uk\" + article.find(\"a\", {\"class\": \"tracking-standard-link\"})[\"href\"][: article.find(\"a\", {\"class\": \"tracking-standard-link\"})[\"href\"].find(\"?\")]\n",
    "                                car[\"price\"] = article.find(\"div\", {\"class\": \"product-card-pricing__price\"}).text.strip()\n",
    "                                car[\"mpg\"] = divide[0]\n",
    "                                car[\"drivertrain\"] = divide[1]\n",
    "                                car[\"make\"] = divide[2]\n",
    "                                seller = article.find_all(\"span\", {\"class\": \"product-card-seller-info__spec-item-copy\"})#.text.strip()\n",
    "                                seller=reversed(seller)\n",
    "                                for count,sel in enumerate(seller):\n",
    "                                    car[\"seller\"+str(count)] = sel.text.strip()\n",
    "                                for a in article.find_all(\"a\", {\"class\": \"product-card-seller-info__review-count dealer-profile-link\", \"href\":True}):\n",
    "                                    seller_href.append(a[\"href\"])\n",
    "                                for count,ref in enumerate(seller_href):\n",
    "                                    car[\"href\"+str(count)] = ref\n",
    "\n",
    "                                # The key specs are in a bulleted list\n",
    "                                key_specs_bs_list = article.find(\"ul\", {\"class\": \"listing-key-specs\"}).find_all(\"li\")\n",
    "\n",
    "                                for key_spec_bs_li in key_specs_bs_list:\n",
    "\n",
    "                                    key_spec_bs = key_spec_bs_li.text                              \n",
    "\n",
    "                                    if any(keyword in key_spec_bs for keyword in keywords[\"mileage\"]):\n",
    "                                        car[\"mileage\"] = int(key_spec_bs[:key_spec_bs.find(\" miles\")].replace(\",\",\"\"))\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"BHP\"]):\n",
    "                                        car[\"BHP\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"transmission\"]):\n",
    "                                        car[\"transmission\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"fuel\"]):\n",
    "                                        car[\"fuel\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"owners\"]):\n",
    "                                        car[\"owners\"] = int(key_spec_bs[:key_spec_bs.find(\" own\")])\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"body\"]):\n",
    "                                        car[\"body\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"ULEZ\"]):\n",
    "                                        car[\"ULEZ\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"year\"]):\n",
    "                                        car[\"year\"] = key_spec_bs\n",
    "                                    elif key_spec_bs[1] == \".\" and key_spec_bs[3] == \"L\":\n",
    "                                        car[\"engine\"] = key_spec_bs\n",
    "                                \n",
    "                                # Set any missing dictionary keys to NA for complete car details.\n",
    "                                for key in keywords.keys():\n",
    "                                    if key in car.keys():\n",
    "                                        pass\n",
    "                                    else:\n",
    "                                        car[key]='NA'\n",
    "                                # Add complete car dictionary to results list.\n",
    "                                results.append(car)\n",
    "                                n_this_year_results = n_this_year_results + 1\n",
    "\n",
    "                            page = page + 1\n",
    "                            attempt = 1\n",
    "\n",
    "                            if verbose:\n",
    "                                print(\"Car count: \", len(results))\n",
    "                                print(\"---------------------------------\")\n",
    "\n",
    "                except KeyboardInterrupt:\n",
    "                    break\n",
    "\n",
    "                except:\n",
    "                    traceback.print_exc()\n",
    "                    attempt = attempt + 1\n",
    "                    if attempt <= max_attempts_per_page:\n",
    "                        if verbose:\n",
    "                            print(\"Exception. Starting attempt #\", attempt, \"and keeping at page #\", page)\n",
    "                    else:\n",
    "                        page = page + 1\n",
    "                        attempt = 1\n",
    "                        if verbose:\n",
    "                            print(\"Exception. All attempts exhausted for this page. Skipping to next page #\", page)\n",
    "\n",
    "        except KeyboardInterrupt:\n",
    "            pass\n",
    "    \n",
    "    # Make df of results and output to csv\n",
    "    results = pd.DataFrame(results)\n",
    "    now = (datetime.datetime.now().strftime(\"%d%B_%I%M%p\"))\n",
    "    filepathdf=f'/raw_data_used/{now}_{min_year}-{max_year}_used.csv'\n",
    "    results.to_csv(filepathdf, index=False, header=results.columns )\n",
    "\n",
    "    return results, list(set(extra_make_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9dc2c6c0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-04T14:16:00.206414Z",
     "start_time": "2022-03-04T14:16:00.172378Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# AutoTrader used electric car scraping function \n",
    "\n",
    "def get_used_electric_cars(\n",
    "    postcode=\"KT12\",\n",
    "    radius=1500,\n",
    "    min_year=1995,\n",
    "    max_year=1995,\n",
    "    include_writeoff=\"include\",\n",
    "    max_attempts_per_page=5,\n",
    "    verbose=False):\n",
    "\n",
    "    # To bypass Cloudflare protection\n",
    "    scraper = cloudscraper.create_scraper()\n",
    "\n",
    "    # Basic variables\n",
    "    results = []\n",
    "    n_this_year_results = 0\n",
    "    # If a search returns > 1000 results then only the first 1000 are made available.\n",
    "    # These such searches are recorded in the extra_make_list for later attention \n",
    "    extra_make_list = []\n",
    "    url = \"https://www.autotrader.co.uk/results-car-search\"\n",
    "\n",
    "    # Keywords commonly used on Autotrader in each of the fields being scraped\n",
    "    keywords = {}\n",
    "    keywords[\"mileage\"] = [\"miles\"]\n",
    "    keywords[\"BHP\"] = [\"BHP\",\"PS\"]\n",
    "    keywords[\"transmission\"] = [\"Automatic\", \"Manual\"]\n",
    "    keywords[\"fuel\"] = [\"Petrol\", \"Diesel\", \"Electric\", \"Hybrid – Diesel/Electric Plug-in\", \n",
    "                        \"Hybrid – Petrol/Electric\", \"Hybrid – Petrol/Electric Plug-in\", \"Bi Fuel\",\n",
    "                        \"Diesel Hybrid\",\"Diesel Plug-in Hybrid\",\"Hydrogen\",\"Natural Gas\",\"Petrol Hybrid\"\n",
    "                        \"Petrol Plug-in Hybrid\"]\n",
    "    keywords[\"owners\"] = [\"owners\", \"owner\",\"own\"]\n",
    "    keywords[\"body\"] = [\"Coupe\", \"Convertible\", \"Estate\", \"Hatchback\", \"MPV\", \"Pickup\", \"SUV\", \"Saloon\"]\n",
    "    keywords[\"ULEZ\"] = [\"ULEZ\"]\n",
    "    keywords[\"year\"] = [\" reg)\",\"new\"]\n",
    "    keywords[\"engine\"] = [\"engine\"]\n",
    "\n",
    "    # Set up parameters for query to autotrader.co.uk\n",
    "    mpgall = ['OVER_60','OVER_50','OVER_40','OVER_30']\n",
    "    drivetrainall = ['Four Wheel Drive','Front Wheel Drive','Rear Wheel Drive']\n",
    "    makeall = ['AC','Abarth','Aixam','Alfa Romeo','Alpine','Ariel','Aston Martin','Audi','Austin',\n",
    "        'BAC','BMW','Beauford','Bentley','Bowler','Bugatti','Buick',\n",
    "        'CUPRA','Cadillac','Carbodies','Caterham','Chesil','Chevrolet','Chrysler','Citroen','Corvette',\n",
    "        'DAF','DFSK','DS Automobiles','Dacia','Daewoo','Daihatsu','Daimler','Datsun','Delorean','Dodge',\n",
    "        'Ferrari','Fiat','Ford','GMC','Great Wall',\n",
    "        'Hillman','Honda','Hummer','Hyundai','Infiniti','Isuzu','Iveco','Jaguar','Jeep','Jensen','KIA',\n",
    "        'LEVC','Lada','Lamborghini','Lancia','Land Rover','Lexus','Lincoln','London Taxis International','Lotus',\n",
    "        'MG','MINI','Mahindra','Maserati','Maybach','Mazda','McLaren','Mercedes-Benz','Microcar','Mitsubishi',\n",
    "        'Mitsuoka','Morgan','Morris','Nissan','Noble','Opel','Packard','Perodua','Peugeot','Pilgrim','Polestar',\n",
    "        'Pontiac','Porsche','Proton','REO','Radical','Rage','Raptor','Reliant','Renault','Replica','Reva','Riley',\n",
    "        'Rolls-Royce','Rover','SEAT','SKODA','Saab','Sebring','Singer','Smart','Spyker','Ssangyong','Subaru',\n",
    "        'Sunbeam','Suzuki','TVR','Tesla','Tiger','Toyota','Triumph','Ultima','Vauxhall','Venturi','Volkswagen',\n",
    "        'Volvo','Westfield','Wolseley','Yamaha','Zenos']\n",
    "    body = [\"Coupe\", \"Convertible\", \"Estate\", \"Hatchback\", \"MPV\", \"Pickup\", \"SUV\", \"Saloon\"]\n",
    "    trans = [\"Automatic\", \"Manual\"]\n",
    "    sort = [\"price-asc\",\"price-desc\"]\n",
    "    \n",
    "    # Iterate over combinations of the search key words. We make such specific searches in an aim\n",
    "    # to return <1000 results per search\n",
    "    for divide in tqdm(list(itertools.product(drivetrainall, makeall, body, trans, sort))):  \n",
    "        params = {\n",
    "            \"sort\": divide[4],\n",
    "            \"postcode\": postcode,\n",
    "            \"radius\": radius,\n",
    "            \"make\": divide[1],\n",
    "            \"search-results-price-type\": \"total-price\",\n",
    "            \"search-results-year\": \"select-year\",\n",
    "            \"exclude-writeoff-categories\":\"on\",\n",
    "            \"fuel-type\":'Electric',\n",
    "            \"drivetrain\":divide[0],\n",
    "            \"body-type\":divide[2],\n",
    "            \"transmission\":divide[3]\n",
    "             }    \n",
    "    \n",
    "        # Set up writeoff parameters for query. Included by default args. \n",
    "        if (include_writeoff == \"include\"):\n",
    "            params[\"writeoff-categories\"] = \"on\"\n",
    "        elif (include_writeoff == \"exclude\"):\n",
    "            params[\"exclude-writeoff-categories\"] = \"on\"\n",
    "        elif (include_writeoff == \"writeoff-only\"):\n",
    "            params[\"only-writeoff-categories\"] = \"on\"\n",
    "\n",
    "        # Set up year parameters for query. Start at year_min and grow to year_max  \n",
    "        year = min_year\n",
    "        page = 1\n",
    "        attempt = 1\n",
    "\n",
    "        try:\n",
    "            while year <= max_year:\n",
    "                params[\"year-from\"] = year\n",
    "                params[\"year-to\"] = year\n",
    "                params[\"page\"] = page\n",
    "\n",
    "                #Sleep timer was not required with VPN cloudscraper combination.\n",
    "                #time.sleep(random.randint(1,9))\n",
    "                r = scraper.get(url, params=params)\n",
    "                \n",
    "                if verbose:\n",
    "                    print(\"Year:     \", year,\"\\t Page:     \", page,\"\\t Response: \", r)\n",
    "\n",
    "                try:\n",
    "                    if r.status_code != 200: # if not successful (e.g. due to bot protection), log as an attempt\n",
    "                        attempt = attempt + 1\n",
    "                        if attempt <= max_attempts_per_page: # break if max_attempts reached\n",
    "                            if verbose:\n",
    "                                print(\"Exception. Starting attempt #\", attempt, \"and keeping at page #\", page)\n",
    "                        if r.status_code == 500: # break if internal website error\n",
    "                            break\n",
    "                        else:\n",
    "                            page = page + 1\n",
    "                            attempt = 1\n",
    "                            if verbose:\n",
    "                                print(\"Exception. All attempts exhausted for this page. Skipping to next page #\", page)\n",
    "                            if page > 100: # Autotrader does not return sensible results beyond 100 pages\n",
    "                                break\n",
    "                    else:\n",
    "\n",
    "                        j = r.json()\n",
    "                        s = BeautifulSoup(j[\"html\"], features=\"html.parser\")\n",
    "\n",
    "                        # Use beautifulsoup to generate list of cars on each page - called articles \n",
    "                        articles = s.find_all(\"article\", attrs={\"data-standout-type\":\"\"})\n",
    "\n",
    "                        # if no results or reached end of results, report this then increment for next search\n",
    "                        if len(articles) == 0 or r.url[r.url.find(\"page=\")+5:] != str(page):\n",
    "                            if verbose:\n",
    "                                print(\"Found total\", n_this_year_results, \"results for year\", year, \"and brand\", divide, \"across\", page-1, \"pages\")\n",
    "                                # If search returnd > 1000 cars, add car band to the list.\n",
    "                                if n_this_year_results == 1000:\n",
    "                                    extra_make_list.append(params[\"make\"])\n",
    "                                if year+1 <= max_year:\n",
    "                                    print(\"Moving on to year\", year + 1)\n",
    "                                    print(\"---------------------------------\")\n",
    "\n",
    "                            # Increment year and reset relevant variables\n",
    "                            year = year + 1\n",
    "                            page = 1\n",
    "                            attempt = 1\n",
    "                            n_this_year_results = 0\n",
    "                        else:\n",
    "                            # For each car, build a dictionary. Some from seach parameters, some from produced article\n",
    "                            for article in articles:\n",
    "                                car = {}\n",
    "                                seller_href=[]\n",
    "                                car[\"name\"] = article.find(\"h3\", {\"class\": \"product-card-details__title\"}).text.strip()\n",
    "                                car[\"name_subtitle\"] = article.find(\"h3\", {\"class\": \"product-card-details__title\"}).findNext('p').text.strip()                       \n",
    "                                car[\"link\"] = \"https://www.autotrader.co.uk\" + article.find(\"a\", {\"class\": \"tracking-standard-link\"})[\"href\"][: article.find(\"a\", {\"class\": \"tracking-standard-link\"})[\"href\"].find(\"?\")]\n",
    "                                car[\"price\"] = article.find(\"div\", {\"class\": \"product-card-pricing__price\"}).text.strip()\n",
    "                                car[\"mpg\"] = 'electric'\n",
    "                                car[\"drivertrain\"] = divide[0]\n",
    "                                car[\"make\"] = divide[1]\n",
    "                                seller = article.find_all(\"span\", {\"class\": \"product-card-seller-info__spec-item-copy\"})#.text.strip()\n",
    "                                seller=reversed(seller)\n",
    "                                for count,sel in enumerate(seller):\n",
    "                                    car[\"seller\"+str(count)] = sel.text.strip()\n",
    "                                for a in article.find_all(\"a\", {\"class\": \"product-card-seller-info__review-count dealer-profile-link\", \"href\":True}):\n",
    "                                    seller_href.append(a[\"href\"])\n",
    "                                for count,ref in enumerate(seller_href):\n",
    "                                    car[\"href\"+str(count)] = ref\n",
    "\n",
    "                                # The key specs are in a bulleted list\n",
    "                                key_specs_bs_list = article.find(\"ul\", {\"class\": \"listing-key-specs\"}).find_all(\"li\")\n",
    "\n",
    "                                for key_spec_bs_li in key_specs_bs_list:\n",
    "\n",
    "                                    key_spec_bs = key_spec_bs_li.text                              \n",
    "\n",
    "                                    if any(keyword in key_spec_bs for keyword in keywords[\"mileage\"]):\n",
    "                                        car[\"mileage\"] = int(key_spec_bs[:key_spec_bs.find(\" miles\")].replace(\",\",\"\"))\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"BHP\"]):\n",
    "                                        car[\"BHP\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"transmission\"]):\n",
    "                                        car[\"transmission\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"fuel\"]):\n",
    "                                        car[\"fuel\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"owners\"]):\n",
    "                                        car[\"owners\"] = int(key_spec_bs[:key_spec_bs.find(\" own\")])\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"body\"]):\n",
    "                                        car[\"body\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"ULEZ\"]):\n",
    "                                        car[\"ULEZ\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"year\"]):\n",
    "                                        car[\"year\"] = key_spec_bs\n",
    "                                    elif key_spec_bs[1] == \".\" and key_spec_bs[3] == \"L\":\n",
    "                                        car[\"engine\"] = key_spec_bs\n",
    "                                \n",
    "                                # Set any missing dictionary keys to NA for complete car details.\n",
    "                                for key in keywords.keys():\n",
    "                                    if key in car.keys():\n",
    "                                        pass\n",
    "                                    else:\n",
    "                                        car[key]='NA'\n",
    "                                # Add complete car dictionary to results list.\n",
    "                                results.append(car)\n",
    "                                n_this_year_results = n_this_year_results + 1\n",
    "\n",
    "                            page = page + 1\n",
    "                            attempt = 1\n",
    "\n",
    "                            if verbose:\n",
    "                                print(\"Car count: \", len(results))\n",
    "                                print(\"---------------------------------\")\n",
    "\n",
    "                except KeyboardInterrupt:\n",
    "                    break\n",
    "\n",
    "                except:\n",
    "                    traceback.print_exc()\n",
    "                    attempt = attempt + 1\n",
    "                    if attempt <= max_attempts_per_page:\n",
    "                        if verbose:\n",
    "                            print(\"Exception. Starting attempt #\", attempt, \"and keeping at page #\", page)\n",
    "                    else:\n",
    "                        page = page + 1\n",
    "                        attempt = 1\n",
    "                        if verbose:\n",
    "                            print(\"Exception. All attempts exhausted for this page. Skipping to next page #\", page)\n",
    "\n",
    "        except KeyboardInterrupt:\n",
    "            pass\n",
    "    \n",
    "    # Make df of results and output to csv\n",
    "    results = pd.DataFrame(results)\n",
    "    now = (datetime.datetime.now().strftime(\"%d%B_%I%M%p\"))\n",
    "    filepathdf=f'/raw_data_used_electric/{now}_{min_year}-{max_year}_used.csv'\n",
    "    results.to_csv(filepathdf, index=False, header=results.columns )\n",
    "\n",
    "    return results, list(set(extra_make_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7c47f94b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-04T14:16:00.242268Z",
     "start_time": "2022-03-04T14:16:00.210025Z"
    },
    "code_folding": [
     0,
     2
    ]
   },
   "outputs": [],
   "source": [
    "# AutoTrader new car scraping function \n",
    "\n",
    "def get_new_cars(\n",
    "    postcode=\"KT12\",\n",
    "    radius=1500,\n",
    "    include_writeoff=\"include\",\n",
    "    max_attempts_per_page=5,\n",
    "    verbose=False):\n",
    "\n",
    "    # To bypass Cloudflare protection\n",
    "    scraper = cloudscraper.create_scraper()\n",
    "\n",
    "    # Basic variables\n",
    "\n",
    "    results = []\n",
    "    extra_brand_list=[]\n",
    "    n_this_year_results = 0\n",
    "    # If a search returns > 1000 results then only the first 1000 are made available.\n",
    "    # These such searches are recorded in the extra_make_list for later attention \n",
    "    extra_make_list = []\n",
    "    url = \"https://www.autotrader.co.uk/results-car-search\"\n",
    "\n",
    "    # Keywords commonly used on Autotrader in each of the fields being scraped\n",
    "    keywords = {}\n",
    "    keywords[\"mileage\"] = [\"miles\"]\n",
    "    keywords[\"BHP\"] = [\"BHP\",\"PS\"]\n",
    "    keywords[\"transmission\"] = [\"Automatic\", \"Manual\"]\n",
    "    keywords[\"fuel\"] = [\"Petrol\", \"Diesel\", \"Electric\", \"Hybrid – Diesel/Electric Plug-in\", \n",
    "                        \"Hybrid – Petrol/Electric\", \"Hybrid – Petrol/Electric Plug-in\", \"Bi Fuel\",\n",
    "                        \"Diesel Hybrid\",\"Diesel Plug-in Hybrid\",\"Hydrogen\",\"Natural Gas\",\"Petrol Hybrid\"\n",
    "                        \"Petrol Plug-in Hybrid\"]\n",
    "    keywords[\"owners\"] = [\"owners\", \"owner\",\"own\"]\n",
    "    keywords[\"body\"] = [\"Coupe\", \"Convertible\", \"Estate\", \"Hatchback\", \"MPV\", \"Pickup\", \"SUV\", \"Saloon\"]\n",
    "    keywords[\"ULEZ\"] = [\"ULEZ\"]\n",
    "    keywords[\"year\"] = [\" reg)\",\"new\"]\n",
    "    keywords[\"engine\"] = [\"engine\"]\n",
    "\n",
    "    # Set up parameters for query to autotrader.co.uk\n",
    "    mpgall = ['OVER_60','OVER_50','OVER_40','OVER_30']\n",
    "    drivetrainall = ['Four Wheel Drive','Front Wheel Drive','Rear Wheel Drive']\n",
    "    makeall = ['AC','Abarth','Aixam','Alfa Romeo','Alpine','Ariel','Aston Martin','Audi','Austin',\n",
    "        'BAC','BMW','Beauford','Bentley','Bowler','Bugatti','Buick',\n",
    "        'CUPRA','Cadillac','Carbodies','Caterham','Chesil','Chevrolet','Chrysler','Citroen','Corvette',\n",
    "        'DAF','DFSK','DS Automobiles','Dacia','Daewoo','Daihatsu','Daimler','Datsun','Delorean','Dodge',\n",
    "        'Ferrari','Fiat','Ford','GMC','Great Wall',\n",
    "        'Hillman','Honda','Hummer','Hyundai','Infiniti','Isuzu','Iveco','Jaguar','Jeep','Jensen','KIA',\n",
    "        'LEVC','Lada','Lamborghini','Lancia','Land Rover','Lexus','Lincoln','London Taxis International','Lotus',\n",
    "        'MG','MINI','Mahindra','Maserati','Maybach','Mazda','McLaren','Mercedes-Benz','Microcar','Mitsubishi',\n",
    "        'Mitsuoka','Morgan','Morris','Nissan','Noble','Opel','Packard','Perodua','Peugeot','Pilgrim','Polestar',\n",
    "        'Pontiac','Porsche','Proton','REO','Radical','Rage','Raptor','Reliant','Renault','Replica','Reva','Riley',\n",
    "        'Rolls-Royce','Rover','SEAT','SKODA','Saab','Sebring','Singer','Smart','Spyker','Ssangyong','Subaru',\n",
    "        'Sunbeam','Suzuki','TVR','Tesla','Tiger','Toyota','Triumph','Ultima','Vauxhall','Venturi','Volkswagen',\n",
    "        'Volvo','Westfield','Wolseley','Yamaha','Zenos']\n",
    "    body=[\"Coupe\", \"Convertible\", \"Estate\", \"Hatchback\", \"MPV\", \"Pickup\", \"SUV\", \"Saloon\"]\n",
    "    trans = [\"Automatic\", \"Manual\"]\n",
    "    sort = [\"price-asc\",\"price-desc\"]\n",
    "\n",
    "    # Iterate over combinations of the search key words. We make such specific searches in an aim\n",
    "    # to return <1000 results per search    \n",
    "    for divide in tqdm(list(itertools.product(mpgall, drivetrainall, makeall, body, trans, sort))): \n",
    "        params = {\n",
    "            \"sort\": divide[5],\n",
    "            \"postcode\": postcode,\n",
    "            \"radius\": radius,\n",
    "            \"make\": divide[2],\n",
    "            \"search-results-price-type\": \"total-price\",\n",
    "            # Search year not used in new car search\n",
    "            # \"search-results-year\": \"select-year\",\n",
    "            \"exclude-writeoff-categories\":\"on\",\n",
    "            \"fuel-consumption\":divide[0],\n",
    "            \"drivetrain\":divide[1],\n",
    "            \"body-type\":divide[3],\n",
    "            \"transmission\":divide[4]\n",
    "             }\n",
    "\n",
    "        # Set up writeoff parameters for query. Included by default args. \n",
    "        if (include_writeoff == \"include\"):\n",
    "            params[\"writeoff-categories\"] = \"on\"\n",
    "        elif (include_writeoff == \"exclude\"):\n",
    "            params[\"exclude-writeoff-categories\"] = \"on\"\n",
    "        elif (include_writeoff == \"writeoff-only\"):\n",
    "            params[\"only-writeoff-categories\"] = \"on\"\n",
    "\n",
    "        # Year parameters are hardcoded for new car search. Notice that the 'year-from' paraneter is hard\n",
    "        # coded to be 'new' below so we don't only search for new cars from 2022.\n",
    "        year = 2022\n",
    "        max_year = 2022\n",
    "        page = 1\n",
    "        attempt = 1\n",
    "\n",
    "\n",
    "        try:\n",
    "            \n",
    "            # This only occurs once as year=max_year to start with\n",
    "            while year <= max_year:\n",
    "\n",
    "                # Search for new cars only\n",
    "                params[\"year-from\"] = 'new'\n",
    "                params[\"page\"] = page\n",
    "\n",
    "                # Sleep timer was not required with VPN cloudscraper combination.\n",
    "                # time.sleep(random.randint(1,9))\n",
    "                r = scraper.get(url, params=params)\n",
    "                if verbose:\n",
    "                    print(\"Year:     \", year)\n",
    "                    print(\"Page:     \", page)\n",
    "                    print(\"Response: \", r)\n",
    "\n",
    "                try:\n",
    "                    if r.status_code != 200: # if not successful (e.g. due to bot protection), log as an attempt\n",
    "                        attempt = attempt + 1\n",
    "                        if attempt <= max_attempts_per_page: # break if max_attempts reached\n",
    "                            if verbose:\n",
    "                                print(\"Exception. Starting attempt #\", attempt, \"and keeping at page #\", page)\n",
    "                        if r.status_code == 500: # break if internal website error\n",
    "                            break\n",
    "                        else:\n",
    "                            page = page + 1\n",
    "                            attempt = 1\n",
    "                            if verbose:\n",
    "                                print(\"Exception. All attempts exhausted for this page. Skipping to next page #\", page)\n",
    "                            if page > 100: # Autotrader does not return sensible results beyond 100 pages\n",
    "                                break\n",
    "                    else:\n",
    "\n",
    "                        j = r.json()\n",
    "                        s = BeautifulSoup(j[\"html\"], features=\"html.parser\")\n",
    "\n",
    "                        # Use beautifulsoup to generate list of cars on each page - called articles                        \n",
    "                        articles = s.find_all(\"article\", attrs={\"data-standout-type\":\"\"})\n",
    "\n",
    "                        # if no results or reached end of results, report this then increment for next search\n",
    "                        if len(articles) == 0 or r.url[r.url.find(\"page=\")+5:] != str(page):\n",
    "                            if verbose:\n",
    "                                print(\"Found total\", n_this_year_results, \"results for year\", year, \"and brand\", divide, \"across\", page-1, \"pages\")\n",
    "                                # If search returnd > 1000 cars, add car band to the list.\n",
    "                                if n_this_year_results == 1000:\n",
    "                                    extra_brand_list.append(params[\"make\"])                                \n",
    "                                if year+1 <= max_year:\n",
    "                                    print(\"Moving on to year\", year + 1)\n",
    "                                    print(\"---------------------------------\")\n",
    "\n",
    "                            # Increment year and reset relevant variables\n",
    "                            year = year + 1\n",
    "                            page = 1\n",
    "                            attempt = 1\n",
    "                            n_this_year_results = 0\n",
    "                        else:\n",
    "                            # For each car, build a dictionary. Some from seach parameters, some from produced article\n",
    "                            for article in articles:\n",
    "                                car = {}\n",
    "                                seller_href=[]\n",
    "                                car[\"name\"] = article.find(\"h3\", {\"class\": \"product-card-details__title\"}).text.strip()\n",
    "                                car[\"name_subtitle\"] = article.find(\"h3\", {\"class\": \"product-card-details__title\"}).findNext('p').text.strip()                       \n",
    "                                car[\"link\"] = \"https://www.autotrader.co.uk\" + article.find(\"a\", {\"class\": \"tracking-standard-link\"})[\"href\"][: article.find(\"a\", {\"class\": \"tracking-standard-link\"})[\"href\"].find(\"?\")]\n",
    "                                car[\"price\"] = article.find(\"div\", {\"class\": \"product-card-pricing__price\"}).text.strip()\n",
    "                                car[\"mpg\"] = divide[0]\n",
    "                                car[\"drivertrain\"] = divide[1]\n",
    "                                car[\"make\"] = divide[2]\n",
    "                                seller = article.find_all(\"span\", {\"class\": \"product-card-seller-info__spec-item-copy\"})#.text.strip()\n",
    "                                seller=reversed(seller)\n",
    "                                for count,sel in enumerate(seller):\n",
    "                                    car[\"seller\"+str(count)] = sel.text.strip()\n",
    "                                for a in article.find_all(\"a\", {\"class\": \"product-card-seller-info__review-count dealer-profile-link\", \"href\":True}):\n",
    "                                    seller_href.append(a[\"href\"])\n",
    "                                for count,ref in enumerate(seller_href):\n",
    "                                    car[\"href\"+str(count)] = ref\n",
    "\n",
    "                                # The key specs are in a bulleted list\n",
    "                                key_specs_bs_list = article.find(\"ul\", {\"class\": \"listing-key-specs\"}).find_all(\"li\")\n",
    "\n",
    "                                for key_spec_bs_li in key_specs_bs_list:\n",
    "\n",
    "                                    key_spec_bs = key_spec_bs_li.text                              \n",
    "\n",
    "                                    if any(keyword in key_spec_bs for keyword in keywords[\"mileage\"]):\n",
    "                                        car[\"mileage\"] = int(key_spec_bs[:key_spec_bs.find(\" miles\")].replace(\",\",\"\"))\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"BHP\"]):\n",
    "                                        car[\"BHP\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"transmission\"]):\n",
    "                                        car[\"transmission\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"fuel\"]):\n",
    "                                        car[\"fuel\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"owners\"]):\n",
    "                                        car[\"owners\"] = int(key_spec_bs[:key_spec_bs.find(\" own\")])\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"body\"]):\n",
    "                                        car[\"body\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"ULEZ\"]):\n",
    "                                        car[\"ULEZ\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"year\"]):\n",
    "                                        car[\"year\"] = key_spec_bs\n",
    "                                    elif key_spec_bs[1] == \".\" and key_spec_bs[3] == \"L\":\n",
    "                                        car[\"engine\"] = key_spec_bs\n",
    "\n",
    "                                # Set any missing dictionary keys to NA for complete car details.\n",
    "                                for key in keywords.keys():\n",
    "                                    if key in car.keys():\n",
    "                                        pass\n",
    "                                    else:\n",
    "                                        car[key]='NA'\n",
    "                                # Add complete car dictionary to results list.\n",
    "                                results.append(car)\n",
    "                                n_this_year_results = n_this_year_results + 1\n",
    "\n",
    "                            page = page + 1\n",
    "                            attempt = 1\n",
    "\n",
    "                            if verbose:\n",
    "                                print(\"Car count: \", len(results))\n",
    "                                print(\"---------------------------------\")\n",
    "\n",
    "                except KeyboardInterrupt:\n",
    "                    break\n",
    "\n",
    "                except:\n",
    "                    traceback.print_exc()\n",
    "                    attempt = attempt + 1\n",
    "                    if attempt <= max_attempts_per_page:\n",
    "                        if verbose:\n",
    "                            print(\"Exception. Starting attempt #\", attempt, \"and keeping at page #\", page)\n",
    "                    else:\n",
    "                        page = page + 1\n",
    "                        attempt = 1\n",
    "                        if verbose:\n",
    "                            print(\"Exception. All attempts exhausted for this page. Skipping to next page #\", page)\n",
    "\n",
    "        except KeyboardInterrupt:\n",
    "            pass\n",
    "    \n",
    "    # Make df of results and output to csv\n",
    "    results = pd.DataFrame(results)\n",
    "    now = (datetime.datetime.now().strftime(\"%d%B_%I%M%p\"))\n",
    "    filepathdf=f'/raw_data_new/{now}_{year}_new.csv'\n",
    "    results.to_csv(filepathdf, index=False, header=results.columns )\n",
    "\n",
    "    return results, list(set(extra_brand_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "437cf0dc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-04T14:16:00.280728Z",
     "start_time": "2022-03-04T14:16:00.245161Z"
    },
    "code_folding": [
     0,
     2
    ]
   },
   "outputs": [],
   "source": [
    "# AutoTrader new electric car scraping function \n",
    "\n",
    "def get_new_electric_cars(\n",
    "    postcode=\"KT12\",\n",
    "    radius=1500,\n",
    "    include_writeoff=\"include\",\n",
    "    max_attempts_per_page=5,\n",
    "    verbose=False):\n",
    "\n",
    "    # To bypass Cloudflare protection\n",
    "    scraper = cloudscraper.create_scraper()\n",
    "\n",
    "    # Basic variables\n",
    "\n",
    "    results = []\n",
    "    extra_brand_list=[]\n",
    "    n_this_year_results = 0\n",
    "    # If a search returns > 1000 results then only the first 1000 are made available.\n",
    "    # These such searches are recorded in the extra_make_list for later attention \n",
    "    extra_make_list = []\n",
    "    url = \"https://www.autotrader.co.uk/results-car-search\"\n",
    "\n",
    "    # Keywords commonly used on Autotrader in each of the fields being scraped\n",
    "    keywords = {}\n",
    "    keywords[\"mileage\"] = [\"miles\"]\n",
    "    keywords[\"BHP\"] = [\"BHP\",\"PS\"]\n",
    "    keywords[\"transmission\"] = [\"Automatic\", \"Manual\"]\n",
    "    keywords[\"fuel\"] = [\"Petrol\", \"Diesel\", \"Electric\", \"Hybrid – Diesel/Electric Plug-in\", \n",
    "                        \"Hybrid – Petrol/Electric\", \"Hybrid – Petrol/Electric Plug-in\", \"Bi Fuel\",\n",
    "                        \"Diesel Hybrid\",\"Diesel Plug-in Hybrid\",\"Hydrogen\",\"Natural Gas\",\"Petrol Hybrid\"\n",
    "                        \"Petrol Plug-in Hybrid\"]\n",
    "    keywords[\"owners\"] = [\"owners\", \"owner\",\"own\"]\n",
    "    keywords[\"body\"] = [\"Coupe\", \"Convertible\", \"Estate\", \"Hatchback\", \"MPV\", \"Pickup\", \"SUV\", \"Saloon\"]\n",
    "    keywords[\"ULEZ\"] = [\"ULEZ\"]\n",
    "    keywords[\"year\"] = [\" reg)\",\"new\"]\n",
    "    keywords[\"engine\"] = [\"engine\"]\n",
    "\n",
    "    # Set up parameters for query to autotrader.co.uk\n",
    "    mpgall = ['OVER_60','OVER_50','OVER_40','OVER_30']\n",
    "    drivetrainall = ['Four Wheel Drive','Front Wheel Drive','Rear Wheel Drive']\n",
    "    makeall = ['AC','Abarth','Aixam','Alfa Romeo','Alpine','Ariel','Aston Martin','Audi','Austin',\n",
    "        'BAC','BMW','Beauford','Bentley','Bowler','Bugatti','Buick',\n",
    "        'CUPRA','Cadillac','Carbodies','Caterham','Chesil','Chevrolet','Chrysler','Citroen','Corvette',\n",
    "        'DAF','DFSK','DS Automobiles','Dacia','Daewoo','Daihatsu','Daimler','Datsun','Delorean','Dodge',\n",
    "        'Ferrari','Fiat','Ford','GMC','Great Wall',\n",
    "        'Hillman','Honda','Hummer','Hyundai','Infiniti','Isuzu','Iveco','Jaguar','Jeep','Jensen','KIA',\n",
    "        'LEVC','Lada','Lamborghini','Lancia','Land Rover','Lexus','Lincoln','London Taxis International','Lotus',\n",
    "        'MG','MINI','Mahindra','Maserati','Maybach','Mazda','McLaren','Mercedes-Benz','Microcar','Mitsubishi',\n",
    "        'Mitsuoka','Morgan','Morris','Nissan','Noble','Opel','Packard','Perodua','Peugeot','Pilgrim','Polestar',\n",
    "        'Pontiac','Porsche','Proton','REO','Radical','Rage','Raptor','Reliant','Renault','Replica','Reva','Riley',\n",
    "        'Rolls-Royce','Rover','SEAT','SKODA','Saab','Sebring','Singer','Smart','Spyker','Ssangyong','Subaru',\n",
    "        'Sunbeam','Suzuki','TVR','Tesla','Tiger','Toyota','Triumph','Ultima','Vauxhall','Venturi','Volkswagen',\n",
    "        'Volvo','Westfield','Wolseley','Yamaha','Zenos']\n",
    "    body=[\"Coupe\", \"Convertible\", \"Estate\", \"Hatchback\", \"MPV\", \"Pickup\", \"SUV\", \"Saloon\"]\n",
    "    trans = [\"Automatic\", \"Manual\"]\n",
    "    sort = [\"price-asc\",\"price-desc\"]\n",
    "\n",
    "    # Iterate over combinations of the search key words. We make such specific searches in an aim\n",
    "    # to return <1000 results per search    \n",
    "    for divide in tqdm(list(itertools.product(drivetrainall, makeall, body, trans, sort))): \n",
    "        params = {\n",
    "            \"sort\": divide[4],\n",
    "            \"postcode\": postcode,\n",
    "            \"radius\": radius,\n",
    "            \"make\": divide[1],\n",
    "            \"search-results-price-type\": \"total-price\",\n",
    "            # Search year not used in new car search\n",
    "            # \"search-results-year\": \"select-year\",\n",
    "            \"fuel-type\":'Electric',\n",
    "            \"exclude-writeoff-categories\":\"on\",\n",
    "            \"drivetrain\":divide[0],\n",
    "            \"body-type\":divide[2],\n",
    "            \"transmission\":divide[3]\n",
    "             }\n",
    "\n",
    "        # Set up writeoff parameters for query. Included by default args. \n",
    "        if (include_writeoff == \"include\"):\n",
    "            params[\"writeoff-categories\"] = \"on\"\n",
    "        elif (include_writeoff == \"exclude\"):\n",
    "            params[\"exclude-writeoff-categories\"] = \"on\"\n",
    "        elif (include_writeoff == \"writeoff-only\"):\n",
    "            params[\"only-writeoff-categories\"] = \"on\"\n",
    "\n",
    "        # Year parameters are hardcoded for new car search. Notice that the 'year-from' paraneter is hard\n",
    "        # coded to be 'new' below so we don't only search for new cars from 2022.\n",
    "        year = 2022\n",
    "        max_year = 2022\n",
    "        page = 1\n",
    "        attempt = 1\n",
    "\n",
    "\n",
    "        try:\n",
    "            \n",
    "            # This only occurs once as year=max_year to start with\n",
    "            while year <= max_year:\n",
    "\n",
    "                # Search for new cars only\n",
    "                params[\"year-from\"] = 'new'\n",
    "                params[\"page\"] = page\n",
    "\n",
    "                # Sleep timer was not required with VPN cloudscraper combination.\n",
    "                # time.sleep(random.randint(1,9))\n",
    "                r = scraper.get(url, params=params)\n",
    "                if verbose:\n",
    "                    print(\"Year:     \", year)\n",
    "                    print(\"Page:     \", page)\n",
    "                    print(\"Response: \", r)\n",
    "\n",
    "                try:\n",
    "                    if r.status_code != 200: # if not successful (e.g. due to bot protection), log as an attempt\n",
    "                        attempt = attempt + 1\n",
    "                        if attempt <= max_attempts_per_page: # break if max_attempts reached\n",
    "                            if verbose:\n",
    "                                print(\"Exception. Starting attempt #\", attempt, \"and keeping at page #\", page)\n",
    "                        if r.status_code == 500: # break if internal website error\n",
    "                            break\n",
    "                        else:\n",
    "                            page = page + 1\n",
    "                            attempt = 1\n",
    "                            if verbose:\n",
    "                                print(\"Exception. All attempts exhausted for this page. Skipping to next page #\", page)\n",
    "                            if page > 100: # Autotrader does not return sensible results beyond 100 pages\n",
    "                                break\n",
    "                    else:\n",
    "\n",
    "                        j = r.json()\n",
    "                        s = BeautifulSoup(j[\"html\"], features=\"html.parser\")\n",
    "\n",
    "                        # Use beautifulsoup to generate list of cars on each page - called articles                        \n",
    "                        articles = s.find_all(\"article\", attrs={\"data-standout-type\":\"\"})\n",
    "\n",
    "                        # if no results or reached end of results, report this then increment for next search\n",
    "                        if len(articles) == 0 or r.url[r.url.find(\"page=\")+5:] != str(page):\n",
    "                            if verbose:\n",
    "                                print(\"Found total\", n_this_year_results, \"results for year\", year, \"and brand\", divide, \"across\", page-1, \"pages\")\n",
    "                                # If search returnd > 1000 cars, add car band to the list.\n",
    "                                if n_this_year_results == 1000:\n",
    "                                    extra_brand_list.append(params[\"make\"])                                \n",
    "                                if year+1 <= max_year:\n",
    "                                    print(\"Moving on to year\", year + 1)\n",
    "                                    print(\"---------------------------------\")\n",
    "\n",
    "                            # Increment year and reset relevant variables\n",
    "                            year = year + 1\n",
    "                            page = 1\n",
    "                            attempt = 1\n",
    "                            n_this_year_results = 0\n",
    "                        else:\n",
    "                            # For each car, build a dictionary. Some from seach parameters, some from produced article\n",
    "                            for article in articles:\n",
    "                                car = {}\n",
    "                                seller_href=[]\n",
    "                                car[\"name\"] = article.find(\"h3\", {\"class\": \"product-card-details__title\"}).text.strip()\n",
    "                                car[\"name_subtitle\"] = article.find(\"h3\", {\"class\": \"product-card-details__title\"}).findNext('p').text.strip()                       \n",
    "                                car[\"link\"] = \"https://www.autotrader.co.uk\" + article.find(\"a\", {\"class\": \"tracking-standard-link\"})[\"href\"][: article.find(\"a\", {\"class\": \"tracking-standard-link\"})[\"href\"].find(\"?\")]\n",
    "                                car[\"price\"] = article.find(\"div\", {\"class\": \"product-card-pricing__price\"}).text.strip()\n",
    "                                car[\"mpg\"] = 'electric'\n",
    "                                car[\"drivertrain\"] = divide[0]\n",
    "                                car[\"make\"] = divide[1]\n",
    "                                seller = article.find_all(\"span\", {\"class\": \"product-card-seller-info__spec-item-copy\"})#.text.strip()\n",
    "                                seller=reversed(seller)\n",
    "                                for count,sel in enumerate(seller):\n",
    "                                    car[\"seller\"+str(count)] = sel.text.strip()\n",
    "                                for a in article.find_all(\"a\", {\"class\": \"product-card-seller-info__review-count dealer-profile-link\", \"href\":True}):\n",
    "                                    seller_href.append(a[\"href\"])\n",
    "                                for count,ref in enumerate(seller_href):\n",
    "                                    car[\"href\"+str(count)] = ref\n",
    "\n",
    "                                # The key specs are in a bulleted list\n",
    "                                key_specs_bs_list = article.find(\"ul\", {\"class\": \"listing-key-specs\"}).find_all(\"li\")\n",
    "\n",
    "                                for key_spec_bs_li in key_specs_bs_list:\n",
    "\n",
    "                                    key_spec_bs = key_spec_bs_li.text                              \n",
    "\n",
    "                                    if any(keyword in key_spec_bs for keyword in keywords[\"mileage\"]):\n",
    "                                        car[\"mileage\"] = int(key_spec_bs[:key_spec_bs.find(\" miles\")].replace(\",\",\"\"))\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"BHP\"]):\n",
    "                                        car[\"BHP\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"transmission\"]):\n",
    "                                        car[\"transmission\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"fuel\"]):\n",
    "                                        car[\"fuel\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"owners\"]):\n",
    "                                        car[\"owners\"] = int(key_spec_bs[:key_spec_bs.find(\" own\")])\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"body\"]):\n",
    "                                        car[\"body\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"ULEZ\"]):\n",
    "                                        car[\"ULEZ\"] = key_spec_bs\n",
    "                                    elif any(keyword in key_spec_bs for keyword in keywords[\"year\"]):\n",
    "                                        car[\"year\"] = key_spec_bs\n",
    "                                    elif key_spec_bs[1] == \".\" and key_spec_bs[3] == \"L\":\n",
    "                                        car[\"engine\"] = key_spec_bs\n",
    "\n",
    "                                # Set any missing dictionary keys to NA for complete car details.\n",
    "                                for key in keywords.keys():\n",
    "                                    if key in car.keys():\n",
    "                                        pass\n",
    "                                    else:\n",
    "                                        car[key]='NA'\n",
    "                                # Add complete car dictionary to results list.\n",
    "                                results.append(car)\n",
    "                                n_this_year_results = n_this_year_results + 1\n",
    "\n",
    "                            page = page + 1\n",
    "                            attempt = 1\n",
    "\n",
    "                            if verbose:\n",
    "                                print(\"Car count: \", len(results))\n",
    "                                print(\"---------------------------------\")\n",
    "\n",
    "                except KeyboardInterrupt:\n",
    "                    break\n",
    "\n",
    "                except:\n",
    "                    traceback.print_exc()\n",
    "                    attempt = attempt + 1\n",
    "                    if attempt <= max_attempts_per_page:\n",
    "                        if verbose:\n",
    "                            print(\"Exception. Starting attempt #\", attempt, \"and keeping at page #\", page)\n",
    "                    else:\n",
    "                        page = page + 1\n",
    "                        attempt = 1\n",
    "                        if verbose:\n",
    "                            print(\"Exception. All attempts exhausted for this page. Skipping to next page #\", page)\n",
    "\n",
    "        except KeyboardInterrupt:\n",
    "            pass\n",
    "    \n",
    "    # Make df of results and output to csv\n",
    "    results = pd.DataFrame(results)\n",
    "    now = (datetime.datetime.now().strftime(\"%d%B_%I%M%p\"))\n",
    "    filepathdf=f'/raw_data_new_electric/{now}_{year}_new.csv'\n",
    "    results.to_csv(filepathdf, index=False, header=results.columns )\n",
    "\n",
    "    return results, list(set(extra_brand_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bc092749",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-04T14:16:00.286011Z",
     "start_time": "2022-03-04T14:16:00.282905Z"
    }
   },
   "outputs": [],
   "source": [
    "# Sample function calls. In practise these were looped over years 1990-2022, scraping a single year at a time\n",
    "\n",
    "# ucars, extra_used_car_list = get_used_cars(1995,2021, Verbose=True)\n",
    "# uecars, extra_used_ecar_list = get_used_electric_cars(2005,2021, Verbose=True)\n",
    "# ncars, extra_new_car_list = get_new_cars(Verbose=True)\n",
    "# necars, extra_new_ecar_list = get_new_electric_cars(Verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6aa66a15",
   "metadata": {},
   "source": [
    "No data cleaning is done in this notebook. It is purely here to define the functions which would be used to scrape the data. Note that due to the size of the dataset it will not be possible to include it in the GitHub repo."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "used_cars_env",
   "language": "python",
   "name": "used_cars_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "346.974px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "toc-autonumbering": true,
  "toc-showtags": false
 },
 "nbformat": 4,
 "nbformat_minor": 5
}