111B Pulling Images via the Twitter API v2.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step #1 Define Functions to Interact with the Twitter API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "import json \n",
    "import pandas as pd\n",
    "import urllib\n",
    "import os\n",
    "from os import path\n",
    "from datetime import datetime as dt\n",
    "\n",
    "# imports the twitter_secrets python file in which we store the twitter API keys\n",
    "# place the twitter_secrets file under <User>/anaconda3/Lib\n",
    "from twitter_secrets import twitter_secrets as ts\n",
    "\n",
    "# puts the bearer token in the request header\n",
    "def create_headers(bearer_token):\n",
    "    headers = {\"Authorization\": \"Bearer {}\".format(bearer_token)}\n",
    "    return headers\n",
    "        \n",
    "# sets the rules on which tweets to retrieve   \n",
    "def set_rules(headers, delete, bearer_token, rules):\n",
    "    payload = {\"add\": rules}\n",
    "    response = requests.post(\n",
    "        \"https://api.twitter.com/2/tweets/search/stream/rules\",\n",
    "        headers=headers,\n",
    "        json=payload,\n",
    "    )\n",
    "    if response.status_code != 201:\n",
    "        raise Exception(\n",
    "            \"Cannot add rules (HTTP {}): {}\".format(response.status_code, response.text)\n",
    "        )\n",
    "    print(json.dumps(response.json()))\n",
    "\n",
    "# retrieves the current set of rules from the API  \n",
    "def get_rules(headers, bearer_token):\n",
    "    response = requests.get(\n",
    "        \"https://api.twitter.com/2/tweets/search/stream/rules\", headers=headers\n",
    "    )\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(\n",
    "            \"Cannot get rules (HTTP {}): {}\".format(response.status_code, response.text)\n",
    "        )\n",
    "    print(json.dumps(response.json()))\n",
    "    return response.json()\n",
    "\n",
    "# tells the API to delete our current rule configuration \n",
    "def delete_all_rules(headers, bearer_token, rules):\n",
    "    if rules is None or \"data\" not in rules:\n",
    "        return None\n",
    "\n",
    "    ids = list(map(lambda rule: rule[\"id\"], rules[\"data\"]))\n",
    "    payload = {\"delete\": {\"ids\": ids}}\n",
    "    response = requests.post(\n",
    "        \"https://api.twitter.com/2/tweets/search/stream/rules\",\n",
    "        headers=headers,\n",
    "        json=payload\n",
    "    )\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(\n",
    "            \"Cannot delete rules (HTTP {}): {}\".format(\n",
    "                response.status_code, response.text\n",
    "            )\n",
    "        )\n",
    "    print(json.dumps(response.json()))\n",
    "\n",
    "# starts the stream, iterates through the lines of the response and for each line calls the save_tweets function and the save_media_to_disk function\n",
    "def get_stream(headers, set, bearer_token, expansions, fields, save_to_disk, save_path):\n",
    "    data = []\n",
    "    response = requests.get(\n",
    "        \"https://api.twitter.com/2/tweets/search/stream\" + expansions + fields, headers=headers, stream=True,\n",
    "    )\n",
    "    print(response.status_code)\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(\n",
    "            \"Cannot get stream (HTTP {}): {}\".format(\n",
    "                response.status_code, response.text\n",
    "            )\n",
    "        )\n",
    "    i = 0\n",
    "    for response_line in response.iter_lines():\n",
    "        i += 1\n",
    "        if i == max_results:\n",
    "            break\n",
    "        else:\n",
    "            json_response = json.loads(response_line)\n",
    "            #print(json.dumps(json_response, indent=4, sort_keys=True))\n",
    "            try:\n",
    "                save_tweets(json_response)\n",
    "                if save_to_disk == True:\n",
    "                    save_media_to_disk(json_response, save_path)\n",
    "            except (json.JSONDecodeError, KeyError) as err:\n",
    "                # In case the JSON fails to decode, we skip this tweet\n",
    "                print(f\"{i}/{max_results}: ERROR: encountered a problem with a line of data... \\n\")\n",
    "                continue\n",
    "\n",
    "# appends information from tweets to a dataframe           \n",
    "def save_tweets(tweet):\n",
    "    #print(json.dumps(tweet, indent=4, sort_keys=True))\n",
    "    data = tweet['data']\n",
    "    includes = tweet['includes']\n",
    "    media = includes['media']\n",
    "    for line in media:\n",
    "        tweet_list.append([data['id'], line['url']])  \n",
    "\n",
    "# iterates through the media attached to a tweet and saves each media file to the specified directory\n",
    "def save_media_to_disk(tweet, save_path):\n",
    "    data = tweet['data']\n",
    "    #print(json.dumps(data, indent=4, sort_keys=True))\n",
    "    includes = tweet['includes']\n",
    "    media = includes['media']\n",
    "    for line in media:\n",
    "        media_url = line['url']\n",
    "        media_key = line['media_key']\n",
    "        pic = urllib.request.urlopen(media_url)\n",
    "        file_path = save_path + \"\\\\\" + media_key + \".jpg\"\n",
    "        \n",
    "        if not path.isfile(file_path):\n",
    "            print(file_path)\n",
    "            try:\n",
    "                with open(file_path, 'wb') as localFile:\n",
    "                    localFile.write(pic.read())\n",
    "                tweet_list.append(media_key, media_url)\n",
    "            except Exception as e:\n",
    "                print('exception when saving media url ' + media_url + ' to path: ' + file_path)\n",
    "\n",
    "# creates a new directory\n",
    "def createDir(save_path):\n",
    "    try:\n",
    "        os.makedirs(save_path)\n",
    "    except OSError:\n",
    "        print (\"Creation of the directory %s failed\" % save_path)\n",
    "        if path.exists(savepath):\n",
    "            print(\"file already exists\")\n",
    "    else:\n",
    "        print (\"Successfully created the directory %s \" % save_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step #2 Define the Folder Structure for the Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to disk true/false\n",
    "save_to_disk = True\n",
    " \n",
    "# saves the images to disk in a new folder path that will be created with the createDir function\n",
    "if save_to_disk == True: \n",
    "    # detect the current working directory and print it\n",
    "    base_path = os.getcwd()\n",
    "    print (\"The current working directory is %s\" % base_path)\n",
    "    img_dir = '\\\\twitter\\\\downloaded_media\\\\'\n",
    "    # the write path in which the data will be stored. If it does not yet exist, it will be created\n",
    "    now = dt.now()\n",
    "    dt_string = now.strftime(\"%d%m%Y-%H%M%S\")# ddmmYY-HMS\n",
    "    save_path = base_path + img_dir + dt_string\n",
    "    createDir(save_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step #3 Subscribe to the Tweet Streaming Service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the max number of tweets that will be returned\n",
    "max_results = 10\n",
    "\n",
    "# you can adjust the rules if needed\n",
    "search_rules = [\n",
    "    {\"value\": \"dog has:images\", \n",
    "     \"tag\": \"dog pictures\", \n",
    "     \"lang\": \"en\"},\n",
    "]\n",
    "\n",
    "# these are the fields that will be delivered with the response\n",
    "media_fields = \"&media.fields=duration_ms,height,media_key,preview_image_url,public_metrics,type,url,width\"\n",
    "\n",
    "# we will retrieve the tweet object extended by the media object\n",
    "expansions = \"?expansions=attachments.media_keys\"\n",
    "tweet_list = []\n",
    "\n",
    "bearer_token = ts.BEARER_TOKEN\n",
    "headers = create_headers(bearer_token)\n",
    "rules = get_rules(headers, bearer_token)\n",
    "delete = delete_all_rules(headers, bearer_token, rules)\n",
    "set = set_rules(headers, delete, bearer_token, search_rules)\n",
    "get_stream(headers, set, bearer_token, expansions, media_fields, save_to_disk, save_path)\n",
    "\n",
    "df = pd.DataFrame (tweet_list, columns = ['tweetid', 'preview_image_url'])\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}