get_meetings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dd751e07",
   "metadata": {},
   "source": [
    "## Scrape Meetings (Youtube)\n",
    "\n",
    "This is using the YoutubeIngestionScraper to pull live streamed / published Board of Education meetings for different school districts in Colorado. The function accepts strings in Y, M , D format to then be converted into Datetime for scraping within that time period."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "146dbc4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from cdp_scrapers.youtube_utils import YoutubeIngestionScraper\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a2e7a67b",
   "metadata": {},
   "outputs": [],
   "source": [
    "## dictionary syntax: \"Channel Name\" : {\"School District Name\":\"Body Searc\"}\n",
    "\n",
    "## This dictionary is ordered by size of the school district\n",
    "schools_and_body_terms = {\n",
    "    \"DougCoSchools\" : {\"Douglas County School District\": \"BOE Meeting\"},\n",
    "    \"adams12fivestarschools-esc86\" : {\"Adams 12 Five Star Schools\":\"Board of Education Meeting\"},\n",
    "    \"psdondemand3088\" : {\"Poudre School District R-1\":\"Board of Education Meeting\"},\n",
    "    \"bouldervalleyschooldistric5781\" : {\"Boulder Valley School District RE-2\":\"Board of Education Meeting\"},\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "657c58e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrape_meetings(start_datetime,end_datetime):\n",
    "    ## storing all of the events in a list\n",
    "    events = []\n",
    "    \n",
    "    ## creating the datetime to be passed into the begin and end params\n",
    "    start = datetime.strptime(start_datetime, \"%Y,%m,%d\")\n",
    "    last = datetime.strptime(end_datetime, \"%Y,%m,%d\")\n",
    "    \n",
    "    ## iterating over the list above\n",
    "    \n",
    "    for channels_name, body_search_terms in schools_and_body_terms.items():\n",
    "        scraper = YoutubeIngestionScraper(channel_name=channels_name,\n",
    "                                          body_search_terms=body_search_terms,\n",
    "                                          timezone=\"MST\")\n",
    "        \n",
    "        curr_channel_events = scraper.get_events(begin=start,end=last)\n",
    "        print(\"Finished\",channels_name,\"\\n\")\n",
    "        \n",
    "        events = [\n",
    "            *events,\n",
    "            *curr_channel_events\n",
    "        ]\n",
    "    print (\"\\nLIST OF EVENTS SCRAPED\\n\")    \n",
    "    return events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a97cf66e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[youtube:tab] Extracting URL: https://www.youtube.com/@DougCoSchools/search?query=BOE+Meeting+after%3A2023-05-01+before%3A2023-05-10\n",
      "[youtube:tab] @DougCoSchools/search: Downloading webpage\n",
      "[download] Downloading playlist: DougCoSchools - Search - BOE Meeting after:2023-05-01 before:2023-05-10\n",
      "[youtube:tab] Playlist DougCoSchools - Search - BOE Meeting after:2023-05-01 before:2023-05-10: Downloading 2 items of 2\n",
      "[download] Downloading item 1 of 2\n",
      "[youtube] Extracting URL: https://www.youtube.com/watch?v=6rOyEnCxXys\n",
      "[youtube] Sleeping 0.5 seconds ...\n",
      "[youtube] 6rOyEnCxXys: Downloading webpage\n",
      "[youtube] Sleeping 0.5 seconds ...\n",
      "[youtube] 6rOyEnCxXys: Downloading android player API JSON\n",
      "[download] Downloading item 2 of 2\n",
      "[youtube] Extracting URL: https://www.youtube.com/watch?v=OlcTKhs7Iu8\n",
      "[youtube] Sleeping 0.5 seconds ...\n",
      "[youtube] OlcTKhs7Iu8: Downloading webpage\n",
      "[youtube] Sleeping 0.5 seconds ...\n",
      "[youtube] OlcTKhs7Iu8: Downloading android player API JSON\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: [youtube] OlcTKhs7Iu8: nsig extraction failed: You may experience throttling for some formats\n",
      "         Install PhantomJS to workaround the issue. Please download it from https://phantomjs.org/download.html\n",
      "         n = zkF6dGMAA453ZbWf14 ; player = https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[download] Finished downloading playlist: DougCoSchools - Search - BOE Meeting after:2023-05-01 before:2023-05-10\n",
      "Finished DougCoSchools \n",
      "\n",
      "[youtube:tab] Extracting URL: https://www.youtube.com/@psdondemand3088/search?query=Board+of+Education+Meeting+after%3A2023-05-01+before%3A2023-05-10\n",
      "[youtube:tab] @psdondemand3088/search: Downloading webpage\n",
      "[download] Downloading playlist: PSD On Demand - Search - Board of Education Meeting after:2023-05-01 before:2023-05-10\n",
      "[youtube:tab] Playlist PSD On Demand - Search - Board of Education Meeting after:2023-05-01 before:2023-05-10: Downloading 0 items of 0\n",
      "[download] Finished downloading playlist: PSD On Demand - Search - Board of Education Meeting after:2023-05-01 before:2023-05-10\n",
      "Finished psdondemand3088 \n",
      "\n",
      "[youtube:tab] Extracting URL: https://www.youtube.com/@bouldervalleyschooldistric5781/search?query=Board+of+Education+Meeting+a...+before%3A2023-05-10\n",
      "[youtube:tab] @bouldervalleyschooldistric5781/search: Downloading webpage\n",
      "[download] Downloading playlist: Boulder Valley School District Board of Education Meetings - Search - Board of Education Meeting after:2023-05-01 before:2023-05-10\n",
      "[youtube:tab] Playlist Boulder Valley School District Board of Education Meetings - Search - Board of Education Meeting after:2023-05-01 before:2023-05-10: Downloading 0 items of 0\n",
      "[download] Finished downloading playlist: Boulder Valley School District Board of Education Meetings - Search - Board of Education Meeting after:2023-05-01 before:2023-05-10\n",
      "Finished bouldervalleyschooldistric5781 \n",
      "\n",
      "\n",
      "LIST OF EVENTS SCRAPED\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[EventIngestionModel(body=Body(name='Douglas County School District', is_active=True, start_datetime=None, description=None, end_datetime=None, external_source_id=None), sessions=[Session(session_datetime=datetime.datetime(2023, 5, 9, 0, 0, tzinfo=<StaticTzInfo 'MST'>), video_uri='https://www.youtube.com/watch?v=6rOyEnCxXys', session_index=1, video_start_time=None, video_end_time=None, caption_uri=None, external_source_id='6rOyEnCxXys')], event_minutes_items=None, agenda_uri=None, minutes_uri=None, static_thumbnail_uri=None, hover_thumbnail_uri=None, external_source_id=None),\n",
       " EventIngestionModel(body=Body(name='Douglas County School District', is_active=True, start_datetime=None, description=None, end_datetime=None, external_source_id=None), sessions=[Session(session_datetime=datetime.datetime(2023, 5, 8, 0, 0, tzinfo=<StaticTzInfo 'MST'>), video_uri='https://www.youtube.com/watch?v=OlcTKhs7Iu8', session_index=2, video_start_time=None, video_end_time=None, caption_uri=None, external_source_id='OlcTKhs7Iu8')], event_minutes_items=None, agenda_uri=None, minutes_uri=None, static_thumbnail_uri=None, hover_thumbnail_uri=None, external_source_id=None)]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scrape_meetings(\"2023,5,1\",\"2023,5,10\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ba6f1e2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}