booking_scrappy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scrapy import Spider\n",
    "from scrapy.crawler import CrawlerProcess\n",
    "\n",
    "import logging\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def try_remove(filename):\n",
    "    try:\n",
    "        os.remove(filename)\n",
    "    except OSError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BookingComOpinions(Spider):\n",
    "    name = 'BookingComOpinionsSpider'\n",
    "    \n",
    "    custom_settings = {\n",
    "        'FEED_URI': '%(json_file)s', \n",
    "        'FEED_FORMAT': 'json',\n",
    "        'DOWNLOAD_DELAY': 3,\n",
    "        'LOG_LEVEL': logging.DEBUG,\n",
    "        'EXTENSIONS': {\n",
    "            'scrapy.extensions.closespider.CloseSpider': 1\n",
    "        },\n",
    "        'CLOSESPIDER_ITEMCOUNT': 10\n",
    "    }\n",
    "    \n",
    "    def parse(self, response):\n",
    "        hotels = response.css('div.sr_item')\n",
    "        for hotel in hotels:\n",
    "            hotel_name = hotel.css('span.sr-hotel__name::text').get().strip()\n",
    "            for hotel_link in hotel.css('.hotel_name_link.url'):\n",
    "                yield response.follow(hotel_link, self.parse_hotel, meta={'hotel': hotel_name})\n",
    "        \n",
    "        for link in response.css('a[data-page-next]'):\n",
    "            yield response.follow(link)\n",
    "            \n",
    "    def parse_hotel(self, response):\n",
    "        hotel_name = response.meta['hotel']\n",
    "        for reviews_link in response.css('a.show_all_reviews_btn'):\n",
    "            yield response.follow(reviews_link, self.parse_reviews, meta={'hotel': hotel_name})\n",
    "        \n",
    "    def parse_reviews(self, response):\n",
    "        hotel_name = response.meta['hotel']\n",
    "        items = response.css('li.review_item')\n",
    "        for item in items:\n",
    "            publish_date = item.css('meta[itemprop=\"datePublished\"]'\n",
    "                                    '::attr(content)').get('')\n",
    "            reviewer = item.css('div.review_item_reviewer')\n",
    "            rev_count = reviewer.css('div.review_item_user_review_count::text').get('')\n",
    "            \n",
    "            review = item.css('div.review_item_review')\n",
    "            rating = review.css('meta[itemprop=\"ratingValue\"]'\n",
    "                                    '::attr(content)').get('')\n",
    "            raw_tags = review.css('li.review_info_tag::text').getall()\n",
    "            tags = list(filter(None, map(str.strip, raw_tags)))\n",
    "\n",
    "            yield {\n",
    "                'hotel': hotel_name,\n",
    "                'publish_date': publish_date,\n",
    "                'rev_count': rev_count,\n",
    "                'rating': rating,\n",
    "                'tags': tags\n",
    "            }\n",
    "            \n",
    "                    \n",
    "        for next_page in response.css('a#review_next_page_link'):\n",
    "            yield response.follow(next_page, self.parse_reviews,\n",
    "                                 meta = response.meta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENTRY_URL = 'https://www.booking.com/searchresults.pl.html?ss=Podstrana'\n",
    "json_file = 'hotels.json'\n",
    "try_remove(json_file) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-04-10 23:48:34 [scrapy.utils.log] INFO: Scrapy 1.5.2 started (bot: scrapybot)\n",
      "2019-04-10 23:48:34 [scrapy.utils.log] INFO: Versions: lxml 4.3.1.0, libxml2 2.9.9, cssselect 1.0.3, parsel 1.5.1, w3lib 1.20.0, Twisted 18.9.0, Python 3.7.2 (default, Dec 29 2018, 06:19:36) - [GCC 7.3.0], pyOpenSSL 19.0.0 (OpenSSL 1.1.1b  26 Feb 2019), cryptography 2.6.1, Platform Linux-4.9.0-8-amd64-x86_64-with-debian-9.8\n",
      "2019-04-10 23:48:34 [scrapy.crawler] INFO: Overridden settings: {'CLOSESPIDER_ITEMCOUNT': 10, 'DOWNLOAD_DELAY': 3, 'FEED_FORMAT': 'json', 'FEED_URI': '%(json_file)s', 'LOG_LEVEL': 10}\n",
      "2019-04-10 23:48:34 [scrapy.extensions.telnet] INFO: Telnet Password: ef9d2a3651d06c75\n",
      "2019-04-10 23:48:34 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.memusage.MemoryUsage',\n",
      " 'scrapy.extensions.feedexport.FeedExporter',\n",
      " 'scrapy.extensions.logstats.LogStats',\n",
      " 'scrapy.extensions.closespider.CloseSpider']\n",
      "2019-04-10 23:48:34 [scrapy.middleware] INFO: Enabled downloader middlewares:\n",
      "['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n",
      " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n",
      " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n",
      " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n",
      " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n",
      " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n",
      " 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n",
      "2019-04-10 23:48:34 [scrapy.middleware] INFO: Enabled spider middlewares:\n",
      "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n",
      " 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n",
      " 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n",
      " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n",
      " 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n",
      "2019-04-10 23:48:34 [scrapy.middleware] INFO: Enabled item pipelines:\n",
      "[]\n",
      "2019-04-10 23:48:34 [scrapy.core.engine] INFO: Spider opened\n",
      "2019-04-10 23:48:34 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-04-10 23:48:34 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Deferred at 0x7f6c0fc9b518>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process = CrawlerProcess()\n",
    "process.crawl(BookingComOpinions, start_urls=[ENTRY_URL], json_file=json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-04-10 23:48:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/searchresults.pl.html?ss=Podstrana> (referer: None)\n",
      "2019-04-10 23:48:38 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/sanantoniopodstrana.pl.html> from <GET https://www.booking.com/hotel/hr/sanantoniopodstrana.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:48:43 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/searchresults.pl.html?dest_id=-92454&dest_type=city&ss=Podstrana&offset=15> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:48:45 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartments-johnny.pl.html> from <GET https://www.booking.com/hotel/hr/apartments-johnny.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:48:47 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/villa-senator.pl.html> from <GET https://www.booking.com/hotel/hr/villa-senator.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:48:51 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartments-9.pl.html> from <GET https://www.booking.com/hotel/hr/apartments-9.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:48:54 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/beach-bozikovina.pl.html> from <GET https://www.booking.com/hotel/hr/beach-bozikovina.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:48:58 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartman-jelic.pl.html> from <GET https://www.booking.com/hotel/hr/apartman-jelic.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:02 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartments-diamond.pl.html> from <GET https://www.booking.com/hotel/hr/apartments-diamond.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:06 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartmani-nera.pl.html> from <GET https://www.booking.com/hotel/hr/apartmani-nera.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:09 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/hoteledensplit.pl.html> from <GET https://www.booking.com/hotel/hr/hoteledensplit.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:14 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/pansion-pitomcia.pl.html> from <GET https://www.booking.com/hotel/hr/pansion-pitomcia.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:15 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartment-sur-la-plage.pl.html> from <GET https://www.booking.com/hotel/hr/apartment-sur-la-plage.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:19 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartment-andrina.pl.html> from <GET https://www.booking.com/hotel/hr/apartment-andrina.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:23 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/split.pl.html> from <GET https://www.booking.com/hotel/hr/split.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:26 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/le-meridien-lav-split.pl.html> from <GET https://www.booking.com/hotel/hr/le-meridien-lav-split.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:30 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartment-brankica-podstrana.pl.html> from <GET https://www.booking.com/hotel/hr/apartment-brankica-podstrana.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:34 [scrapy.extensions.logstats] INFO: Crawled 2 pages (at 2 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-04-10 23:49:40 [scrapy.downloadermiddlewares.retry] DEBUG: Retrying <GET https://www.booking.com/hotel/hr/apartments-johnny.pl.html> (failed 1 times): No route to host: 101: Network is unreachable.\n",
      "2019-04-10 23:49:43 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartment-mares.pl.html> from <GET https://www.booking.com/hotel/hr/apartment-mares.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:49:46 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/villa-senator.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:49:50 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartments-9.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:49:53 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/beach-bozikovina.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:49:57 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartman-jelic.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:00 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartments-diamond.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:04 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartmani-nera.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:08 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/hoteledensplit.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:08 [scrapy.dupefilters] DEBUG: Filtered duplicate request: <GET https://www.booking.com/hotel/hr/hoteledensplit.pl.html#blockdisplay4> - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)\n",
      "2019-04-10 23:50:12 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/pansion-pitomcia.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:16 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartment-sur-la-plage.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartment-andrina.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:24 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/split.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/le-meridien-lav-split.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:30 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartment-brankica-podstrana.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:33 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartments-johnny.pl.html> (referer: https://www.booking.com/searchresults.pl.html?ss=Podstrana)\n",
      "2019-04-10 23:50:34 [scrapy.extensions.logstats] INFO: Crawled 16 pages (at 14 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-04-10 23:50:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartment-mares.pl.html> (referer: https://www.booking.com/searchresults.pl.html?dest_id=-92454&dest_type=city&ss=Podstrana&offset=15)\n",
      "2019-04-10 23:50:39 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/searchresults.pl.html?dest_id=-92454&dest_type=city&ss=Podstrana&offset=30> (referer: https://www.booking.com/searchresults.pl.html?dest_id=-92454&dest_type=city&ss=Podstrana&offset=15)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-04-10 23:50:42 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartments-mate-podstrana.pl.html> from <GET https://www.booking.com/hotel/hr/apartments-mate-podstrana.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:50:45 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/reviews/hr/hotel/apartments-9.pl.html> (referer: https://www.booking.com/hotel/hr/apartments-9.pl.html)\n",
      "2019-04-10 23:50:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-9.pl.html>\n",
      "{'hotel': 'Apartments 9', 'publish_date': '2018-08-31', 'rev_count': '\\n3 opinie\\n', 'rating': '8.8', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament z widokiem na morze', '8 noclegów']}\n",
      "2019-04-10 23:50:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-9.pl.html>\n",
      "{'hotel': 'Apartments 9', 'publish_date': '2018-08-16', 'rev_count': '\\n2 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament z widokiem na morze', '7 noclegów']}\n",
      "2019-04-10 23:50:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-9.pl.html>\n",
      "{'hotel': 'Apartments 9', 'publish_date': '2018-08-11', 'rev_count': '\\n1 opinia\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Podróżujący ze znajomymi', 'Apartament dwupoziomowy', '7 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:50:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-9.pl.html>\n",
      "{'hotel': 'Apartments 9', 'publish_date': '2018-08-11', 'rev_count': '\\n4 opinie\\n', 'rating': '7.5', 'tags': ['Wyjazd wakacyjny', 'Podróżujący ze znajomymi', 'Apartament z widokiem na morze', '6 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:50:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-9.pl.html>\n",
      "{'hotel': 'Apartments 9', 'publish_date': '2018-06-12', 'rev_count': '\\n16 opinii\\n', 'rating': '2.5', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Apartament z widokiem na morze', '2 noclegi', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:50:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-9.pl.html>\n",
      "{'hotel': 'Apartments 9', 'publish_date': '2018-07-22', 'rev_count': '\\n1 opinia\\n', 'rating': '8.8', 'tags': ['Wyjazd wakacyjny', 'Podróżujący w pojedynkę', 'Apartament dwupoziomowy', '10 noclegów']}\n",
      "2019-04-10 23:50:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-9.pl.html>\n",
      "{'hotel': 'Apartments 9', 'publish_date': '2017-07-19', 'rev_count': '\\n9 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Grupa', 'Apartament dwupoziomowy', '7 noclegów']}\n",
      "2019-04-10 23:50:49 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/reviews/hr/hotel/beach-bozikovina.pl.html> (referer: https://www.booking.com/hotel/hr/beach-bozikovina.pl.html)\n",
      "2019-04-10 23:50:49 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/beach-bozikovina.pl.html>\n",
      "{'hotel': 'Beach Hotel Bozikovina', 'publish_date': '2017-09-01', 'rev_count': '\\n3 opinie\\n', 'rating': '9.6', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament typu Deluxe z widokiem na morze', '6 noclegów']}\n",
      "2019-04-10 23:50:49 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/beach-bozikovina.pl.html>\n",
      "{'hotel': 'Beach Hotel Bozikovina', 'publish_date': '2017-08-30', 'rev_count': '\\n1 opinia\\n', 'rating': '8.3', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój trzyosobowy', '1 nocleg']}\n",
      "2019-04-10 23:50:49 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/beach-bozikovina.pl.html>\n",
      "{'hotel': 'Beach Hotel Bozikovina', 'publish_date': '2017-05-08', 'rev_count': '\\n1 opinia\\n', 'rating': '9.6', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Pokój trzyosobowy', '4 noclegi']}\n",
      "2019-04-10 23:50:49 [scrapy.core.engine] INFO: Closing spider (closespider_itemcount)\n",
      "2019-04-10 23:50:49 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/beach-bozikovina.pl.html>\n",
      "{'hotel': 'Beach Hotel Bozikovina', 'publish_date': '2016-05-27', 'rev_count': '\\n11 opinii\\n', 'rating': '9.6', 'tags': ['Wyjazd wakacyjny', 'Grupa']}\n",
      "2019-04-10 23:50:49 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/beach-bozikovina.pl.html>\n",
      "{'hotel': 'Beach Hotel Bozikovina', 'publish_date': '2015-06-21', 'rev_count': '\\n1 opinia\\n', 'rating': '8.3', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi']}\n",
      "2019-04-10 23:50:49 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/beach-bozikovina.pl.html>\n",
      "{'hotel': 'Beach Hotel Bozikovina', 'publish_date': '2016-05-10', 'rev_count': '\\n3 opinie\\n', 'rating': '9.2', 'tags': ['Grupa']}\n",
      "2019-04-10 23:50:49 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/beach-bozikovina.pl.html>\n",
      "{'hotel': 'Beach Hotel Bozikovina', 'publish_date': '2015-06-02', 'rev_count': '\\n7 opinii\\n', 'rating': '8.3', 'tags': ['Wyjazd wakacyjny', 'Grupa']}\n",
      "2019-04-10 23:50:49 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/beach-bozikovina.pl.html>\n",
      "{'hotel': 'Beach Hotel Bozikovina', 'publish_date': '2015-05-01', 'rev_count': '\\n13 opinii\\n', 'rating': '9.6', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n",
      "2019-04-10 23:50:53 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/villa-maris-podstrana.pl.html> from <GET https://www.booking.com/hotel/hr/villa-maris-podstrana.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:50:57 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/reviews/hr/hotel/apartments-diamond.pl.html> (referer: https://www.booking.com/hotel/hr/apartments-diamond.pl.html)\n",
      "2019-04-10 23:50:57 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-diamond.pl.html>\n",
      "{'hotel': 'Apartments Diamond', 'publish_date': '2018-08-17', 'rev_count': '\\n2 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament z 1 sypialnią', '4 noclegi']}\n",
      "2019-04-10 23:50:57 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-diamond.pl.html>\n",
      "{'hotel': 'Apartments Diamond', 'publish_date': '2018-07-17', 'rev_count': '\\n3 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament z 1 sypialnią', '2 noclegi', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:50:57 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-diamond.pl.html>\n",
      "{'hotel': 'Apartments Diamond', 'publish_date': '2018-07-05', 'rev_count': '\\n4 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament z 1 sypialnią', '9 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:50:57 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-diamond.pl.html>\n",
      "{'hotel': 'Apartments Diamond', 'publish_date': '2017-06-18', 'rev_count': '\\n14 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Podróżujący w pojedynkę', 'Apartament z 1 sypialnią', '6 noclegów']}\n",
      "2019-04-10 23:50:57 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-diamond.pl.html>\n",
      "{'hotel': 'Apartments Diamond', 'publish_date': '2015-09-25', 'rev_count': '\\n1 opinia\\n', 'rating': '9.6', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi']}\n",
      "2019-04-10 23:51:01 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/reviews/hr/hotel/apartmani-nera.pl.html> (referer: https://www.booking.com/hotel/hr/apartmani-nera.pl.html)\n",
      "2019-04-10 23:51:01 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartmani-nera.pl.html>\n",
      "{'hotel': 'Apartments Nera', 'publish_date': '2018-08-23', 'rev_count': '\\n9 opinii\\n', 'rating': '9.6', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Studio z balkonem i widokiem na morze', '6 noclegów']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-04-10 23:51:01 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartmani-nera.pl.html>\n",
      "{'hotel': 'Apartments Nera', 'publish_date': '2018-06-09', 'rev_count': '\\n5 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Studio', '4 noclegi']}\n",
      "2019-04-10 23:51:01 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartmani-nera.pl.html>\n",
      "{'hotel': 'Apartments Nera', 'publish_date': '2018-07-17', 'rev_count': '\\n38 opinii\\n', 'rating': '10', 'tags': ['Rodzina z małymi dziećmi', '2 pokoje', '7 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:01 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartmani-nera.pl.html>\n",
      "{'hotel': 'Apartments Nera', 'publish_date': '2017-09-21', 'rev_count': '\\n5 opinii\\n', 'rating': '9.2', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Studio z balkonem i widokiem na morze', '13 noclegów']}\n",
      "2019-04-10 23:51:01 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartmani-nera.pl.html>\n",
      "{'hotel': 'Apartments Nera', 'publish_date': '2017-08-31', 'rev_count': '\\n3 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Apartament z 2 sypialniami i balkonem oraz widokiem na morze', '6 noclegów']}\n",
      "2019-04-10 23:51:01 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartmani-nera.pl.html>\n",
      "{'hotel': 'Apartments Nera', 'publish_date': '2015-09-12', 'rev_count': '\\n3 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n",
      "2019-04-10 23:51:01 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartmani-nera.pl.html>\n",
      "{'hotel': 'Apartments Nera', 'publish_date': '2015-08-25', 'rev_count': '\\n7 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi']}\n",
      "2019-04-10 23:51:01 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartmani-nera.pl.html>\n",
      "{'hotel': 'Apartments Nera', 'publish_date': '2015-08-17', 'rev_count': '\\n1 opinia\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n",
      "2019-04-10 23:51:05 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/vlasic.pl.html> from <GET https://www.booking.com/hotel/hr/vlasic.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:51:09 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html> (referer: https://www.booking.com/hotel/hr/pansion-pitomcia.pl.html)\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2018-05-07', 'rev_count': '\\n22 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój Dwuosobowy z balkonem i widokiem na morze', '5 noclegów']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2017-09-11', 'rev_count': '\\n49 opinii\\n', 'rating': '8.8', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój Dwuosobowy z balkonem i widokiem na morze', '2 noclegi']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2017-08-07', 'rev_count': '\\n8 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój Dwuosobowy z balkonem i widokiem na morze', '3 noclegi']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2017-06-17', 'rev_count': '\\n20 opinii\\n', 'rating': '7.9', 'tags': ['Wyjazd wakacyjny', 'Grupa', 'Pokój Dwuosobowy z balkonem i widokiem na morze', '9 noclegów']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2018-08-25', 'rev_count': '\\n4 opinie\\n', 'rating': '8.8', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój Dwuosobowy z balkonem i widokiem na morze', '6 noclegów']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2018-06-12', 'rev_count': '\\n7 opinii\\n', 'rating': '9.2', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój dwuosobowy z tarasem i widokiem na morze.', '2 noclegi']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2017-07-02', 'rev_count': '\\n5 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój dwuosobowy z tarasem i widokiem na morze.', '3 noclegi', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2017-06-01', 'rev_count': '\\n12 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój dwuosobowy z tarasem i widokiem na morze.', '4 noclegi', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2015-10-13', 'rev_count': '\\n22 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Grupa']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2015-08-03', 'rev_count': '\\n7 opinii\\n', 'rating': '6.7', 'tags': ['Wyjazd wakacyjny', 'Grupa']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2015-10-04', 'rev_count': '\\n2 opinie\\n', 'rating': '8.3', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2015-07-04', 'rev_count': '\\n22 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n",
      "2019-04-10 23:51:09 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/pansion-pitomcia.pl.html>\n",
      "{'hotel': 'Villa Pitomcia', 'publish_date': '2015-06-16', 'rev_count': '\\n18 opinii\\n', 'rating': '7.9', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n",
      "2019-04-10 23:51:13 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/villa-lenka-apartments.pl.html> from <GET https://www.booking.com/hotel/hr/villa-lenka-apartments.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:51:17 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartments-goro.pl.html> from <GET https://www.booking.com/hotel/hr/apartments-goro.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:51:21 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/reviews/hr/hotel/split.pl.html> (referer: https://www.booking.com/hotel/hr/split.pl.html)\n",
      "2019-04-10 23:51:21 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/split.pl.html>\n",
      "{'hotel': 'Hotel Split', 'publish_date': '2018-05-15', 'rev_count': '\\n1 opinia\\n', 'rating': '8.8', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój Dwuosobowy typu Deluxe z widokiem na morze i tarasem', '4 noclegi']}\n",
      "2019-04-10 23:51:21 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/split.pl.html>\n",
      "{'hotel': 'Hotel Split', 'publish_date': '2017-08-06', 'rev_count': '\\n9 opinii\\n', 'rating': '5.4', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój dwuosobowy typu Superior z 1 lub 2 łóżkami i widokiem na morze i balkonem', '7 noclegów']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-04-10 23:51:21 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/split.pl.html>\n",
      "{'hotel': 'Hotel Split', 'publish_date': '2017-06-12', 'rev_count': '\\n50 opinii\\n', 'rating': '5.8', 'tags': ['Wyjazd służbowy', 'Grupa', 'Pokój dwuosobowy typu Superior z 1 lub 2 łóżkami i widokiem na morze i balkonem', '2 noclegi']}\n",
      "2019-04-10 23:51:21 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/split.pl.html>\n",
      "{'hotel': 'Hotel Split', 'publish_date': '2017-10-09', 'rev_count': '\\n3 opinie\\n', 'rating': '9.5', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój Dwuosobowy typu Deluxe z widokiem na morze i tarasem', '5 noclegów']}\n",
      "2019-04-10 23:51:21 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/split.pl.html>\n",
      "{'hotel': 'Hotel Split', 'publish_date': '2017-05-08', 'rev_count': '\\n32 opinie\\n', 'rating': '8.8', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Pokój dwuosobowy typu Superior z 1 lub 2 łóżkami i widokiem na morze i balkonem', '1 nocleg']}\n",
      "2019-04-10 23:51:21 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/split.pl.html>\n",
      "{'hotel': 'Hotel Split', 'publish_date': '2015-10-04', 'rev_count': '\\n9 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n",
      "2019-04-10 23:51:21 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/split.pl.html>\n",
      "{'hotel': 'Hotel Split', 'publish_date': '2016-05-24', 'rev_count': '\\n27 opinii\\n', 'rating': '9.6', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n",
      "2019-04-10 23:51:21 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/split.pl.html>\n",
      "{'hotel': 'Hotel Split', 'publish_date': '2016-04-29', 'rev_count': '\\n15 opinii\\n', 'rating': '6.7', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n",
      "2019-04-10 23:51:21 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/split.pl.html>\n",
      "{'hotel': 'Hotel Split', 'publish_date': '2015-11-12', 'rev_count': '\\n4 opinie\\n', 'rating': '9.6', 'tags': ['Wyjazd służbowy', 'Grupa']}\n",
      "2019-04-10 23:51:23 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/villa-19.pl.html> from <GET https://www.booking.com/hotel/hr/villa-19.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:51:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/reviews/hr/hotel/apartment-brankica-podstrana.pl.html> (referer: https://www.booking.com/hotel/hr/apartment-brankica-podstrana.pl.html)\n",
      "2019-04-10 23:51:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartment-brankica-podstrana.pl.html>\n",
      "{'hotel': 'Apartment Brankica', 'publish_date': '2018-07-30', 'rev_count': '\\n4 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', '2 pokoje', '7 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartment-brankica-podstrana.pl.html>\n",
      "{'hotel': 'Apartment Brankica', 'publish_date': '2017-09-15', 'rev_count': '\\n10 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament – parter', '8 noclegów']}\n",
      "2019-04-10 23:51:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartment-brankica-podstrana.pl.html>\n",
      "{'hotel': 'Apartment Brankica', 'publish_date': '2017-09-02', 'rev_count': '\\n8 opinii\\n', 'rating': '9.2', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament – parter', '5 noclegów']}\n",
      "2019-04-10 23:51:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartment-brankica-podstrana.pl.html>\n",
      "{'hotel': 'Apartment Brankica', 'publish_date': '2017-08-14', 'rev_count': '\\n4 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament – parter', '3 noclegi']}\n",
      "2019-04-10 23:51:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartment-brankica-podstrana.pl.html>\n",
      "{'hotel': 'Apartment Brankica', 'publish_date': '2017-07-18', 'rev_count': '\\n5 opinii\\n', 'rating': '9.2', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament z 3 sypialniami', '11 noclegów']}\n",
      "2019-04-10 23:51:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartment-brankica-podstrana.pl.html>\n",
      "{'hotel': 'Apartment Brankica', 'publish_date': '2018-08-22', 'rev_count': '\\n5 opinii\\n', 'rating': '8.8', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament z 3 sypialniami', '10 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartment-brankica-podstrana.pl.html>\n",
      "{'hotel': 'Apartment Brankica', 'publish_date': '2018-07-25', 'rev_count': '\\n9 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament – parter', '10 noclegów']}\n",
      "2019-04-10 23:51:27 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartment-brankica-podstrana.pl.html>\n",
      "{'hotel': 'Apartment Brankica', 'publish_date': '2017-08-05', 'rev_count': '\\n1 opinia\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament – parter', '7 noclegów']}\n",
      "2019-04-10 23:51:32 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/reviews/hr/hotel/apartments-johnny.pl.html> (referer: https://www.booking.com/hotel/hr/apartments-johnny.pl.html)\n",
      "2019-04-10 23:51:32 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-johnny.pl.html>\n",
      "{'hotel': 'Apartments Johnny', 'publish_date': '2018-07-18', 'rev_count': '\\n2 opinie\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Apartament z balkonem i widokiem na morze', '7 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:32 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-johnny.pl.html>\n",
      "{'hotel': 'Apartments Johnny', 'publish_date': '2017-09-19', 'rev_count': '\\n5 opinii\\n', 'rating': '8.8', 'tags': ['Wyjazd wakacyjny', 'Rodzina z małymi dziećmi', 'Apartament z częściowym widokiem na morze (3 osoby dorosłe)', '7 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:32 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-johnny.pl.html>\n",
      "{'hotel': 'Apartments Johnny', 'publish_date': '2017-09-07', 'rev_count': '\\n19 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Apartament typu Studio z tarasem i widokiem na morze (2 osoby dorosłe)', '7 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:32 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-johnny.pl.html>\n",
      "{'hotel': 'Apartments Johnny', 'publish_date': '2018-08-14', 'rev_count': '\\n8 opinii\\n', 'rating': '8.3', 'tags': ['Wyjazd wakacyjny', 'Podróżujący ze znajomymi', 'Apartament typu Studio z częściowym widokiem na morze (4 osoby dorosłe)', '2 noclegi', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:32 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-johnny.pl.html>\n",
      "{'hotel': 'Apartments Johnny', 'publish_date': '2018-06-11', 'rev_count': '\\n1 opinia\\n', 'rating': '8.3', 'tags': ['Wyjazd wakacyjny', 'Grupa', 'Pokój z balkonem i widokiem na morze (2 osoby dorosłe)', '2 noclegi', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:32 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-johnny.pl.html>\n",
      "{'hotel': 'Apartments Johnny', 'publish_date': '2018-05-28', 'rev_count': '\\n10 opinii\\n', 'rating': '9.2', 'tags': ['Wyjazd wakacyjny', 'W parze', 'Pokój z balkonem i widokiem na morze (2 osoby dorosłe)', '5 noclegów', 'Wysłana przez urządzenie mobilne']}\n",
      "2019-04-10 23:51:32 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-johnny.pl.html>\n",
      "{'hotel': 'Apartments Johnny', 'publish_date': '2015-08-18', 'rev_count': '\\n16 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'W parze']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-04-10 23:51:32 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.booking.com/reviews/hr/hotel/apartments-johnny.pl.html>\n",
      "{'hotel': 'Apartments Johnny', 'publish_date': '2015-08-13', 'rev_count': '\\n6 opinii\\n', 'rating': '10', 'tags': ['Wyjazd wakacyjny', 'Grupa']}\n",
      "2019-04-10 23:51:34 [scrapy.extensions.logstats] INFO: Crawled 26 pages (at 10 pages/min), scraped 66 items (at 66 items/min)\n",
      "2019-04-10 23:51:35 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/apartment-rino.pl.html> from <GET https://www.booking.com/hotel/hr/apartment-rino.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:51:41 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/hotel/hr/apartments-mate-podstrana.pl.html> (referer: https://www.booking.com/searchresults.pl.html?dest_id=-92454&dest_type=city&ss=Podstrana&offset=15)\n",
      "2019-04-10 23:51:44 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.booking.com/searchresults.pl.html?dest_id=-92454&dest_type=city&ss=Podstrana&offset=45> (referer: https://www.booking.com/searchresults.pl.html?dest_id=-92454&dest_type=city&ss=Podstrana&offset=30)\n",
      "2019-04-10 23:51:47 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.booking.com/hotel/hr/the-one-resort.pl.html> from <GET https://www.booking.com/hotel/hr/the-one-resort.pl.html?from=searchresults%0A#hotelTmpl>\n",
      "2019-04-10 23:52:33 [scrapy.downloadermiddlewares.retry] DEBUG: Retrying <GET https://www.booking.com/hotel/hr/sanantoniopodstrana.pl.html> (failed 1 times): User timeout caused connection failure: Getting https://www.booking.com/hotel/hr/sanantoniopodstrana.pl.html took longer than 180.0 seconds..\n",
      "2019-04-10 23:52:33 [scrapy.extensions.feedexport] INFO: Stored json feed (66 items) in: hotels.json\n",
      "2019-04-10 23:52:33 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n",
      "{'downloader/exception_count': 2,\n",
      " 'downloader/exception_type_count/twisted.internet.error.NoRouteError': 1,\n",
      " 'downloader/exception_type_count/twisted.internet.error.TimeoutError': 1,\n",
      " 'downloader/request_bytes': 30035,\n",
      " 'downloader/request_count': 54,\n",
      " 'downloader/request_method_count/GET': 54,\n",
      " 'downloader/response_bytes': 4185968,\n",
      " 'downloader/response_count': 52,\n",
      " 'downloader/response_status_count/200': 28,\n",
      " 'downloader/response_status_count/301': 24,\n",
      " 'dupefilter/filtered': 5,\n",
      " 'finish_reason': 'closespider_itemcount',\n",
      " 'finish_time': datetime.datetime(2019, 4, 10, 21, 52, 33, 660168),\n",
      " 'item_scraped_count': 66,\n",
      " 'log_count/DEBUG': 122,\n",
      " 'log_count/INFO': 12,\n",
      " 'memusage/max': 181645312,\n",
      " 'memusage/startup': 74309632,\n",
      " 'request_depth_max': 4,\n",
      " 'response_received_count': 28,\n",
      " 'retry/count': 2,\n",
      " 'retry/reason_count/twisted.internet.error.NoRouteError': 1,\n",
      " 'retry/reason_count/twisted.internet.error.TimeoutError': 1,\n",
      " 'scheduler/dequeued': 54,\n",
      " 'scheduler/dequeued/memory': 54,\n",
      " 'scheduler/enqueued': 100,\n",
      " 'scheduler/enqueued/memory': 100,\n",
      " 'start_time': datetime.datetime(2019, 4, 10, 21, 48, 34, 245747)}\n",
      "2019-04-10 23:52:33 [scrapy.core.engine] INFO: Spider closed (closespider_itemcount)\n"
     ]
    }
   ],
   "source": [
    "process.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_json('hotels.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['rev_count'] = df['rev_count'].str.strip()\n",
    "df['rev_count'] = df['rev_count'].str.strip(' opinea')\n",
    "df['rev_count'] = df['rev_count'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "subdf = df[['rating', 'tags']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "dum = subdf['tags'].map(lambda tags: '|'.join(tags)).str.get_dummies()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "indicators = pd.concat([subdf['rating'], dum], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "melted = pd.melt(indicators, id_vars=['rating'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "positive = melted['value'] == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected = melted[positive]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "aggregated = selected.groupby(['variable'])['rating'].agg(['mean', 'count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>variable</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Pokój Dwuosobowy z balkonem i widokiem na morze</th>\n",
       "      <td>9.100000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Apartament – parter</th>\n",
       "      <td>9.840000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6 noclegów</th>\n",
       "      <td>9.250000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2 noclegi</th>\n",
       "      <td>7.557143</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7 noclegów</th>\n",
       "      <td>9.420000</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Grupa</th>\n",
       "      <td>8.672727</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wysłana przez urządzenie mobilne</th>\n",
       "      <td>8.962500</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rodzina z małymi dziećmi</th>\n",
       "      <td>9.557143</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>W parze</th>\n",
       "      <td>9.013793</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wyjazd wakacyjny</th>\n",
       "      <td>9.153226</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     mean  count\n",
       "variable                                                        \n",
       "Pokój Dwuosobowy z balkonem i widokiem na morze  9.100000      5\n",
       "Apartament – parter                              9.840000      5\n",
       "6 noclegów                                       9.250000      6\n",
       "2 noclegi                                        7.557143      7\n",
       "7 noclegów                                       9.420000     10\n",
       "Grupa                                            8.672727     11\n",
       "Wysłana przez urządzenie mobilne                 8.962500     16\n",
       "Rodzina z małymi dziećmi                         9.557143     21\n",
       "W parze                                          9.013793     29\n",
       "Wyjazd wakacyjny                                 9.153226     62"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggregated.sort_values(by=['count']).tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}