Web scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scrapy implementation\n",
    "\n",
    "import scrapy # Web Crawling and Web scraping modules\n",
    "import logging # \n",
    "import pandas as pd # Dataframe processing\n",
    "from scrapy.crawler import CrawlerProcess # A class to run multiple scrapy crawlers in a process simultaneously\n",
    "import json # setting pipeline\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class JsonWriterPipeline(object):\n",
    "    \"\"\"\n",
    "    To write extracted data to a JSON-line file.\n",
    "    \"\"\"\n",
    "\n",
    "    def open_spider(self, spider):\n",
    "        self.file = open('result.jl', 'w')\n",
    "\n",
    "    def close_spider(self, spider):\n",
    "        self.file.close()\n",
    "\n",
    "    def process_item(self, item, spider):\n",
    "        line = json.dumps(dict(item)) + \"\\n\"\n",
    "        self.file.write(line)\n",
    "        return item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StudiosSpider(scrapy.Spider):\n",
    "    name = \"studios\"\n",
    "    allowed_domains = [\"myanimelist.net\"]\n",
    "    start_urls = [\"https://myanimelist.net/anime/producer\"]\n",
    "    custom_settings = {\n",
    "        'LOG_LEVEL': logging.WARNING, #Logging level set to warning to avoid overload with DEBUG messages about the retrieved data\n",
    "        'ITEM_PIPELINES': {'__main__.JsonWriterPipeline': 1}, # Used for pipeline 1\n",
    "        #'FEED_FORMAT':'json',                                 # Used for pipeline 2\n",
    "        #'FEED_URI': 'result.json'                        # Used for pipeline 2\n",
    "    }\n",
    "    \n",
    "    def parse(self, response):\n",
    "        \"\"\"\n",
    "        Crawls the initial URL, that is where all producers are listed.\n",
    "        \"\"\"\n",
    "        for href in response.xpath(\"//div[@class='genre-list al']/a/@href\").extract():\n",
    "            url = response.urljoin(href)\n",
    "            #time.sleep(4) #only request pages every n seconds\n",
    "            #req.meta['proxy'] = \"http://yourproxy.com:80\"\n",
    "            time.sleep(4)\n",
    "            req = scrapy.Request(url, callback=self.check_subpages)\n",
    "            yield req\n",
    "                \n",
    "    def check_subpages(self, response):\n",
    "        \"\"\"\n",
    "        Checks if there are subpages for producers with more than 100 anime productions\n",
    "        to crawl through subpages.\n",
    "        \"\"\"\n",
    "        subpages = response.xpath('//div[@class=\"pagination ac\"]/a/@href').getall()\n",
    "        if subpages:\n",
    "            for subpage in subpages:\n",
    "                next_page = response.urljoin(subpage)\n",
    "                req = scrapy.Request(next_page, callback=self.parse_titles)\n",
    "                yield req\n",
    "        else:\n",
    "            req = scrapy.Request(response.url, callback=self.parse_titles, dont_filter=True)\n",
    "            yield req\n",
    "            \n",
    "           \n",
    "    def parse_titles(self, response):\n",
    "        \"\"\"\n",
    "        Get data of anime titles per studio, one title per record object.\n",
    "        \"\"\"\n",
    "        for div in response.xpath('//div[@class=\"seasonal-anime js-seasonal-anime\"]'):\n",
    "            data = {}\n",
    "            data['studio'] = response.xpath(\"//span[@class='di-ib mt4']/text()\").get()\n",
    "            data['title'] = div.xpath(\".//p[@class='title-text']/a/text()\").get()\n",
    "            data['genre'] = div.xpath(\".//div[@class='genres-inner js-genre-inner']/span/a/text()\").getall()\n",
    "            yield data \n",
    "        \n",
    "                \n",
    "                \n",
    "    \"\"\"     \n",
    "    def parse_titles(self, response):\n",
    "    \"\"\"\n",
    "    #Get data of anime titles per studio, all titles per record object.\n",
    "    \"\"\"\n",
    "        for sel in response.css('html').getall():\n",
    "        #for title in response.xpath(\"//p[@class='title-text']/a/text()\").getall():\n",
    "            data = {}\n",
    "            data['studio'] = response.xpath(\"//span[@class='di-ib mt4']/text()\").get()\n",
    "            data['title'] = response.xpath(\"//p[@class='title-text']/a/text()\").getall()\n",
    "            #data['genre'] = response.xpath(\"//span[@class='genre']/a/text()\").getall()\n",
    "        yield data\n",
    "    \n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 02:53:00 [scrapy.utils.log] INFO: Scrapy 2.2.0 started (bot: scrapybot)\n",
      "2020-08-02 02:53:00 [scrapy.utils.log] INFO: Versions: lxml 4.5.1.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 20.3.0, Python 3.8.1 (default, Jan  8 2020, 22:29:32) - [GCC 7.3.0], pyOpenSSL 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020), cryptography 2.9.2, Platform Linux-5.4.0-42-generic-x86_64-with-glibc2.10\n",
      "2020-08-02 02:53:00 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor\n",
      "2020-08-02 02:53:00 [scrapy.crawler] INFO: Overridden settings:\n",
      "{'LOG_LEVEL': 30, 'USER_AGENT': 'Mozilla/78.0.1'}\n"
     ]
    }
   ],
   "source": [
    "process = CrawlerProcess({\n",
    "    'USER_AGENT': 'Mozilla/78.0.1' #add user agents to avoid getting banned\n",
    "})\n",
    "\n",
    "process.crawl(StudiosSpider)\n",
    "process.start()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For around two hours of execution with time.sleep(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"studio\": \"Viz Media \", \"title\": \"K: Seven Stories Movie 1 - R:B - Blaze\", \"genre\": [\"Action\", \"Super Power\", \"Supernatural\", \"Drama\"]}\n",
      "{\"studio\": \"Viz Media \", \"title\": \"Ayashi no Ceres\", \"genre\": [\"Adventure\", \"Comedy\", \"Horror\", \"Psychological\", \"Supernatural\", \"Drama\", \"Romance\", \"Shoujo\"]}\n"
     ]
    }
   ],
   "source": [
    "!tail -n 2 result.jl #show two last records in the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rwxrwxrwx 1 alejandro 2830739 ago  2 00:55 \u001b[0m\u001b[01;32mresult.jl\u001b[0m*\n",
      "-rw-rw-r-- 1 alejandro       0 jul 30 22:51 result.json\n"
     ]
    }
   ],
   "source": [
    "ll result.* #check that the file exists in the directory"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}