selenium-crawler-NSTL-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "\"\"\"\n",
    "@Author: Shuyue Jia\n",
    "@Date: Arg 8, 2020\n",
    "\n",
    "Usage Notice:\n",
    "1. It's just a demo, which I don't recommend to run because of selenium's slow time \n",
    "   (5 - 20 seconds for one website, typically at least 5 seconds, TOO SLOW!!!)\n",
    "2. I'd like to recommend \"fast-crawler-NSTL-data.ipynb\" script instead\n",
    "3. If you have few websites, e.g., 100 - 1000, you can definitely use selenium to capture the \n",
    "   source info (contents) of the website.\n",
    "4. If you want fast capturing, don't use too many \"print()\" functions since they take a lot time\n",
    "5. Download the Chrome Driver first by using the below code\n",
    "   driver_path = ChromeDriverManager().install()\n",
    "   Then, comment this line and change the blow to your Driver's path:\n",
    "   driver_path = '/Users/shuyuej/.wdm/drivers/chromedriver/mac64/84.0.4147.30/chromedriver'\n",
    "6. I don't recommend you to download the Google Chrome Driver from the website by yourself,\n",
    "   as it might lead to the problem of version compatibility\n",
    "   So, just use \"driver_path = ChromeDriverManager().install()\"\n",
    "   to download it automatically by the system\n",
    "   Then, you comment it and change the path of the driver\n",
    "7. If you have new dynamic webs to capture, you can change the string matching rules to fit the needs\n",
    "\"\"\"\n",
    "\n",
    "# import necessary packages\n",
    "import os\n",
    "import requests\n",
    "import time\n",
    "import numpy as np\n",
    "import sys\n",
    "import re\n",
    "from selenium import webdriver\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "import xml\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_url(url: str, driver_path: str):\n",
    "    \"\"\"\n",
    "    Read the website and return the contents of the website\n",
    "    :param url: The url of the website\n",
    "    :param driver_path: The path of the Google Chrome Driver\n",
    "    :return soup.text: The contents of the website\n",
    "    \"\"\"\n",
    "    start_time = time.time()\n",
    "    option = webdriver.ChromeOptions()\n",
    "    option.add_argument(\n",
    "        'user-agent=\"MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7)'\n",
    "        'AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1\"')\n",
    "    option.add_argument(\n",
    "        'user-agent=\"Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko)'\n",
    "        'Version/9.0 Mobile/13B143 Safari/601.1\"')\n",
    "\n",
    "    option.add_argument('--disable-infobars')\n",
    "    option.add_argument('--incognito')\n",
    "    option.add_argument('headless')\n",
    "    option.add_argument('blink-settings=imagesEnabled=false')\n",
    "    \n",
    "    prefs = {\n",
    "        \"profile.managed_default_content_settings.images\": 2,\n",
    "        'profile.default_content_settings.popups': 0\n",
    "    }\n",
    "    option.add_experimental_option(\"prefs\", prefs)\n",
    "\n",
    "    # If you don't have Google Chrome Driver installed, uncomment this line\n",
    "    # driver_path = ChromeDriverManager().install()\n",
    "    \n",
    "    try:\n",
    "        driver = webdriver.Chrome(driver_path, chrome_options=option)\n",
    "    except Exception as e:\n",
    "        driver_path = ChromeDriverManager().install()\n",
    "        print('You have successfully installed the Google Chrome Driver!')\n",
    "        print('Now change the path of driver to your own path in your laptop!')\n",
    "        driver = webdriver.Chrome(driver_path, chrome_options=option)\n",
    "    \n",
    "    driver.get(url)\n",
    "    contents = driver.page_source\n",
    "    \n",
    "    START = contents.find('serverContent')\n",
    "    END = contents.find('QRcodebox')\n",
    "    contents_cut = contents[START:END]\n",
    "    end_time = time.time()\n",
    "    print('Time used for get the website was %7f' % (end_time - start_time))\n",
    "    return contents, contents_cut, driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_English_term(content: str):\n",
    "    \"\"\"\n",
    "    Find the English Term from the contents\n",
    "    :param content: The contents of the website\n",
    "    :return Eng_term: The found English term\n",
    "    :return content: The contents that cut the English term part\n",
    "    \"\"\"\n",
    "    mark = content.find('detail_content')\n",
    "    temp_cont = content[mark-100:mark]\n",
    "    START = temp_cont.find('\">')\n",
    "    END = temp_cont.find('</a></h3>')\n",
    "    Eng_term = temp_cont[START+2:END]\n",
    "    content = content[mark+len('detail_content'):]\n",
    "    return Eng_term, content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_Chinese_term(content: str):\n",
    "    \"\"\"\n",
    "    Find the Chinese Term from the contents\n",
    "    :param content: The contents of the website\n",
    "    :return Chi_term: The found Chinese Term\n",
    "    :return content: The contents that cut the Chinese term part\n",
    "    \"\"\"\n",
    "    if '中文名称' not in content:\n",
    "        Chi_term = ''\n",
    "    else:\n",
    "        mark = content.find('target')\n",
    "        temp_cont = content[mark:mark+100]\n",
    "        START = temp_cont.find('target')\n",
    "        END = temp_cont.find('</a>')\n",
    "        Chi_term = temp_cont[START+len('target=\"_blank\">'):END]\n",
    "        chi_loc = content.find(Chi_term)\n",
    "        content = content[chi_loc+len(Chi_term):]\n",
    "    return Chi_term, content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_English_definition(content: str):\n",
    "    \"\"\"\n",
    "    Find the English Definition from the content\n",
    "    :param content: The contents of the website\n",
    "    :return Eng_def: The found English definition\n",
    "    :return content: The contents that cut the English definition part\n",
    "    \"\"\"\n",
    "    if '释义' not in content:\n",
    "        Eng_def = ''\n",
    "    else:\n",
    "        START = content.find('释义')\n",
    "        END = content.find('</i>')\n",
    "        Eng_def = content[START+len('释义：<span><i>'):END]\n",
    "        content = content[END+len('</i></span></div>'):]\n",
    "    return Eng_def, content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def synonym(content: str):\n",
    "    \"\"\"\n",
    "    Find all the Synonym words w.r.t. the English term\n",
    "    :param content: The contents of the website\n",
    "    :return synonym_words: The found synonym words\n",
    "    \"\"\"\n",
    "    if '同义词' not in content:\n",
    "        synonym_words = ''\n",
    "    else:\n",
    "        START = content.find('同义词')\n",
    "        END = content.find('范畴')\n",
    "        main_content = content[START:END]\n",
    "\n",
    "        key_word = 'target'\n",
    "        synonym_words = []\n",
    "        cur_content = main_content\n",
    "        while key_word in cur_content:\n",
    "            start = cur_content.find('target') + len('target')\n",
    "            ite_content = cur_content[start:start+100]\n",
    "\n",
    "            new_start = ite_content.find(\">\")\n",
    "            end = ite_content.find('</a></span>')\n",
    "            synonym_word = ite_content[new_start+1:end]\n",
    "            synonym_words.append(synonym_word)\n",
    "\n",
    "            cur_content = cur_content[start+1:]\n",
    "\n",
    "        synonym_words = np.array(synonym_words)\n",
    "        synonym_words = np.squeeze(synonym_words)\n",
    "        synonym_words = str(synonym_words).replace('[', '')\n",
    "        synonym_words = [str(synonym_words).replace(']', '')]\n",
    "\n",
    "        content = content[END:]\n",
    "    return synonym_words, content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def field(content: str):\n",
    "    \"\"\"\n",
    "    Find and save all the Fields of this particular term\n",
    "    :param content: The contents of the website\n",
    "    :return content: The Fields contents\n",
    "    \"\"\"\n",
    "    if '范畴' not in content:\n",
    "        field = ''\n",
    "    else:\n",
    "        content.replace(\"title=\"\"\", '')\n",
    "        START = content.find('target') + len('target')\n",
    "        content = content[START:]\n",
    "\n",
    "        field = []\n",
    "        new_content = content\n",
    "        while 'title' in new_content:\n",
    "            start = new_content.find('title=') + len('title=')\n",
    "            end = new_content.find('><span')\n",
    "            temp_field = new_content[start+1:end-1]\n",
    "            if temp_field != '':\n",
    "                field.append(temp_field)\n",
    "\n",
    "            new_content = new_content[start:]\n",
    "\n",
    "        field = np.array(field)\n",
    "        field = np.squeeze(field)\n",
    "        field = str(field).replace('[', '')\n",
    "        field = [str(field).replace(']', '')]\n",
    "    return field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The main function\n",
    "if __name__ == \"__main__\":\n",
    "    # Set an index to observe the final number of captured webs\n",
    "    index = 0\n",
    "    \n",
    "    # Initialize the saved contents\n",
    "    English_terms = []\n",
    "    Chinese_terms = []\n",
    "    English_definition = []\n",
    "    Synonym_words = []\n",
    "    Fileds_summary = []\n",
    "    \n",
    "    # If you want to capture 0 - 100 webs\n",
    "    start = '0'\n",
    "    end = '100'\n",
    "    save_file = start + '-' + end\n",
    "    \n",
    "    # Change this if you want to capture more websites\n",
    "    start_index = int(0)\n",
    "    end_index = int(100)\n",
    "    \n",
    "    for i in range(start_index, end_index):\n",
    "        if i < 10:\n",
    "            i = '00000' + str(i)\n",
    "        elif 10 <= i < 100:\n",
    "            i = '0000' + str(i)\n",
    "        elif 100 <= i < 1000:\n",
    "            i = '000' + str(i)\n",
    "        elif 1000 <= i < 10000:\n",
    "            i = '00' + str(i)\n",
    "        elif 10000 <= i < 100000:\n",
    "            i = '0' + str(i)\n",
    "        else:\n",
    "            i = str(i)\n",
    "        \n",
    "        url = 'https://www.nstl.gov.cn/stkos_detail.html?id=C019' + i\n",
    "        driver_path = '/Users/shuyuej/.wdm/drivers/chromedriver/mac64/84.0.4147.30/chromedriver'\n",
    "        save_path = 'NSTD_data/'\n",
    "        if not os.path.exists(save_path):\n",
    "            os.mkdir(save_path)\n",
    "        \n",
    "        # Read the URL and see whether there was contents\n",
    "        contents, contents_cut, driver = read_url(url=url, driver_path=driver_path)\n",
    "        \n",
    "        # If there was no contents in this website, skip and continue\n",
    "        if '暂无相关资源' in contents:\n",
    "            print('There is no data in this webpage! Skip and continue......')\n",
    "            continue\n",
    "        else:\n",
    "            Eng_term, con_cut_eng = find_English_term(content=contents_cut)\n",
    "            English_terms.append(Eng_term)\n",
    "\n",
    "            Chi_term, con_cut_chi = find_Chinese_term(content=con_cut_eng)\n",
    "            Chinese_terms.append(Chi_term)\n",
    "\n",
    "            Eng_def, con_cut_def = find_English_definition(content=con_cut_chi)\n",
    "            English_definition.append(Eng_def)\n",
    "\n",
    "            synonym_word, synonym_cut_con = synonym(content=con_cut_def)\n",
    "            Synonym_words.append(synonym_word)\n",
    "\n",
    "            fields = field(content=synonym_cut_con)\n",
    "            Fileds_summary.append(fields)\n",
    "            \n",
    "            index += 1\n",
    "            print('It\\'s ' + str(i) + ' Website, saved its data, and continue......')\n",
    "    \n",
    "    rows = np.shape(English_terms)[0]\n",
    "    English_terms = np.reshape(English_terms, [rows, 1])\n",
    "    Chinese_terms = np.reshape(Chinese_terms, [rows, 1])\n",
    "    English_definition = np.reshape(English_definition, [rows, 1])\n",
    "    Synonym_words = np.reshape(Synonym_words, [rows, 1])\n",
    "    Fileds_summary = np.reshape(Fileds_summary, [rows, 1])\n",
    "\n",
    "    save_data = np.concatenate([English_terms, Chinese_terms, English_definition, Synonym_words, Fileds_summary], axis=1)\n",
    "    save_data = pd.DataFrame(save_data)\n",
    "    save_data.to_csv(save_path + '%s.csv' % save_file, sep=',', index=False, header=['English Term', 'Chinese Term', 'English Definition', 'Synonym', 'Field'])\n",
    "    \n",
    "    driver.close()\n",
    "    print('Cheers! %s\\'s NSTL data (%s terms) has been successfully saved!' % (save_file, str(index)))\n",
    "    driver.quit()"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}