Upgrade to python3

Author: 梁睿坤

requirements.txt

@@ -1,67 +1,49 @@
-adblockparser==0.7
-aliyun-python-sdk-core-v3==2.13.0
-aliyun-python-sdk-kms==2.5.0
-asn1crypto==0.24.0
-astroid==2.1.0
-attrs==18.2.0
-Automat==0.7.0
-autopep8==1.4.3
-certifi==2018.11.29
-cffi==1.12.1
-chardet==3.0.4
+aliyun-python-sdk-core==2.13.30
+aliyun-python-sdk-core-v3==2.13.11
+aliyun-python-sdk-kms==2.13.0
+attrs==20.3.0
+Automat==20.2.0
+certifi==2020.12.5
+cffi==1.14.4
+chardet==4.0.0
 constantly==15.1.0
 crcmod==1.7
-cryptography==2.5
-cssselect==1.0.3
-dateparser==0.7.1
-funcparserlib==0.3.6
-hyperlink==18.0.0
-idna==2.8
+cryptography==3.2.1
+cssselect==1.1.0
+dateparser==1.0.0
+hyperlink==20.0.1
+idna==2.10
 incremental==17.5.0
-isort==4.3.4
-jmespath==0.9.3
-lazy-object-proxy==1.3.1
-lxml==4.3.1
-mccabe==0.6.1
-oss2==2.6.1
-parsel==1.5.1
-Pillow==5.4.1
-psutil==5.5.1
-psycopg2==2.7.7
-psycopg2-binary==2.7.7
-pyasn1==0.4.5
-pyasn1-modules==0.2.4
-pycodestyle==2.5.0
-pycparser==2.19
-pycryptodome==3.7.3
+itemadapter==0.2.0
+itemloaders==1.0.4
+jmespath==0.10.0
+lxml==4.6.2
+oss2==2.13.1
+parsel==1.6.0
+Protego==0.1.16
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pycryptodome==3.9.9
 PyDispatcher==2.0.5
-PyHamcrest==1.9.0
-pylint==2.2.2
-pymongo==3.7.2
-pyOpenSSL==19.0.0
-pyquery==1.4.0
-python-dateutil==2.8.0
-pytz==2018.9
-qt5reactor==0.5
+PyHamcrest==2.0.2
+pymongo==3.11.2
+pyOpenSSL==20.0.1
+python-dateutil==2.8.1
+pytz==2020.4
 queuelib==1.5.0
-redis==3.2.1
-regex==2019.2.21
-requests==2.21.0
-Scrapy==1.6.0
+redis==3.5.3
+regex==2020.11.13
+requests==2.25.1
+Scrapy==2.4.1
 scrapy-splash==0.7.2
-scrapyd==1.2.0
-scrapyd-client==1.1.0
 selenium==3.141.0
 service-identity==18.1.0
-six==1.12.0
-splash==3.3.1
-SQLAlchemy==1.2.18
-stem==1.7.1
-Twisted==18.9.0
-typed-ast==1.3.1
-tzlocal==1.5.1
-urllib3==1.24.1
-w3lib==1.20.0
-wrapt==1.11.1
-xvfbwrapper==0.2.9
-zope.interface==4.6.0
+six==1.15.0
+SQLAlchemy==1.3.20
+stem==1.8.0
+Twisted==20.3.0
+tzlocal==2.1
+urllib3==1.26.2
+w3lib==1.22.0
+zope.interface==5.2.0

scrapy_plus/middlewares/ua.py

@@ -2,17 +2,39 @@
 
 import random
 
+_agents = [
+    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
+    'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
+    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)'
+    'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0',
+    'Mozilla/5.0 (Linux; U; Android 2.2) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1',
+    'Mozilla/5.0 (Windows NT 5.1; rv:7.0.1) Gecko/20100101 Firefox/7.0.1',
+    'Mozilla/5.0 (Linux; Android 6.0.1; SM-G532G Build/MMB29T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.83 Mobile Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0.3 Safari/604.5.6',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
+    'MAC：Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36',
+    'Windows：Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
+    'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',
+    'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',
+    'Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',
+]
+
 
 class RandomUserAgentMiddleware(object):
     """
     随机User Agent 中间件
     """
+
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(user_agents=crawler.settings.getlist('USER_AGENTS'))
+        return cls(user_agents=crawler.settings.getlist('USER_AGENTS', None))
 
-    def __init__(self, user_agents=[]):
-        self.user_agents = user_agents
+    def __init__(self, user_agents=None):
+        self.user_agents = user_agents if user_agents is not None else _agents
 
     def process_request(self, request, spider):
         if self.user_agents != None and len(self.user_agents) > 0:

scrapy_plus/processors.py

@@ -2,7 +2,7 @@
 import re
 import six
 
-from six.moves.urllib.parse import urljoin, urlparse, urlunparse
+from urllib.parse import urljoin, urlparse, urlunparse
 
 from copy import deepcopy
 from itertools import chain

setup.py

@@ -10,7 +10,7 @@
 
 setup(
     name='scrapy_plus',
-    version='1.0.4',
+    version='1.0.5',
     packages=find_packages(exclude=["tests"]),
     install_requires=REQS,
     url='http://www.github.com/dotnetage/scrapy_plus',

tests/test_processors.py

@@ -1,6 +1,6 @@
 # coding:utf-8
 import unittest
-from douban.processors import Number, Text, Date, Price
+from scrapy_plus.processors import Number, Text, Date, Price
 
 
 class ProcessorTestCase(unittest.TestCase):