[Bug Fix - Added item validation support for dataclasses, fixed timeout]

Author: ian-kerins

scrapeops_scrapy/__init__.py

@@ -1 +1 @@
-__version__ = "0.3"
+__version__ = "0.3.1"

scrapeops_scrapy/core/api.py

@@ -7,7 +7,7 @@
 
 class SOPSRequest(object):
 
-    TIMEOUT = 5
+    TIMEOUT = 15
     RETRY_LIMIT = 3
     API_KEY = None
     JOB_GROUP_ID = None

scrapeops_scrapy/core/core.py

@@ -36,8 +36,6 @@ def close_sdk(self, spider=None, reason=None):
             self.spider_close_stats(reason=reason, crawler=self.crawler)
             self.send_stats(periodic_stats=self._periodic_stats, overall_stats=self._overall_stats, stats_type='finished', reason=reason)
             self.close_periodic_monitor()
-            if self._scrapeops_debug_mode:
-                self.display_stats()
 
 
     def request_stats(self, request=None):
@@ -66,7 +64,8 @@ def exception_stats(self, request=None, exception_class=None):
     def item_stats(self, signal_type=None, item=None, response=None, spider=None):
         if self.sdk_enabled():
             request_response_object = RequestResponse(response=response)
-            self.request_response_middleware.normalise_domain_proxy_data(request_response_object) 
+            if response is not None:
+                self.request_response_middleware.normalise_domain_proxy_data(request_response_object) 
             if signal_type == 'item_scraped':
                 self.item_validation_middleware.validate(request_response_object, item)
             self.generate_item_stats(request_response_object, signal=signal_type, response=response)

scrapeops_scrapy/core/model.py

@@ -214,7 +214,7 @@ def get_settings(self, spider):
                 self.spider_settings[key] = value
 
     def include_setting(self, key):
-        exclusion_terms = ['API_KEY', 'APIKEY', 'SECRET_KEY', 'SECRETKEY']
+        exclusion_terms = ['API_KEY', 'APIKEY', 'SECRET_KEY', 'SECRETKEY', 'PASSWORD', 'CONNECTION_STRING']
         if key in self._scrapeops_settings_exclusion_list:
             return False
         for term in exclusion_terms:

scrapeops_scrapy/normalizer/request_response.py

@@ -63,16 +63,16 @@ def __init__(self):
     """
 
     def get_proxy_name(self):
-        return self._proxy_name
+        return self._proxy_name or 'unknown'
 
     def get_proxy_setup(self):
-        return self._proxy_setup
+        return self._proxy_setup or 'unknown'
 
     def get_domain(self):
-        return self._domain
+        return self._domain or 'unknown'
 
     def get_page_type(self):
-        return self._page_type
+        return self._page_type or 'unknown'
 
     def get_proxy_api_name(self):
         return self._proxy_api_name
@@ -84,7 +84,7 @@ def get_raw_proxy(self):
         return self.raw_proxy_port
 
     def get_real_url(self):
-        return self._real_url
+        return self._real_url or 'unknown'
 
     def get_validation_test(self):
         return self._validation_test or 'pass'
@@ -193,12 +193,13 @@ class RequestResponse(BaseRequestResponse):
     def __init__(self, signal_type=None, request=None, response=None):
         BaseRequestResponse.__init__(self)
         self.signal_type = signal_type
-        self.request = response.request if request is None else request
-        self.raw_url = request.url if response is None else response.url
-        self.raw_proxy_port = self.request.meta.get('proxy') 
-        self.raw_domain = DomainNormalizer.get_domain(self.raw_url)
-        self._active_proxy = self._active_porxy_port = False if self.raw_proxy_port is None else True
-        self.raw_headers = self.request.headers
+        if request is not None or response is not None:
+            self.request = response.request if request is None else request
+            self.raw_url = request.url if response is None else response.url
+            self.raw_proxy_port = self.request.meta.get('proxy') 
+            self.raw_domain = DomainNormalizer.get_domain(self.raw_url)
+            self._active_proxy = self._active_porxy_port = False if self.raw_proxy_port is None else True
+            self.raw_headers = self.request.headers
 
     """
         Domain Normalization

scrapeops_scrapy/stats/logger.py

@@ -98,7 +98,12 @@ def generate_response_stats(self, request_response_object, request=None, respons
 
 
     def generate_item_stats(self, request_response_object, signal=None, response=None):
-        request = response.request
+        if response is not None:
+             request = response.request
+             request_method = request.method
+             status = response.status
+        else: 
+            request_method = status = 'unknown'
         proxy_name = request_response_object.get_proxy_name()
         proxy_setup = request_response_object.get_proxy_setup()
         domain_name = request_response_object.get_domain()
@@ -110,16 +115,16 @@ def generate_item_stats(self, request_response_object, signal=None, response=Non
         self.check_periodic_stats()
 
         if signal == 'item_scraped':
-            self.inc_value(self._periodic_stats, f'responses|{request.method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{response.status}|{validation}|{geo}|{custom_tag}|{custom_signal}|items')
-            self.inc_value(self._overall_stats, f'responses|{request.method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{response.status}|{validation}|{geo}|{custom_tag}|{custom_signal}|items')
+            self.inc_value(self._periodic_stats, f'responses|{request_method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{status}|{validation}|{geo}|{custom_tag}|{custom_signal}|items')
+            self.inc_value(self._overall_stats, f'responses|{request_method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{status}|{validation}|{geo}|{custom_tag}|{custom_signal}|items')
 
         elif signal == 'item_dropped':
-            self.inc_value(self._periodic_stats, f'responses|{request.method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{response.status}|{validation}|{geo}|{custom_tag}|{custom_signal}|items_dropped')
-            self.inc_value(self._overall_stats, f'responses|{request.method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{response.status}|{validation}|{geo}|{custom_tag}|{custom_signal}|items_dropped')
+            self.inc_value(self._periodic_stats, f'responses|{request_method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{status}|{validation}|{geo}|{custom_tag}|{custom_signal}|items_dropped')
+            self.inc_value(self._overall_stats, f'responses|{request_method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{status}|{validation}|{geo}|{custom_tag}|{custom_signal}|items_dropped')
 
         elif signal == 'item_error':
-            self.inc_value(self._periodic_stats, f'responses|{request.method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{response.status}|{validation}|{geo}|{custom_tag}|{custom_signal}|item_errors')
-            self.inc_value(self._overall_stats, f'responses|{request.method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{response.status}|{validation}|{geo}|{custom_tag}|{custom_signal}|item_errors')
+            self.inc_value(self._periodic_stats, f'responses|{request_method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{status}|{validation}|{geo}|{custom_tag}|{custom_signal}|item_errors')
+            self.inc_value(self._overall_stats, f'responses|{request_method}|{proxy_name}|{proxy_setup}|{domain_name}|{page_type}|{status}|{validation}|{geo}|{custom_tag}|{custom_signal}|item_errors')
 
 
 

scrapeops_scrapy/validators/item_validator.py

@@ -1,5 +1,4 @@
-from typing import Dict
-from scrapy.item import Item
+from itemadapter import ItemAdapter, is_item
 
 class ItemValidator(object):
 
@@ -25,35 +24,35 @@ def extract_name_fields_item(item):
         return 
 
     def validate(self, request_response_object, item):
-        if ItemValidator.ITEM_COVERAGE_ENABLED:
-            self.increment_items()
-            if isinstance(item, Item):
+        if ItemValidator.ITEM_COVERAGE_ENABLED and is_item(item):
+            try:
+                self.increment_items()
+                adapter = ItemAdapter(item)
                 item_name = ItemValidator.get_item_name(item)
-                fields = ItemValidator.get_item_fields(item)
-                field_keys = fields.keys()
-            if isinstance(item, Dict):
-                item_name = 'Unknown'
-                field_keys = item.keys()
-            if item_name is not None and field_keys is not None:
-                domain = request_response_object.get_domain()
-                invalid_fields = []
-                valid_item = True
-                self.check_item_exists(domain, item_name, field_keys)
-                self.item_coverage[domain][item_name]['num_items'] += 1
-                self.increment_total_fields(field_keys)
-                for k in field_keys:
-                    if(item.get(k) is not None and item.get(k) != ''):
-                        self.item_coverage[domain][item_name]['coverage'][k] += 1
-                    else:
-                        valid_item = False
-                        self.increment_invalid_fields()
-                        invalid_fields.append(k)
-                
-                if valid_item is False:
-                    self.item_coverage[domain][item_name]['num_invalid_items'] += 1
-                    self.increment_invalid_items()
-                if ItemValidator.INVALID_ITEM_URLS_LOGGING_ENABLED and len(invalid_fields) > 0:
-                    self.log_invalid_item_url(request_response_object.get_real_url(), item_name, invalid_fields)
+                dict_item = adapter.asdict()
+                field_keys = dict_item.keys()
+                if item_name is not None and field_keys is not None:
+                    domain = request_response_object.get_domain()
+                    invalid_fields = []
+                    valid_item = True
+                    self.check_item_exists(domain, item_name, field_keys)
+                    self.item_coverage[domain][item_name]['num_items'] += 1
+                    self.increment_total_fields(field_keys)
+                    for k in field_keys:
+                        if(dict_item.get(k) is not None and dict_item.get(k) != ''):
+                            self.item_coverage[domain][item_name]['coverage'][k] += 1
+                        else:
+                            valid_item = False
+                            self.increment_invalid_fields()
+                            invalid_fields.append(k)
+                    
+                    if valid_item is False:
+                        self.item_coverage[domain][item_name]['num_invalid_items'] += 1
+                        self.increment_invalid_items()
+                    if ItemValidator.INVALID_ITEM_URLS_LOGGING_ENABLED and len(invalid_fields) > 0:
+                        self.log_invalid_item_url(request_response_object.get_real_url(), item_name, invalid_fields)
+            except Exception:
+                pass
 
 
     def check_item_exists(self, domain, item_name, field_keys):

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.3'
+VERSION = '0.3.1'
 DESCRIPTION = 'Scrapeops Scrapy SDK, is a monitoring tool for your Scrapy spiders.'
 
 setup(name='scrapeops_scrapy',
@@ -14,10 +14,11 @@
       url="https://github.com/ScrapeOps/scrapeops-scrapy-sdk",
       packages=find_packages(),
       install_requires=[
-          "tld==0.12.4",
-          "requests==2.24.0",
-          "json5==0.9.5",
-          "urllib3==1.25.10",
+          "tld>=0.12.4",
+          "requests>=2.24.0",
+          "json5>=0.9.5",
+          "urllib3>=1.25.10",
+          "itemadapter>=0.4.0",
           ],
       classifiers=[
         "Programming Language :: Python",
@@ -26,6 +27,7 @@
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Intended Audience :: Developers",