Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
@monitors.name("A Test")
def test_with_name(self):
pass
pass
class D(DummyMonitorSuite):
pass
class Ordered:
@monitors.order(1)
class A(DummyMonitor):
pass
@monitors.order(2)
class B(DummyMonitor):
pass
@monitors.order(3)
class C(DummyMonitorSuite):
pass
@monitors.order(4)
class D(DummyMonitorSuite):
pass
# ----------------------------------
# Methods ordering
# ----------------------------------
class UnorderedMethodsMonitor(Monitor):
def test_a(self):
pass
def test_b(self):
# ----------------------------------
# Suites
# ----------------------------------
class NotDescriptedSuite(MonitorSuite):
pass
class DocstringDescriptedSuite(MonitorSuite):
"""docstring suite description"""
pass
@monitors.description("decorator suite description")
class DescoratedDescriptedSuite(MonitorSuite):
pass
@monitors.description("decorator suite description")
class DescoratedDescriptedSuite2(MonitorSuite):
"""docstring suite description"""
# ----------------------------------
# Methods
# ----------------------------------
class DescriptedMethodsMonitor(Monitor):
def test_not_descripted(self):
pass
# ----------------------------------
# Suites
# ----------------------------------
class Suites:
class NoLevelSuite(MonitorSuite):
pass
@monitors.level.high
class HighLevelSuite(MonitorSuite):
pass
@monitors.level.normal
class NormalLevelSuite(MonitorSuite):
pass
@monitors.level.low
class LowLevelSuite(MonitorSuite):
pass
SPIDERMON_UNWANTED_HTTP_CODES = "SPIDERMON_UNWANTED_HTTP_CODES"
SPIDERMON_UNWANTED_HTTP_CODES_MAX_COUNT = "SPIDERMON_UNWANTED_HTTP_CODES_MAX_COUNT"
SPIDERMON_MAX_ITEM_VALIDATION_ERRORS = "SPIDERMON_MAX_ITEM_VALIDATION_ERRORS"
class BaseScrapyMonitor(Monitor, SpiderMonitorMixin):
longMessage = False
@property
def monitor_description(self):
if self.__class__.__doc__:
return self.__class__.__doc__.split("\n")[0]
return super(BaseScrapyMonitor, self).monitor_description
@monitors.name("Extracted Items Monitor")
class ItemCountMonitor(BaseScrapyMonitor):
"""Check if spider extracted the minimum number of items.
You can configure it using ``SPIDERMON_MIN_ITEMS`` setting.
There's **NO** default value for this setting, if you try to use this
monitor without setting it, it'll raise a ``NotConfigured`` exception.
"""
def run(self, result):
self.minimum_threshold = self.crawler.settings.getint(SPIDERMON_MIN_ITEMS, 0)
if not self.minimum_threshold:
raise NotConfigured(
"You should specify a minimum number of items " "to check against."
)
return super(ItemCountMonitor, self).run(result)
from spidermon import Monitor, MonitorSuite, monitors
@monitors.name('Distrito Federal')
class DistritoFederalMonitor(Monitor):
@monitors.name('Minimum number of items')
def test_minimum_number_of_items(self):
items_extracted = self.data.stats.get('item_scraped_count', 0)
self.assertGreaterEqual(items_extracted, 433)
class DistritoFederalMonitorSuite(MonitorSuite):
monitors = [DistritoFederalMonitor]
@monitors.name('Minimum number of items')
def test_minimum_number_of_items(self):
spider = self.data['spider']
items_extracted = self.data.stats.get('item_scraped_count', 0)
minimum_expected = self.minimum_expected.get(spider.name, 0)
self.assertGreaterEqual(items_extracted, minimum_expected)
# monitors.py
from spidermon import Monitor, MonitorSuite, monitors
from spidermon.contrib.actions.slack.notifiers import SendSlackMessageSpiderFinished
from spidermon.contrib.monitors.mixins import StatsMonitorMixin
from tutorial.actions import CloseSpiderAction
@monitors.name("Item count")
class ItemCountMonitor(Monitor):
@monitors.name("Minimum items extracted")
def test_minimum_number_of_items_extracted(self):
minimum_threshold = 1000
item_extracted = getattr(self.data.stats, "item_scraped_count", 0)
self.assertFalse(
item_extracted < minimum_threshold,
msg="Extracted less than {} items".format(minimum_threshold),
)
@monitors.name("Item validation")
class ItemValidationMonitor(Monitor, StatsMonitorMixin):
@monitors.name("No item validation errors")
def test_no_item_validation_errors(self):
validation_errors = getattr(
@monitors.name('Minimum number of items')
def test_minimum_number_of_items(self):
items_extracted = self.data.stats.get('item_scraped_count', 0)
self.assertGreaterEqual(items_extracted, 433)
@monitors.level.high
def test_repeated_ips(self):
"""Checks for repeated ips."""
pass