Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# ----------------------------------
# Base Monitors
# ----------------------------------
class NoLevelMethodMonitor(Monitor):
def test(self):
pass
class HighLevelMethodMonitor(Monitor):
@monitors.level.high
def test(self):
pass
class NormalLevelMethodMonitor(Monitor):
@monitors.level.normal
def test(self):
pass
class LowLevelMethodMonitor(Monitor):
@monitors.level.low
def test(self):
pass
# ----------------------------------
# Monitors
# ----------------------------------
class Monitors:
class NoLevelMonitor:
from __future__ import absolute_import
from spidermon import Monitor
class EmptyMonitor(Monitor):
pass
class Monitor01(Monitor):
def test_a(self):
pass
def test_b(self):
pass
def test_c(self):
pass
class Monitor02(Monitor):
def test_d(self):
pass
def test_e(self):
pass
from __future__ import absolute_import
from spidermon import Monitor
class EmptyMonitor(Monitor):
pass
class Monitor01(Monitor):
def test_a(self):
pass
def test_b(self):
pass
def test_c(self):
pass
class Monitor02(Monitor):
def test_d(self):
# ----------------------------------
# Monitors
# ----------------------------------
class UnnamedMonitor(Monitor):
def test_without_name(self):
pass
@monitors.name("A Test")
def test_with_name(self):
pass
@monitors.name("Class Monitor")
class NamedMonitor(Monitor):
def test_without_name(self):
pass
@monitors.name("A Test")
def test_with_name(self):
pass
# ----------------------------------
# Child Suites
# ----------------------------------
class BaseChildSuite(MonitorSuite):
monitors = [
UnnamedMonitor,
NamedMonitor,
("Instance Monitor", UnnamedMonitor),
rules=[
('Rule 1', rule_as_function),
('Rule 2', rule_as_function),
],
actions=[
('Action 1', DummyAction()),
]
)
MONITOR_E.add_rule(rule=rule_as_function, name='Rule 3')
MONITOR_E.add_action(action=DummyAction(), name='Action 2')
#---------------------------------------------------------------
# F. Rule levels and action triggers
#---------------------------------------------------------------
MONITOR_F = Monitor(
name='F. Rule levels and action triggers',
rules=[
('Rule High', 'stats.finish_reason == "finished"', 'HIGH'),
('Rule Normal', 'stats.finish_reason != "finished"', 'NORMAL'),
('Rule Low', 'stats.finish_reason != "finished"', 'LOW'),
],
actions=[
('Action runs always', DummyAction()), # trigger defaults to ALWAYS
('Action runs always', DummyAction(), 'ALWAYS'),
('Action runs on passed', DummyAction(), 'PASSED'),
('Action runs on failed', DummyAction(), 'FAILED'),
('Action runs on error', DummyAction(), 'ERROR'),
]
)
MONITOR_F.add_rule(rule=rule_as_function, name='Rule Low 2', level='LOW')
MONITOR_F.add_action(action=DummyAction(), name='Action on passed', trigger='PASSED')
]
)
#---------------------------------------------------------------
# B. Adding rules and actions
#---------------------------------------------------------------
MONITOR_B = Monitor(name='B. Adding rules and actions')
MONITOR_B.add_rule(rule=lambda stats: stats.finish_reason == 'finished')
MONITOR_B.add_action(action=MessageAction('finish reason is ok!'))
#---------------------------------------------------------------
# C. Monitor without rules
#---------------------------------------------------------------
MONITOR_C = Monitor(
name='C. Monitor without rules',
actions=[
MessageAction('hi there!'),
]
)
#---------------------------------------------------------------
# D. All rule type definitions
#---------------------------------------------------------------
def rule_as_function(stats):
return stats.finish_reason == 'finished'
class ARule(Rule):
def check(self, stats):
lambda stats: stats.finish_reason == 'finished', # lambda
'stats.finish_reason == "finished"', # python expression string
rule_as_function, # function
ARule(), # Rule
ATestCase(), # TestCase
],
actions=[
DummyAction(),
]
)
#---------------------------------------------------------------
# E. Naming rules and actions
#---------------------------------------------------------------
MONITOR_E = Monitor(
name='E. Naming rules and actions',
rules=[
('Rule 1', rule_as_function),
('Rule 2', rule_as_function),
],
actions=[
('Action 1', DummyAction()),
]
)
MONITOR_E.add_rule(rule=rule_as_function, name='Rule 3')
MONITOR_E.add_action(action=DummyAction(), name='Action 2')
#---------------------------------------------------------------
# F. Rule levels and action triggers
#---------------------------------------------------------------
from spidermon import Monitor, MonitorSuite, monitors
class MinimumItemsMonitor(Monitor):
minimum_expected = {
'distrito_federal': 433,
'tjsp_numbers': 826
}
@monitors.name('Minimum number of items')
def test_minimum_number_of_items(self):
spider = self.data['spider']
items_extracted = self.data.stats.get('item_scraped_count', 0)
minimum_expected = self.minimum_expected.get(spider.name, 0)
self.assertGreaterEqual(items_extracted, minimum_expected)
class JustaMonitorSuite(MonitorSuite):
monitors = [MinimumItemsMonitor]
# monitors.py
from spidermon import Monitor, MonitorSuite, monitors
from spidermon.contrib.actions.slack.notifiers import SendSlackMessageSpiderFinished
from spidermon.contrib.monitors.mixins import StatsMonitorMixin
from tutorial.actions import CloseSpiderAction
@monitors.name("Item count")
class ItemCountMonitor(Monitor):
@monitors.name("Minimum items extracted")
def test_minimum_number_of_items_extracted(self):
minimum_threshold = 1000
item_extracted = getattr(self.data.stats, "item_scraped_count", 0)
self.assertFalse(
item_extracted < minimum_threshold,
msg="Extracted less than {} items".format(minimum_threshold),
)
@monitors.name("Item validation")
class ItemValidationMonitor(Monitor, StatsMonitorMixin):
@monitors.name("No item validation errors")
def test_no_item_validation_errors(self):
validation_errors = getattr(
self.data.stats, "spidermon/validation/fields/errors", 0
@monitors.name("No item validation errors")
def test_no_item_validation_errors(self):
validation_errors = getattr(
self.data.stats, "spidermon/validation/fields/errors", 0
)
self.assertEqual(
validation_errors,
0,
msg="Found validation errors in {} fields".format(validation_errors),
)
self.data.stats
@monitors.name("Periodic job stats monitor")
class PeriodicJobStatsMonitor(Monitor, StatsMonitorMixin):
@monitors.name("Maximum number of errors exceeded")
def test_number_of_errors(self):
accepted_num_errors = 20
num_errors = self.data.stats.get("log_count/ERROR", 0)
msg = "The job has exceeded the maximum number of errors"
self.assertLessEqual(num_errors, accepted_num_errors, msg=msg)
class PeriodicMonitorSuite(MonitorSuite):
monitors = [PeriodicJobStatsMonitor]
monitors_failed_actions = [CloseSpiderAction]
class SpiderCloseMonitorSuite(MonitorSuite):
monitors = [ItemCountMonitor, ItemValidationMonitor, PeriodicJobStatsMonitor]