How to use the spidermon.monitors function in spidermon

To help you get started, we’ve selected a few spidermon examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github scrapinghub / spidermon / tests / fixtures / names.py View on Github external
    @monitors.name("A Test")
    def test_with_name(self):
        pass
github scrapinghub / spidermon / tests / fixtures / ordering.py View on Github external
pass

    class D(DummyMonitorSuite):
        pass


class Ordered:
    @monitors.order(1)
    class A(DummyMonitor):
        pass

    @monitors.order(2)
    class B(DummyMonitor):
        pass

    @monitors.order(3)
    class C(DummyMonitorSuite):
        pass

    @monitors.order(4)
    class D(DummyMonitorSuite):
        pass


# ----------------------------------
# Methods ordering
# ----------------------------------
class UnorderedMethodsMonitor(Monitor):
    def test_a(self):
        pass

    def test_b(self):
github scrapinghub / spidermon / tests / fixtures / descriptions.py View on Github external
# ----------------------------------
# Suites
# ----------------------------------
class NotDescriptedSuite(MonitorSuite):
    pass


class DocstringDescriptedSuite(MonitorSuite):
    """docstring suite description"""

    pass


@monitors.description("decorator suite description")
class DescoratedDescriptedSuite(MonitorSuite):
    pass


@monitors.description("decorator suite description")
class DescoratedDescriptedSuite2(MonitorSuite):
    """docstring suite description"""


# ----------------------------------
# Methods
# ----------------------------------
class DescriptedMethodsMonitor(Monitor):
    def test_not_descripted(self):
        pass
github scrapinghub / spidermon / tests / fixtures / levels.py View on Github external
# ----------------------------------
# Suites
# ----------------------------------
class Suites:
    class NoLevelSuite(MonitorSuite):
        pass

    @monitors.level.high
    class HighLevelSuite(MonitorSuite):
        pass

    @monitors.level.normal
    class NormalLevelSuite(MonitorSuite):
        pass

    @monitors.level.low
    class LowLevelSuite(MonitorSuite):
        pass
github scrapinghub / spidermon / spidermon / contrib / scrapy / monitors.py View on Github external
SPIDERMON_UNWANTED_HTTP_CODES = "SPIDERMON_UNWANTED_HTTP_CODES"
SPIDERMON_UNWANTED_HTTP_CODES_MAX_COUNT = "SPIDERMON_UNWANTED_HTTP_CODES_MAX_COUNT"
SPIDERMON_MAX_ITEM_VALIDATION_ERRORS = "SPIDERMON_MAX_ITEM_VALIDATION_ERRORS"


class BaseScrapyMonitor(Monitor, SpiderMonitorMixin):
    longMessage = False

    @property
    def monitor_description(self):
        if self.__class__.__doc__:
            return self.__class__.__doc__.split("\n")[0]
        return super(BaseScrapyMonitor, self).monitor_description


@monitors.name("Extracted Items Monitor")
class ItemCountMonitor(BaseScrapyMonitor):
    """Check if spider extracted the minimum number of items.

    You can configure it using ``SPIDERMON_MIN_ITEMS`` setting.
    There's **NO** default value for this setting, if you try to use this
    monitor without setting it, it'll raise a ``NotConfigured`` exception.
    """

    def run(self, result):
        self.minimum_threshold = self.crawler.settings.getint(SPIDERMON_MIN_ITEMS, 0)
        if not self.minimum_threshold:
            raise NotConfigured(
                "You should specify a minimum number of items " "to check against."
            )
        return super(ItemCountMonitor, self).run(result)
github okfn-brasil / open-jus / justa / crawler / justa / monitors / distrito_federal.py View on Github external
from spidermon import Monitor, MonitorSuite, monitors


@monitors.name('Distrito Federal')
class DistritoFederalMonitor(Monitor):

    @monitors.name('Minimum number of items')
    def test_minimum_number_of_items(self):
        items_extracted = self.data.stats.get('item_scraped_count', 0)
        self.assertGreaterEqual(items_extracted, 433)


class DistritoFederalMonitorSuite(MonitorSuite):
    monitors = [DistritoFederalMonitor]
github okfn-brasil / open-jus / justa / crawler / justa / monitors.py View on Github external
    @monitors.name('Minimum number of items')
    def test_minimum_number_of_items(self):
        spider = self.data['spider']
        items_extracted = self.data.stats.get('item_scraped_count', 0)
        minimum_expected = self.minimum_expected.get(spider.name, 0)
        self.assertGreaterEqual(items_extracted, minimum_expected)
github scrapinghub / spidermon / examples / tutorial / tutorial / monitors.py View on Github external
# monitors.py
from spidermon import Monitor, MonitorSuite, monitors
from spidermon.contrib.actions.slack.notifiers import SendSlackMessageSpiderFinished
from spidermon.contrib.monitors.mixins import StatsMonitorMixin

from tutorial.actions import CloseSpiderAction


@monitors.name("Item count")
class ItemCountMonitor(Monitor):
    @monitors.name("Minimum items extracted")
    def test_minimum_number_of_items_extracted(self):
        minimum_threshold = 1000
        item_extracted = getattr(self.data.stats, "item_scraped_count", 0)
        self.assertFalse(
            item_extracted < minimum_threshold,
            msg="Extracted less than {} items".format(minimum_threshold),
        )


@monitors.name("Item validation")
class ItemValidationMonitor(Monitor, StatsMonitorMixin):
    @monitors.name("No item validation errors")
    def test_no_item_validation_errors(self):
        validation_errors = getattr(
github okfn-brasil / open-jus / justa / crawler / justa / monitors / distrito_federal.py View on Github external
    @monitors.name('Minimum number of items')
    def test_minimum_number_of_items(self):
        items_extracted = self.data.stats.get('item_scraped_count', 0)
        self.assertGreaterEqual(items_extracted, 433)
github scrapinghub / spidermon / examples / example_01.py View on Github external
    @monitors.level.high
    def test_repeated_ips(self):
        """Checks for repeated ips."""
        pass