Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def execute(self, data):
"""Execute the crawler and create a database record of having done
so."""
if Crawl.is_aborted(self.crawler, self.run_id):
return
try:
Crawl.operation_start(self.crawler, self.stage, self.run_id)
self.log.info('[%s->%s(%s)]: %s',
self.crawler.name,
self.stage.name,
self.stage.method_name,
self.run_id)
return self.stage.method(self, data)
except Exception as exc:
self.emit_exception(exc)
finally:
Crawl.operation_end(self.crawler, self.run_id)
shutil.rmtree(self.work_path)
def execute(self, data):
"""Execute the crawler and create a database record of having done
so."""
if Crawl.is_aborted(self.crawler, self.run_id):
return
try:
Crawl.operation_start(self.crawler, self.stage, self.run_id)
self.log.info('[%s->%s(%s)]: %s',
self.crawler.name,
self.stage.name,
self.stage.method_name,
self.run_id)
return self.stage.method(self, data)
except Exception as exc:
self.emit_exception(exc)
finally:
Crawl.operation_end(self.crawler, self.run_id)
shutil.rmtree(self.work_path)
def op_count(self):
"""Total operations performed for this stage"""
return Crawl.op_count(self.crawler, self)
def flush(self):
"""Delete all run-time data generated by this crawler."""
Queue.flush(self)
Event.delete(self)
Crawl.flush(self)
def cancel(self):
Crawl.abort_all(self)
Queue.flush(self)
def op_count(self):
"""Total operations performed for this crawler"""
return Crawl.op_count(self)