Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def cleanup():
yield
clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
clean_dir('/tmp/file_urls')
clean_dir('/tmp/WSP')
def cleanup():
yield
clean_dir('/tmp/last_runs/')
def remove_generated_files(package_location):
clean_dir()
clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
_, dirs, files = next(os.walk(package_location))
for dir_name in dirs:
clean_dir(os.path.join(package_location, dir_name))
for file_name in files:
if not file_name.endswith('.zip'):
os.unlink(os.path.join(package_location, file_name))
def remove_generated_files(package_location):
clean_dir()
clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
_, dirs, files = next(os.walk(package_location))
for dir_name in dirs:
clean_dir(os.path.join(package_location, dir_name))
for file_name in files:
if not file_name.endswith('.zip'):
os.unlink(os.path.join(package_location, file_name))
crawler = Crawler(spidercls=arxiv_spider.ArxivSpider)
spider = arxiv_spider.ArxivSpider.from_crawler(crawler)
fake_response = fake_response_from_file(
'arxiv/sample_arxiv_record0.xml',
response_type=TextResponse,
)
test_selectors = fake_response.xpath('.//record')
parsed_items = [spider.parse_record(sel) for sel in test_selectors]
pipeline = InspireCeleryPushPipeline()
pipeline.open_spider(spider)
yield [_get_record_from_processed_item(parsed_item, spider) for parsed_item in parsed_items]
clean_dir()
return crawl_result['record']
crawler = Crawler(spidercls=crossref_spider.CrossrefSpider)
spider = crossref_spider.CrossrefSpider.from_crawler(crawler, 'fakedoi')
fake_response = fake_response_from_file(
'crossref/sample_crossref_record.json',
response_type=TextResponse,
)
parsed_items = spider.parse(fake_response)
pipeline = InspireCeleryPushPipeline()
pipeline.open_spider(spider)
yield _get_record_from_processed_item(parsed_items, spider)
clean_dir()
fake_response = fake_response_from_file(
'arxiv/sample_arxiv_record.xml',
response_type=TextResponse,
)
test_selectors = fake_response.xpath('.//record')
parsed_items = [spider.parse_record(sel) for sel in test_selectors]
pipeline = InspireCeleryPushPipeline()
pipeline.open_spider(spider)
yield [
_get_processed_record(parsed_item, spider)
for parsed_item in parsed_items
]
clean_dir()
def cleanup():
yield
clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
clean_dir('/tmp/file_urls')
clean_dir('/tmp/WSP')
def cleanup():
clean_dir()
clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
yield
clean_dir()
clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
def cleanup():
clean_dir()
clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))
yield
clean_dir()
clean_dir(path=os.path.join(os.getcwd(), '.scrapy'))