How to use the comiccrawler.core.__init__.EpisodeList function in comiccrawler

To help you get started, we’ve selected a few comiccrawler examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github eight04 / ComicCrawler / comiccrawler / core / __init__.py View on Github external
def analyze_pages(self):
		"""Crawl for each pages"""
		url = self.mission.url
		old_eps = EpisodeList(self.mission.episodes or ())
		new_eps = EpisodeList()
		
		while True:
			try:
				eps = self.mission.module.get_episodes(self.html, url)
			except SkipPageError:
				pass
			else:
				if not eps:
					print("Warning: get_episodes returns an empty list")
				self.transform_title(eps)
				
				eps = EpisodeList(eps)
				
				# add result episodes into new_eps in new to old order.
				for ep in reversed(eps):
github eight04 / ComicCrawler / comiccrawler / core / __init__.py View on Github external
"""Crawl for each pages"""
		url = self.mission.url
		old_eps = EpisodeList(self.mission.episodes or ())
		new_eps = EpisodeList()
		
		while True:
			try:
				eps = self.mission.module.get_episodes(self.html, url)
			except SkipPageError:
				pass
			else:
				if not eps:
					print("Warning: get_episodes returns an empty list")
				self.transform_title(eps)
				
				eps = EpisodeList(eps)
				
				# add result episodes into new_eps in new to old order.
				for ep in reversed(eps):
					new_eps.add(ep)
					
				# FIXME: do we really need this check?
				# one-time mission?
				if self.is_onetime(new_eps):
					break
					
				# duplicate with old_eps
				if any(e in old_eps for e in eps):
					break
				
			# get next page
			next_url = self.get_next_page(self.html, url)
github eight04 / ComicCrawler / comiccrawler / core / __init__.py View on Github external
def analyze_pages(self):
		"""Crawl for each pages"""
		url = self.mission.url
		old_eps = EpisodeList(self.mission.episodes or ())
		new_eps = EpisodeList()
		
		while True:
			try:
				eps = self.mission.module.get_episodes(self.html, url)
			except SkipPageError:
				pass
			else:
				if not eps:
					print("Warning: get_episodes returns an empty list")
				self.transform_title(eps)
				
				eps = EpisodeList(eps)
				
				# add result episodes into new_eps in new to old order.
				for ep in reversed(eps):
					new_eps.add(ep)