Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def setUpClass(cls):
if os.environ.get('USE_UVLOOP'):
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
cls.loop = loop
def handle(handle_func):
log.log("entering handle")
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
format_def = os.environ.get(constants.FN_FORMAT)
lsnr = os.environ.get(constants.FN_LISTENER)
log.log("format: {0}".format(format_def))
if format_def == constants.HTTPSTREAM:
if lsnr is None:
log.log("{0} is not set".format(constants.FN_LISTENER))
sys.exit(1)
log.log("{0} is set, value: {1}".
format(constants.FN_LISTENER, lsnr))
http_stream.start(handle_func, lsnr, loop=loop)
else:
log.log("incompatible function format!")
print("incompatible function format!", file=sys.stderr, flush=True)
def parse_args():
parser = argparse.ArgumentParser(description="Wallabag-Kindle-Consumer")
parser.add_argument("--cfg", help="config file", required=False)
parser.add_argument("--env", help="Read config from env", action="store_true")
parser.add_argument("--refresher", help="Start token refresher", action="store_true")
parser.add_argument("--interface", help="Start web interface", action="store_true")
parser.add_argument("--consumer", help="Start article consumer", action="store_true")
parser.add_argument("--create_db", help="Try to create the db", action="store_true")
parser.add_argument("--debug", help="Enable debug logging", action="store_true")
return parser.parse_args()
if __name__ == "__main__":
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
args = parse_args()
level = logbook.INFO
if args.debug:
level = logbook.DEBUG
StreamHandler(sys.stdout, level=level).push_application()
config = Config.from_file("config.ini")
if 'cfg' in args and args.cfg is not None:
new = Config.from_file(args.cfg)
if new is not None:
def main():
try:
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
asyncio.get_event_loop().run_until_complete(_main(int(os.environ.get('PORT', '5000'))))
asyncio.get_event_loop().run_forever()
except KeyboardInterrupt:
print('terminating...')
from acrawler.http import Request
from acrawler.item import DefaultItem
from acrawler.middleware import middleware
from acrawler.scheduler import RedisDupefilter, RedisPQ, Scheduler
from acrawler.task import SpecialTask, Task
from acrawler.utils import (
config_from_setting,
merge_config,
sync_coroutine,
to_asyncgen,
)
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
except ImportError:
pass
_Config = Dict[str, Any]
_Response = acrawler.http.Response
logger = logging.getLogger(__name__)
class Worker:
"""Worker execute the task.
One :class:`Crawler` will create many workers. A worker will
- calls its scheduler's methods
- implements task's retry mechanism
def get_running_loop(loop=None, force_fresh=False):
#if not force_fresh and loop and not loop.is_closed() or loop.is_running():
# return loop
try:
import uvloop
except ImportError:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
else:
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
if sys.platform == 'win32':
if not force_fresh and isinstance(asyncio.get_event_loop(), asyncio.ProactorEventLoop) and not asyncio.get_event_loop().is_closed():
return asyncio.get_event_loop()
return asyncio.ProactorEventLoop()
if force_fresh or asyncio.get_event_loop().is_closed():
return asyncio.new_event_loop()
return asyncio.get_event_loop()
def main():
import sys
import asyncio
if sys.implementation.name == 'cpython':
try:
import uvloop
except ImportError:
pass
else:
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
try:
# Python 3.7+
asyncio.run(amain())
except (AttributeError):
loop.run_until_complete(amain())
except KeyboardInterrupt:
pass
finally:
loop.close()
args = parser.parse_args()
latest = args.latest
limit_entities = int(args.limit_entities)
limit_target = int(args.limit_target)
orion = args.orion
timeout = int(args.timeout)
if 'path' in args:
path = args.path
if 'service' in args:
service = args.service
logger, logger_req = setup_logger()
set_event_loop_policy(EventLoopPolicy())
res = setup_stations_config(args.config)
stations = setup_stations(res, args.station_file)
reply_status()
while True:
res = collect()
if res:
res = run(prepare_schema(res))
run(post(res))
if timeout == -1:
break
else:
logger.debug('Sleeping for the %s seconds', timeout)
sleep(timeout)
def enable_uvloop(): # pragma: no cover
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
except ImportError:
return False
else:
return True
def listen(self, port=8000, host="127.0.0.1", workers=multiprocessing.cpu_count()):
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
pid = os.getpid()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setblocking(False)
sock.bind((host, port))
os.set_inheritable(sock.fileno(), True)
try:
print(f'[{pid}] Listening at: http://{host}:{port}')
print(f'[{pid}] Workers: {workers}')
for _ in range(workers):
worker = multiprocessing.Process(target=self.serve, kwargs=dict(sock=sock))
worker.daemon = True
worker.start()
print(f'[{pid}] Starting worker with pid: {worker.pid}')