Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
endpoint_id: str
Usually a uuid4 as string that identifies the executor
logdir: str
Path to logdir
logging_level : int
Logging level as defined in the logging module. Default: logging.INFO (20)
"""
super().__init__()
self.logdir = logdir
os.makedirs(self.logdir, exist_ok=True)
global logger
logger = set_file_logger(os.path.join(self.logdir, "forwarder.{}.log".format(endpoint_id)),
level=logging_level)
logger.info("Initializing forwarder for endpoint:{}".format(endpoint_id))
self.task_q = task_q
self.result_q = result_q
self.executor = executor
self.endpoint_id = endpoint_id
self.internal_q = Queue()
self.client_ports = None
parser.add_argument("--scheduler_mode", default="soft",
help=("Choose the mode of scheduler "
"(hard, soft"))
parser.add_argument("-r", "--result_url", required=True,
help="REQUIRED: ZMQ url for posting results")
args = parser.parse_args()
try:
os.makedirs(os.path.join(args.logdir, args.uid))
except FileExistsError:
pass
try:
global logger
logger = set_file_logger('{}/{}/manager.log'.format(args.logdir, args.uid),
level=logging.DEBUG if args.debug is True else logging.INFO)
logger.info("Python version: {}".format(sys.version))
logger.info("Debug logging: {}".format(args.debug))
logger.info("Log dir: {}".format(args.logdir))
logger.info("Manager ID: {}".format(args.uid))
logger.info("Block ID: {}".format(args.block_id))
logger.info("cores_per_worker: {}".format(args.cores_per_worker))
logger.info("task_url: {}".format(args.task_url))
logger.info("result_url: {}".format(args.result_url))
logger.info("hb_period: {}".format(args.hb_period))
logger.info("hb_threshold: {}".format(args.hb_threshold))
logger.info("max_workers: {}".format(args.max_workers))
logger.info("poll_period: {}".format(args.poll))
logger.info("worker_mode: {}".format(args.worker_mode))
logger.info("scheduler_mode: {}".format(args.scheduler_mode))
parser.add_argument("--mode", default="singularity_reuse",
help=("Choose the mode of operation from "
"(no_container, singularity_reuse, singularity_single_use"))
parser.add_argument("-r", "--result_url", required=True,
help="REQUIRED: ZMQ url for posting results")
args = parser.parse_args()
try:
os.makedirs(os.path.join(args.logdir, args.uid))
except FileExistsError:
pass
try:
global logger
logger = set_file_logger('{}/{}/manager.log'.format(args.logdir, args.uid),
level=logging.DEBUG if args.debug is True else logging.INFO)
logger.info("Python version: {}".format(sys.version))
logger.info("Debug logging: {}".format(args.debug))
logger.info("Log dir: {}".format(args.logdir))
logger.info("Manager ID: {}".format(args.uid))
logger.info("Block ID: {}".format(args.block_id))
logger.info("cores_per_worker: {}".format(args.cores_per_worker))
logger.info("task_url: {}".format(args.task_url))
logger.info("result_url: {}".format(args.result_url))
logger.info("hb_period: {}".format(args.hb_period))
logger.info("hb_threshold: {}".format(args.hb_threshold))
logger.info("max_workers: {}".format(args.max_workers))
logger.info("poll_period: {}".format(args.poll))
logger.info("mode: {}".format(args.mode))
logger.info("container_image: {}".format(args.container_image))
def executor_starter(htex, logdir, endpoint_id, logging_level=logging.DEBUG):
from funcx import set_file_logger
stdout = open(os.path.join(logdir, "executor.{}.stdout".format(endpoint_id)), 'w')
stderr = open(os.path.join(logdir, "executor.{}.stderr".format(endpoint_id)), 'w')
logdir = os.path.abspath(logdir)
with daemon.DaemonContext(stdout=stdout, stderr=stderr):
global logger
print("cwd: ", os.getcwd())
logger = set_file_logger(os.path.join(logdir, "executor.{}.log".format(endpoint_id)),
level=logging_level)
htex.start()
stdout.close()
stderr.close()
def executor_starter(htex, logdir, endpoint_id, logging_level=logging.DEBUG):
from funcx import set_file_logger
stdout = open(os.path.join(logdir, "executor.{}.stdout".format(endpoint_id)), 'w')
stderr = open(os.path.join(logdir, "executor.{}.stderr".format(endpoint_id)), 'w')
logdir = os.path.abspath(logdir)
with daemon.DaemonContext(stdout=stdout, stderr=stderr):
global logger
print("cwd: ", os.getcwd())
logger = set_file_logger(os.path.join(logdir, "executor.{}.log".format(endpoint_id)),
level=logging_level)
htex.start()
stdout.close()
stderr.close()
def __init__(self, worker_id, address, port, logdir, debug=False, worker_type='RAW'):
self.worker_id = worker_id
self.address = address
self.port = port
self.logdir = logdir
self.debug = debug
self.worker_type = worker_type
self.serializer = FuncXSerializer()
self.serialize = self.serializer.serialize
self.deserialize = self.serializer.deserialize
global logger
logger = set_file_logger('{}/funcx_worker_{}.log'.format(logdir, worker_id),
name="worker_log",
level=logging.DEBUG if debug else logging.INFO)
logger.info('Initializing worker {}'.format(worker_id))
logger.info('Worker is of type: {}'.format(worker_type))
if debug:
logger.debug('Debug logging enabled')
self.context = zmq.Context()
self.poller = zmq.Poller()
self.identity = worker_id.encode()
self.task_socket = self.context.socket(zmq.DEALER)
self.task_socket.setsockopt(zmq.IDENTITY, self.identity)