Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Run with Executor --------------------------------------------------
task = exctr.submit(calc_type='sim', machinefile=machinefilename,
stdout=outfile, stderr=errfile, hyperthreads=True)
while(not task.finished):
time.sleep(0.2)
task.poll()
task_states.append(task.state)
H_o['f'][i] = six_hump_camel_func(x)
# v = np.random.uniform(0, 10)
# print('About to sleep for :' + str(v))
# time.sleep(v)
calc_status = UNSET_TAG # Returns to worker
if all(t == 'FINISHED' for t in task_states):
calc_status = WORKER_DONE
elif any(t == 'FAILED' for t in task_states):
calc_status = TASK_FAILED
return H_o, persis_info, calc_status
elif calc_type == EVAL_SIM_TAG and not self.loc_stack:
self.loc_stack = Worker._make_sim_worker_dir(self.sim_specs, self.workerID)
with self.loc_stack.loc(calc_type):
out = calc(calc_in, Work['persis_info'], Work['libE_info'])
else:
out = calc(calc_in, Work['persis_info'], Work['libE_info'])
logger.debug("Return from calc call")
assert isinstance(out, tuple), \
"Calculation output must be a tuple."
assert len(out) >= 2, \
"Calculation output must be at least two elements."
calc_status = out[2] if len(out) >= 3 else UNSET_TAG
return out[0], out[1], calc_status
except Exception:
logger.debug("Re-raising exception from calc")
calc_status = CALC_EXCEPTION
raise
finally:
# This was meant to be handled by calc_stats module.
if job_timing and JobController.controller.list_of_jobs:
# Initially supporting one per calc. One line output.
job = JobController.controller.list_of_jobs[-1]
calc_msg = "Calc {:5d}: {} {} {} Status: {}".\
format(calc_id,
calc_type_strings[calc_type],
timer,
job.timer,
calc_status_strings.get(calc_status, "Completed"))
with timer:
logger.debug("Calling calc {}".format(calc_type))
if calc_type == EVAL_SIM_TAG:
out = self._determine_dir_then_calc(Work, calc_type, calc_in, calc)
else:
out = calc(calc_in, Work['persis_info'], Work['libE_info'])
logger.debug("Return from calc call")
assert isinstance(out, tuple), \
"Calculation output must be a tuple."
assert len(out) >= 2, \
"Calculation output must be at least two elements."
calc_status = out[2] if len(out) >= 3 else UNSET_TAG
# Check for buffered receive
if self.comm.recv_buffer:
tag, message = self.comm.recv()
if tag in [STOP_TAG, PERSIS_STOP]:
if message is MAN_SIGNAL_FINISH:
calc_status = MAN_SIGNAL_FINISH
return out[0], out[1], calc_status
except Exception:
logger.debug("Re-raising exception from calc")
calc_status = CALC_EXCEPTION
raise
finally:
# This was meant to be handled by calc_stats module.
if task_timing and Executor.executor.list_of_tasks:
print('Task {} still running on worker {} ....'.format(task.id, exctr.workerID))
# Check output file for error
if task.stdout_exists():
if 'Error' in task.read_stdout():
print("Found (deliberate) Error in ouput file - cancelling "
"task {} on worker {}".format(task.id, exctr.workerID))
exctr.kill(task)
calc_status = WORKER_KILL_ON_ERR
break
# After exiting loop
if task.finished:
print('Task {} done on worker {}'.format(task.id, exctr.workerID))
# Fill in calc_status if not already
if calc_status == UNSET_TAG:
if task.state == 'FINISHED': # Means finished succesfully
calc_status = WORKER_DONE
elif task.state == 'FAILED':
calc_status = TASK_FAILED
else:
print("Task {} timed out - killing on worker {}".format(task.id, exctr.workerID))
exctr.kill(task)
if task.finished:
print('Task {} done on worker {}'.format(task.id, exctr.workerID))
calc_status = WORKER_KILL_ON_TIMEOUT
return task, calc_status
def polling_loop(comm, exctr, task, timeout_sec=3.0, delay=0.3):
import time
calc_status = UNSET_TAG # Sim func determines status of libensemble calc - returned to worker
while task.runtime < timeout_sec:
time.sleep(delay)
# print('Probing manager at time: ', task.runtime)
exctr.manager_poll(comm)
if exctr.manager_signal == 'finish':
exctr.kill(task)
calc_status = MAN_SIGNAL_FINISH # Worker will pick this up and close down
print('Task {} killed by manager on worker {}'.format(task.id, exctr.workerID))
break
# print('Polling task at time', task.runtime)
task.poll()
if task.finished:
break
def send_mgr_worker_msg(comm, O):
"""Send message from worker to manager.
"""
D = {'calc_out': O,
'libE_info': {'persistent': True},
'calc_status': UNSET_TAG,
'calc_type': EVAL_GEN_TAG
}
comm.send(EVAL_GEN_TAG, D)
# Check output file for error
# print('Checking output file for error at time:', task.runtime)
if task.stdout_exists():
if 'Error' in task.read_stdout():
print("Found (deliberate) Error in ouput file - cancelling "
"task {} on worker {}".format(task.id, exctr.workerID))
exctr.kill(task)
calc_status = WORKER_KILL_ON_ERR
break
# After exiting loop
if task.finished:
print('Task {} done on worker {}'.format(task.id, exctr.workerID))
# Fill in calc_status if not already
if calc_status == UNSET_TAG:
if task.state == 'FINISHED': # Means finished succesfully
calc_status = WORKER_DONE
elif task.state == 'FAILED':
calc_status = TASK_FAILED
# elif task.state == 'USER_KILLED':
# calc_status = WORKER_KILL
else:
# assert task.state == 'RUNNING', "task.state expected to be RUNNING. Returned: " + str(task.state)
print("Task {} timed out - killing on worker {}".format(task.id, exctr.workerID))
exctr.kill(task)
if task.finished:
print('Task {} done on worker {}'.format(task.id, exctr.workerID))
calc_status = WORKER_KILL_ON_TIMEOUT
return task, calc_status
print('Job {} still running on worker {} ....'.format(job.id, jobctl.workerID))
# Check output file for error
# print('Checking output file for error at time:', job.runtime)
if job.stdout_exists():
if 'Error' in job.read_stdout():
print("Found (deliberate) Error in ouput file - cancelling job {} on worker {}".format(job.id, jobctl.workerID))
jobctl.kill(job)
calc_status = WORKER_KILL_ON_ERR
break
# After exiting loop
if job.finished:
print('Job {} done on worker {}'.format(job.id, jobctl.workerID))
# Fill in calc_status if not already
if calc_status == UNSET_TAG:
if job.state == 'FINISHED': # Means finished succesfully
calc_status = WORKER_DONE
elif job.state == 'FAILED':
calc_status = JOB_FAILED
# elif job.state == 'USER_KILLED':
# calc_status = WORKER_KILL
else:
# assert job.state == 'RUNNING', "job.state expected to be RUNNING. Returned: " + str(job.state)
print("Job {} timed out - killing on worker {}".format(job.id, jobctl.workerID))
jobctl.kill(job)
if job.finished:
print('Job {} done on worker {}'.format(job.id, jobctl.workerID))
calc_status = WORKER_KILL_ON_TIMEOUT
return job, calc_status
def polling_loop(comm, exctr, task, timeout_sec=3.0, delay=0.3):
import time
calc_status = UNSET_TAG # Sim func determines status of libensemble calc - returned to worker
while task.runtime < timeout_sec:
time.sleep(delay)
# print('Probing manager at time: ', task.runtime)
exctr.manager_poll(comm)
if exctr.manager_signal == 'finish':
exctr.kill(task)
calc_status = MAN_SIGNAL_FINISH # Worker will pick this up and close down
print('Task {} killed by manager on worker {}'.format(task.id, exctr.workerID))
break
task.poll()
if task.finished:
break
elif task.state == 'RUNNING':
def _check_received_calc(D_recv):
"Checks the type and status fields on a receive calculation"
calc_type = D_recv['calc_type']
calc_status = D_recv['calc_status']
assert calc_type in [EVAL_SIM_TAG, EVAL_GEN_TAG], \
"Aborting, Unknown calculation type received. " \
"Received type: {}".format(calc_type)
assert calc_status in [FINISHED_PERSISTENT_SIM_TAG,
FINISHED_PERSISTENT_GEN_TAG,
UNSET_TAG,
PERSIS_STOP,
MAN_SIGNAL_FINISH,
MAN_SIGNAL_KILL,
WORKER_KILL_ON_ERR,
WORKER_KILL_ON_TIMEOUT,
WORKER_KILL,
TASK_FAILED,
WORKER_DONE], \
"Aborting: Unknown calculation status received. " \
"Received status: {}".format(calc_status)