Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def launch(self, force_no_qsub=False):
"""
Method to launch a job on the grid
:raises: cluster.ClusterLaunchException if the jobid is 0 or empty
as returned by pbs.submit() method
:return: True if the job failed
"""
batch_path = self.batch_path()
outlog = self.outlog_path()
jobid, job_failed = cluster.submit_job(batch_path, outlog=outlog,
force_no_qsub=force_no_qsub)
if jobid == '' or jobid == '0':
LOGGER.error('failed to launch job on cluster')
raise ClusterLaunchException
else:
self.set_launch(jobid)
cmd = DAX_SETTINGS.get_cmd_submit()
if (force_no_qsub or not cluster.command_found(cmd)) and \
job_failed:
self.set_status(JOB_FAILED)
return True
force_no_qsub=False):
"""
Launch tasks from the passed list until the queue is full or
the list is empty
:param task_list: list of task to launch
:param writeonly: write the job files without submitting them
:param pbsdir: folder to store the pbs file
:param force_no_qsub: run the job locally on the computer (serial mode)
:return: None
"""
if force_no_qsub:
LOGGER.info('No qsub - Running job locally on your computer.')
else:
# Check number of jobs on cluster
cjobs = cluster.count_jobs()
if cjobs == -1:
LOGGER.error('cannot get count of jobs from cluster')
return
if cluster.command_found(cmd=DAX_SETTINGS.get_cmd_submit()):
LOGGER.info('%s jobs currently in queue' % str(cjobs))
# Launch until we reach cluster limit or no jobs left to launch
while (cjobs < self.queue_limit or writeonly) and len(task_list) > 0:
cur_task = task_list.pop()
if writeonly:
msg = "+Writing PBS file for job:%s, currently %s jobs in \
cluster queue"
LOGGER.info(msg % (cur_task.assessor_label,
str(cjobs)))
as returned by pbs.submit() method
:return: True if the job failed
"""
batch_path = self.batch_path()
outlog = self.outlog_path()
jobid, job_failed = cluster.submit_job(batch_path, outlog=outlog,
force_no_qsub=force_no_qsub)
if jobid == '' or jobid == '0':
LOGGER.error('failed to launch job on cluster')
raise ClusterLaunchException
else:
self.set_launch(jobid)
cmd = DAX_SETTINGS.get_cmd_submit()
if (force_no_qsub or not cluster.command_found(cmd)) and \
job_failed:
self.set_status(JOB_FAILED)
return True
:param task_list: list of task to launch
:param writeonly: write the job files without submitting them
:param pbsdir: folder to store the pbs file
:param force_no_qsub: run the job locally on the computer (serial mode)
:return: None
"""
if force_no_qsub:
LOGGER.info('No qsub - Running job locally on your computer.')
else:
# Check number of jobs on cluster
cjobs = cluster.count_jobs()
if cjobs == -1:
LOGGER.error('cannot get count of jobs from cluster')
return
if cluster.command_found(cmd=DAX_SETTINGS.get_cmd_submit()):
LOGGER.info('%s jobs currently in queue' % str(cjobs))
# Launch until we reach cluster limit or no jobs left to launch
while (cjobs < self.queue_limit or writeonly) and len(task_list) > 0:
cur_task = task_list.pop()
if writeonly:
msg = "+Writing PBS file for job:%s, currently %s jobs in \
cluster queue"
LOGGER.info(msg % (cur_task.assessor_label,
str(cjobs)))
else:
msg = '+Launching job:%s, currently %s jobs in cluster queue'
LOGGER.info(msg % (cur_task.assessor_label, str(cjobs)))
try:
"""
[memused, walltime, jobid, jobnode, jobstrdate] = self.get_job_usage()
if walltime:
if memused and jobnode:
LOGGER.debug('memused and walltime already set, skipping')
else:
if memused == '':
self.set_memused('NotFound')
if jobnode == '':
self.set_jobnode('NotFound')
return
# We can't get info from cluster if job too old
if not cluster.is_traceable_date(jobstrdate):
self.set_walltime('NotFound')
self.set_memused('NotFound')
self.set_jobnode('NotFound')
return
# Get usage with tracejob
jobinfo = cluster.tracejob_info(jobid, jobstrdate)
if jobinfo['mem_used'].strip():
self.set_memused(jobinfo['mem_used'])
else:
self.set_memused('NotFound')
if jobinfo['walltime_used'].strip():
self.set_walltime(jobinfo['walltime_used'])
else:
self.set_walltime('NotFound')
if jobinfo['jobnode'].strip():
def get_job_status(self, jobid=None):
"""
Get the status of a job given its jobid as assigned by the scheduler
:param jobid: job id assigned by the scheduler
:return: string from call to cluster.job_status or UNKNOWN.
"""
jobstatus = 'UNKNOWN'
if jobid is None:
jobid = self.get_jobid()
if jobid != '' and jobid != '0':
jobstatus = cluster.job_status(jobid)
LOGGER.debug('jobid,jobstatus='+str(jobid)+','+str(jobstatus))
return jobstatus
"""
[memused, walltime, jobid, jobnode, jobstrdate] = self.get_job_usage()
if walltime:
if memused and jobnode:
LOGGER.debug('memused and walltime already set, skipping')
else:
if memused == '':
self.set_memused('NotFound')
if jobnode == '':
self.set_jobnode('NotFound')
return
# We can't get info from cluster if job too old
if not cluster.is_traceable_date(jobstrdate):
self.set_walltime('NotFound')
self.set_memused('NotFound')
self.set_jobnode('NotFound')
return
# Get usage with tracejob
jobinfo = cluster.tracejob_info(jobid, jobstrdate)
if jobinfo['mem_used'].strip():
self.set_memused(jobinfo['mem_used'])
else:
self.set_memused('NotFound')
if jobinfo['walltime_used'].strip():
self.set_walltime(jobinfo['walltime_used'])
else:
self.set_walltime('NotFound')
if jobinfo['jobnode'].strip():
else:
if memused == '':
self.set_memused('NotFound')
if jobnode == '':
self.set_jobnode('NotFound')
return
# We can't get info from cluster if job too old
if not cluster.is_traceable_date(jobstrdate):
self.set_walltime('NotFound')
self.set_memused('NotFound')
self.set_jobnode('NotFound')
return
# Get usage with tracejob
jobinfo = cluster.tracejob_info(jobid, jobstrdate)
if jobinfo['mem_used'].strip():
self.set_memused(jobinfo['mem_used'])
else:
self.set_memused('NotFound')
if jobinfo['walltime_used'].strip():
self.set_walltime(jobinfo['walltime_used'])
else:
self.set_walltime('NotFound')
if jobinfo['jobnode'].strip():
self.set_jobnode(jobinfo['jobnode'])
else:
self.set_jobnode('NotFound')