Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def master_bulk_resubmit(self, rjobs):
'''GLITE bulk resubmission'''
from GangaCore.Utility.logging import log_user_exception
job = self.getJobObject()
# compose master JDL for collection job
node_jdls = []
for sj in rjobs:
jdlpath = os.path.join(sj.inputdir, '__jdlfile__')
node_jdls.append(jdlpath)
if config['MatchBeforeSubmit']:
matches = Grid.list_match(node_jdls[-1], self.credential_requirements, ce=self.CE)
if not matches:
self.__print_no_resource_error__(node_jdls[-1])
return False
max_node = config['GliteBulkJobSize']
results = self.__mt_bulk_submit__(node_jdls, max_node=max_node)
status = False
if results:
offsets = sorted(results.keys())
self.__refresh_jobinfo__(job)
self.id = []
self.status = {}
for ibeg in offsets:
def resubmit(self):
'''Resubmit the job'''
job = self.getJobObject()
jdlpath = job.getInputWorkspace().getPath("__jdlfile__")
# fix for savannah 76289, update the excludedCEs in jdl file
self.updateExcudedCEsInJdl(jdlpath)
if config['MatchBeforeSubmit']:
matches = Grid.list_match(jdlpath, self.credential_requirements, ce=self.CE)
if not matches:
self.__print_no_resource_error__(jdlpath)
return None
self.id = Grid.submit(jdlpath, self.credential_requirements, ce=self.CE, perusable=self.perusable)
self.parent_id = self.id
if self.id:
# refresh the lcg job information
self.__refresh_jobinfo__(job)
return not self.id is None
def submit(self, subjobconfig, master_job_sandbox):
'''Submit the job to the grid'''
jdlpath = self.preparejob(subjobconfig, master_job_sandbox)
if config['MatchBeforeSubmit']:
matches = Grid.list_match(jdlpath, self.credential_requirements, ce=self.CE)
if not matches:
self.__print_no_resource_error__(jdlpath)
return None
self.id = Grid.submit(jdlpath, self.credential_requirements, ce=self.CE, perusable=self.perusable)
self.parent_id = self.id
return not self.id is None
app, appmasterconfig)
# prepare the subjobs with the runtime handler
jobsubconfig = [rtHandler.prepare(j.application, s, appmasterconfig, jobmasterconfig) for (
j, s) in zip(rjobs, appsubconfig)]
# prepare masterjob's inputsandbox
master_input_sandbox = self.master_prepare(jobmasterconfig)
# prepare JDL
jdlpath = self.preparejob(
jobsubconfig[0], master_input_sandbox)
logger.debug('JDL used for match-making: %s' % jdlpath)
matches = Grid.list_match(jdlpath, self.credential_requirements, ce=self.CE)
except Exception as x:
logger.warning('job match failed: %s', str(x))
# clean up the job's inputdir
if doPrepareEmulation:
logger.debug('clean up job inputdir')
files = os.listdir(job.inputdir)
for f in files:
if f not in existing_files:
os.remove(os.path.join(job.inputdir, f))
return matches
profiler.start()
assert(implies(rjobs, len(subjobconfigs) == len(rjobs)))
# prepare the subjobs, jdl repository before bulk submission
node_jdls = self.__mt_job_prepare__(
rjobs, subjobconfigs, masterjobconfig)
if not node_jdls:
logger.error('Some jobs not successfully prepared')
return False
profiler.checkAndStart('job preparation elapsed time')
if config['MatchBeforeSubmit']:
matches = Grid.list_match(node_jdls[-1], self.credential_requirements, ce=self.CE)
if not matches:
self.__print_no_resource_error__(node_jdls[-1])
return False
profiler.checkAndStart('job list-match elapsed time')
# set all subjobs to submitting status
for sj in rjobs:
sj.updateStatus('submitting')
profiler.checkAndStart(
'job state transition (submitting) elapsed time')
max_node = config['GliteBulkJobSize']
results = self.__mt_bulk_submit__(node_jdls, max_node=max_node)
jdl_file2 = jdl_file
if not jdl_file:
# create a dummy jdl file from the given requirements
import tempfile
jdl = {'VirtualOrganisation': config['VirtualOrganisation'],
'Executable': os.path.basename(__file__),
'InputSandbox': [__file__],
'Requirements': self.requirements.convert()}
jdl_file_txt = Grid.expandjdl(jdl)
jdl_file2 = tempfile.mktemp('.jdl')
with open(jdl_file2, 'w') as file_:
file_.write(jdl_file_txt)
matches = Grid.list_match(jdl_file2, self.credential_requirements, ce=spec_ce)
# clean up
if not jdl_file:
os.remove(jdl_file2)
return matches