Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
lines = []
for line in stdout.split('\n'):
if len(line) == 0:
continue
if line.startswith(self._CONTINUATION_LINE_START):
lines[-1] += line[len(self._CONTINUATION_LINE_START):]
else:
lines.append(line)
# now rebuild stdout by joining the reconstructed lines
stdout = str.join('\n', lines)
jobstatus = gc3libs.utils.Struct()
# XXX: this only works if the current status is the first one
# reported in STDOUT ...
match = LsfLrms._status_re.search(stdout)
if match:
stat = match.group('state')
jobstatus.state = LsfLrms._lsf_state_to_gc3pie_state(stat)
if stat == 'DONE':
# DONE = success
jobstatus.exit_status = 0
elif stat == 'EXIT':
# EXIT = job exited with exit code != 0
match = LsfLrms._unsuccessful_exit_re.search(stdout)
if match:
jobstatus.exit_status = int(match.group('exit_status'))
if 'state' not in jobstatus:
jobstatus.state = Run.State.UNKNOWN
return jobstatus
match = LsfLrms._RESOURCE_USAGE_RE.match(line)
if match:
# actual resource usage is on next line
rusage = lines.next()
cpu_t, wait, turnaround, status, hog_factor, mem, swap = \
rusage.split()
# common backend attrs (see Issue 78)
if 'lsf_completion_time' in data and 'lsf_start_time' in data:
data['duration'] = Duration(
data['lsf_completion_time'] - data['lsf_start_time'])
else:
# XXX: what should we use for jobs that did not run at all?
data['duration'] = Duration(0, unit=seconds)
data['used_cpu_time'] = Duration(float(cpu_t), unit=seconds)
data['max_used_memory'] = LsfLrms._parse_memspec(mem)\
+ LsfLrms._parse_memspec(swap)
# the resource usage line is the last interesting line
break
return data
def __parse_acct_output_w_bjobs(stdout):
data = dict()
# Try to parse used cputime
match = LsfLrms._cpu_time_re.search(stdout)
if match:
cpu_time = match.group('cputime')
data['used_cpu_time'] = Duration(float(cpu_time), unit=seconds)
# Parse memory usage
match = LsfLrms._mem_used_re.search(stdout)
if match:
mem_used = match.group('mem_used')
# mem_unit should always be Mbytes
data['max_used_memory'] = Memory(float(mem_used), unit=MB)
# Find submission time and completion time
lines = iter(stdout.split('\n'))
for line in lines:
match = LsfLrms._EVENT_RE.match(line)
if match:
# mem_unit should always be Mbytes
data['max_used_memory'] = Memory(float(mem_used), unit=MB)
# Find submission time and completion time
lines = iter(stdout.split('\n'))
for line in lines:
match = LsfLrms._EVENT_RE.match(line)
if match:
timestamp = line.split(': ')[0]
event = match.group('event')
if event == 'Submitted':
data['lsf_submission_time'] = \
LsfLrms._parse_timespec(timestamp)
elif event in ['Dispatched', 'Started']:
data['lsf_start_time'] = \
LsfLrms._parse_timespec(timestamp)
elif event in ['Completed', 'Done successfully']:
data['lsf_completion_time'] = \
LsfLrms._parse_timespec(timestamp)
continue
if 'lsf_completion_time' in data and 'lsf_start_time' in data:
data['duration'] = Duration(
data['lsf_completion_time'] - data['lsf_start_time'])
else:
# XXX: what should we use for jobs that did not run at all?
data['duration'] = Duration(0, unit=seconds)
return data
def __parse_acct_output_w_bjobs(stdout):
data = dict()
# Try to parse used cputime
match = LsfLrms._cpu_time_re.search(stdout)
if match:
cpu_time = match.group('cputime')
data['used_cpu_time'] = Duration(float(cpu_time), unit=seconds)
# Parse memory usage
match = LsfLrms._mem_used_re.search(stdout)
if match:
mem_used = match.group('mem_used')
# mem_unit should always be Mbytes
data['max_used_memory'] = Memory(float(mem_used), unit=MB)
# Find submission time and completion time
lines = iter(stdout.split('\n'))
for line in lines:
match = LsfLrms._EVENT_RE.match(line)
if match:
timestamp = line.split(': ')[0]
event = match.group('event')
if event == 'Submitted':
data['lsf_submission_time'] = \
LsfLrms._parse_timespec(timestamp)
elif event in ['Dispatched', 'Started']:
for line in lines:
match = LsfLrms._EVENT_RE.match(line)
if match:
timestamp = line.split(': ')[0]
event = match.group('event')
if event == 'Submitted':
data['lsf_submission_time'] = \
LsfLrms._parse_timespec(timestamp)
elif event == 'Dispatched':
data['lsf_start_time'] = \
LsfLrms._parse_timespec(timestamp)
elif event == 'Completed':
data['lsf_completion_time'] = \
LsfLrms._parse_timespec(timestamp)
continue
match = LsfLrms._RESOURCE_USAGE_RE.match(line)
if match:
# actual resource usage is on next line
rusage = lines.next()
cpu_t, wait, turnaround, status, hog_factor, mem, swap = \
rusage.split()
# common backend attrs (see Issue 78)
if 'lsf_completion_time' in data and 'lsf_start_time' in data:
data['duration'] = Duration(
data['lsf_completion_time'] - data['lsf_start_time'])
else:
# XXX: what should we use for jobs that did not run at all?
data['duration'] = Duration(0, unit=seconds)
data['used_cpu_time'] = Duration(float(cpu_t), unit=seconds)
data['max_used_memory'] = LsfLrms._parse_memspec(mem)\
+ LsfLrms._parse_memspec(swap)
# the resource usage line is the last interesting line
# now rebuild stdout by joining the reconstructed lines
stdout = str.join('\n', lines)
jobstatus = gc3libs.utils.Struct()
# XXX: this only works if the current status is the first one
# reported in STDOUT ...
match = LsfLrms._status_re.search(stdout)
if match:
stat = match.group('state')
jobstatus.state = LsfLrms._lsf_state_to_gc3pie_state(stat)
if stat == 'DONE':
# DONE = success
jobstatus.exit_status = 0
elif stat == 'EXIT':
# EXIT = job exited with exit code != 0
match = LsfLrms._unsuccessful_exit_re.search(stdout)
if match:
jobstatus.exit_status = int(match.group('exit_status'))
if 'state' not in jobstatus:
jobstatus.state = Run.State.UNKNOWN
return jobstatus
# try "with year" format first, as it has all the info we need
try:
return datetime.datetime.strptime(
ts, LsfLrms._TIMESTAMP_FMT_WITH_YEAR)
except ValueError:
pass # ignore and try again without year
try:
# XXX: since we do not have a year, we resort to the
# following heuristics: if the month in the timespec is
# less than or equal to the current month, the timestamp
# is for an event occurred during this year; if it's in a
# month later than the current one, the timestamp refers
# to an event occurred in the *past* year.
today = datetime.date.today()
# XXX: datetime.strptime() only available starting Py 2.5
tm = time.strptime(ts, LsfLrms._TIMESTAMP_FMT_NO_YEAR)
if tm[1] <= today.month:
return datetime.datetime(today.year, *(tm[1:6]))
else:
return datetime.datetime(today.year - 1, *(tm[1:6]))
except ValueError as err:
gc3libs.log.error(
"Cannot parse '%s' as an LSF timestamp: %s: %s",
ts, err.__class__.__name__, err)
raise
continue
if line.startswith(self._CONTINUATION_LINE_START):
lines[-1] += line[len(self._CONTINUATION_LINE_START):]
else:
lines.append(line)
# now rebuild stdout by joining the reconstructed lines
stdout = str.join('\n', lines)
jobstatus = gc3libs.utils.Struct()
# XXX: this only works if the current status is the first one
# reported in STDOUT ...
match = LsfLrms._status_re.search(stdout)
if match:
stat = match.group('state')
jobstatus.state = LsfLrms._lsf_state_to_gc3pie_state(stat)
if stat == 'DONE':
# DONE = success
jobstatus.exit_status = 0
elif stat == 'EXIT':
# EXIT = job exited with exit code != 0
match = LsfLrms._unsuccessful_exit_re.search(stdout)
if match:
jobstatus.exit_status = int(match.group('exit_status'))
if 'state' not in jobstatus:
jobstatus.state = Run.State.UNKNOWN
return jobstatus
def _parse_timespec(ts):
"""Parse a timestamp as it appears in LSF bjobs/bacct logs."""
# try "with year" format first, as it has all the info we need
try:
return datetime.datetime.strptime(
ts, LsfLrms._TIMESTAMP_FMT_WITH_YEAR)
except ValueError:
pass # ignore and try again without year
try:
# XXX: since we do not have a year, we resort to the
# following heuristics: if the month in the timespec is
# less than or equal to the current month, the timestamp
# is for an event occurred during this year; if it's in a
# month later than the current one, the timestamp refers
# to an event occurred in the *past* year.
today = datetime.date.today()
# XXX: datetime.strptime() only available starting Py 2.5
tm = time.strptime(ts, LsfLrms._TIMESTAMP_FMT_NO_YEAR)
if tm[1] <= today.month:
return datetime.datetime(today.year, *(tm[1:6]))
else:
return datetime.datetime(today.year - 1, *(tm[1:6]))