How to use the gc3pie.gc3libs.backends.lsf.LsfLrms function in gc3pie

To help you get started, we’ve selected a few gc3pie examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
lines = []
        for line in stdout.split('\n'):
            if len(line) == 0:
                continue
            if line.startswith(self._CONTINUATION_LINE_START):
                lines[-1] += line[len(self._CONTINUATION_LINE_START):]
            else:
                lines.append(line)

        # now rebuild stdout by joining the reconstructed lines
        stdout = str.join('\n', lines)

        jobstatus = gc3libs.utils.Struct()
        # XXX: this only works if the current status is the first one
        # reported in STDOUT ...
        match = LsfLrms._status_re.search(stdout)
        if match:
            stat = match.group('state')
            jobstatus.state = LsfLrms._lsf_state_to_gc3pie_state(stat)
            if stat == 'DONE':
                # DONE = success
                jobstatus.exit_status = 0
            elif stat == 'EXIT':
                # EXIT = job exited with exit code != 0
                match = LsfLrms._unsuccessful_exit_re.search(stdout)
                if match:
                    jobstatus.exit_status = int(match.group('exit_status'))

        if 'state' not in jobstatus:
            jobstatus.state = Run.State.UNKNOWN
        return jobstatus
github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
match = LsfLrms._RESOURCE_USAGE_RE.match(line)
            if match:
                # actual resource usage is on next line
                rusage = lines.next()
                cpu_t, wait, turnaround, status, hog_factor, mem, swap = \
                    rusage.split()
                # common backend attrs (see Issue 78)
                if 'lsf_completion_time' in data and 'lsf_start_time' in data:
                    data['duration'] = Duration(
                        data['lsf_completion_time'] - data['lsf_start_time'])
                else:
                    # XXX: what should we use for jobs that did not run at all?
                    data['duration'] = Duration(0, unit=seconds)
                data['used_cpu_time'] = Duration(float(cpu_t), unit=seconds)
                data['max_used_memory'] = LsfLrms._parse_memspec(mem)\
                    + LsfLrms._parse_memspec(swap)
                # the resource usage line is the last interesting line
                break
        return data
github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
def __parse_acct_output_w_bjobs(stdout):
        data = dict()

        # Try to parse used cputime
        match = LsfLrms._cpu_time_re.search(stdout)
        if match:
            cpu_time = match.group('cputime')
            data['used_cpu_time'] = Duration(float(cpu_time), unit=seconds)

        # Parse memory usage
        match = LsfLrms._mem_used_re.search(stdout)
        if match:
            mem_used = match.group('mem_used')
            # mem_unit should always be Mbytes
            data['max_used_memory'] = Memory(float(mem_used), unit=MB)

        # Find submission time and completion time
        lines = iter(stdout.split('\n'))
        for line in lines:
            match = LsfLrms._EVENT_RE.match(line)
            if match:
github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
# mem_unit should always be Mbytes
            data['max_used_memory'] = Memory(float(mem_used), unit=MB)

        # Find submission time and completion time
        lines = iter(stdout.split('\n'))
        for line in lines:
            match = LsfLrms._EVENT_RE.match(line)
            if match:
                timestamp = line.split(': ')[0]
                event = match.group('event')
                if event == 'Submitted':
                    data['lsf_submission_time'] = \
                        LsfLrms._parse_timespec(timestamp)
                elif event in ['Dispatched', 'Started']:
                    data['lsf_start_time'] = \
                        LsfLrms._parse_timespec(timestamp)
                elif event in ['Completed', 'Done successfully']:
                    data['lsf_completion_time'] = \
                        LsfLrms._parse_timespec(timestamp)
                continue
        if 'lsf_completion_time' in data and 'lsf_start_time' in data:
            data['duration'] = Duration(
                data['lsf_completion_time'] - data['lsf_start_time'])
        else:
            # XXX: what should we use for jobs that did not run at all?
            data['duration'] = Duration(0, unit=seconds)

        return data
github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
def __parse_acct_output_w_bjobs(stdout):
        data = dict()

        # Try to parse used cputime
        match = LsfLrms._cpu_time_re.search(stdout)
        if match:
            cpu_time = match.group('cputime')
            data['used_cpu_time'] = Duration(float(cpu_time), unit=seconds)

        # Parse memory usage
        match = LsfLrms._mem_used_re.search(stdout)
        if match:
            mem_used = match.group('mem_used')
            # mem_unit should always be Mbytes
            data['max_used_memory'] = Memory(float(mem_used), unit=MB)

        # Find submission time and completion time
        lines = iter(stdout.split('\n'))
        for line in lines:
            match = LsfLrms._EVENT_RE.match(line)
            if match:
                timestamp = line.split(': ')[0]
                event = match.group('event')
                if event == 'Submitted':
                    data['lsf_submission_time'] = \
                        LsfLrms._parse_timespec(timestamp)
                elif event in ['Dispatched', 'Started']:
github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
for line in lines:
            match = LsfLrms._EVENT_RE.match(line)
            if match:
                timestamp = line.split(': ')[0]
                event = match.group('event')
                if event == 'Submitted':
                    data['lsf_submission_time'] = \
                        LsfLrms._parse_timespec(timestamp)
                elif event == 'Dispatched':
                    data['lsf_start_time'] = \
                        LsfLrms._parse_timespec(timestamp)
                elif event == 'Completed':
                    data['lsf_completion_time'] = \
                        LsfLrms._parse_timespec(timestamp)
                continue
            match = LsfLrms._RESOURCE_USAGE_RE.match(line)
            if match:
                # actual resource usage is on next line
                rusage = lines.next()
                cpu_t, wait, turnaround, status, hog_factor, mem, swap = \
                    rusage.split()
                # common backend attrs (see Issue 78)
                if 'lsf_completion_time' in data and 'lsf_start_time' in data:
                    data['duration'] = Duration(
                        data['lsf_completion_time'] - data['lsf_start_time'])
                else:
                    # XXX: what should we use for jobs that did not run at all?
                    data['duration'] = Duration(0, unit=seconds)
                data['used_cpu_time'] = Duration(float(cpu_t), unit=seconds)
                data['max_used_memory'] = LsfLrms._parse_memspec(mem)\
                    + LsfLrms._parse_memspec(swap)
                # the resource usage line is the last interesting line
github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
# now rebuild stdout by joining the reconstructed lines
        stdout = str.join('\n', lines)

        jobstatus = gc3libs.utils.Struct()
        # XXX: this only works if the current status is the first one
        # reported in STDOUT ...
        match = LsfLrms._status_re.search(stdout)
        if match:
            stat = match.group('state')
            jobstatus.state = LsfLrms._lsf_state_to_gc3pie_state(stat)
            if stat == 'DONE':
                # DONE = success
                jobstatus.exit_status = 0
            elif stat == 'EXIT':
                # EXIT = job exited with exit code != 0
                match = LsfLrms._unsuccessful_exit_re.search(stdout)
                if match:
                    jobstatus.exit_status = int(match.group('exit_status'))

        if 'state' not in jobstatus:
            jobstatus.state = Run.State.UNKNOWN
        return jobstatus
github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
# try "with year" format first, as it has all the info we need
        try:
            return datetime.datetime.strptime(
                ts, LsfLrms._TIMESTAMP_FMT_WITH_YEAR)
        except ValueError:
            pass  # ignore and try again without year
        try:
            # XXX: since we do not have a year, we resort to the
            # following heuristics: if the month in the timespec is
            # less than or equal to the current month, the timestamp
            # is for an event occurred during this year; if it's in a
            # month later than the current one, the timestamp refers
            # to an event occurred in the *past* year.
            today = datetime.date.today()
            # XXX: datetime.strptime() only available starting Py 2.5
            tm = time.strptime(ts, LsfLrms._TIMESTAMP_FMT_NO_YEAR)
            if tm[1] <= today.month:
                return datetime.datetime(today.year, *(tm[1:6]))
            else:
                return datetime.datetime(today.year - 1, *(tm[1:6]))
        except ValueError as err:
            gc3libs.log.error(
                "Cannot parse '%s' as an LSF timestamp: %s: %s",
                ts, err.__class__.__name__, err)
            raise
github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
continue
            if line.startswith(self._CONTINUATION_LINE_START):
                lines[-1] += line[len(self._CONTINUATION_LINE_START):]
            else:
                lines.append(line)

        # now rebuild stdout by joining the reconstructed lines
        stdout = str.join('\n', lines)

        jobstatus = gc3libs.utils.Struct()
        # XXX: this only works if the current status is the first one
        # reported in STDOUT ...
        match = LsfLrms._status_re.search(stdout)
        if match:
            stat = match.group('state')
            jobstatus.state = LsfLrms._lsf_state_to_gc3pie_state(stat)
            if stat == 'DONE':
                # DONE = success
                jobstatus.exit_status = 0
            elif stat == 'EXIT':
                # EXIT = job exited with exit code != 0
                match = LsfLrms._unsuccessful_exit_re.search(stdout)
                if match:
                    jobstatus.exit_status = int(match.group('exit_status'))

        if 'state' not in jobstatus:
            jobstatus.state = Run.State.UNKNOWN
        return jobstatus
github gc3pie / gc3pie / gc3pie / gc3libs / backends / lsf.py View on Github external
def _parse_timespec(ts):
        """Parse a timestamp as it appears in LSF bjobs/bacct logs."""
        # try "with year" format first, as it has all the info we need
        try:
            return datetime.datetime.strptime(
                ts, LsfLrms._TIMESTAMP_FMT_WITH_YEAR)
        except ValueError:
            pass  # ignore and try again without year
        try:
            # XXX: since we do not have a year, we resort to the
            # following heuristics: if the month in the timespec is
            # less than or equal to the current month, the timestamp
            # is for an event occurred during this year; if it's in a
            # month later than the current one, the timestamp refers
            # to an event occurred in the *past* year.
            today = datetime.date.today()
            # XXX: datetime.strptime() only available starting Py 2.5
            tm = time.strptime(ts, LsfLrms._TIMESTAMP_FMT_NO_YEAR)
            if tm[1] <= today.month:
                return datetime.datetime(today.year, *(tm[1:6]))
            else:
                return datetime.datetime(today.year - 1, *(tm[1:6]))