How to use the dax.cluster function in dax

To help you get started, we’ve selected a few dax examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github VUIIS / dax / dax / task.py View on Github external
def launch(self, force_no_qsub=False):
        """
        Method to launch a job on the grid

        :raises: cluster.ClusterLaunchException if the jobid is 0 or empty
         as returned by pbs.submit() method
        :return: True if the job failed

        """
        batch_path = self.batch_path()
        outlog = self.outlog_path()
        jobid, job_failed = cluster.submit_job(batch_path, outlog=outlog,
                                               force_no_qsub=force_no_qsub)

        if jobid == '' or jobid == '0':
            LOGGER.error('failed to launch job on cluster')
            raise ClusterLaunchException
        else:
            self.set_launch(jobid)
            cmd = DAX_SETTINGS.get_cmd_submit()
            if (force_no_qsub or not cluster.command_found(cmd)) and \
               job_failed:
                self.set_status(JOB_FAILED)
            return True
github VUIIS / dax / dax / launcher.py View on Github external
force_no_qsub=False):
        """
        Launch tasks from the passed list until the queue is full or
         the list is empty

        :param task_list: list of task to launch
        :param writeonly: write the job files without submitting them
        :param pbsdir: folder to store the pbs file
        :param force_no_qsub: run the job locally on the computer (serial mode)
        :return: None
        """
        if force_no_qsub:
            LOGGER.info('No qsub - Running job locally on your computer.')
        else:
            # Check number of jobs on cluster
            cjobs = cluster.count_jobs()
            if cjobs == -1:
                LOGGER.error('cannot get count of jobs from cluster')
                return

            if cluster.command_found(cmd=DAX_SETTINGS.get_cmd_submit()):
                LOGGER.info('%s jobs currently in queue' % str(cjobs))

        # Launch until we reach cluster limit or no jobs left to launch
        while (cjobs < self.queue_limit or writeonly) and len(task_list) > 0:
            cur_task = task_list.pop()

            if writeonly:
                msg = "+Writing PBS file for job:%s, currently %s jobs in \
cluster queue"
                LOGGER.info(msg % (cur_task.assessor_label,
                                   str(cjobs)))
github VUIIS / dax / dax / task.py View on Github external
as returned by pbs.submit() method
        :return: True if the job failed

        """
        batch_path = self.batch_path()
        outlog = self.outlog_path()
        jobid, job_failed = cluster.submit_job(batch_path, outlog=outlog,
                                               force_no_qsub=force_no_qsub)

        if jobid == '' or jobid == '0':
            LOGGER.error('failed to launch job on cluster')
            raise ClusterLaunchException
        else:
            self.set_launch(jobid)
            cmd = DAX_SETTINGS.get_cmd_submit()
            if (force_no_qsub or not cluster.command_found(cmd)) and \
               job_failed:
                self.set_status(JOB_FAILED)
            return True
github VUIIS / dax / dax / launcher.py View on Github external
:param task_list: list of task to launch
        :param writeonly: write the job files without submitting them
        :param pbsdir: folder to store the pbs file
        :param force_no_qsub: run the job locally on the computer (serial mode)
        :return: None
        """
        if force_no_qsub:
            LOGGER.info('No qsub - Running job locally on your computer.')
        else:
            # Check number of jobs on cluster
            cjobs = cluster.count_jobs()
            if cjobs == -1:
                LOGGER.error('cannot get count of jobs from cluster')
                return

            if cluster.command_found(cmd=DAX_SETTINGS.get_cmd_submit()):
                LOGGER.info('%s jobs currently in queue' % str(cjobs))

        # Launch until we reach cluster limit or no jobs left to launch
        while (cjobs < self.queue_limit or writeonly) and len(task_list) > 0:
            cur_task = task_list.pop()

            if writeonly:
                msg = "+Writing PBS file for job:%s, currently %s jobs in \
cluster queue"
                LOGGER.info(msg % (cur_task.assessor_label,
                                   str(cjobs)))
            else:
                msg = '+Launching job:%s, currently %s jobs in cluster queue'
                LOGGER.info(msg % (cur_task.assessor_label, str(cjobs)))

            try:
github VUIIS / dax / dax / task.py View on Github external
"""
        [memused, walltime, jobid, jobnode, jobstrdate] = self.get_job_usage()

        if walltime:
            if memused and jobnode:
                LOGGER.debug('memused and walltime already set, skipping')
            else:
                if memused == '':
                    self.set_memused('NotFound')
                if jobnode == '':
                    self.set_jobnode('NotFound')
            return

        # We can't get info from cluster if job too old
        if not cluster.is_traceable_date(jobstrdate):
            self.set_walltime('NotFound')
            self.set_memused('NotFound')
            self.set_jobnode('NotFound')
            return

        # Get usage with tracejob
        jobinfo = cluster.tracejob_info(jobid, jobstrdate)
        if jobinfo['mem_used'].strip():
            self.set_memused(jobinfo['mem_used'])
        else:
            self.set_memused('NotFound')
        if jobinfo['walltime_used'].strip():
            self.set_walltime(jobinfo['walltime_used'])
        else:
            self.set_walltime('NotFound')
        if jobinfo['jobnode'].strip():
github VUIIS / dax / dax / task.py View on Github external
def get_job_status(self, jobid=None):
        """
        Get the status of a job given its jobid as assigned by the scheduler

        :param jobid: job id assigned by the scheduler
        :return: string from call to cluster.job_status or UNKNOWN.

        """
        jobstatus = 'UNKNOWN'
        if jobid is None:
            jobid = self.get_jobid()

        if jobid != '' and jobid != '0':
            jobstatus = cluster.job_status(jobid)

        LOGGER.debug('jobid,jobstatus='+str(jobid)+','+str(jobstatus))

        return jobstatus
github VUIIS / dax / dax / task.py View on Github external
"""
        [memused, walltime, jobid, jobnode, jobstrdate] = self.get_job_usage()

        if walltime:
            if memused and jobnode:
                LOGGER.debug('memused and walltime already set, skipping')
            else:
                if memused == '':
                    self.set_memused('NotFound')
                if jobnode == '':
                    self.set_jobnode('NotFound')
            return

        # We can't get info from cluster if job too old
        if not cluster.is_traceable_date(jobstrdate):
            self.set_walltime('NotFound')
            self.set_memused('NotFound')
            self.set_jobnode('NotFound')
            return

        # Get usage with tracejob
        jobinfo = cluster.tracejob_info(jobid, jobstrdate)
        if jobinfo['mem_used'].strip():
            self.set_memused(jobinfo['mem_used'])
        else:
            self.set_memused('NotFound')
        if jobinfo['walltime_used'].strip():
            self.set_walltime(jobinfo['walltime_used'])
        else:
            self.set_walltime('NotFound')
        if jobinfo['jobnode'].strip():
github VUIIS / dax / dax / task.py View on Github external
else:
                if memused == '':
                    self.set_memused('NotFound')
                if jobnode == '':
                    self.set_jobnode('NotFound')
            return

        # We can't get info from cluster if job too old
        if not cluster.is_traceable_date(jobstrdate):
            self.set_walltime('NotFound')
            self.set_memused('NotFound')
            self.set_jobnode('NotFound')
            return

        # Get usage with tracejob
        jobinfo = cluster.tracejob_info(jobid, jobstrdate)
        if jobinfo['mem_used'].strip():
            self.set_memused(jobinfo['mem_used'])
        else:
            self.set_memused('NotFound')
        if jobinfo['walltime_used'].strip():
            self.set_walltime(jobinfo['walltime_used'])
        else:
            self.set_walltime('NotFound')
        if jobinfo['jobnode'].strip():
            self.set_jobnode(jobinfo['jobnode'])
        else:
            self.set_jobnode('NotFound')