How to use the toil.jobStores.abstractJobStore.NoSuchFileException function in toil

To help you get started, we’ve selected a few toil examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github DataBiosphere / toil / src / toil / jobStores / googleClientJobStore.py View on Github external
def getPublicUrl(self, fileName):
        blob = self.bucket.get_blob(bytes(fileName))
        if blob is None:
            raise NoSuchFileException(fileName)
        return blob.generate_signed_url(self.publicUrlExpiration)
github DataBiosphere / toil / src / toil / jobStores / azureJobStore.py View on Github external
def getPublicUrl(self, jobStoreFileID):
        try:
            self.files.get_blob_properties(blob_name=str(jobStoreFileID))
        except AzureMissingResourceHttpError:
            raise NoSuchFileException(jobStoreFileID)
        startTime = (datetime.utcnow() - timedelta(minutes=5))
        endTime = datetime.utcnow() + self.publicUrlExpiration
        sas_token = self.files.generate_blob_shared_access_signature(blob_name=str(jobStoreFileID),
                                                                     permission=BlobPermissions.READ,
                                                                     start=startTime,
                                                                     expiry=endTime)
        return self.files.make_blob_url(blob_name=str(jobStoreFileID)) + '?' + sas_token
github DataBiosphere / toil / src / toil / jobStores / fileJobStore.py View on Github external
def fileExists(self, jobStoreFileID):
        absPath = self._getAbsPath(jobStoreFileID)
        try:
            st = os.stat(absPath)
        except os.error:
            return False
        if not stat.S_ISREG(st.st_mode):
            raise NoSuchFileException("Path %s is not a file in the jobStore" % jobStoreFileID)
        return True
github DataBiosphere / toil / src / toil / jobStores / googleJobStore.py View on Github external
def _writeFile(self, jobStoreID, fileObj, update=False, encrypt=True):
        blob = self.bucket.blob(compat_bytes(jobStoreID), encryption_key=self.sseKey if encrypt else None)
        if not update:
            # TODO: should probably raise a special exception and be added to all jobStores
            assert not blob.exists()
        else:
            if not blob.exists():
                raise NoSuchFileException(jobStoreID)
        blob.upload_from_file(fileObj)
github DataBiosphere / toil / src / toil / common.py View on Github external
with self._jobStore.writeSharedFileStream('userScript') as f:
                        pickle.dump(userScript, f, protocol=pickle.HIGHEST_PROTOCOL)
                else:
                    from toil.batchSystems.singleMachine import SingleMachineBatchSystem
                    if not isinstance(self._batchSystem, SingleMachineBatchSystem):
                        logger.warn('Batch system does not support auto-deployment. The user '
                                    'script %s will have to be present at the same location on '
                                    'every worker.', userScript)
                    userScript = None
        else:
            # This branch is hit on restarts
            from toil.jobStores.abstractJobStore import NoSuchFileException
            try:
                with self._jobStore.readSharedFileStream('userScript') as f:
                    userScript = safeUnpickleFromStream(f)
            except NoSuchFileException:
                logger.debug('User script neither set explicitly nor present in the job store.')
                userScript = None
        if userScript is None:
            logger.debug('No user script to auto-deploy.')
        else:
            logger.debug('Saving user script %s as a resource', userScript)
            userScriptResource = userScript.saveAsResourceTo(self._jobStore)
            logger.debug('Injecting user script %s into batch system.', userScriptResource)
            self._batchSystem.setUserScript(userScriptResource)
github DataBiosphere / toil / src / toil / utils / toilStatus.py View on Github external
jobstore = Toil.resumeJobStore(jobStoreName)
        except NoSuchJobStoreException:
            return 'QUEUED'
        except NoSuchFileException:
            return 'QUEUED'

        try:
            with jobstore.readSharedFileStream('succeeded.log') as successful:
                pass
            return 'COMPLETED'
        except NoSuchFileException:
            try:
                with jobstore.readSharedFileStream('failed.log') as failed:
                    pass
                return 'ERROR'
            except NoSuchFileException:
                pass
        return 'RUNNING'
github DataBiosphere / toil / src / toil / common.py View on Github external
with self._jobStore.writeSharedFileStream('userScript') as f:
                        pickle.dump(userScript, f, protocol=pickle.HIGHEST_PROTOCOL)
                else:
                    from toil.batchSystems.singleMachine import SingleMachineBatchSystem
                    if not isinstance(self._batchSystem, SingleMachineBatchSystem):
                        logger.warn('Batch system does not support auto-deployment. The user '
                                    'script %s will have to be present at the same location on '
                                    'every worker.', userScript)
                    userScript = None
        else:
            # This branch is hit on restarts
            from toil.jobStores.abstractJobStore import NoSuchFileException
            try:
                with self._jobStore.readSharedFileStream('userScript') as f:
                    userScript = safeUnpickleFromStream(f)
            except NoSuchFileException:
                logger.debug('User script neither set explicitly nor present in the job store.')
                userScript = None
        if userScript is None:
            logger.debug('No user script to auto-deploy.')
        else:
            logger.debug('Saving user script %s as a resource', userScript)
            userScriptResource = userScript.saveAsResourceTo(self._jobStore)
            logger.debug('Injecting user script %s into batch system.', userScriptResource)
            self._batchSystem.setUserScript(userScriptResource)
github DataBiosphere / toil / src / toil / jobStores / fileJobStore.py View on Github external
def fileExists(self, jobStoreFileID):
        absPath = self._getFilePathFromId(jobStoreFileID)

        if (not absPath.startswith(self.jobsDir) and
            not absPath.startswith(self.filesDir) and
            not absPath.startswith(self.jobFilesDir)):
            # Don't even look for it, it is out of bounds.
            raise NoSuchFileException(jobStoreFileID)
            
        try:
            st = os.stat(absPath)
        except os.error:
            return False
        if not stat.S_ISREG(st.st_mode):
            raise NoSuchFileException(jobStoreFileID)
        return True