Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def getPublicUrl(self, fileName):
blob = self.bucket.get_blob(bytes(fileName))
if blob is None:
raise NoSuchFileException(fileName)
return blob.generate_signed_url(self.publicUrlExpiration)
def getPublicUrl(self, jobStoreFileID):
try:
self.files.get_blob_properties(blob_name=str(jobStoreFileID))
except AzureMissingResourceHttpError:
raise NoSuchFileException(jobStoreFileID)
startTime = (datetime.utcnow() - timedelta(minutes=5))
endTime = datetime.utcnow() + self.publicUrlExpiration
sas_token = self.files.generate_blob_shared_access_signature(blob_name=str(jobStoreFileID),
permission=BlobPermissions.READ,
start=startTime,
expiry=endTime)
return self.files.make_blob_url(blob_name=str(jobStoreFileID)) + '?' + sas_token
def fileExists(self, jobStoreFileID):
absPath = self._getAbsPath(jobStoreFileID)
try:
st = os.stat(absPath)
except os.error:
return False
if not stat.S_ISREG(st.st_mode):
raise NoSuchFileException("Path %s is not a file in the jobStore" % jobStoreFileID)
return True
def _writeFile(self, jobStoreID, fileObj, update=False, encrypt=True):
blob = self.bucket.blob(compat_bytes(jobStoreID), encryption_key=self.sseKey if encrypt else None)
if not update:
# TODO: should probably raise a special exception and be added to all jobStores
assert not blob.exists()
else:
if not blob.exists():
raise NoSuchFileException(jobStoreID)
blob.upload_from_file(fileObj)
with self._jobStore.writeSharedFileStream('userScript') as f:
pickle.dump(userScript, f, protocol=pickle.HIGHEST_PROTOCOL)
else:
from toil.batchSystems.singleMachine import SingleMachineBatchSystem
if not isinstance(self._batchSystem, SingleMachineBatchSystem):
logger.warn('Batch system does not support auto-deployment. The user '
'script %s will have to be present at the same location on '
'every worker.', userScript)
userScript = None
else:
# This branch is hit on restarts
from toil.jobStores.abstractJobStore import NoSuchFileException
try:
with self._jobStore.readSharedFileStream('userScript') as f:
userScript = safeUnpickleFromStream(f)
except NoSuchFileException:
logger.debug('User script neither set explicitly nor present in the job store.')
userScript = None
if userScript is None:
logger.debug('No user script to auto-deploy.')
else:
logger.debug('Saving user script %s as a resource', userScript)
userScriptResource = userScript.saveAsResourceTo(self._jobStore)
logger.debug('Injecting user script %s into batch system.', userScriptResource)
self._batchSystem.setUserScript(userScriptResource)
jobstore = Toil.resumeJobStore(jobStoreName)
except NoSuchJobStoreException:
return 'QUEUED'
except NoSuchFileException:
return 'QUEUED'
try:
with jobstore.readSharedFileStream('succeeded.log') as successful:
pass
return 'COMPLETED'
except NoSuchFileException:
try:
with jobstore.readSharedFileStream('failed.log') as failed:
pass
return 'ERROR'
except NoSuchFileException:
pass
return 'RUNNING'
with self._jobStore.writeSharedFileStream('userScript') as f:
pickle.dump(userScript, f, protocol=pickle.HIGHEST_PROTOCOL)
else:
from toil.batchSystems.singleMachine import SingleMachineBatchSystem
if not isinstance(self._batchSystem, SingleMachineBatchSystem):
logger.warn('Batch system does not support auto-deployment. The user '
'script %s will have to be present at the same location on '
'every worker.', userScript)
userScript = None
else:
# This branch is hit on restarts
from toil.jobStores.abstractJobStore import NoSuchFileException
try:
with self._jobStore.readSharedFileStream('userScript') as f:
userScript = safeUnpickleFromStream(f)
except NoSuchFileException:
logger.debug('User script neither set explicitly nor present in the job store.')
userScript = None
if userScript is None:
logger.debug('No user script to auto-deploy.')
else:
logger.debug('Saving user script %s as a resource', userScript)
userScriptResource = userScript.saveAsResourceTo(self._jobStore)
logger.debug('Injecting user script %s into batch system.', userScriptResource)
self._batchSystem.setUserScript(userScriptResource)
def fileExists(self, jobStoreFileID):
absPath = self._getFilePathFromId(jobStoreFileID)
if (not absPath.startswith(self.jobsDir) and
not absPath.startswith(self.filesDir) and
not absPath.startswith(self.jobFilesDir)):
# Don't even look for it, it is out of bounds.
raise NoSuchFileException(jobStoreFileID)
try:
st = os.stat(absPath)
except os.error:
return False
if not stat.S_ISREG(st.st_mode):
raise NoSuchFileException(jobStoreFileID)
return True