Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
filename = f.get_filename()
dest = paths.get_content_storage_file_path(filename)
# if the file already exists, add its size to our overall progress, and skip
if os.path.isfile(dest) and os.path.getsize(dest) == f.file_size:
overall_progress_update(f.file_size)
file_checksums_to_annotate.append(f.id)
continue
# determine where we're downloading/copying from, and create appropriate transfer object
if method == DOWNLOAD_METHOD:
url = paths.get_content_storage_remote_url(filename, baseurl=baseurl)
filetransfer = transfer.FileDownload(url, dest, session=session)
elif method == COPY_METHOD:
srcpath = paths.get_content_storage_file_path(filename, datafolder=path)
filetransfer = transfer.FileCopy(srcpath, dest)
finished = False
try:
while not finished:
finished, increment = self._start_file_transfer(
f, filetransfer, overall_progress_update)
if self.is_cancelled():
break
if increment == 2:
file_checksums_to_annotate.append(f.id)
else:
number_of_skipped_files += increment
except Exception as e:
def export_file(self, f, data_dir, overall_progress_update):
filename = f.get_filename()
try:
srcpath = paths.get_content_storage_file_path(filename)
dest = paths.get_content_storage_file_path(filename, datafolder=data_dir)
except InvalidStorageFilenameError:
# If any files have an invalid storage file name, don't export them.
overall_progress_update(f.file_size)
return
# if the file already exists, add its size to our overall progress, and skip
if os.path.isfile(dest) and os.path.getsize(dest) == f.file_size:
overall_progress_update(f.file_size)
return
copy = transfer.FileCopy(srcpath, dest, cancel_check=self.is_cancelled)
with copy, self.start_progress(
total=copy.total_size
) as file_cp_progress_update:
try:
number_of_skipped_files = 0
file_checksums_to_annotate = []
with self.start_progress(total=total_bytes_to_transfer) as overall_progress_update:
exception = None # Exception that is not caught by the retry logic
if method == DOWNLOAD_METHOD:
session = requests.Session()
for f in files_to_download:
if self.is_cancelled():
break
filename = f.get_filename()
dest = paths.get_content_storage_file_path(filename)
# if the file already exists, add its size to our overall progress, and skip
if os.path.isfile(dest) and os.path.getsize(dest) == f.file_size:
overall_progress_update(f.file_size)
file_checksums_to_annotate.append(f.id)
continue
# determine where we're downloading/copying from, and create appropriate transfer object
if method == DOWNLOAD_METHOD:
url = paths.get_content_storage_remote_url(filename, baseurl=baseurl)
filetransfer = transfer.FileDownload(url, dest, session=session)
elif method == COPY_METHOD:
srcpath = paths.get_content_storage_file_path(filename, datafolder=path)
filetransfer = transfer.FileCopy(srcpath, dest)
finished = False
def get_path_or_404(zipped_filename):
try:
# calculate the local file path to the zip file
zipped_path = get_content_storage_file_path(zipped_filename)
# if the zipfile does not exist on disk, return a 404
if not os.path.exists(zipped_path):
raise InvalidStorageFilenameError()
return zipped_path
except InvalidStorageFilenameError:
raise Http404(
'"%(filename)s" is not a valid zip file' % {"filename": zipped_filename}
)
def export_file(self, f, data_dir, overall_progress_update):
filename = f.get_filename()
try:
srcpath = paths.get_content_storage_file_path(filename)
dest = paths.get_content_storage_file_path(filename, datafolder=data_dir)
except InvalidStorageFilenameError:
# If any files have an invalid storage file name, don't export them.
overall_progress_update(f.file_size)
return
# if the file already exists, add its size to our overall progress, and skip
if os.path.isfile(dest) and os.path.getsize(dest) == f.file_size:
overall_progress_update(f.file_size)
return
copy = transfer.FileCopy(srcpath, dest, cancel_check=self.is_cancelled)
with copy, self.start_progress(
total=copy.total_size
) as file_cp_progress_update:
)
else:
logger.info(
"Setting availability of LocalFile object with checksum {checksum} based on disk availability".format(
checksum=checksums
)
)
files = [bridge.session.query(LocalFileClass).get(checksums)]
checksums_to_set_available = []
checksums_to_set_unavailable = []
for file in files:
try:
# Update if the file exists, *and* the localfile is set as unavailable.
if os.path.exists(
get_content_storage_file_path(get_content_file_name(file))
):
if not file.available:
checksums_to_set_available.append(file.id)
# Update if the file does not exist, *and* the localfile is set as available.
else:
if file.available:
checksums_to_set_unavailable.append(file.id)
except InvalidStorageFilenameError:
continue
bridge.end()
mark_local_files_as_available(checksums_to_set_available, destination=destination)
mark_local_files_as_unavailable(
checksums_to_set_unavailable, destination=destination
)
def delete_unused_files(self):
for file in self.get_unused_files():
try:
os.remove(paths.get_content_storage_file_path(file.get_filename()))
yield True, file
except (IOError, OSError, InvalidStorageFilenameError):
yield False, file
self.get_unused_files().update(available=False)