Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
Files are uploaded separately to the destination storage (e.g. s3,gc,file) and then
a link to the uploaded model is stored in the model object
Notice: credentials for the upload destination will be pooled from the
global configuration file (i.e. ~/trains.conf)
:param uri: upload destination (string). example: 's3://bucket/directory/' or 'file:///tmp/debug/'
:return: True if destination scheme is supported (i.e. s3:// file:// gc:// etc...)
"""
if not uri:
return
# Test if we can update the model.
self._validate_update()
# Create the storage helper
storage = StorageHelper.get(uri)
# Verify that we can upload to this destination
try:
uri = storage.verify_upload(folder_uri=uri)
except Exception:
raise ValueError("Could not set destination uri to: %s [Check write permissions]" % uri)
# store default uri
self._get_base_model().upload_storage_uri = uri
def download_model_weights(self):
""" Download the model weights into a local file in our cache """
uri = self.data.uri
if not uri or not uri.strip():
return None
helper = StorageHelper.get(uri)
filename = uri.split('/')[-1]
ext = '.'.join(filename.split('.')[1:])
fd, local_filename = mkstemp(suffix='.'+ext)
os.close(fd)
local_download = helper.download_to_file(uri, local_path=local_filename, overwrite_existing=True, verbose=True)
# if we ended up without any local copy, delete the temp file
if local_download != local_filename:
try:
Path(local_filename).unlink()
except Exception:
pass
# save local model, so we can later query what was the original one
Model._local_model_to_id_uri[str(local_download)] = (self.model_id, uri)
return local_download
def _get_storage(self, storage_uri=None):
""" Storage helper used to upload files """
try:
# use a lock since this storage object will be requested by thread pool threads, so we need to make sure
# any singleton initialization will occur only once
self._storage_lock.acquire()
storage_uri = storage_uri or self._storage_uri
return StorageHelper.get(storage_uri)
except Exception as e:
log.error('Failed getting storage helper for %s: %s' % (storage_uri, str(e)))
finally:
self._storage_lock.release()
if len([x for x in (matrix, image, local_path) if x is not None]) != 1:
raise ValueError('Expected only one of [image, matrix, local_path]')
if image is None:
image = matrix
if image is not None and not isinstance(image, (np.ndarray, Image.Image)):
raise ValueError("Supported 'image' types are: numpy.ndarray or PIL.Image")
# if task was not started, we have to start it
self._start_task_if_needed()
upload_uri = self.get_default_upload_destination()
if not upload_uri:
upload_uri = Path(get_cache_dir()) / 'debug_images'
upload_uri.mkdir(parents=True, exist_ok=True)
# Verify that we can upload to this destination
upload_uri = str(upload_uri)
storage = StorageHelper.get(upload_uri)
upload_uri = storage.verify_upload(folder_uri=upload_uri)
self._touch_title_series(title, series)
if isinstance(image, Image.Image):
image = np.array(image)
self._task.reporter.report_image_and_upload(
title=title,
series=series,
path=local_path,
image=image,
iter=iteration,
upload_uri=upload_uri,
max_image_history=max_image_history,
delete_after_upload=delete_after_upload,
)