Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Ask synapse for a signed URL for S3 upload
(_, base_filename) = os.path.split(filename)
data = {'md5':md5.hexdigest(), 'path':base_filename, 'contentType':mimetype}
uri = '/entity/%s/s3Token' % id_of(entity)
response_json = self.restPOST(uri, body=json.dumps(data))
location_path = response_json['path']
headers = { 'Content-MD5' : base64.b64encode(md5.digest()),
'Content-Type' : mimetype,
'x-amz-acl' : 'bucket-owner-full-control'}
headers.update(synapseclient.USER_AGENT)
# PUT file to S3
with open(filename, 'rb') as f:
response = requests.put(response_json['presignedUrl'], headers=headers, data=f)
exceptions._raise_for_status(response, verbose=self.debug)
# Add location to entity. Path will get converted to a signed S3 URL.
locations = [{'path': location_path, 'type': 'awss3'}]
return {'locations':locations, 'md5':md5.hexdigest()}
if response.text:
chunk_record['response-body'] = response.text
# Is requests closing response stream? Let's make sure:
# "Note that connections are only released back to
# the pool for reuse once all body data has been
# read; be sure to either set stream to False or
# read the content property of the Response object."
# see: http://docs.python-requests.org/en/latest/user/advanced/#keep-alive
try:
if response:
throw_away = response.content
except Exception as ex:
sys.stderr.write('error reading response: '+str(ex))
exceptions._raise_for_status(response, verbose=self.debug)
status = self._startCompleteUploadDaemon(chunkedFileToken=token, chunkNumbers=[a+1 for a in range(i)])
# Poll until concatenating chunks is complete
while (status['state']=='PROCESSING'):
time.sleep(CHUNK_UPLOAD_POLL_INTERVAL)
status = self._completeUploadDaemonStatus(status)
if status['state'] == 'FAILED':
raise SynapseError(status['errorMessage'])
# Return a fileHandle
fileHandle = self._getFileHandle(status['fileHandleId'])
except Exception as ex:
raise sys.exc_info()[0], ex, sys.exc_info()[2]
def restDELETE(self, uri, endpoint=None, headers=None, retryPolicy={}, **kwargs):
"""
Performs a REST DELETE operation to the Synapse server.
:param uri: URI of resource to be deleted
:param endpoint: Server endpoint, defaults to self.repoEndpoint
:param headers: Dictionary of headers to use rather than the API-key-signed default set of headers
:param kwargs: Any other arguments taken by a `requests `_ method
"""
uri, headers = self._build_uri_and_headers(uri, endpoint, headers)
retryPolicy = self._build_retry_policy(retryPolicy)
response = _with_retry(lambda: requests.delete(uri, headers=headers, **kwargs), **retryPolicy)
exceptions._raise_for_status(response, verbose=self.debug)
response = requests.get(url, headers=self._generateSignedHeaders(url), allow_redirects=False)
if response.status_code in [301,302,303,307,308]:
url = response.headers['location']
# If it's a file URL, turn it into a path and return it
if url.startswith('file:'):
pathinfo = utils.file_url_to_path(url, verify_exists=True)
if 'path' not in pathinfo:
raise IOError("Could not download non-existent file (%s)." % url)
else:
raise NotImplementedError("File can already be accessed. Consider setting downloadFile to False")
response = requests.get(url, headers=self._generateSignedHeaders(url, {}), stream=True)
try:
exceptions._raise_for_status(response, verbose=self.debug)
except SynapseHTTPError as err:
if err.response.status_code == 404:
raise SynapseError("Could not download the file at %s" % url)
raise
# Stream the file to disk
with open(destination, "wb") as f:
data = response.raw.read(FILE_BUFFER_SIZE)
while data:
f.write(data)
data = response.raw.read(FILE_BUFFER_SIZE)
destination = os.path.abspath(destination)
return {
'path': destination,
'files': [os.path.basename(destination)],
"""
Performs a REST GET operation to the Synapse server.
:param uri: URI on which get is performed
:param endpoint: Server endpoint, defaults to self.repoEndpoint
:param headers: Dictionary of headers to use rather than the API-key-signed default set of headers
:param kwargs: Any other arguments taken by a `requests `_ method
:returns: JSON encoding of response
"""
uri, headers = self._build_uri_and_headers(uri, endpoint, headers)
retryPolicy = self._build_retry_policy(retryPolicy)
response = _with_retry(lambda: requests.get(uri, headers=headers, **kwargs), **retryPolicy)
exceptions._raise_for_status(response, verbose=self.debug)
return self._return_rest_body(response)
Performs a REST POST operation to the Synapse server.
:param uri: URI on which get is performed
:param endpoint: Server endpoint, defaults to self.repoEndpoint
:param body: The payload to be delivered
:param headers: Dictionary of headers to use rather than the API-key-signed default set of headers
:param kwargs: Any other arguments taken by a `requests `_ method
:returns: JSON encoding of response
"""
uri, headers = self._build_uri_and_headers(uri, endpoint, headers)
retryPolicy = self._build_retry_policy(retryPolicy)
response = _with_retry(lambda: requests.post(uri, data=body, headers=headers, **kwargs), **retryPolicy)
exceptions._raise_for_status(response, verbose=self.debug)
return self._return_rest_body(response)