Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_is_url():
"""test the ability to determine whether a string is a URL"""
assert_true(utils.is_url("http://mydomain.com/foo/bar/bat?asdf=1234&qewr=ooo"))
assert_true(utils.is_url("http://xkcd.com/1193/"))
assert_false(utils.is_url("syn123445"))
assert_false(utils.is_url("wasssuuuup???"))
assert_true(utils.is_url('file://foo.com/path/to/file.xyz'))
assert_true(utils.is_url('file:///path/to/file.xyz'))
assert_true(utils.is_url('file:/path/to/file.xyz'))
assert_true(utils.is_url('file:///c:/WINDOWS/clock.avi'))
assert_true(utils.is_url('file:c:/WINDOWS/clock.avi'))
assert_false(utils.is_url('c:/WINDOWS/ugh/ugh.ugh'))
def test_is_url():
"""test the ability to determine whether a string is a URL"""
assert_true(utils.is_url("http://mydomain.com/foo/bar/bat?asdf=1234&qewr=ooo"))
assert_true(utils.is_url("http://xkcd.com/1193/"))
assert_false(utils.is_url("syn123445"))
assert_false(utils.is_url("wasssuuuup???"))
assert_true(utils.is_url('file://foo.com/path/to/file.xyz'))
assert_true(utils.is_url('file:///path/to/file.xyz'))
assert_true(utils.is_url('file:/path/to/file.xyz'))
assert_true(utils.is_url('file:///c:/WINDOWS/clock.avi'))
assert_true(utils.is_url('file:c:/WINDOWS/clock.avi'))
assert_false(utils.is_url('c:/WINDOWS/ugh/ugh.ugh'))
if 'versionNumber' in target:
reference['targetVersionNumber'] = target['versionNumber']
if targetVersion:
reference['targetVersionNumber'] = int(targetVersion)
resource = {'reference': reference,
'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedEntity'}
# -- URL parameter
elif url:
badargs = _get_any_bad_args(['target', 'targetVersion'], locals())
_raise_incorrect_used_usage(badargs, 'URL')
resource = {'url': url, 'name': name if name else target,
'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedURL'}
# -- URL as a string
elif is_url(target):
badargs = _get_any_bad_args(['targetVersion'], locals())
_raise_incorrect_used_usage(badargs, 'URL')
resource = {'url': target, 'name': name if name else target,
'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedURL'}
# -- Synapse Entity ID (assuming the string is an ID)
elif isinstance(target, six.string_types):
badargs = _get_any_bad_args(['url', 'name'], locals())
_raise_incorrect_used_usage(badargs, 'Synapse entity')
vals = target.split('.') # Handle synapseIds of from syn234.4
if not is_synapse_id(vals[0]):
raise ValueError('%s is not a valid Synapse id' % target)
if len(vals) == 2:
if targetVersion and int(targetVersion) != int(vals[1]):
raise ValueError('Two conflicting versions for %s were specified' % target)
targetVersion = int(vals[1])
args.type = 'FileEntity' if args.type == 'File' else args.type
if args.id is not None:
entity = syn.get(args.id, downloadFile=False)
else:
entity = {'concreteType': 'org.sagebionetworks.repo.model.%s' % args.type,
'name': utils.guess_file_name(args.file) if args.file and not args.name else None,
'parentId' : None,
'description' : None,
'path': args.file}
#Overide setting for parameters included in args
entity['name'] = args.name if args.name is not None else entity['name']
entity['description'] = args.description if args.description is not None else entity.get('description', None)
entity['parentId'] = args.parentid if args.parentid is not None else entity['parentId']
entity['path'] = args.file if args.file is not None else None
entity['synapseStore'] = not utils.is_url(args.file)
used = _convertProvenanceList(args.used, args.limitSearch, syn)
executed = _convertProvenanceList(args.executed, args.limitSearch, syn)
entity = syn.store(entity, used=used, executed=executed)
print('Created/Updated entity: %s\t%s' %(entity['id'], entity['name']))
# After creating/updating, if there are annotations to add then
# add them
if args.annotations is not None:
# Need to override the args id parameter
setattr(args, 'id', entity['id'])
setAnnotations(args, syn)
def _uploadToFileHandleService(self, filename, synapseStore=True, mimetype=None):
"""
Create and return a fileHandle, by either uploading a local file or
linking to an external URL.
:param synapseStore: Indicates whether the file should be stored or just the URL.
Defaults to True.
"""
if filename is None:
raise ValueError('No filename given')
elif utils.is_url(filename):
if synapseStore:
raise NotImplementedError('Automatic downloading and storing of external files is not supported. Please try downloading the file locally first before storing it.')
return self._addURLtoFileHandleService(filename)
# For local files, we default to uploading the file unless explicitly instructed otherwise
else:
if synapseStore:
return self._chunkedUploadFile(filename, mimetype=mimetype)
else:
return self._addURLtoFileHandleService(filename)
contentType content type of file to overload defaults text/html
=============== ========================================== ============
**Example manifest file**
=============== ======== ======= ======= =========================== ============================
path parent annot1 annot2 used executed
=============== ======== ======= ======= =========================== ============================
/path/file1.txt syn1243 "bar" 3.1415 "syn124; /path/file2.txt" "https://github.org/foo/bar"
/path/file2.txt syn12433 "baz" 2.71 "" "https://github.org/foo/baz"
=============== ======== ======= ======= =========================== ============================
"""
df = readManifestFile(syn, manifestFile)
sizes = [os.stat(os.path.expandvars(os.path.expanduser(f))).st_size for f in df.path if not is_url(f)]
# Write output on what is getting pushed and estimated times - send out message.
sys.stdout.write('='*50+'\n')
sys.stdout.write('We are about to upload %i files with a total size of %s.\n '
% (len(df), utils.humanizeBytes(sum(sizes))))
sys.stdout.write('='*50+'\n')
if dryRun:
return
sys.stdout.write('Starting upload...\n')
if sendMessages:
notify_decorator = notifyMe(syn, 'Upload of %s' % manifestFile, retries=retries)
upload = notify_decorator(_manifest_upload)
upload(syn, df)
else:
_manifest_upload(syn, df)