Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _storage_list(args, _):
""" List the buckets or the contents of a bucket.
This command is a bit different in that we allow wildchars in the bucket name and will list
the buckets that match.
"""
target = args['object'] if args['object'] else args['bucket']
project = args['project']
if target is None:
return _storage_list_buckets(project, '*') # List all buckets.
bucket_name, key = datalab.storage._bucket.parse_name(target)
if bucket_name is None:
raise Exception('Cannot list %s; not a valid bucket name' % target)
if key or not re.search('\?|\*|\[', target):
# List the contents of the bucket
if not key:
key = '*'
if project:
# Only list if the bucket is in the project
for bucket in datalab.storage.Buckets(project_id=project):
if bucket.name == bucket_name:
break
else:
raise Exception('%s does not exist in project %s' % (target, project))
else:
bucket = datalab.storage.Bucket(bucket_name)
def verify_permitted_to_read(gs_path):
"""Check if the user has permissions to read from the given path.
Args:
gs_path: the GCS path to check if user is permitted to read.
Raises:
Exception if user has no permissions to read.
"""
# TODO(qimingj): Storage APIs need to be modified to allow absence of project
# or credential on Items. When that happens we can move the function
# to Items class.
from . import _bucket
bucket, prefix = _bucket.parse_name(gs_path)
credentials = None
if datalab.context.Context.is_signed_in():
credentials = datalab.context._utils.get_credentials()
args = {
'maxResults': Api._MAX_RESULTS,
'projection': 'noAcl'
}
if prefix is not None:
args['prefix'] = prefix
url = Api._ENDPOINT + (Api._OBJECT_PATH % (bucket, ''))
try:
datalab.utils.Http.request(url, args=args, credentials=credentials)
except datalab.utils.RequestException as e:
if e.status == 401:
raise Exception('Not permitted to read from specified path. '
'Please sign in and make sure you have read access.')
def _get_item_contents(source_name):
source_bucket, source_key = datalab.storage._bucket.parse_name(source_name)
if source_bucket is None:
raise Exception('Invalid source object name %s; no bucket specified.' % source_name)
if source_key is None:
raise Exception('Invalid source object name %si; source cannot be a bucket.' % source_name)
source = datalab.storage.Item(source_bucket, source_key)
if not source.exists():
raise Exception('Source object %s does not exist' % source_name)
return source.read_from()
def _storage_copy(args, _):
target = args['destination']
target_bucket, target_key = datalab.storage._bucket.parse_name(target)
if target_bucket is None and target_key is None:
raise Exception('Invalid copy target name %s' % target)
sources = _expand_list(args['source'])
if len(sources) > 1:
# Multiple sources; target must be a bucket
if target_bucket is None or target_key is not None:
raise Exception('More than one source but target %s is not a bucket' % target)
errs = []
for source in sources:
source_bucket, source_key = datalab.storage._bucket.parse_name(source)
if source_bucket is None or source_key is None:
raise Exception('Invalid source object name %s' % source)
destination_bucket = target_bucket if target_bucket else source_bucket
def _expand_list(names):
""" Do a wildchar name expansion of object names in a list and return expanded list.
The items are expected to exist as this is used for copy sources or delete targets.
Currently we support wildchars in the key name only.
"""
if names is None:
names = []
elif isinstance(names, basestring):
names = [names]
results = [] # The expanded list.
items = {} # Cached contents of buckets; used for matching.
for name in names:
bucket, key = datalab.storage._bucket.parse_name(name)
results_len = len(results) # If we fail to add any we add name and let caller deal with it.
if bucket:
if not key:
# Just a bucket; add it.
results.append('gs://%s' % bucket)
elif datalab.storage.Item(bucket, key).exists():
results.append('gs://%s/%s' % (bucket, key))
else:
# Expand possible key values.
if bucket not in items and key[:1] == '*':
# We need the full list; cache a copy for efficiency.
items[bucket] = [item.metadata.name
for item in list(datalab.storage.Bucket(bucket).items())]
# If we have a cached copy use it
if bucket in items:
candidates = items[bucket]