Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
if not localize:
all_keys = [k for k in all_keys if 'frame.pb' in k]
fetch_count = 0
fetch_tuples = []
for s3_key in all_keys:
obj_basename = os.path.basename(s3_key)
obj_suffix = s3_key.replace(remote_obj_dir,'')
obj_suffix_dir = os.path.dirname(obj_suffix).strip('/') # remote_obj_dir won't have a trailing slash
local_uuid_dir = os.path.join(data_context.get_object_dir(), obj_suffix_dir)
local_object_path = os.path.join(local_uuid_dir, obj_basename)
if not os.path.exists(local_object_path):
fetch_count += 1
fetch_tuples.append((s3_bucket, s3_key, local_object_path))
_logger.info("Fast pull fetching {} objects...".format(fetch_count))
results = aws_s3.get_s3_key_many(fetch_tuples)
_logger.info("Fast pull completed {} transfers -- process pool closed and joined.".format(len(results)))
import botocore
s3 = b3.resource('s3')
exists = True
try:
s3.meta.client.head_bucket(Bucket=bucket)
except botocore.exceptions.ClientError as e:
error_code = int(e.response['Error']['Code'])
if error_code == 404:
exists = False
elif error_code == 403:
# for buckets you can get a forbidden instead of resource not found
# if you have the s3:ListBucket permission on the bucket, Amazon S3 will return a
# HTTP status code 404 ("no such key") error. If you don't have the s3:ListBucket permission,
# Amazon S3 will return a HTTP status code 403 ("access denied") error.
_logger.info("aws_s3: bucket {} raised a 403 (access forbidden), do you have ListBucket permission?".format(bucket))
exists = False
else:
raise
return exists
os.remove(os.path.join(DisdatConfig.instance().get_meta_dir(), META_FS_FILE))
if local_context in self._all_contexts:
dc = self._all_contexts[local_context]
remote_context_url = dc.get_remote_object_dir()
dc.delete_context(force=force)
del self._all_contexts[local_context]
if os.path.exists(ctxt_dir):
shutil.rmtree(ctxt_dir)
_logger.info("Disdat deleted local data context {}.".format(local_context))
if remote:
aws_s3.delete_s3_dir(remote_context_url)
_logger.info("Disdat deleted remote data context {}.".format(remote_context_url))
else:
_logger.info("Disdat local data context {} appears to already have been deleted.".format(local_context))
_logger.debug("Skipping a db file on bundle add")
elif o.scheme == 'file':
if dst_scheme == 's3':
# local to s3
aws_s3.put_s3_file(o.path, os.path.dirname(dst_file))
elif dst_scheme != 'db': # assume 'file'
# local to local
shutil.copy(o.path, os.path.dirname(dst_file))
else:
raise Exception("copy_in_files: copy local file to unsupported scheme {}".format(dst_scheme))
else:
raise Exception("DataContext copy-in-file found bad scheme: {} from {}".format(o.scheme, o))
else:
_logger.info("DataContext copy-in-file: Not adding files in directory {}".format(src_path))
except (IOError, os.error) as why:
_logger.error("Disdat add error: {} {} {}".format(src_path, dst_dir, str(why)))
if return_one_file:
return file_set[0]
else:
return file_set
try:
if type(pb_tbls) is dict:
assert (isinstance(pb_rows, dict) or isinstance(pb_rows, defaultdict))
results = []
for k, tbl in pb_tbls.items(): # dict of tables
for r in pb_rows[k]: # dict of list of rows
ins = tbl.insert()
results.append(db_conn.execute(ins, r))
return results
else:
assert (type(pb_tbls) is not list)
assert (type(pb_tbls) is not tuple)
ins = pb_tbls.insert()
return db_conn.execute(ins, pb_rows)
except IntegrityError as ie:
_logger.info("Writing class pb {} to table encountered error {}".format(self._pb_type(), ie))
return None
# At the moment, this is all or none. There are cases where you could localize and pull
# only some of the files, in which case we could mix local and remote. However, that may
# indicate that something else is wrong. We should probably indicate that in the future.
if all(os.path.isfile(lf) for lf in local_file_set):
if strip_file_scheme:
append = ''
else:
append = 'file://'
file_set = ["{}{}".format(append, lf) for lf in local_file_set]
else:
# Note that remote_dir already includes the URL scheme
remote_dir = self.get_remote_object_dir()
if remote_dir is not None:
file_set = [ "{}".format(os.path.join(remote_dir, fr.hframe_uuid, f.replace(common.BUNDLE_URI_SCHEME,''))) for f in urls]
else:
_logger.info("actualize_link_urls: Files are not local, and no remote context bound.")
raise Exception("actualize_link_urls: Files are not local, and no remote context bound.")
return file_set
Switch to a different local context.
Args:
local_context_name (str): May be / or
save (bool): Whether to record context change on disk.
Returns:
"""
assert local_context_name is not None
repo, local_context = DisdatFS._parse_fq_context_name(local_context_name)
if self.curr_context is not None and self.curr_context_name == local_context_name:
assert(local_context in self._all_contexts)
assert(self.curr_context == self._all_contexts[local_context_name])
_logger.info("Disdat already within a valid data context_name {}".format(local_context))
new_context = self.get_context(local_context_name)
if new_context is not None:
self.curr_context = new_context
print("Switched to context {}".format(self.curr_context_name))
else:
print("In context {}".format(self.curr_context_name))