Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
elif o.scheme == 'file':
if dst_scheme == 's3':
# local to s3
aws_s3.put_s3_file(o.path, os.path.dirname(dst_file))
elif dst_scheme != 'db': # assume 'file'
# local to local
shutil.copy(o.path, os.path.dirname(dst_file))
else:
raise Exception("copy_in_files: copy local file to unsupported scheme {}".format(dst_scheme))
else:
raise Exception("DataContext copy-in-file found bad scheme: {} from {}".format(o.scheme, o))
else:
_logger.info("DataContext copy-in-file: Not adding files in directory {}".format(src_path))
except (IOError, os.error) as why:
_logger.error("Disdat add error: {} {} {}".format(src_path, dst_dir, str(why)))
if return_one_file:
return file_set[0]
else:
return file_set
def assert_or_log(cli, msg):
if cli:
_logger.error(msg)
else:
assert False, msg
# First, test whether any db links are used to back the current view
no_force_required = self.rm_db_links(hfr[0])
if no_force_required or force:
hyperframe.update_hfr_db(self.local_engine, hyperframe.RecordState.deleted, uuid=hfr_uuid)
self.rm_db_links(hfr[0], dry_run=False)
shutil.rmtree(self.implicit_hframe_path(hfr_uuid))
hyperframe.delete_hfr_db(self.local_engine, uuid=hfr_uuid)
hyperframe.delete_fr_db(self.local_engine, hfr_uuid)
else:
print ("Disdat: Looks like you're trying to remove a committed bundle with a db link backing a view.")
print ("Disdat: Removal of this bundle with db links that back a view requires '--force'")
return False
return True
except (IOError, os.error) as why:
_logger.error("Removal of hyperframe directory {} failed with error {}.".format(self.implicit_hframe_path(hfr_uuid), why))
# Must clean up db if directory removal failed, only delete same record if state marked for removal
hyperframe.delete_hfr_db(self.local_engine, uuid=hfr_uuid, state=hyperframe.RecordState.deleted)
return False
# if the user erroneously passes in the directory of the bundle, return same
if dir_name == self._local_dir:
return self._local_dir
fqp = os.path.join(self._local_dir, dir_name.lstrip('/'))
try:
os.makedirs(fqp)
except OSError as why:
if not why.errno == errno.EEXIST:
_logger.error("Creating directory in bundle directory failed errno {}".format(why.strerror))
raise
# else directory exists is OK and fall through
except IOError as why:
_logger.error("Creating directory in bundle directory failed {}".format(why))
raise
return fqp
""" NOTE: If a user changes a task param in run(), and that param parameterizes a dependency in requires(),
then running requires() post run() will give different tasks. To be safe we record the inputs before run()
"""
cached_bundle_inputs = self.bundle_inputs()
try:
start = time.time() # P3 datetime.now().timestamp()
user_rtn_val = self.pipe_run(**kwargs)
stop = time.time() # P3 datetime.now().timestamp()
except Exception as error:
""" If user's pipe fails for any reason, remove bundle dir and raise """
try:
_logger.error("User pipe_run encountered exception: {}".format(error))
pce.bundle.abandon()
except OSError as ose:
_logger.error("User pipe_run encountered error, and error on remove bundle: {}".format(ose))
raise
try:
# Add any output tags to the user tag dict
if self.output_tags:
self.user_tags.update(self.output_tags)
# If this is the root_task, identify it as so in the tag dict
if isinstance(self.calling_task, DriverTask):
self.user_tags.update({'root_task': 'True'})
""" if we have a pce, we have a new bundle that we need to add info to and close """
pce.bundle.add_data(user_rtn_val)
pce.bundle.add_timing(start, stop)