Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def checkout_file_before_last_merge(fname):
cmd = 'git checkout @{1} -- ' + fname
Logger.debug('[dvc-git] Checkout file {} before last merge. Command: {}'.format(fname, cmd))
Executor.exec_cmd_only_success(cmd.split())
def commit_all_changes(message):
Logger.debug('[dvc-git] Commit all changes. Commands: {} && {} && {}'.format(
'git add --all', 'git status --porcelain', 'git commit -m'))
Executor.exec_cmd_only_success(['git', 'add', '--all'])
out_status = Executor.exec_cmd_only_success(['git', 'status', '--porcelain'])
changes = GitWrapper.parse_porcelain_files(out_status)
if len(changes) == 0:
return []
Executor.exec_cmd_only_success(['git', 'commit', '-m', '\'{}\''.format(message)])
Logger.debug('[dvc-git] Commit all changes. Success.')
return changes
def _skip_part(multipart, part_num, size):
"""
Skip part of multipart upload if it has been already uploaded to the server.
"""
for part in multipart.get_all_parts():
if part.part_number == part_num and part.size == size:# and p.etag and p.last_modified
Logger.debug("Skipping part #{}".format(str(part_num)))
return True
return False
def _verify_md5(self, req, fname):
"""
Verify md5 of a downloaded file if server supports 'content-md5' header.
"""
md5 = file_md5(fname)[0]
content_md5 = self._get_header(req, 'content-md5')
if content_md5 == None:
return True
if md5 != content_md5:
Logger.error('Checksum mismatch')
return False
Logger.debug('Checksum matches')
return True
def push(self, path):
""" push, aws version """
aws_key = self.cache_file_key(path)
bucket = self._get_bucket_aws(self.storage_bucket)
key = bucket.get_key(aws_key)
if key:
Logger.debug('File already uploaded to the cloud. Checksum validation...')
if self._cmp_checksum(key, path):
Logger.debug('File checksum matches. No uploading is needed.')
return path
Logger.debug('Checksum miss-match. Re-uploading is required.')
key = bucket.new_key(aws_key)
try:
self._push_multipart(key, path)
except Exception as exc:
Logger.error('Failed to upload "{}": {}'.format(path, exc))
return None
progress.finish_target(os.path.basename(path))
return path
def push(self, path):
key = self._get_key(path)
if key:
Logger.debug("File '{}' already uploaded to the cloud. Validating checksum...".format(path))
if self._cmp_checksum(key, path):
Logger.debug('File checksum matches. No uploading is needed.')
return []
Logger.debug('Checksum mismatch. Reuploading is required.')
key = self._new_key(path)
return self._push_key(key, path)
def push(self, path):
""" push, gcp version """
bucket = self._get_bucket_gc(self.storage_bucket)
blob_name = self.cache_file_key(path)
name = os.path.basename(path)
blob = bucket.get_blob(blob_name)
if blob is not None and blob.exists():
if self._cmp_checksum(blob, path):
Logger.debug('checksum %s matches. Skipping upload' % path)
return path
Logger.debug('checksum %s mismatch. re-uploading' % path)
# same as in _import
progress.update_target(name, 0, None)
blob = bucket.blob(blob_name)
blob.upload_from_filename(path)
progress.finish_target(name)
Logger.debug('uploading %s completed' % path)
return path
def _get_header(req, name):
"""
Get header value from request.
"""
val = req.headers.get(name)
if val == None:
Logger.debug('\'{}\' not supported by the server'.format(name))
return val
def _unlink_upload_tracker(self, fname):
"""
Remove upload tracker file.
"""
try:
os.unlink(self._upload_tracker(fname))
except Exception as exc:
Logger.debug("Failed to unlink upload tracker file for {}: {}".format(fname, exc))
def _resume_multipart(self, key, fname):
"""
Try resuming multipart upload.
"""
try:
mp_id = open(self._upload_tracker(fname), 'r').read()
except Exception as exc:
Logger.debug("Failed to read upload tracker file for {}: {}".format(fname, exc))
return None
for part in key.bucket.get_all_multipart_uploads():
if part.id != mp_id:
continue
Logger.debug("Found existing multipart {}".format(mp_id))
return part
return None