How to use the dvc.logger.Logger.debug function in dvc

To help you get started, we’ve selected a few dvc examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github iterative / dvc / dvc / git_wrapper.py View on Github external
def checkout_file_before_last_merge(fname):
        cmd = 'git checkout @{1} -- ' + fname
        Logger.debug('[dvc-git] Checkout file {} before last merge. Command: {}'.format(fname, cmd))
        Executor.exec_cmd_only_success(cmd.split())
github iterative / dvc / dvc / git_wrapper.py View on Github external
def commit_all_changes(message):
        Logger.debug('[dvc-git] Commit all changes. Commands: {} && {} && {}'.format(
            'git add --all', 'git status --porcelain', 'git commit -m'))

        Executor.exec_cmd_only_success(['git', 'add', '--all'])
        out_status = Executor.exec_cmd_only_success(['git', 'status', '--porcelain'])
        changes = GitWrapper.parse_porcelain_files(out_status)
        if len(changes) == 0:
            return []

        Executor.exec_cmd_only_success(['git', 'commit', '-m', '\'{}\''.format(message)])
        Logger.debug('[dvc-git] Commit all changes. Success.')
        return changes
github iterative / dvc / dvc / data_cloud.py View on Github external
def _skip_part(multipart, part_num, size):
        """
        Skip part of multipart upload if it has been already uploaded to the server.
        """
        for part in multipart.get_all_parts():
            if part.part_number == part_num and part.size == size:# and p.etag and p.last_modified
                Logger.debug("Skipping part #{}".format(str(part_num)))
                return True
        return False
github iterative / dvc / dvc / data_cloud.py View on Github external
def _verify_md5(self, req, fname):
        """
        Verify md5 of a downloaded file if server supports 'content-md5' header.
        """
        md5 = file_md5(fname)[0]
        content_md5 = self._get_header(req, 'content-md5')

        if content_md5 == None:
            return True

        if md5 != content_md5:
            Logger.error('Checksum mismatch')
            return False

        Logger.debug('Checksum matches')
        return True
github iterative / dvc / dvc / data_cloud.py View on Github external
def push(self, path):
        """ push, aws version """

        aws_key = self.cache_file_key(path)
        bucket = self._get_bucket_aws(self.storage_bucket)
        key = bucket.get_key(aws_key)
        if key:
            Logger.debug('File already uploaded to the cloud. Checksum validation...')

            if self._cmp_checksum(key, path):
                Logger.debug('File checksum matches. No uploading is needed.')
                return path

            Logger.debug('Checksum miss-match. Re-uploading is required.')

        key = bucket.new_key(aws_key)

        try:
            self._push_multipart(key, path)
        except Exception as exc:
            Logger.error('Failed to upload "{}": {}'.format(path, exc))
            return None

        progress.finish_target(os.path.basename(path))

        return path
github iterative / dvc / dvc / cloud / base.py View on Github external
def push(self, path):
        key = self._get_key(path)
        if key:
            Logger.debug("File '{}' already uploaded to the cloud. Validating checksum...".format(path))
            if self._cmp_checksum(key, path):
                Logger.debug('File checksum matches. No uploading is needed.')
                return []
            Logger.debug('Checksum mismatch. Reuploading is required.')

        key = self._new_key(path)
        return self._push_key(key, path)
github iterative / dvc / dvc / data_cloud.py View on Github external
def push(self, path):
        """ push, gcp version """

        bucket = self._get_bucket_gc(self.storage_bucket)
        blob_name = self.cache_file_key(path)
        name = os.path.basename(path)

        blob = bucket.get_blob(blob_name)
        if blob is not None and blob.exists():
            if self._cmp_checksum(blob, path):
                Logger.debug('checksum %s matches.  Skipping upload' % path)
                return path
            Logger.debug('checksum %s mismatch.  re-uploading' % path)

        # same as in _import
        progress.update_target(name, 0, None)

        blob = bucket.blob(blob_name)
        blob.upload_from_filename(path)

        progress.finish_target(name)
        Logger.debug('uploading %s completed' % path)

        return path
github iterative / dvc / dvc / data_cloud.py View on Github external
def _get_header(req, name):
        """
        Get header value from request.
        """
        val = req.headers.get(name)
        if val == None:
            Logger.debug('\'{}\' not supported by the server'.format(name))

        return val
github iterative / dvc / dvc / data_cloud.py View on Github external
def _unlink_upload_tracker(self, fname):
        """
        Remove upload tracker file.
        """
        try:
            os.unlink(self._upload_tracker(fname))
        except Exception as exc:
            Logger.debug("Failed to unlink upload tracker file for {}: {}".format(fname, exc))
github iterative / dvc / dvc / data_cloud.py View on Github external
def _resume_multipart(self, key, fname):
        """
        Try resuming multipart upload.
        """
        try:
            mp_id = open(self._upload_tracker(fname), 'r').read()
        except Exception as exc:
            Logger.debug("Failed to read upload tracker file for {}: {}".format(fname, exc))
            return None

        for part in key.bucket.get_all_multipart_uploads():
            if part.id != mp_id:
                continue

            Logger.debug("Found existing multipart {}".format(mp_id))
            return part

        return None