How to use the danlp.download.TqdmUpTo function in danlp

To help you get started, we’ve selected a few danlp examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github alexandrainst / danlp / danlp / download.py View on Github external
def _download_file(meta_info: dict, destination: str, verbose: bool = False):
    """
    :param meta_info:
    :param destination:
    :param verbose:
    """
    file_name = os.path.split(destination)[1]

    expected_size = meta_info['size']
    expected_hash = meta_info['md5_checksum']
    url = meta_info['url']

    if not os.path.isfile(destination):
        if verbose:
            with TqdmUpTo(unit='B', unit_scale=True, miniters=1) as t:
                t.set_description("Downloading file {}".format(destination))
                urllib.request.urlretrieve(url, destination, reporthook=t.update_to)
        else:
            print("Downloading file {}".format(destination))
            urllib.request.urlretrieve(url, destination)

    else:
        if verbose:
            print("The file {} exists here: {}".format(file_name, destination))
            
    assert _check_file(destination) == (expected_size, expected_hash), \
        "Downloaded file does not match the expected checksum! Remove the file: {} and try again.".format(destination)