Source code for

import tarfile
from pathlib import Path

from labml import monit, logger
from labml.logger import Text

_TAR_BUFFER_SIZE = 100_000

[docs]def download_file(url: str, path: Path): """ Download a file from ``url``. Arguments: url (str): URL to download the file from path (Path): The location to save the downloaded file """ if path.exists(): return import urllib.request if not path.parent.exists(): path.parent.mkdir(parents=True) with monit.section("Download"): def reporthook(count, block_size, total_size): monit.progress(count * block_size / total_size) urllib.request.urlretrieve(url, path, reporthook=reporthook)
def _extract_tar_file(tar: tarfile.TarFile, f: tarfile.TarInfo, path: Path): if not path.parent.exists(): path.parent.mkdir(parents=True) with tar.extractfile(f) as ef: with open(str(path), 'wb') as o: while True: r = if r: o.write(r) else: break
[docs]def extract_tar(tar_file: Path, to_path: Path): """ Extract a ``.tar.gz`` file. Arguments: tar_file (Path): ``.tar.gz`` file to_path (Path): location to extract the contents """ with, 'r:gz') as tar: files = tar.getmembers() for f in monit.iterate('Extract tar', files): if f.isdir(): pass elif f.isfile(): _extract_tar_file(tar, f, to_path / else: logger.log(f'Unknown file type {}', Text.warning)