Coverage for torchapp/download.py : 29.03%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from pathlib import Path
2from typing import Union
3import ssl, certifi
4import urllib.request
5from rich.progress import Progress
7class DownloadError(Exception):
8 pass
11def download_file(url:str, local_path:Path, chunk_size:int=16 * 1024, context=None):
12 """ adapted from https://stackoverflow.com/a/1517728 """
13 local_path = Path(local_path)
14 local_path.parent.mkdir(parents=True, exist_ok=True)
16 with urllib.request.urlopen(url, context=context) as response, open(local_path, 'wb') as f, Progress() as progress:
17 task = progress.add_task("[red]Downloading", total=response.length)
18 while True:
19 chunk = response.read(chunk_size)
20 progress.update(task, advance=chunk_size)
21 if not chunk:
22 break
23 f.write(chunk)
25 return local_path
28def cached_download(url: str, local_path: Union[str, Path], force: bool = False) -> None:
29 """
30 Downloads a file if a local file does not already exist.
32 Args:
33 url (str): The url of the file to download.
34 local_path (str, Path): The local path of where the file should be.
35 If this file isn't there or the file size is zero then this function downloads it to this location.
36 force (bool): Whether or not the file should be forced to download again even if present in the local path.
37 Default False.
39 Raises:
40 DownloadError: Raises an exception if it cannot download the file.
41 IOError: Raises an exception if the file does not exist or is empty after downloading.
42 """
43 local_path = Path(local_path)
44 if (not local_path.exists() or local_path.stat().st_size == 0) or force:
45 try:
46 print(f"Downloading {url} to {local_path}")
47 download_file(url, local_path)
48 except Exception:
49 try:
50 download_file(url, local_path, context=ssl.create_default_context(cafile=certifi.where()))
51 except Exception as err:
52 raise DownloadError(f"Error downloading {url} to {local_path}:\n{err}")
54 if not local_path.exists() or local_path.stat().st_size == 0:
55 raise IOError(f"Error reading {local_path}")