Commit 5be4cfab authored by Eric Duminil's avatar Eric Duminil
Browse files

Moving to requests, to check if cert is still a problem

parent ae277e50
...@@ -13,8 +13,8 @@ from dataclasses import dataclass ...@@ -13,8 +13,8 @@ from dataclasses import dataclass
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Callable, Sequence from typing import Callable, Sequence
from urllib.error import URLError
from urllib.request import urlopen import requests
SCRIPT_DIR = Path(__file__).resolve().parent SCRIPT_DIR = Path(__file__).resolve().parent
TMP_DIR = SCRIPT_DIR / "tmp" TMP_DIR = SCRIPT_DIR / "tmp"
...@@ -62,16 +62,13 @@ def download_file(url: str, dest_path: Path, sleep: int = 0, chunk_size: int = 8 ...@@ -62,16 +62,13 @@ def download_file(url: str, dest_path: Path, sleep: int = 0, chunk_size: int = 8
try: try:
if sleep: if sleep:
time.sleep(sleep) time.sleep(sleep)
with urlopen(url, timeout=30) as response: with requests.get(url, stream=True) as response:
dest_path.parent.mkdir(parents=True, exist_ok=True) dest_path.parent.mkdir(parents=True, exist_ok=True)
with open(dest_path, "wb") as f: with open(dest_path, "wb") as f:
while True: for chunk in response.iter_content(chunk_size=chunk_size):
chunk = response.read(chunk_size)
if not chunk:
break
f.write(chunk) f.write(chunk)
return True return True
except (URLError, OSError) as e: except requests.exceptions.RequestException as e:
print(f" Error downloading from {url}: {e}") print(f" Error downloading from {url}: {e}")
return False return False
...@@ -79,8 +76,8 @@ def download_file(url: str, dest_path: Path, sleep: int = 0, chunk_size: int = 8 ...@@ -79,8 +76,8 @@ def download_file(url: str, dest_path: Path, sleep: int = 0, chunk_size: int = 8
@dataclass @dataclass
class Bundesland: class Bundesland:
name: str name: str
source: str ="Unknown" source: str = "Unknown"
info: str ="Unknown" info: str = "Unknown"
license: str = "CC BY 4.0." license: str = "CC BY 4.0."
# TODO: Add Sequence[CityGML] # TODO: Add Sequence[CityGML]
# TODO: Get size # TODO: Get size
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment