Moving util file

citygml.py → citygml_download.py

@@ -5,18 +5,61 @@ CityGML dataclass for German federal states building data.
 
 import hashlib
 import os
+import sys
+import time
 from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import Callable
+from typing import Callable, Sequence
 from urllib.error import URLError
 from urllib.request import urlopen
 
+SCRIPT_DIR = Path(__file__).resolve().parent
 
-def download_file(url: str, dest_path: Path, chunk_size: int = 8192) -> bool:
+
+def download_all_files(
+    files: Sequence["CityGML"], tmp_dir: Path = SCRIPT_DIR / "tmp", jobs: int = 3, retries: int = 2, sleep: int = 5
+):
+    """Download all CityGML files with parallel downloads."""
+    # Create directories
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"Found {len(files)} files to download\n")
+
+    # Download files in parallel
+    successful = 0
+    failed = 0
+
+    with ThreadPoolExecutor(max_workers=jobs) as executor:
+        futures = {executor.submit(file.download, tmp_dir, retries, sleep): file for file in files}
+
+        for future in as_completed(futures):
+            file = futures[future]
+            try:
+                if future.result():
+                    successful += 1
+                else:
+                    failed += 1
+            except Exception as e:
+                print(f"✗ Exception while processing {file.filename}: {e}")
+                failed += 1
+
+    # Summary
+    print(f"\n{'='*60}")
+    print(f"Download complete: {successful} successful, {failed} failed")
+    print(f"{'='*60}")
+
+    if failed > 0:
+        sys.exit(1)
+
+
+def download_file(url: str, dest_path: Path, sleep: int = 0, chunk_size: int = 8192) -> bool:
     """Download file from URL to destination path."""
     try:
+        if sleep:
+            time.sleep(sleep)
         with urlopen(url, timeout=30) as response:
             dest_path.parent.mkdir(parents=True, exist_ok=True)
             with open(dest_path, "wb") as f:
@@ -98,7 +141,7 @@ class CityGML(ABC):
 
         return self.verify(self.path)
 
-    def download(self, tmp_dir: Path, max_retries: int = 3) -> bool:
+    def download(self, tmp_dir: Path, max_retries: int = 3, sleep: int = 0) -> bool:
         """
         Download and verify the file.
 
@@ -126,7 +169,7 @@ class CityGML(ABC):
             print(f"  Trying {self.url}")
 
             # Try to download
-            if not download_file(self.url, tmp_path):
+            if not download_file(self.url, tmp_path, sleep):
                 print(f"  Failed to download")
                 if tmp_path.exists():
                     tmp_path.unlink()