refactor

02_bayern.py

@@ -10,60 +10,15 @@
 # Weitere Informationen: https://geodaten.bayern.de/odd/m/3/pdf/hinweise_daten_lod2_download.pdf
 
 import random
-import sys
 import xml.etree.ElementTree as ET
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
-from typing import Sequence
 
-from citygml import CityGML, CityGMLWithHash, download_file
+from citygml_download import CityGMLWithHash, download_all_files, download_file
 
 SCRIPT_DIR = Path(__file__).resolve().parent
 BAYERN_METALINK = "https://geodaten.bayern.de/odd/a/lod2/citygml/meta/metalink/09.meta4"
 
 
-def download_all_files(
-    files: Sequence[CityGML],
-    tmp_dir: Path = SCRIPT_DIR / "tmp",
-    jobs: int = 4,
-    retries: int = 3,
-):
-    """Download all CityGML files with parallel downloads."""
-    # Create directories
-    tmp_dir.mkdir(parents=True, exist_ok=True)
-
-    print(f"Found {len(files)} files to download\n")
-
-    # Download files in parallel
-    successful = 0
-    failed = 0
-
-    with ThreadPoolExecutor(max_workers=jobs) as executor:
-        futures = {
-            executor.submit(file.download, tmp_dir, retries): file
-            for file in files
-        }
-
-        for future in as_completed(futures):
-            file = futures[future]
-            try:
-                if future.result():
-                    successful += 1
-                else:
-                    failed += 1
-            except Exception as e:
-                print(f"✗ Exception while processing {file.filename}: {e}")
-                failed += 1
-
-    # Summary
-    print(f"\n{'='*60}")
-    print(f"Download complete: {successful} successful, {failed} failed")
-    print(f"{'='*60}")
-
-    if failed > 0:
-        sys.exit(1)
-
-
 def download_metalink(metalink_url: str, tmp_path: Path) -> Path:
     """Download metalink file to temporary directory."""
     print(f"Downloading {metalink_url}")
@@ -86,7 +41,7 @@ def parse_metalink(metalink_path: Path, download_dir: Path) -> list[CityGMLWithH
     for file_elem in root.findall("ml:file", ns):
         filename = file_elem.get("name")
         hash_node = file_elem.find('ml:hash[@type="sha-256"]', ns)
-        
+
         if hash_node is None:
             raise ValueError(f"{filename} has no sha256")
 
@@ -112,12 +67,12 @@ if __name__ == "__main__":
     # Download and parse metalink
     tmp_dir = SCRIPT_DIR / "tmp"
     download_dir = SCRIPT_DIR / "citygml" / "bayern"
-    
+
     tmp_dir.mkdir(parents=True, exist_ok=True)
     download_dir.mkdir(parents=True, exist_ok=True)
-    
+
     metalink_path = download_metalink(BAYERN_METALINK, tmp_dir)
     files = parse_metalink(metalink_path, download_dir)
-    
+
     # Download all files
-    download_all_files(files, tmp_dir)
+    download_all_files(files, tmp_dir, sleep=5, retries=1, jobs=1)