Bayern refactor

02_bayern.py

+#!/usr/bin/env python3
 # https://geodaten.bayern.de/opengeodata/OpenDataDetail.html?pn=lod2
 
-# Lizenz	CC BY 4.0
-# Koordinatensystem(e)	UTM32 (EPSG:25832)
-# Datenformat(e)	CITYGML
-# Abgabeeinteilung	Kachelung 2km x 2km
-# Aktualisierung	wöchentlich
-# Datenmenge	ca. 150 GB
-# Weitere Informationen	Hinweise zur Datenabgabe
-
-# More info : https://geodaten.bayern.de/odd/m/3/pdf/hinweise_daten_lod2_download.pdf
+# Lizenz: CC BY 4.0
+# Koordinatensystem(e): UTM32 (EPSG:25832)
+# Datenformat(e): CITYGML
+# Abgabeeinteilung: Kachelung 2km x 2km
+# Aktualisierung: wöchentlich
+# Datenmenge: ca. 150 GB
+# Weitere Informationen: https://geodaten.bayern.de/odd/m/3/pdf/hinweise_daten_lod2_download.pdf
 
+import random
+import sys
+import xml.etree.ElementTree as ET
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
+from typing import Sequence
 
-import download_metalink
+from citygml import CityGML, CityGMLWithHash, download_file
 
 SCRIPT_DIR = Path(__file__).resolve().parent
+BAYERN_METALINK = "https://geodaten.bayern.de/odd/a/lod2/citygml/meta/metalink/09.meta4"
 
 
+def download_all_files(
+    files: Sequence[CityGML],
+    tmp_dir: Path = SCRIPT_DIR / "tmp",
+    jobs: int = 4,
+    retries: int = 3,
+):
+    """Download all CityGML files with parallel downloads."""
+    # Create directories
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"Found {len(files)} files to download\n")
+
+    # Download files in parallel
+    successful = 0
+    failed = 0
+
+    with ThreadPoolExecutor(max_workers=jobs) as executor:
+        futures = {
+            executor.submit(file.download, tmp_dir, retries): file
+            for file in files
+        }
+
+        for future in as_completed(futures):
+            file = futures[future]
+            try:
+                if future.result():
+                    successful += 1
+                else:
+                    failed += 1
+            except Exception as e:
+                print(f"✗ Exception while processing {file.filename}: {e}")
+                failed += 1
+
+    # Summary
+    print(f"\n{'='*60}")
+    print(f"Download complete: {successful} successful, {failed} failed")
+    print(f"{'='*60}")
+
+    if failed > 0:
+        sys.exit(1)
+
+
+def download_metalink(metalink_url: str, tmp_path: Path) -> Path:
+    """Download metalink file to temporary directory."""
+    print(f"Downloading {metalink_url}")
+    basename = metalink_url.split("/")[-1] or "metalink.xml"
+    metalink_path = tmp_path / basename
+    download_file(metalink_url, metalink_path)
+    return metalink_path
+
+
+def parse_metalink(metalink_path: Path, download_dir: Path) -> list[CityGMLWithHash]:
+    """Parse metalink XML file and return CityGMLWithHash objects."""
+    print(f"Parsing metalink file: {metalink_path}")
+    tree = ET.parse(metalink_path)
+    root = tree.getroot()
+
+    # Handle namespace
+    ns = {"ml": "urn:ietf:params:xml:ns:metalink"}
+
+    files = []
+    for file_elem in root.findall("ml:file", ns):
+        filename = file_elem.get("name")
+        hash_node = file_elem.find('ml:hash[@type="sha-256"]', ns)
+        
+        if hash_node is None:
+            raise ValueError(f"{filename} has no sha256")
+
+        urls = [url.text for url in file_elem.findall("ml:url", ns)]
+        # Randomize URL order to distribute load across servers
+        random.shuffle(urls)
+
+        files.append(
+            CityGMLWithHash(
+                url=urls[0],
+                path=download_dir / filename,
+                coordinate_reference_system="EPSG:25832",
+                source="https://geodaten.bayern.de",
+                bundesland="Bayern",
+                expected_sha256=hash_node.text,
+            )
+        )
+
+    return files
 
-BAYERN_METALINK = "https://geodaten.bayern.de/odd/a/lod2/citygml/meta/metalink/09.meta4"
 
 if __name__ == "__main__":
-    download_metalink.download_all_files(BAYERN_METALINK, SCRIPT_DIR / "citygml" / "bayern")
+    # Download and parse metalink
+    tmp_dir = SCRIPT_DIR / "tmp"
+    download_dir = SCRIPT_DIR / "citygml" / "bayern"
+    
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+    download_dir.mkdir(parents=True, exist_ok=True)
+    
+    metalink_path = download_metalink(BAYERN_METALINK, tmp_dir)
+    files = parse_metalink(metalink_path, download_dir)
+    
+    # Download all files
+    download_all_files(files, tmp_dir)