NRW Refactor

10_nordrhein_westfalen.py

-# https://www.opengeodata.nrw.de/produkte/geobasis/3dg/lod2_gml/
-
 import json
-import os
-import sys
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from pathlib import Path
-
-from download_metalink import SCRIPT_DIR, download_file
-
-NRW_SERVER = "https://www.opengeodata.nrw.de/produkte/geobasis/3dg/lod2_gml/lod2_gml/"
-NRW_JSON = SCRIPT_DIR / "tmp" / "nrw_lod2.json"
-
-
-# TODO: Dry with 02_bayern.py, and move some functions to utils.py?
-def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_retries=3) -> bool:
-    """Download a file, verify its size, and move to final location."""
-    filename = file_info["name"]
-    expected_size = int(file_info["size"])
-    urls = [NRW_SERVER + filename]
-
-    final_path = download_dir / filename
-
-    # Check if file already exists and is valid
-    if final_path.exists():
-        print(f"Checking existing file: {filename}")
-        if os.path.getsize(final_path) == expected_size:
-            print(f"✓ {filename} already downloaded and verified")
-            return True
-        else:
-            print(f"  Existing file has incorrect hash, will re-download")
-            final_path.unlink()
-
-    tmp_path = tmp_dir / filename
-
-    for attempt in range(max_retries):
-        print(f"Downloading {filename} (attempt {attempt + 1}/{max_retries})")
-
-        # Try each URL until one succeeds
-        download_success = False
-        for url in urls:
-            print(f"  Trying {url}")
-            if download_file(url, tmp_path):
-                download_success = True
-                break
-
-        if not download_success:
-            print(f"  Failed to download from all URLs")
-            if tmp_path.exists():
-                tmp_path.unlink()
-            continue
-
-        print(f"  Verifying size for {filename}")
-        actual_size = os.path.getsize(tmp_path)
 
-        if actual_size == expected_size:
-            # Move to final location
-            final_path.parent.mkdir(parents=True, exist_ok=True)
-            tmp_path.rename(final_path)
-            print(f"✓ {filename} downloaded and verified successfully")
-            return True
-        else:
-            print(f"  Size mismatch for {filename}")
-            print(f"    Expected: {expected_size}")
-            print(f"    Got:      {actual_size}")
-            tmp_path.unlink()
+from citygml_download import TMP_DIR, Bundesland, CityGMLWithSize, download_all_files, download_file
 
-    print(f"✗ Failed to download {filename} after {max_retries} attempts")
-    return False
+NRW_JSON = TMP_DIR / "nrw_lod2.json"
+NORDRHEIN_WESTFALEN = Bundesland(
+    "Nordrhein-Westfalen",
+    source="https://www.opengeodata.nrw.de/produkte/geobasis/3dg/lod2_gml/lod2_gml/",
+    info="https://www.opengeodata.nrw.de/produkte/geobasis/3dg/lod2_gml",
+)
 
 
-def download_all_files(
-    json_url: str, download_path: Path, tmp_path: Path = SCRIPT_DIR / "tmp", jobs: int = 4, retries: int = 3
-):
-    # Create directories
-    tmp_path.mkdir(parents=True, exist_ok=True)
+if __name__ == "__main__":
     download_file(
-        json_url,
+        NORDRHEIN_WESTFALEN.source + "index.json",
         NRW_JSON,
     )
 
     with open(NRW_JSON) as json_file:
         data = json.load(json_file)
 
-    # Parse metalink file
-    files = [file for sets in data["datasets"] for file in sets["files"]]
-
-    print(f"Found {len(files)} files to download\n")
-
-    download_path.mkdir(parents=True, exist_ok=True)
-
-    # Download files in parallel
-    successful = 0
-    failed = 0
-
-    with ThreadPoolExecutor(max_workers=jobs) as executor:
-        futures = {
-            executor.submit(download_and_verify, file_info, tmp_path, download_path, retries): file_info["name"]
-            for file_info in files
-        }
-
-        for future in as_completed(futures):
-            filename = futures[future]
-            try:
-                if future.result():
-                    successful += 1
-                else:
-                    failed += 1
-            except Exception as e:
-                print(f"✗ Exception while processing {filename}: {e}")
-                failed += 1
-
-    # Summary
-    print(f"\n{'='*60}")
-    print(f"Download complete: {successful} successful, {failed} failed")
-    print(f"{'='*60}")
-
-    if failed > 0:
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    download_all_files(NRW_SERVER + "index.json", SCRIPT_DIR / "citygml" / "nordrhein_westfalen", retries=2)
+    citygmls = []
+    for sets in data["datasets"]:
+        for file in sets["files"]:
+            name = file["name"]
+            citygmls.append(
+                CityGMLWithSize(
+                    url=NORDRHEIN_WESTFALEN.source + name,
+                    bundesland=NORDRHEIN_WESTFALEN,
+                    expected_size=int(file["size"]),
+                    # coordinate_reference_system ?
+                )
+            )
+
+    download_all_files(citygmls)