QuickNDirty NRW

10_nordrhein_westfalen.py

 # https://www.opengeodata.nrw.de/produkte/geobasis/3dg/lod2_gml/
 
 import json
-import random
+import os
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 
 from download_metalink import SCRIPT_DIR, download_file
@@ -9,33 +11,20 @@ from download_metalink import SCRIPT_DIR, download_file
 NRW_SERVER = "https://www.opengeodata.nrw.de/produkte/geobasis/3dg/lod2_gml/lod2_gml/"
 NRW_JSON = SCRIPT_DIR / "tmp" / "nrw_lod2.json"
 
-download_file(
-    NRW_SERVER + "index.json",
-    NRW_JSON,
-)
-
-with open(NRW_JSON) as json_file:
-    data = json.load(json_file)
-
-import rich
-
-rich.print(data)
-
 
+# TODO: Dry with 02_bayern.py, and move some functions to utils.py?
 def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_retries=3) -> bool:
-    """Download a file, verify its hash, and move to final location."""
+    """Download a file, verify its size, and move to final location."""
     filename = file_info["name"]
-    expected_hash = file_info["hash"]
-    urls = file_info["urls"]
-    # In order to not always download from the same server
-    random.shuffle(urls)
+    expected_size = int(file_info["size"])
+    urls = [NRW_SERVER + filename]
 
     final_path = download_dir / filename
 
     # Check if file already exists and is valid
     if final_path.exists():
         print(f"Checking existing file: {filename}")
-        if calculate_sha256(final_path) == expected_hash:
+        if os.path.getsize(final_path) == expected_size:
             print(f"✓ {filename} already downloaded and verified")
             return True
         else:
@@ -61,11 +50,10 @@ def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_
                 tmp_path.unlink()
             continue
 
-        # Verify hash
-        print(f"  Verifying hash for {filename}")
-        actual_hash = calculate_sha256(tmp_path)
+        print(f"  Verifying size for {filename}")
+        actual_size = os.path.getsize(tmp_path)
 
-        if actual_hash == expected_hash:
+        if actual_size == expected_size:
             # Move to final location
             final_path.parent.mkdir(parents=True, exist_ok=True)
             tmp_path.rename(final_path)
@@ -73,8 +61,8 @@ def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_
             return True
         else:
             print(f"  Hash mismatch for {filename}")
-            print(f"    Expected: {expected_hash}")
-            print(f"    Got:      {actual_hash}")
+            print(f"    Expected: {expected_size}")
+            print(f"    Got:      {actual_size}")
             tmp_path.unlink()
 
     print(f"✗ Failed to download {filename} after {max_retries} attempts")
@@ -82,22 +70,21 @@ def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_
 
 
 def download_all_files(
-    metalink: str, download_path: Path, tmp_path: Path = SCRIPT_DIR / "tmp", jobs: int = 4, retries: int = 3
+    json_filename: str, download_path: Path, tmp_path: Path = SCRIPT_DIR / "tmp", jobs: int = 4, retries: int = 3
 ):
     # Create directories
     tmp_path.mkdir(parents=True, exist_ok=True)
+    download_file(
+        json_filename,
+        NRW_JSON,
+    )
 
-    if metalink.startswith("https://"):
-        metalink_path = download_metalink(metalink, tmp_path)
-    else:
-        metalink_path = Path(metalink)
-
-    if not metalink_path.exists():
-        print(f"Error: Metalink file not found: {metalink_path}")
-        sys.exit(1)
+    with open(NRW_JSON) as json_file:
+        data = json.load(json_file)
 
     # Parse metalink file
-    files = parse_metalink(metalink_path)
+    files = [file for sets in data["datasets"] for file in sets["files"]]
+
     print(f"Found {len(files)} files to download\n")
 
     download_path.mkdir(parents=True, exist_ok=True)
@@ -130,3 +117,7 @@ def download_all_files(
 
     if failed > 0:
         sys.exit(1)
+
+
+if __name__ == "__main__":
+    download_all_files(NRW_SERVER + "index.json", SCRIPT_DIR / "citygml" / "nordrhein_westfalen")