ifmain

02_bayern.py

@@ -6,4 +6,6 @@ SCRIPT_DIR = Path(__file__).resolve().parent
 
 
 BAYERN_METALINK = "https://geodaten.bayern.de/odd/a/lod2/citygml/meta/metalink/09.meta4"
-download_metalink.download_all_files(BAYERN_METALINK, SCRIPT_DIR / "citygml" / "bayern")
+
+if __name__ == "__main__":
+    download_metalink.download_all_files(BAYERN_METALINK, SCRIPT_DIR / "citygml" / "bayern")

10_nordrhein_westfalen.py

+# https://www.opengeodata.nrw.de/produkte/geobasis/3dg/lod2_gml/
+
+import json
+import random
+from pathlib import Path
+
+from download_metalink import SCRIPT_DIR, download_file
+
+NRW_SERVER = "https://www.opengeodata.nrw.de/produkte/geobasis/3dg/lod2_gml/lod2_gml/"
+NRW_JSON = SCRIPT_DIR / "tmp" / "nrw_lod2.json"
+
+download_file(
+    NRW_SERVER + "index.json",
+    NRW_JSON,
+)
+
+with open(NRW_JSON) as json_file:
+    data = json.load(json_file)
+
+import rich
+
+rich.print(data)
+
+
+def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_retries=3) -> bool:
+    """Download a file, verify its hash, and move to final location."""
+    filename = file_info["name"]
+    expected_hash = file_info["hash"]
+    urls = file_info["urls"]
+    # In order to not always download from the same server
+    random.shuffle(urls)
+
+    final_path = download_dir / filename
+
+    # Check if file already exists and is valid
+    if final_path.exists():
+        print(f"Checking existing file: {filename}")
+        if calculate_sha256(final_path) == expected_hash:
+            print(f"✓ {filename} already downloaded and verified")
+            return True
+        else:
+            print(f"  Existing file has incorrect hash, will re-download")
+            final_path.unlink()
+
+    tmp_path = tmp_dir / filename
+
+    for attempt in range(max_retries):
+        print(f"Downloading {filename} (attempt {attempt + 1}/{max_retries})")
+
+        # Try each URL until one succeeds
+        download_success = False
+        for url in urls:
+            print(f"  Trying {url}")
+            if download_file(url, tmp_path):
+                download_success = True
+                break
+
+        if not download_success:
+            print(f"  Failed to download from all URLs")
+            if tmp_path.exists():
+                tmp_path.unlink()
+            continue
+
+        # Verify hash
+        print(f"  Verifying hash for {filename}")
+        actual_hash = calculate_sha256(tmp_path)
+
+        if actual_hash == expected_hash:
+            # Move to final location
+            final_path.parent.mkdir(parents=True, exist_ok=True)
+            tmp_path.rename(final_path)
+            print(f"✓ {filename} downloaded and verified successfully")
+            return True
+        else:
+            print(f"  Hash mismatch for {filename}")
+            print(f"    Expected: {expected_hash}")
+            print(f"    Got:      {actual_hash}")
+            tmp_path.unlink()
+
+    print(f"✗ Failed to download {filename} after {max_retries} attempts")
+    return False
+
+
+def download_all_files(
+    metalink: str, download_path: Path, tmp_path: Path = SCRIPT_DIR / "tmp", jobs: int = 4, retries: int = 3
+):
+    # Create directories
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    if metalink.startswith("https://"):
+        metalink_path = download_metalink(metalink, tmp_path)
+    else:
+        metalink_path = Path(metalink)
+
+    if not metalink_path.exists():
+        print(f"Error: Metalink file not found: {metalink_path}")
+        sys.exit(1)
+
+    # Parse metalink file
+    files = parse_metalink(metalink_path)
+    print(f"Found {len(files)} files to download\n")
+
+    download_path.mkdir(parents=True, exist_ok=True)
+
+    # Download files in parallel
+    successful = 0
+    failed = 0
+
+    with ThreadPoolExecutor(max_workers=jobs) as executor:
+        futures = {
+            executor.submit(download_and_verify, file_info, tmp_path, download_path, retries): file_info["name"]
+            for file_info in files
+        }
+
+        for future in as_completed(futures):
+            filename = futures[future]
+            try:
+                if future.result():
+                    successful += 1
+                else:
+                    failed += 1
+            except Exception as e:
+                print(f"✗ Exception while processing {filename}: {e}")
+                failed += 1
+
+    # Summary
+    print(f"\n{'='*60}")
+    print(f"Download complete: {successful} successful, {failed} failed")
+    print(f"{'='*60}")
+
+    if failed > 0:
+        sys.exit(1)