Niedersachsen

09_niedersachsen.py

-# https://www.lgln.niedersachsen.de/startseite/geodaten_karten/3d_geobasisdaten/3d_gebaudemodelle/3d-gebaudemodelle-lod1-und-lod2-142891.html
-
 # Für alle offenen Geodaten des LGLN gilt die Lizenz CC BY 4.0.
 
 import json
-import os
-import sys
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
-from pathlib import Path
 
-from download_metalink import SCRIPT_DIR, download_file
+from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_all_files, download_file
 
-NIEDERSACHSEN_GEOJSON_URL = (
-    "https://arcgis-geojson.s3.eu-de.cloud-object-storage.appdomain.cloud/lod2/lgln-opengeodata-lod2.geojson"
+NIEDERSACHSEN = Bundesland(
+    "Niedersachsen",
+    source="https://arcgis-geojson.s3.eu-de.cloud-object-storage.appdomain.cloud/lod2/lgln-opengeodata-lod2.geojson",
+    info="https://www.lgln.niedersachsen.de/startseite/geodaten_karten/3d_geobasisdaten/3d_gebaudemodelle/3d-gebaudemodelle-lod1-und-lod2-142891.html",
 )
 
 
@@ -40,125 +36,22 @@ NIEDERSACHSEN_GEOJSON_URL = (
 # },
 
 
-# TODO: Dry with 02_bayern.py and 10_nordrhein_westfalen.py, and move some functions to utils.py?
-# NOTE: Could define a CityGML class: URL, Path, Size, SHA, Date, UTM, Source, Bundesland. And "Up to date?", "Check" method, as well as "Download". "Select", "Compress" too?
-def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_retries=3) -> bool:
-    """Download a file, verify its age, and move to final location."""
-    url = file_info["xml"]
-    filename = url.split("/")[-1]
-    last_updated_server = datetime.strptime(file_info["Aktualitaet"], "%Y-%m-%d %H:%M:%S")
-    urls = [url]
-
-    final_path = download_dir / filename
-
-    # Check if file already exists and is valid
-    if final_path.exists():
-        print(f"Checking existing file: {filename}")
-        if (
-            os.path.getsize(final_path) > 0
-            and datetime.fromtimestamp(os.path.getmtime(final_path)) > last_updated_server
-        ):
-            print(f"✓ {filename} already downloaded and verified")
-            return True
-        else:
-            print(f"  Existing file is too old, will re-download")
-            final_path.unlink()
-
-    tmp_path = tmp_dir / filename
-
-    for attempt in range(max_retries):
-        print(f"Downloading {filename} (attempt {attempt + 1}/{max_retries})")
-
-        # Try each URL until one succeeds
-        download_success = False
-        for url in urls:
-            print(f"  Trying {url}")
-            if download_file(url, tmp_path):
-                download_success = True
-                break
-
-        if not download_success:
-            print(f"  Failed to download from all URLs")
-            if tmp_path.exists():
-                tmp_path.unlink()
-            continue
-
-        print(f"  Verifying size for {filename}")
-        actual_size = os.path.getsize(tmp_path)
-
-        if actual_size > 0:
-            # Move to final location
-            final_path.parent.mkdir(parents=True, exist_ok=True)
-            tmp_path.rename(final_path)
-            print(f"✓ {filename} downloaded and verified successfully")
-            return True
-        else:
-            print("Empty GML")
-            tmp_path.unlink()
-
-    print(f"✗ Failed to download {filename} after {max_retries} attempts")
-    return False
-
-
-def download_all_files(
-    json_url: str,
-    local_json: str,
-    download_path: Path,
-    tmp_path: Path = SCRIPT_DIR / "tmp",
-    jobs: int = 4,
-    retries: int = 3,
-):
-    # Create directories
-    tmp_path.mkdir(parents=True, exist_ok=True)
-    local_json_path = tmp_path / local_json
+if __name__ == "__main__":
+    local_json_path = TMP_DIR / "niedersachsen_lod2.geojson"
     if not local_json_path.exists():
-        download_file(json_url, local_json_path)
-
+        download_file(NIEDERSACHSEN.source, local_json_path)
     with open(local_json_path) as json_file:
         data = json.load(json_file)
 
-    features = data["features"]
-
-    files = [feature["properties"] for feature in features]
-
-    print(f"Found {len(files)} files to download\n")
-
-    download_path.mkdir(parents=True, exist_ok=True)
-
-    # Download files in parallel
-    successful = 0
-    failed = 0
-
-    with ThreadPoolExecutor(max_workers=jobs) as executor:
-        futures = {
-            executor.submit(download_and_verify, file_info, tmp_path, download_path, retries): file_info["xml"]
-            for file_info in files
-        }
-
-        for future in as_completed(futures):
-            filename = futures[future]
-            try:
-                if future.result():
-                    successful += 1
-                else:
-                    failed += 1
-            except Exception as e:
-                print(f"✗ Exception while processing {filename}: {e}")
-                failed += 1
-
-    # Summary
-    print(f"\n{'='*60}")
-    print(f"Download complete: {successful} successful, {failed} failed")
-    print(f"{'='*60}")
-
-    if failed > 0:
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    download_all_files(
-        NIEDERSACHSEN_GEOJSON_URL,
-        "niedersachsen_lod2.geojson",
-        SCRIPT_DIR / "citygml" / "niedersachsen",
-        retries=2,
-    )
+    citygmls = []
+    for feature in data["features"]:
+        citygmls.append(
+            CityGMLWithDate(
+                url=feature["properties"]["xml"],
+                bundesland=NIEDERSACHSEN,
+                source_date=datetime.strptime(feature["properties"]["Aktualitaet"], "%Y-%m-%d %H:%M:%S"),
+                # coordinate_reference_system ?
+            )
+        )
+
+    download_all_files(citygmls)