Working example?

09_niedersachsen.py

@@ -6,6 +6,7 @@ import json
 import os
 import sys
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
 from pathlib import Path
 
 from download_metalink import SCRIPT_DIR, download_file
@@ -13,17 +14,7 @@ from download_metalink import SCRIPT_DIR, download_file
 NIEDERSACHSEN_GEOJSON_URL = (
     "https://arcgis-geojson.s3.eu-de.cloud-object-storage.appdomain.cloud/lod2/lgln-opengeodata-lod2.geojson"
 )
-NIEDERSACHSEN_JSON = SCRIPT_DIR / "tmp" / "niedersachsen_lod2.geojson"
 
-tmp_path = SCRIPT_DIR / "tmp"
-tmp_path.mkdir(parents=True, exist_ok=True)
-if not NIEDERSACHSEN_JSON.exists():
-    download_file(NIEDERSACHSEN_GEOJSON_URL, NIEDERSACHSEN_JSON)
-
-with open(NIEDERSACHSEN_JSON) as json_file:
-    data = json.load(json_file)
-
-features = data["features"]
 
 # {
 #     'type': 'Feature',
@@ -50,19 +41,23 @@ features = data["features"]
 
 
 # TODO: Dry with 02_bayern.py and 10_nordrhein_westfalen.py, and move some functions to utils.py?
-# NOTE: Could define a CityGML class: URL, Path, Size, SHA, Date, UTM, Source, Bundesland. And "Check" method, as well as "Download". "Select", "Compress" too?
+# NOTE: Could define a CityGML class: URL, Path, Size, SHA, Date, UTM, Source, Bundesland. And "Up to date?", "Check" method, as well as "Download". "Select", "Compress" too?
 def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_retries=3) -> bool:
     """Download a file, verify its size, and move to final location."""
-    filename = file_info["name"]
-    expected_size = int(file_info["size"])
-    urls = [NRW_SERVER + filename]
+    url = file_info["xml"]
+    filename = url.split("/")[-1]
+    last_updated_server = datetime.strptime(file_info["Aktualitaet"], "%Y-%m-%d %H:%M:%S")
+    urls = [url]
 
     final_path = download_dir / filename
 
     # Check if file already exists and is valid
     if final_path.exists():
         print(f"Checking existing file: {filename}")
-        if os.path.getsize(final_path) == expected_size:
+        if (
+            os.path.getsize(final_path) > 0
+            and datetime.fromtimestamp(os.path.getmtime(final_path)) > last_updated_server
+        ):
             print(f"✓ {filename} already downloaded and verified")
             return True
         else:
@@ -91,16 +86,14 @@ def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_
         print(f"  Verifying size for {filename}")
         actual_size = os.path.getsize(tmp_path)
 
-        if actual_size == expected_size:
+        if actual_size > 0:
             # Move to final location
             final_path.parent.mkdir(parents=True, exist_ok=True)
             tmp_path.rename(final_path)
             print(f"✓ {filename} downloaded and verified successfully")
             return True
         else:
-            print(f"  Hash mismatch for {filename}")
-            print(f"    Expected: {expected_size}")
-            print(f"    Got:      {actual_size}")
+            print("Empty GML")
             tmp_path.unlink()
 
     print(f"✗ Failed to download {filename} after {max_retries} attempts")
@@ -108,20 +101,25 @@ def download_and_verify(file_info: dict, tmp_dir: Path, download_dir: Path, max_
 
 
 def download_all_files(
-    json_filename: str, download_path: Path, tmp_path: Path = SCRIPT_DIR / "tmp", jobs: int = 4, retries: int = 3
+    json_url: str,
+    local_json: str,
+    download_path: Path,
+    tmp_path: Path = SCRIPT_DIR / "tmp",
+    jobs: int = 4,
+    retries: int = 3,
 ):
     # Create directories
     tmp_path.mkdir(parents=True, exist_ok=True)
-    download_file(
-        json_filename,
-        NRW_JSON,
-    )
+    local_json_path = tmp_path / local_json
+    if not local_json_path.exists():
+        download_file(json_url, local_json_path)
 
-    with open(NRW_JSON) as json_file:
+    with open(local_json_path) as json_file:
         data = json.load(json_file)
 
-    # Parse metalink file
-    files = [file for sets in data["datasets"] for file in sets["files"]]
+    features = data["features"][:20]
+
+    files = [feature["properties"] for feature in features]
 
     print(f"Found {len(files)} files to download\n")
 
@@ -133,7 +131,7 @@ def download_all_files(
 
     with ThreadPoolExecutor(max_workers=jobs) as executor:
         futures = {
-            executor.submit(download_and_verify, file_info, tmp_path, download_path, retries): file_info["name"]
+            executor.submit(download_and_verify, file_info, tmp_path, download_path, retries): file_info["xml"]
             for file_info in files
         }
 
@@ -158,4 +156,8 @@ def download_all_files(
 
 
 if __name__ == "__main__":
-    download_all_files(NRW_SERVER + "index.json", SCRIPT_DIR / "citygml" / "nordrhein_westfalen")
+    download_all_files(
+        NIEDERSACHSEN_GEOJSON_URL,
+        "niedersachsen_lod2.geojson",
+        SCRIPT_DIR / "citygml" / "niedersachsen",
+    )