Trying to download BW zips

01_baden_wuerttemberg.py

-from citygml_download import Bundesland, TMP_DIR, download_file
 import json
+import zipfile
+from datetime import datetime
+from pathlib import Path
+
+from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_file, download_all_files
 
 # Data are available as WFS
 # NOTE: Use geopandas or specific WFS lib?
 # TODO: Try to find data with QGIS or Geopandas
 BW_WFS = "https://owsproxy.lgl-bw.de/owsproxy/wfs/WFS_LGL-BW_LoD2_Aktualitaet?SERVICE=WFS&VERSION=1.1.0&REQUEST=GetFeature&TYPENAME=verm:v_lod2_aktualitaet&MAXFEATURES=50000&OUTPUTFORMAT=application/json"
 
+ZIP_URL_FORMAT = "https://opengeodata.lgl-bw.de/data/lod2/LoD2_32_%s_%s_2_bw.zip"
+
 BADEN_WUERTTEMBERG = Bundesland(
     "Baden-Württemberg",
     source="https://opengeodata.lgl-bw.de/#/(sidenav:product/lod2)",
@@ -13,98 +19,54 @@ BADEN_WUERTTEMBERG = Bundesland(
     license="dl-de/by-2-0",
 )
 
-if __name__ == "__main__":
-    # bw_json = TMP_DIR / "bw_lod2.json"
-    # download_file(
-    #     BW_WFS,
-    #     bw_json
-    # )
-    #
-    # with open(bw_json) as json_file:
-    #     data = json.load(json_file)
-    # xs, ys = [], []
-    # import matplotlib.pyplot as plt
-    # for feature in data["features"]:
-    #     x, y = feature['bbox'][:2]
-    #     date = feature['properties']['produktionsdatum'] or '1970-01-01'
-    #     xs.append(x)
-    #     ys.append(y)
-    #     # citygmls.append(
-    #     #     CityGMLWithDate(
-    #     #         url=feature["properties"]["xml"],
-    #     #         bundesland=NIEDERSACHSEN,
-    #     #         source_date=datetime.strptime(feature["properties"]["Aktualitaet"], "%Y-%m-%d %H:%M:%S"),
-    #     #         # coordinate_reference_system ?
-    #     #     )
-    #     # )
-    # # TODO: Show dates too?
-    # plt.scatter(xs, ys, s=0.1)
-    # plt.show()
-    from owslib.wfs import WebFeatureService
-    import geopandas as gpd
-    import io
-    from citygml_download import TMP_DIR
-
-# 1. Define the WFS URL and Feature Type
-    WFS_URL = "https://owsproxy.lgl-bw.de/owsproxy/wfs/WFS_LGL-BW_LoD2_Aktualitaet?"
-    FEATURE_TYPE = "verm:v_lod2_aktualitaet"
-
-# 2. Connect to the WFS
-    try:
-        # Use version 2.0.0 as it's the modern standard
-        wfs = WebFeatureService(url=WFS_URL, version="2.0.0")
-    except Exception as e:
-        print(f"Error connecting to WFS: {e}")
-        # Handle error or exit
-
-# 3. Request all features in GeoJSON format
-# We specify a large 'count' (MAXIMUM=1000000) to request all tiles.
-# Since the server handles streaming large files, this should work without manual pagination.
-    print(f"Requesting features for {FEATURE_TYPE}...")
-
-    try:
-        # Get the raw response content (GeoJSON string)
-        response = wfs.getfeature(
-            typename=FEATURE_TYPE,
-            outputFormat="application/json",
-            # count=1000000,  # Request a large number to ensure all tiles are returned
-        ).read()
 
-        # 4. Convert the GeoJSON response to a GeoPandas GeoDataFrame
-        # Use io.BytesIO to treat the response string as a file for geopandas
-        gdf = gpd.read_file(io.BytesIO(response), driver="GeoJSON")
+# NOTE: Slightly different than in 07_hessen.py
+class ZipFile(CityGMLWithDate):
+    def download(self, tmp_dir: Path = TMP_DIR, max_retries: int = 3, sleep: int = 0) -> bool:
+        result = super().download(tmp_dir, max_retries, sleep)
+        if result:
+            output_folder = self.download_folder
+            try:
+                print(f"  Extracting {self.filename}")
+                with zipfile.ZipFile(self.path, "r") as main_zip:
+                    main_zip.extractall(output_folder)
+            except zipfile.BadZipfile:
+                print(f"🛑 {self.path} is corrupt. Deleting")
+                self.path.unlink()
+                return False
 
-        # 5. Review and Save the Data
+        return result
 
-        # Print the total number of tiles successfully loaded
-        print(f"\n✅ Download complete. Total tiles loaded: {len(gdf)}")
 
-        # Save the GeoDataFrame to a file
-        file_path = "bw_lod2_aktualitaet_tiles.geojson"
-        gdf.to_file(TMP_DIR / file_path, driver="GeoJSON")
-        print(f"Data saved to: {file_path}")
-
-        print(gdf.columns)
-
-        # Inspect the key columns
-        print("\nFirst 5 tiles and their Kachelnamen:")
-        print(gdf[["kachelname", "produktionsdatum", "dgm_datum"]].head())
-        print(gdf["dgm_datum"].describe())
-        print(gdf["produktionsdatum"].describe())
-        # Assuming your GeoDataFrame is named 'gdf'
-        import pandas as pd
-
-# 1. Ensure 'produktionsdatum' is in datetime format
-        gdf['produktionsdatum'] = pd.to_datetime(gdf['produktionsdatum'])
-
-# 2. Sort the GeoDataFrame by the more recent 'produktionsdatum'
-# This column effectively captures the latest update that led to file re-packaging.
-        most_recent_tiles = gdf.sort_values(by='produktionsdatum', ascending=False)
-
-# 3. Display the top 10 most recently updated tiles
-        print("Top 10 most recently updated tiles, sorted by 'produktionsdatum':")
-        print(most_recent_tiles[['kachelname', 'produktionsdatum', 'dgm_datum']].head(10))
-
-    except Exception as e:
-        print(f"\n❌ An error occurred during the GetFeature request or processing: {e}")
-        print("If the error is related to size, consider manually paginating the requests.")
+if __name__ == "__main__":
+    bw_json = TMP_DIR / "bw_lod2.json"
+    download_file(BW_WFS, bw_json)
+
+    with open(bw_json) as json_file:
+        data = json.load(json_file)
+
+    zip_files = []
+
+    for feature in data["features"]:
+        date = feature["properties"]["produktionsdatum"] or "1970-01-01"
+        name = feature["properties"]["kachelname"]
+        x, y = name.split("-")
+
+        # Zip contains 4 GMLs. X odd, Y even.
+        # example: https://opengeodata.lgl-bw.de/data/lod2/LoD2_32_403_5312_2_bw.zip
+        if int(x) % 2 == 0:
+            continue
+        if int(y) % 2 == 1:
+            continue
+
+        url = ZIP_URL_FORMAT % (x, y)
+        zip_files.append(
+            ZipFile(
+                url=url,
+                bundesland=BADEN_WUERTTEMBERG,
+                source_date=datetime.strptime(date, "%Y-%m-%d"),
+                # coordinate_reference_system ?
+            )
+        )
+
+    download_all_files(zip_files)