Commit c80d906b authored by Eric Duminil's avatar Eric Duminil
Browse files

Moving zipfile to own lib

parent d2e3f935
import json
import zipfile
from datetime import datetime
from pathlib import Path
from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_all_files, download_file
from citygml_download import TMP_DIR, Bundesland, download_all_files, download_file
from zipfile_download import ZipFile
# Data are available as WFS
# TODO: Try to find data with QGIS or Geopandas
......@@ -12,40 +11,12 @@ ZIP_URL_FORMAT = "https://opengeodata.lgl-bw.de/data/lod2/LoD2_32_%s_%s_2_bw.zip
BADEN_WUERTTEMBERG = Bundesland(
"Baden-Württemberg",
source = "https://owsproxy.lgl-bw.de/owsproxy/wfs/WFS_LGL-BW_LoD2_Aktualitaet?SERVICE=WFS&VERSION=1.1.0&REQUEST=GetFeature&TYPENAME=verm:v_lod2_aktualitaet&MAXFEATURES=50000&OUTPUTFORMAT=application/json",
source="https://owsproxy.lgl-bw.de/owsproxy/wfs/WFS_LGL-BW_LoD2_Aktualitaet?SERVICE=WFS&VERSION=1.1.0&REQUEST=GetFeature&TYPENAME=verm:v_lod2_aktualitaet&MAXFEATURES=50000&OUTPUTFORMAT=application/json",
info="https://www.lgl-bw.de/Produkte/Open-Data/",
license="dl-de/by-2-0",
)
# NOTE: Slightly different than in 07_hessen.py
class ZipFile(CityGMLWithDate):
@property
def download_folder(self) -> Path:
tmp_folder = TMP_DIR / self.bundesland.name
tmp_folder.mkdir(exist_ok=True)
return tmp_folder
def download(self, tmp_dir: Path = TMP_DIR, max_retries: int = 3, sleep: int = 0) -> bool:
result = super().download(tmp_dir, max_retries, sleep)
if result:
try:
print(f" Extracting {self.filename}")
with zipfile.ZipFile(self.path, "r") as main_zip:
# Flat extract, without directory
for zip_info in main_zip.infolist():
if zip_info.is_dir():
continue
zip_info.filename = Path(zip_info.filename).name
main_zip.extract(zip_info, self.bundesland.download_folder)
except zipfile.BadZipfile:
print(f"🛑 {self.path} is corrupt. Deleting")
self.path.unlink()
return False
return result
if __name__ == "__main__":
bw_json = TMP_DIR / "bw_lod2.json"
download_file(BADEN_WUERTTEMBERG.source, bw_json)
......
......@@ -3,13 +3,14 @@ from datetime import datetime
import requests
from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_all_files, download_file
from citygml_download import TMP_DIR, Bundesland, download_all_files, download_file
from zipfile_download import ZipFile
SACHSEN = Bundesland(
"Sachsen",
source="https://geodienste.sachsen.de/ags-relay/ArcGISServer/guest/arcgis/rest/services/geosn/rest_geosn_downloadlinks/MapServer/3/query",
info="https://www.geodaten.sachsen.de/",
license="Landesamt für Geobasisinformation Sachsen (GeoSN), DL-DE->BY-2.0"
license="Landesamt für Geobasisinformation Sachsen (GeoSN), DL-DE->BY-2.0",
)
PAGE_SIZE = 1000 # Max amount for Sachsen Server
......@@ -39,7 +40,7 @@ if __name__ == "__main__":
print(f"Starting paged query to retrieve all tiles (page size: {PAGE_SIZE})...")
current_offset = 0
keep_downloading = True
citygmls = []
zip_files = []
KEY = get_lod2_key("https://www.geodaten.sachsen.de/batch-download-4719.html")
DOWNLOAD_SERVER = f"https://geocloud.landesvermessung.sachsen.de/public.php/dav/files/{KEY}/"
......@@ -70,8 +71,8 @@ if __name__ == "__main__":
filename = attributes["Download_CityGML"].split("/")[-1]
year = int(attributes["Stand"].split("(")[0])
citygmls.append(
CityGMLWithDate(
zip_files.append(
ZipFile(
url=DOWNLOAD_SERVER + filename,
bundesland=SACHSEN,
# NOTE: No information is given about month or day, so assume end of year to be sure files get updated
......@@ -87,4 +88,4 @@ if __name__ == "__main__":
print(f"❌ Error in response: {data['error']['message']}")
break
download_all_files(citygmls)
download_all_files(zip_files)
import zipfile
from pathlib import Path
from citygml_download import TMP_DIR, CityGMLWithDate
# NOTE: Slightly different than in 07_hessen.py
class ZipFile(CityGMLWithDate):
@property
def download_folder(self) -> Path:
tmp_folder = TMP_DIR / self.bundesland.name
tmp_folder.mkdir(exist_ok=True)
return tmp_folder
def download(self, tmp_dir: Path = TMP_DIR, max_retries: int = 3, sleep: int = 0) -> bool:
result = super().download(tmp_dir, max_retries, sleep)
if result:
try:
print(f" Extracting {self.filename}")
with zipfile.ZipFile(self.path, "r") as main_zip:
# Flat extract, without directory
for zip_info in main_zip.infolist():
if zip_info.is_dir():
continue
zip_info.filename = Path(zip_info.filename).name
main_zip.extract(zip_info, self.bundesland.download_folder)
except zipfile.BadZipfile:
print(f"🛑 {self.path} is corrupt. Deleting")
self.path.unlink()
return False
return result
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment