Commit c80d906b authored by Eric Duminil's avatar Eric Duminil
Browse files

Moving zipfile to own lib

parent d2e3f935
import json import json
import zipfile
from datetime import datetime from datetime import datetime
from pathlib import Path
from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_all_files, download_file from citygml_download import TMP_DIR, Bundesland, download_all_files, download_file
from zipfile_download import ZipFile
# Data are available as WFS # Data are available as WFS
# TODO: Try to find data with QGIS or Geopandas # TODO: Try to find data with QGIS or Geopandas
...@@ -12,40 +11,12 @@ ZIP_URL_FORMAT = "https://opengeodata.lgl-bw.de/data/lod2/LoD2_32_%s_%s_2_bw.zip ...@@ -12,40 +11,12 @@ ZIP_URL_FORMAT = "https://opengeodata.lgl-bw.de/data/lod2/LoD2_32_%s_%s_2_bw.zip
BADEN_WUERTTEMBERG = Bundesland( BADEN_WUERTTEMBERG = Bundesland(
"Baden-Württemberg", "Baden-Württemberg",
source = "https://owsproxy.lgl-bw.de/owsproxy/wfs/WFS_LGL-BW_LoD2_Aktualitaet?SERVICE=WFS&VERSION=1.1.0&REQUEST=GetFeature&TYPENAME=verm:v_lod2_aktualitaet&MAXFEATURES=50000&OUTPUTFORMAT=application/json", source="https://owsproxy.lgl-bw.de/owsproxy/wfs/WFS_LGL-BW_LoD2_Aktualitaet?SERVICE=WFS&VERSION=1.1.0&REQUEST=GetFeature&TYPENAME=verm:v_lod2_aktualitaet&MAXFEATURES=50000&OUTPUTFORMAT=application/json",
info="https://www.lgl-bw.de/Produkte/Open-Data/", info="https://www.lgl-bw.de/Produkte/Open-Data/",
license="dl-de/by-2-0", license="dl-de/by-2-0",
) )
# NOTE: Slightly different than in 07_hessen.py
class ZipFile(CityGMLWithDate):
@property
def download_folder(self) -> Path:
tmp_folder = TMP_DIR / self.bundesland.name
tmp_folder.mkdir(exist_ok=True)
return tmp_folder
def download(self, tmp_dir: Path = TMP_DIR, max_retries: int = 3, sleep: int = 0) -> bool:
result = super().download(tmp_dir, max_retries, sleep)
if result:
try:
print(f" Extracting {self.filename}")
with zipfile.ZipFile(self.path, "r") as main_zip:
# Flat extract, without directory
for zip_info in main_zip.infolist():
if zip_info.is_dir():
continue
zip_info.filename = Path(zip_info.filename).name
main_zip.extract(zip_info, self.bundesland.download_folder)
except zipfile.BadZipfile:
print(f"🛑 {self.path} is corrupt. Deleting")
self.path.unlink()
return False
return result
if __name__ == "__main__": if __name__ == "__main__":
bw_json = TMP_DIR / "bw_lod2.json" bw_json = TMP_DIR / "bw_lod2.json"
download_file(BADEN_WUERTTEMBERG.source, bw_json) download_file(BADEN_WUERTTEMBERG.source, bw_json)
......
...@@ -3,13 +3,14 @@ from datetime import datetime ...@@ -3,13 +3,14 @@ from datetime import datetime
import requests import requests
from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_all_files, download_file from citygml_download import TMP_DIR, Bundesland, download_all_files, download_file
from zipfile_download import ZipFile
SACHSEN = Bundesland( SACHSEN = Bundesland(
"Sachsen", "Sachsen",
source="https://geodienste.sachsen.de/ags-relay/ArcGISServer/guest/arcgis/rest/services/geosn/rest_geosn_downloadlinks/MapServer/3/query", source="https://geodienste.sachsen.de/ags-relay/ArcGISServer/guest/arcgis/rest/services/geosn/rest_geosn_downloadlinks/MapServer/3/query",
info="https://www.geodaten.sachsen.de/", info="https://www.geodaten.sachsen.de/",
license="Landesamt für Geobasisinformation Sachsen (GeoSN), DL-DE->BY-2.0" license="Landesamt für Geobasisinformation Sachsen (GeoSN), DL-DE->BY-2.0",
) )
PAGE_SIZE = 1000 # Max amount for Sachsen Server PAGE_SIZE = 1000 # Max amount for Sachsen Server
...@@ -39,7 +40,7 @@ if __name__ == "__main__": ...@@ -39,7 +40,7 @@ if __name__ == "__main__":
print(f"Starting paged query to retrieve all tiles (page size: {PAGE_SIZE})...") print(f"Starting paged query to retrieve all tiles (page size: {PAGE_SIZE})...")
current_offset = 0 current_offset = 0
keep_downloading = True keep_downloading = True
citygmls = [] zip_files = []
KEY = get_lod2_key("https://www.geodaten.sachsen.de/batch-download-4719.html") KEY = get_lod2_key("https://www.geodaten.sachsen.de/batch-download-4719.html")
DOWNLOAD_SERVER = f"https://geocloud.landesvermessung.sachsen.de/public.php/dav/files/{KEY}/" DOWNLOAD_SERVER = f"https://geocloud.landesvermessung.sachsen.de/public.php/dav/files/{KEY}/"
...@@ -70,8 +71,8 @@ if __name__ == "__main__": ...@@ -70,8 +71,8 @@ if __name__ == "__main__":
filename = attributes["Download_CityGML"].split("/")[-1] filename = attributes["Download_CityGML"].split("/")[-1]
year = int(attributes["Stand"].split("(")[0]) year = int(attributes["Stand"].split("(")[0])
citygmls.append( zip_files.append(
CityGMLWithDate( ZipFile(
url=DOWNLOAD_SERVER + filename, url=DOWNLOAD_SERVER + filename,
bundesland=SACHSEN, bundesland=SACHSEN,
# NOTE: No information is given about month or day, so assume end of year to be sure files get updated # NOTE: No information is given about month or day, so assume end of year to be sure files get updated
...@@ -87,4 +88,4 @@ if __name__ == "__main__": ...@@ -87,4 +88,4 @@ if __name__ == "__main__":
print(f"❌ Error in response: {data['error']['message']}") print(f"❌ Error in response: {data['error']['message']}")
break break
download_all_files(citygmls) download_all_files(zip_files)
import zipfile
from pathlib import Path
from citygml_download import TMP_DIR, CityGMLWithDate
# NOTE: Slightly different than in 07_hessen.py
class ZipFile(CityGMLWithDate):
@property
def download_folder(self) -> Path:
tmp_folder = TMP_DIR / self.bundesland.name
tmp_folder.mkdir(exist_ok=True)
return tmp_folder
def download(self, tmp_dir: Path = TMP_DIR, max_retries: int = 3, sleep: int = 0) -> bool:
result = super().download(tmp_dir, max_retries, sleep)
if result:
try:
print(f" Extracting {self.filename}")
with zipfile.ZipFile(self.path, "r") as main_zip:
# Flat extract, without directory
for zip_info in main_zip.infolist():
if zip_info.is_dir():
continue
zip_info.filename = Path(zip_info.filename).name
main_zip.extract(zip_info, self.bundesland.download_folder)
except zipfile.BadZipfile:
print(f"🛑 {self.path} is corrupt. Deleting")
self.path.unlink()
return False
return result
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment