Commit db1b113d authored by Eric Duminil's avatar Eric Duminil
Browse files

Downloading zip files instead

parent 9af0712c
# https://gds.hessen.de/INTERSHOP/web/WFS/HLBG-Geodaten-Site/de_DE/-/EUR/ViewDownloadcenter-Start?path=3D-Daten/3D-Geb%C3%A4udemodelle/3D-Geb%C3%A4udemodelle%20LoD2 # https://gds.hessen.de/INTERSHOP/web/WFS/HLBG-Geodaten-Site/de_DE/-/EUR/ViewDownloadcenter-Start?path=3D-Daten/3D-Geb%C3%A4udemodelle/3D-Geb%C3%A4udemodelle%20LoD2
import json import json
from datetime import datetime from datetime import datetime
from pathlib import Path
from urllib.parse import quote from urllib.parse import quote
from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_all_files, download_file from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_all_files, download_file
...@@ -13,13 +14,27 @@ HESSEN = Bundesland( ...@@ -13,13 +14,27 @@ HESSEN = Bundesland(
) )
class ZipFile(CityGMLWithDate):
@property
def download_folder(self) -> Path:
tmp_folder = TMP_DIR / self.bundesland.name
tmp_folder.mkdir(exist_ok=True)
return tmp_folder
def download(self, tmp_dir: Path = TMP_DIR, max_retries: int = 3, sleep: int = 0) -> bool:
result = super().download(tmp_dir, max_retries, sleep)
if result:
print("TODO: Extract zip")
return result
if __name__ == "__main__": if __name__ == "__main__":
local_json_path = TMP_DIR / "hessen_lod2.json" local_json_path = TMP_DIR / "hessen_lod2.json"
download_file(HESSEN.source, local_json_path) download_file(HESSEN.source, local_json_path)
with open(local_json_path) as json_file: with open(local_json_path) as json_file:
data = json.load(json_file) data = json.load(json_file)
citygmls = [] zip_files = []
for kreis_link in data["navigation"]: for kreis_link in data["navigation"]:
if "LoD2" in kreis_link["uri"] and kreis_link["level"] == 4: if "LoD2" in kreis_link["uri"] and kreis_link["level"] == 4:
kreis_info_json_path = TMP_DIR / f"hessen_{kreis_link['name']}.json" kreis_info_json_path = TMP_DIR / f"hessen_{kreis_link['name']}.json"
...@@ -29,12 +44,12 @@ if __name__ == "__main__": ...@@ -29,12 +44,12 @@ if __name__ == "__main__":
creation_date = kreis_data["searchresult"]["packages"][0]["creationDate"] creation_date = kreis_data["searchresult"]["packages"][0]["creationDate"]
# NOTE: Some links may contain spaces # NOTE: Some links may contain spaces
zip_link = quote(kreis_data["searchresult"]["packages"][0]["downloadLink"]["uri"]) zip_link = quote(kreis_data["searchresult"]["packages"][0]["downloadLink"]["uri"])
citygmls.append( zip_files.append(
CityGMLWithDate( ZipFile(
url=SERVER + zip_link, url=SERVER + zip_link,
bundesland=HESSEN, bundesland=HESSEN,
source_date=datetime.strptime(creation_date, "%d.%m.%Y"), source_date=datetime.strptime(creation_date, "%d.%m.%Y"),
) )
) )
download_all_files(citygmls, jobs=1, sleep=20) download_all_files(zip_files, jobs=1, sleep=20)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment