Commit 7c5c7e80 authored by Eric Duminil's avatar Eric Duminil
Browse files

Remove corrupt zips

parent c69d76a9
# https://gds.hessen.de/INTERSHOP/web/WFS/HLBG-Geodaten-Site/de_DE/-/EUR/ViewDownloadcenter-Start?path=3D-Daten/3D-Geb%C3%A4udemodelle/3D-Geb%C3%A4udemodelle%20LoD2 # https://gds.hessen.de/INTERSHOP/web/WFS/HLBG-Geodaten-Site/de_DE/-/EUR/ViewDownloadcenter-Start?path=3D-Daten/3D-Geb%C3%A4udemodelle/3D-Geb%C3%A4udemodelle%20LoD2
import json import json
import zipfile
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from urllib.parse import quote from urllib.parse import quote
import zipfile
from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_all_files, download_file from citygml_download import TMP_DIR, Bundesland, CityGMLWithDate, download_all_files, download_file
...@@ -26,12 +26,21 @@ class ZipFile(CityGMLWithDate): ...@@ -26,12 +26,21 @@ class ZipFile(CityGMLWithDate):
result = super().download(tmp_dir, max_retries, sleep) result = super().download(tmp_dir, max_retries, sleep)
if result: if result:
output_folder = self.download_folder / self.path.stem output_folder = self.download_folder / self.path.stem
with zipfile.ZipFile(self.path, "r") as main_zip: try:
main_zip.extractall(output_folder) print(f" Extracting {self.filename}", end="")
for inner_zip_path in output_folder.glob('*.zip'): with zipfile.ZipFile(self.path, "r") as main_zip:
main_zip.extractall(output_folder)
except zipfile.BadZipfile:
print(f"🛑 {self.path} is corrupt. Deleting")
self.path.unlink()
return False
for inner_zip_path in output_folder.glob("*.zip"):
with zipfile.ZipFile(inner_zip_path, "r") as inner_zip: with zipfile.ZipFile(inner_zip_path, "r") as inner_zip:
inner_zip.extractall(self.bundesland.download_folder) inner_zip.extractall(self.bundesland.download_folder)
print(f" Extracted {inner_zip_path}") print(".", end="", flush=True)
print()
return result return result
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment