From 3de3973265ace2d579d0bf6c66bddf67f0c53851 Mon Sep 17 00:00:00 2001
From: Riegel <alexander.riegel@hft-stuttgart.de>
Date: Mon, 9 Dec 2024 17:11:43 +0100
Subject: [PATCH] Feat: Rework parsing of ZipEntry
---
.../datastructure/LibraryObject.java | 32 +++++++
.../citygml3/Citygml3FeatureMapper.java | 22 ++++-
.../citydoctor2/parser/CityGmlParser.java | 84 ++++++++++---------
.../citydoctor2/utils/ArchivePacker.java | 2 +-
.../citydoctor2/zip/CityGmlZipArchive.java | 11 +--
.../citydoctor2/zip/CityGmlZipEntry.java | 76 ++++++++++-------
.../zip/CityGmlZipInputStream.java | 54 ++++++++++++
.../citydoctor2/zip/ErroneousEntry.java | 4 +-
8 files changed, 200 insertions(+), 85 deletions(-)
create mode 100644 CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipInputStream.java
diff --git a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/datastructure/LibraryObject.java b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/datastructure/LibraryObject.java
index f1e2024..2459a39 100644
--- a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/datastructure/LibraryObject.java
+++ b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/datastructure/LibraryObject.java
@@ -4,9 +4,13 @@ import de.hft.stuttgart.citydoctor2.parser.CityGmlParseException;
import de.hft.stuttgart.citydoctor2.parser.CityGmlParser;
import de.hft.stuttgart.citydoctor2.parser.InvalidGmlFileException;
import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipEntry;
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipInputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.io.IOException;
+import java.io.InputStream;
import java.io.Serial;
import java.nio.file.Path;
import java.util.List;
@@ -42,6 +46,22 @@ public class LibraryObject extends Geometry {
return libOb;
}
+ public static LibraryObject of(CityGmlZipEntry entry, ParserConfiguration config) {
+ String fileName = entry.getFileName();
+ if (libraryObjects.containsKey(fileName)){
+ return libraryObjects.get(fileName);
+ }
+ Geometry protoGeom = parseZipEntry(entry, config);
+ if (protoGeom == null) {
+ return null;
+ }
+ LibraryObject libOb = new LibraryObject(protoGeom.getType(), protoGeom.getLod(), Path.of(fileName), config);
+ protoGeom.getPolygons().forEach(libOb::addPolygon);
+ libOb.updateEdgesAndVertices();
+ libraryObjects.put(fileName, libOb);
+ return libOb;
+ }
+
private LibraryObject(GeometryType type, Lod lod, Path path, ParserConfiguration config) {
super(type, lod);
this.filepath = path.toString();
@@ -78,4 +98,16 @@ public class LibraryObject extends Geometry {
return geo;
}
+ private static Geometry parseZipEntry(CityGmlZipEntry entry, ParserConfiguration config) {
+ Geometry geom = null;
+ try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(entry)) {
+ InputStream is = cgis.getInputStream();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return geom;
+ }
+
}
diff --git a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/mapper/citygml3/Citygml3FeatureMapper.java b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/mapper/citygml3/Citygml3FeatureMapper.java
index b70dcaf..7f8c33a 100644
--- a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/mapper/citygml3/Citygml3FeatureMapper.java
+++ b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/mapper/citygml3/Citygml3FeatureMapper.java
@@ -36,6 +36,7 @@ import de.hft.stuttgart.citydoctor2.datastructure.Vegetation.VegetationType;
import de.hft.stuttgart.citydoctor2.math.graph.KDTree;
import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
import de.hft.stuttgart.citydoctor2.utils.Localization;
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipEntry;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.citygml4j.core.model.CityGMLVersion;
@@ -77,6 +78,7 @@ public class Citygml3FeatureMapper extends ObjectWalker {
private final CityDoctorModel model;
private final ParserConfiguration config;
private final Path directory;
+ private final CityGmlZipEntry zipEntry;
private final double neighborDistance;
private Map<String, ConcretePolygon> polygonMap = new HashMap<>();
private Map<String, CompositeCollection> compositeMap = new HashMap<>();
@@ -88,6 +90,15 @@ public class Citygml3FeatureMapper extends ObjectWalker {
this.directory = path.getParent();
model = new CityDoctorModel(config, path.toFile());
neighborDistance = 1.8d / Math.pow(10, config.getNumberOfRoundingPlaces());
+ zipEntry = null;
+ }
+
+ public Citygml3FeatureMapper(ParserConfiguration config, CityGmlZipEntry entry) {
+ this.config = config;
+ this.directory = null;
+ this.zipEntry = entry;
+ model = new CityDoctorModel(config, Path.of(entry.getFileName()).toFile());
+ neighborDistance = 1.8d / Math.pow(10, config.getNumberOfRoundingPlaces());
}
public static void parseId(AbstractGML gml, GmlElement gmlElement) {
@@ -822,8 +833,15 @@ public class Citygml3FeatureMapper extends ObjectWalker {
private ImplicitGeometryHolder resolveImplicitGeometry(ImplicitGeometry ig, int lodInt) {
ImplicitGeometryHolder igh = null;
if (ig.getLibraryObject() != null) {
- Path libraryObjectPath = directory.resolve(ig.getLibraryObject());
- LibraryObject libObj = LibraryObject.of(libraryObjectPath, config);
+ LibraryObject libObj = null;
+ if (directory != null){
+ Path libraryObjectPath = directory.resolve(ig.getLibraryObject());
+ libObj = LibraryObject.of(libraryObjectPath, config);
+ } else {
+ String libraryObjectPath = ig.getLibraryObject();
+ CityGmlZipEntry libEntry = zipEntry.getArchive().getEntry(libraryObjectPath);
+ libObj = LibraryObject.of(libEntry, config);
+ }
if (libObj != null) {
igh = new ImplicitGeometryHolder(ig, libObj);
}
diff --git a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/parser/CityGmlParser.java b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/parser/CityGmlParser.java
index 3b09332..716a736 100644
--- a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/parser/CityGmlParser.java
+++ b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/parser/CityGmlParser.java
@@ -26,7 +26,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.xml.XMLConstants;
@@ -40,6 +39,8 @@ import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator;
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipEntry;
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipInputStream;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -191,19 +192,53 @@ public class CityGmlParser {
}
}
- public static CityDoctorModel parseCityGmlZipEntry(ZipEntry entry, ZipFile archive, ParserConfiguration config)
+ public static CityDoctorModel parseCityGmlZipEntry(CityGmlZipEntry entry, ParserConfiguration config)
throws CityGmlParseException, InvalidGmlFileException, IOException {
CityGMLContext context = getContext();
if (config.getValidate()) {
- InputStream vis = archive.getInputStream(entry);
- List<String> messages = validateStream(vis,context);
- if (!messages.isEmpty()) {
- throw new InvalidGmlFileException("Invalid GML File. First error: \n" + messages.get(0));
+ try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(entry)){
+ List<String> messages = validateStream(cgis.getInputStream(),context);
+ if (!messages.isEmpty()) {
+ throw new InvalidGmlFileException("Invalid GML File. First error: \n" + messages.get(0));
+ }
+ } catch (Exception e) {
+ throw new CityGmlParseException(e);
+ }
+
+ }
+ return decompressAndParseCityGmlEntry(entry, config, context);
+ }
+
+ public static CityDoctorModel decompressAndParseCityGmlEntry(CityGmlZipEntry entry, ParserConfiguration config, CityGMLContext context)
+ throws CityGmlParseException {
+ return decompressAndParseCityGmlEntry(entry, config, null, context);
+ }
+
+ public static CityDoctorModel decompressAndParseCityGmlEntry(CityGmlZipEntry entry, ParserConfiguration config, ProgressListener l, CityGMLContext context)
+ throws CityGmlParseException {
+
+ try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(entry)){
+ BufferedInputStream bis = new BufferedInputStream(cgis.getInputStream());
+ readEpsgCodeFromInputStream(bis,config);
+ CityGMLInputFactory in = context.createCityGMLInputFactory()
+ .withChunking(ChunkOptions.chunkByProperties(chunkProperties).skipCityModel(false));
+ try(ObservedInputStream ois = new ObservedInputStream(bis, bis.available())){
+ if (l != null){
+ ois.addListener(l::updateProgress);
+ }
+ return readAndKeepFeatures(config, entry, in, ois);
}
- }
- InputStream is = archive.getInputStream(entry);
- return parseCityGmlStream(is, archive, config, context);
+ } catch (CityGMLReadException | IOException e) {
+ throw new CityGmlParseException("Failed to read CityGML file", e);
+ } catch (Exception e) {
+ throw new CityGmlParseException(e);
+ }
+ }
+
+ private static CityDoctorModel readAndKeepFeatures(ParserConfiguration config, CityGmlZipEntry entry,
+ CityGMLInputFactory inputFactory, ObservedInputStream ois) throws CityGMLReadException {
+ return readAndKeepModel(new Citygml3FeatureMapper(config, entry), inputFactory, ois);
}
private static List<String> validateStream(InputStream vis, CityGMLContext context) throws CityGmlParseException {
@@ -239,35 +274,6 @@ public class CityGmlParser {
}
}
- public static CityDoctorModel parseCityGmlStream(InputStream is, ZipFile archive ,ParserConfiguration config, CityGMLContext context)
- throws CityGmlParseException {
- return parseCityGmlStream(is, archive, config, null, context);
- }
-
- public static CityDoctorModel parseCityGmlStream(InputStream is, ZipFile archive, ParserConfiguration config, ProgressListener l, CityGMLContext context)
- throws CityGmlParseException {
-
- try {
- BufferedInputStream bis = new BufferedInputStream(is);
- readEpsgCodeFromInputStream(bis,config);
- CityGMLInputFactory in = context.createCityGMLInputFactory()
- .withChunking(ChunkOptions.chunkByProperties(chunkProperties).skipCityModel(false));
- try(ObservedInputStream ois = new ObservedInputStream(bis, bis.available())){
- if (l != null){
- ois.addListener(l::updateProgress);
- }
- return readAndKeepFeatures(config, Path.of(archive.getName()), in, ois);
- }
- } catch (CityGMLReadException | IOException e) {
- throw new CityGmlParseException("Failed to read CityGML file", e);
- }
- }
-
- private static CityDoctorModel readAndKeepFeatures(ZipFile archive, ParserConfiguration config,
- CityGMLInputFactory inputFactory, ObservedInputStream ois) throws CityGMLReadException {
- return readAndKeepModel(new Citygml3FeatureMapper(config, Path.of(archive.getName())), inputFactory, ois);
- }
-
private static void readEpsgCodeFromInputStream(BufferedInputStream bis, ParserConfiguration config) throws CityGmlParseException {
try{
// Mark start position of GML-"file"
@@ -419,7 +425,7 @@ public class CityGmlParser {
/**
* Suppresses logger output of {@link #readAndKeepFeatures} for the next parse.
- * Used to prevent logging spam while resolving implicit geometries.
+ * Used to prevent logging spam while resolving implicit geometries and zip-files.
*/
public static void gagLogger(boolean value){
gagged = value;
diff --git a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/utils/ArchivePacker.java b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/utils/ArchivePacker.java
index 50f77bf..239a5eb 100644
--- a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/utils/ArchivePacker.java
+++ b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/utils/ArchivePacker.java
@@ -25,7 +25,7 @@ public class ArchivePacker {
public static void packArchive(String targetPath, CityGmlZipArchive archive){
if (!targetPath.endsWith(".zip")){
- throw new IllegalArgumentException("Target zip-filepath must end with '.zip'");
+ targetPath = targetPath.concat(".zip");
}
Path tmpDir = null;
try{
diff --git a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipArchive.java b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipArchive.java
index b967f25..ade36b3 100644
--- a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipArchive.java
+++ b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipArchive.java
@@ -39,7 +39,7 @@ public class CityGmlZipArchive implements Serializable {
continue;
}
if (ze.getName().endsWith(".gml")) {
- archiveEntries.add(CityGmlZipEntry.of(ze, zip, config));
+ archiveEntries.add(CityGmlZipEntry.of(ze, config));
}
}
} catch (IOException e) {
@@ -82,7 +82,7 @@ public class CityGmlZipArchive implements Serializable {
try (ZipFile zip = new ZipFile(archivePath.toFile())) {
zipFile = zip;
for (CityGmlZipEntry entry : entries){
- entry.loadEntry(zip, config);
+ entry.loadEntry(config);
}
} catch (IOException e) {
logger.error(e);
@@ -110,13 +110,6 @@ public class CityGmlZipArchive implements Serializable {
}
- public InputStream getInputStream(CityGmlZipEntry entry) throws IOException {
- if(zipFile == null){
- throw new ZipException("Requested InputStream from unmounted CityGmlArchive");
- }
- return zipFile.getInputStream(zipFile.getEntry(entry.getFileName()));
- }
-
public CityGmlZipEntry getEntry(String fileName) {
for(CityGmlZipEntry entry : entries){
if(entry.getFileName().equals(fileName)){
diff --git a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipEntry.java b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipEntry.java
index 16a1609..78ea6ee 100644
--- a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipEntry.java
+++ b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipEntry.java
@@ -25,61 +25,64 @@ public class CityGmlZipEntry {
private boolean decompressed = false;
private long size = -1L;
private static final long MB = 1024 * 1024L;
+ private ZipEntryErrorType errorType = null;
- public static CityGmlZipEntry of(ZipEntry entry, ZipFile archive, ParserConfiguration config){
+ public static CityGmlZipEntry of(ZipEntry entry, ParserConfiguration config){
CityGmlZipEntry ze = CityGmlZipEntry.register(entry);
- return ze.loadEntry(archive, config);
+ ze.loadEntry(config);
+ return ze;
}
- public CityGmlZipEntry loadEntry(ZipFile zip ,ParserConfiguration config){
+ public void loadEntry(ParserConfiguration config){
if (decompressed){
- return this;
+ return;
}
- ZipEntry ze = zip.getEntry(fileName);
- try {
- if (!entrySizeWithinMemoryLimits(ze,zip)) {
- return new ErroneousEntry(ze, ZipEntryErrorType.EXCESSIVE_FILESIZE);
- }
+ if (errorType != null){
+ logger.warn("Tried loading erroneous CityGmlZipEntry");
+ }
+ try{
CityGmlParser.gagLogger(true);
- this.model = CityGmlParser.parseCityGmlZipEntry(ze, zip, config);
+ this.model = CityGmlParser.parseCityGmlZipEntry(this, config);
this.decompressed = true;
- return this;
} catch (CityGmlParseException | InvalidGmlFileException e) {
logger.error(e);
- return new ErroneousEntry(ze, ZipEntryErrorType.INVALID_CITY_GML_FILE);
+ this.errorType = ZipEntryErrorType.INVALID_CITY_GML_FILE;
} catch (IOException e){
logger.error(e);
- return new ErroneousEntry(ze, ZipEntryErrorType.IO_ERROR);
+ this.errorType = ZipEntryErrorType.IO_ERROR;
}
}
public static CityGmlZipEntry register(ZipEntry entry){
- return new CityGmlZipEntry(entry, false);
+ CityGmlZipEntry cgzEntry = new CityGmlZipEntry(entry, false);
+ try{
+ if (!cgzEntry.entrySizeWithinMemoryLimits()) {
+ cgzEntry.errorType = ZipEntryErrorType.EXCESSIVE_FILESIZE;
+ }
+ } catch (IOException e){
+ logger.error(e);
+ cgzEntry.errorType = ZipEntryErrorType.IO_ERROR;
+ }
+ return cgzEntry;
}
- private boolean entrySizeWithinMemoryLimits(ZipEntry ze, ZipFile zip) throws IOException {
+ private boolean entrySizeWithinMemoryLimits() throws IOException {
long memoryLimit = (long) Math.ceil(((double) Runtime.getRuntime().maxMemory() / MB)*0.9);
- if (size != -1L){
+ if (size != -1L) {
return memoryLimit > size;
}
- if (ze.getSize() == -1L){
- InputStream is = zip.getInputStream(ze);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- for(int i = is.read(); i != -1; i=is.read()) {
- baos.write(i);
- if ((baos.size() / MB) + 1 > memoryLimit) {
- //Entry exceeds memory limit
- return false;
- }
+ try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(this)){
+ long filesize = cgis.getFileSize(memoryLimit);
+ if (filesize != -1){
+ this.size = filesize;
+ return true;
+ } else {
+ return false;
}
- // end of stream reached without exceeding memory limit
- this.size = baos.size();
- return true;
-
- } else {
- size = (long) Math.ceil((double) ze.getSize() / MB);
- return memoryLimit > size;
+ } catch (Exception e) {
+ throw new IOException(e);
}
+
}
protected CityGmlZipEntry(ZipEntry entry, boolean decompressed) {
@@ -88,6 +91,11 @@ public class CityGmlZipEntry {
this.decompressed = decompressed;
}
+ protected CityGmlZipEntry(String fileName, boolean decompressed) {
+ this.fileName = fileName;
+ this.model = null;
+ this.decompressed = decompressed;
+ }
public void setArchive(CityGmlZipArchive archive){
parentArchive = archive;
@@ -106,6 +114,10 @@ public class CityGmlZipEntry {
return fileName;
}
+ public ZipEntryErrorType getErrorType() {
+ return errorType;
+ }
+
public CityDoctorModel getModel() {
return model;
}
diff --git a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipInputStream.java b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipInputStream.java
new file mode 100644
index 0000000..593f364
--- /dev/null
+++ b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipInputStream.java
@@ -0,0 +1,54 @@
+package de.hft.stuttgart.citydoctor2.zip;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+public class CityGmlZipInputStream implements AutoCloseable {
+
+ private ZipFile zip;
+ private ZipEntry zipEntry;
+ private boolean closed = false;
+ private static final long MB = 1024 * 1024L;
+
+ public CityGmlZipInputStream(CityGmlZipEntry entry) throws IOException {
+ CityGmlZipArchive archive = entry.getArchive();
+ zip = new ZipFile(archive.getArchivePath().toFile());
+ zipEntry = zip.getEntry(entry.getFileName());
+ }
+
+ public InputStream getInputStream() throws IOException {
+ if (closed){
+ throw new IOException("Stream closed");
+ }
+ return zip.getInputStream(zipEntry);
+ }
+
+ public long getFileSize(long memoryLimit) throws IOException {
+ if (closed){
+ throw new IOException("Stream closed");
+ }
+ if (zipEntry.getSize() != -1){
+ return (long) Math.ceil((double) zipEntry.getSize() / MB);
+ }
+ InputStream is = this.getInputStream();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for(int i = is.read(); i != -1; i=is.read()) {
+ baos.write(i);
+ if ((baos.size() / MB) + 1 > memoryLimit) {
+ //Entry exceeds memory limit
+ return -1L;
+ }
+ }
+ // end of stream reached without exceeding memory limit
+ return (long) Math.ceil((double) baos.size() / MB);
+ }
+
+ @Override
+ public void close() throws IOException {
+ zip.close();
+ closed = true;
+ }
+}
diff --git a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/ErroneousEntry.java b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/ErroneousEntry.java
index c103ce1..16bdd0c 100644
--- a/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/ErroneousEntry.java
+++ b/CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/ErroneousEntry.java
@@ -8,8 +8,8 @@ public class ErroneousEntry extends CityGmlZipEntry {
private ZipEntryErrorType errorType = null;
- public ErroneousEntry(ZipEntry entry, ZipEntryErrorType errorType){
- super(entry, true);
+ public ErroneousEntry(String filename, ZipEntryErrorType errorType){
+ super(filename, true);
this.errorType = errorType;
}
--
GitLab