Feat: Rework parsing of ZipEntry

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/datastructure/LibraryObject.java

@@ -4,9 +4,13 @@ import de.hft.stuttgart.citydoctor2.parser.CityGmlParseException;
 import de.hft.stuttgart.citydoctor2.parser.CityGmlParser;
 import de.hft.stuttgart.citydoctor2.parser.InvalidGmlFileException;
 import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipEntry;
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipInputStream;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.Serial;
 import java.nio.file.Path;
 import java.util.List;
@@ -42,6 +46,22 @@ public class LibraryObject extends Geometry {
         return libOb;
     }
 
+    public static LibraryObject of(CityGmlZipEntry entry, ParserConfiguration config) {
+        String fileName = entry.getFileName();
+        if (libraryObjects.containsKey(fileName)){
+            return libraryObjects.get(fileName);
+        }
+        Geometry protoGeom = parseZipEntry(entry, config);
+        if (protoGeom == null) {
+            return null;
+        }
+        LibraryObject libOb = new LibraryObject(protoGeom.getType(), protoGeom.getLod(), Path.of(fileName), config);
+        protoGeom.getPolygons().forEach(libOb::addPolygon);
+        libOb.updateEdgesAndVertices();
+        libraryObjects.put(fileName, libOb);
+        return libOb;
+    }
+
     private LibraryObject(GeometryType type, Lod lod, Path path, ParserConfiguration config) {
         super(type, lod);
         this.filepath = path.toString();
@@ -78,4 +98,16 @@ public class LibraryObject extends Geometry {
         return geo;
     }
 
+    private static Geometry parseZipEntry(CityGmlZipEntry entry, ParserConfiguration config) {
+        Geometry geom = null;
+        try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(entry)) {
+            InputStream is = cgis.getInputStream();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return geom;
+    }
+
 }

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/mapper/citygml3/Citygml3FeatureMapper.java

@@ -36,6 +36,7 @@ import de.hft.stuttgart.citydoctor2.datastructure.Vegetation.VegetationType;
 import de.hft.stuttgart.citydoctor2.math.graph.KDTree;
 import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
 import de.hft.stuttgart.citydoctor2.utils.Localization;
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipEntry;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.citygml4j.core.model.CityGMLVersion;
@@ -77,6 +78,7 @@ public class Citygml3FeatureMapper extends ObjectWalker {
     private final CityDoctorModel model;
     private final ParserConfiguration config;
     private final Path directory;
+    private final CityGmlZipEntry zipEntry;
     private final double neighborDistance;
     private Map<String, ConcretePolygon> polygonMap = new HashMap<>();
     private Map<String, CompositeCollection> compositeMap = new HashMap<>();
@@ -88,6 +90,15 @@ public class Citygml3FeatureMapper extends ObjectWalker {
         this.directory = path.getParent();
         model = new CityDoctorModel(config, path.toFile());
         neighborDistance = 1.8d / Math.pow(10, config.getNumberOfRoundingPlaces());
+        zipEntry = null;
+    }
+
+    public Citygml3FeatureMapper(ParserConfiguration config, CityGmlZipEntry entry) {
+        this.config = config;
+        this.directory = null;
+        this.zipEntry = entry;
+        model = new CityDoctorModel(config, Path.of(entry.getFileName()).toFile());
+        neighborDistance = 1.8d / Math.pow(10, config.getNumberOfRoundingPlaces());
     }
 
     public static void parseId(AbstractGML gml, GmlElement gmlElement) {
@@ -822,8 +833,15 @@ public class Citygml3FeatureMapper extends ObjectWalker {
     private ImplicitGeometryHolder resolveImplicitGeometry(ImplicitGeometry ig, int lodInt) {
         ImplicitGeometryHolder igh = null;
         if (ig.getLibraryObject() != null) {
-            Path libraryObjectPath = directory.resolve(ig.getLibraryObject());
-            LibraryObject libObj = LibraryObject.of(libraryObjectPath, config);
+            LibraryObject libObj = null;
+            if (directory != null){
+                Path libraryObjectPath = directory.resolve(ig.getLibraryObject());
+                libObj = LibraryObject.of(libraryObjectPath, config);
+            } else {
+                String libraryObjectPath = ig.getLibraryObject();
+                CityGmlZipEntry libEntry = zipEntry.getArchive().getEntry(libraryObjectPath);
+                libObj = LibraryObject.of(libEntry, config);
+            }
             if (libObj != null) {
                 igh = new ImplicitGeometryHolder(ig, libObj);
             }

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/parser/CityGmlParser.java

@@ -26,7 +26,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
 import javax.xml.XMLConstants;
@@ -40,6 +39,8 @@ import javax.xml.validation.Schema;
 import javax.xml.validation.SchemaFactory;
 import javax.xml.validation.Validator;
 
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipEntry;
+import de.hft.stuttgart.citydoctor2.zip.CityGmlZipInputStream;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -191,19 +192,53 @@ public class CityGmlParser {
 		}
 	}
 
-	public static CityDoctorModel parseCityGmlZipEntry(ZipEntry entry, ZipFile archive, ParserConfiguration config)
+	public static CityDoctorModel parseCityGmlZipEntry(CityGmlZipEntry entry, ParserConfiguration config)
             throws CityGmlParseException, InvalidGmlFileException, IOException {
 		CityGMLContext context = getContext();
 
 		if (config.getValidate()) {
-			InputStream vis = archive.getInputStream(entry);
-			List<String> messages = validateStream(vis,context);
-			if (!messages.isEmpty()) {
-				throw new InvalidGmlFileException("Invalid GML File. First error: \n" + messages.get(0));
+			try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(entry)){
+				List<String> messages = validateStream(cgis.getInputStream(),context);
+				if (!messages.isEmpty()) {
+					throw new InvalidGmlFileException("Invalid GML File. First error: \n" + messages.get(0));
+				}
+			} catch (Exception e) {
+                throw new CityGmlParseException(e);
+            }
+
+        }
+		return decompressAndParseCityGmlEntry(entry, config, context);
+	}
+
+	public static CityDoctorModel decompressAndParseCityGmlEntry(CityGmlZipEntry entry, ParserConfiguration config, CityGMLContext context)
+			throws CityGmlParseException {
+		return decompressAndParseCityGmlEntry(entry, config, null, context);
+	}
+
+	public static CityDoctorModel decompressAndParseCityGmlEntry(CityGmlZipEntry entry, ParserConfiguration config, ProgressListener l, CityGMLContext context)
+			throws CityGmlParseException {
+
+		try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(entry)){
+			BufferedInputStream bis = new BufferedInputStream(cgis.getInputStream());
+			readEpsgCodeFromInputStream(bis,config);
+			CityGMLInputFactory in = context.createCityGMLInputFactory()
+					.withChunking(ChunkOptions.chunkByProperties(chunkProperties).skipCityModel(false));
+			try(ObservedInputStream ois = new ObservedInputStream(bis, bis.available())){
+				if (l != null){
+					ois.addListener(l::updateProgress);
+				}
+				return readAndKeepFeatures(config, entry, in, ois);
 			}
-		}
-		InputStream is = archive.getInputStream(entry);
-		return parseCityGmlStream(is, archive, config, context);
+		} catch (CityGMLReadException | IOException e) {
+			throw new CityGmlParseException("Failed to read CityGML file", e);
+		} catch (Exception e) {
+            throw new CityGmlParseException(e);
+        }
+    }
+
+	private static CityDoctorModel readAndKeepFeatures(ParserConfiguration config, CityGmlZipEntry entry,
+													   CityGMLInputFactory inputFactory, ObservedInputStream ois) throws CityGMLReadException {
+		return readAndKeepModel(new Citygml3FeatureMapper(config, entry), inputFactory, ois);
 	}
 
 	private static List<String> validateStream(InputStream vis, CityGMLContext context) throws CityGmlParseException {
@@ -239,35 +274,6 @@ public class CityGmlParser {
 		}
 	}
 
-	public static CityDoctorModel parseCityGmlStream(InputStream is, ZipFile archive ,ParserConfiguration config, CityGMLContext context)
-			throws CityGmlParseException {
-		return parseCityGmlStream(is, archive, config, null, context);
-	}
-
-	public static CityDoctorModel parseCityGmlStream(InputStream is, ZipFile archive, ParserConfiguration config, ProgressListener l, CityGMLContext context)
-			throws CityGmlParseException {
-
-		try {
-			BufferedInputStream bis = new BufferedInputStream(is);
-			readEpsgCodeFromInputStream(bis,config);
-			CityGMLInputFactory in = context.createCityGMLInputFactory()
-					.withChunking(ChunkOptions.chunkByProperties(chunkProperties).skipCityModel(false));
-			try(ObservedInputStream ois = new ObservedInputStream(bis, bis.available())){
-				if (l != null){
-					ois.addListener(l::updateProgress);
-				}
-				return readAndKeepFeatures(config, Path.of(archive.getName()), in, ois);
-			}
-		} catch (CityGMLReadException | IOException e) {
-			throw new CityGmlParseException("Failed to read CityGML file", e);
-		}
-	}
-
-	private static CityDoctorModel readAndKeepFeatures(ZipFile archive, ParserConfiguration config,
-													   CityGMLInputFactory inputFactory, ObservedInputStream ois) throws CityGMLReadException {
-		return readAndKeepModel(new Citygml3FeatureMapper(config, Path.of(archive.getName())), inputFactory, ois);
-	}
-
 	private static void readEpsgCodeFromInputStream(BufferedInputStream bis, ParserConfiguration config) throws CityGmlParseException {
 		try{
 			// Mark start position of GML-"file"
@@ -419,7 +425,7 @@ public class CityGmlParser {
 
 	/**
 	 * Suppresses logger output of {@link #readAndKeepFeatures} for the next parse.
-	 * Used to prevent logging spam while resolving implicit geometries.
+	 * Used to prevent logging spam while resolving implicit geometries and zip-files.
 	 */
 	public static void gagLogger(boolean value){
 		gagged = value;

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/utils/ArchivePacker.java

@@ -25,7 +25,7 @@ public class ArchivePacker {
 
     public static void packArchive(String targetPath, CityGmlZipArchive archive){
         if (!targetPath.endsWith(".zip")){
-            throw new IllegalArgumentException("Target zip-filepath must end with '.zip'");
+            targetPath = targetPath.concat(".zip");
         }
         Path tmpDir = null;
         try{

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipArchive.java

@@ -39,7 +39,7 @@ public class CityGmlZipArchive implements Serializable {
                     continue;
                 }
                 if (ze.getName().endsWith(".gml")) {
-                    archiveEntries.add(CityGmlZipEntry.of(ze, zip, config));
+                    archiveEntries.add(CityGmlZipEntry.of(ze, config));
                 }
             }
         } catch (IOException e) {
@@ -82,7 +82,7 @@ public class CityGmlZipArchive implements Serializable {
         try (ZipFile zip = new ZipFile(archivePath.toFile())) {
             zipFile = zip;
             for (CityGmlZipEntry entry : entries){
-                entry.loadEntry(zip, config);
+                entry.loadEntry(config);
             }
         } catch (IOException e) {
             logger.error(e);
@@ -110,13 +110,6 @@ public class CityGmlZipArchive implements Serializable {
     }
 
 
-    public InputStream getInputStream(CityGmlZipEntry entry) throws IOException {
-        if(zipFile == null){
-            throw new ZipException("Requested InputStream from unmounted CityGmlArchive");
-        }
-        return zipFile.getInputStream(zipFile.getEntry(entry.getFileName()));
-    }
-
     public CityGmlZipEntry getEntry(String fileName) {
         for(CityGmlZipEntry entry : entries){
             if(entry.getFileName().equals(fileName)){

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipEntry.java

@@ -25,61 +25,64 @@ public class CityGmlZipEntry {
     private boolean decompressed = false;
     private long size = -1L;
     private static final long MB = 1024 * 1024L;
+    private ZipEntryErrorType errorType = null;
 
-    public static CityGmlZipEntry of(ZipEntry entry, ZipFile archive, ParserConfiguration config){
+    public static CityGmlZipEntry of(ZipEntry entry, ParserConfiguration config){
         CityGmlZipEntry ze = CityGmlZipEntry.register(entry);
-        return ze.loadEntry(archive, config);
+        ze.loadEntry(config);
+        return ze;
     }
 
-    public CityGmlZipEntry loadEntry(ZipFile zip ,ParserConfiguration config){
+    public void loadEntry(ParserConfiguration config){
         if (decompressed){
-            return this;
+            return;
         }
-        ZipEntry ze = zip.getEntry(fileName);
-        try {
-            if (!entrySizeWithinMemoryLimits(ze,zip)) {
-                return new ErroneousEntry(ze, ZipEntryErrorType.EXCESSIVE_FILESIZE);
-            }
+        if (errorType != null){
+            logger.warn("Tried loading erroneous CityGmlZipEntry");
+        }
+        try{
             CityGmlParser.gagLogger(true);
-            this.model = CityGmlParser.parseCityGmlZipEntry(ze, zip, config);
+            this.model = CityGmlParser.parseCityGmlZipEntry(this, config);
             this.decompressed = true;
-            return this;
         } catch (CityGmlParseException | InvalidGmlFileException e) {
             logger.error(e);
-            return new ErroneousEntry(ze, ZipEntryErrorType.INVALID_CITY_GML_FILE);
+            this.errorType = ZipEntryErrorType.INVALID_CITY_GML_FILE;
         } catch (IOException e){
             logger.error(e);
-            return new ErroneousEntry(ze, ZipEntryErrorType.IO_ERROR);
+            this.errorType = ZipEntryErrorType.IO_ERROR;
         }
     }
 
     public static CityGmlZipEntry register(ZipEntry entry){
-        return new CityGmlZipEntry(entry, false);
+        CityGmlZipEntry cgzEntry = new CityGmlZipEntry(entry, false);
+        try{
+            if (!cgzEntry.entrySizeWithinMemoryLimits()) {
+                cgzEntry.errorType = ZipEntryErrorType.EXCESSIVE_FILESIZE;
+            }
+        } catch (IOException e){
+            logger.error(e);
+            cgzEntry.errorType = ZipEntryErrorType.IO_ERROR;
+        }
+        return cgzEntry;
     }
 
-    private boolean entrySizeWithinMemoryLimits(ZipEntry ze, ZipFile zip) throws IOException {
+    private boolean entrySizeWithinMemoryLimits() throws IOException {
         long memoryLimit = (long) Math.ceil(((double) Runtime.getRuntime().maxMemory() / MB)*0.9);
-        if (size != -1L){
+        if (size != -1L) {
             return memoryLimit > size;
         }
-        if (ze.getSize() == -1L){
-            InputStream is = zip.getInputStream(ze);
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();
-            for(int i = is.read(); i != -1; i=is.read()) {
-                baos.write(i);
-                if ((baos.size() / MB) + 1 > memoryLimit) {
-                    //Entry exceeds memory limit
-                    return false;
-                }
+        try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(this)){
+            long filesize = cgis.getFileSize(memoryLimit);
+            if (filesize != -1){
+                this.size = filesize;
+                return true;
+            } else {
+                return false;
             }
-            // end of stream reached without exceeding memory limit
-            this.size = baos.size();
-            return true;
-
-        } else {
-            size = (long) Math.ceil((double) ze.getSize() / MB);
-            return memoryLimit > size;
+        } catch (Exception e) {
+            throw new IOException(e);
         }
+
     }
 
     protected CityGmlZipEntry(ZipEntry entry, boolean decompressed) {
@@ -88,6 +91,11 @@ public class CityGmlZipEntry {
         this.decompressed = decompressed;
     }
 
+    protected CityGmlZipEntry(String fileName, boolean decompressed) {
+        this.fileName = fileName;
+        this.model = null;
+        this.decompressed = decompressed;
+    }
 
     public void setArchive(CityGmlZipArchive archive){
         parentArchive = archive;
@@ -106,6 +114,10 @@ public class CityGmlZipEntry {
         return fileName;
     }
 
+    public ZipEntryErrorType getErrorType() {
+        return errorType;
+    }
+
     public CityDoctorModel getModel() {
         return model;
     }

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipInputStream.java

+package de.hft.stuttgart.citydoctor2.zip;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+public class CityGmlZipInputStream implements AutoCloseable {
+
+    private ZipFile zip;
+    private ZipEntry zipEntry;
+    private boolean closed = false;
+    private static final long MB = 1024 * 1024L;
+
+    public CityGmlZipInputStream(CityGmlZipEntry entry) throws IOException {
+        CityGmlZipArchive archive = entry.getArchive();
+        zip = new ZipFile(archive.getArchivePath().toFile());
+        zipEntry = zip.getEntry(entry.getFileName());
+    }
+
+    public InputStream getInputStream() throws IOException {
+        if (closed){
+            throw new IOException("Stream closed");
+        }
+        return zip.getInputStream(zipEntry);
+    }
+
+    public long getFileSize(long memoryLimit) throws IOException {
+        if (closed){
+            throw new IOException("Stream closed");
+        }
+        if (zipEntry.getSize() != -1){
+            return (long) Math.ceil((double) zipEntry.getSize() / MB);
+        }
+        InputStream is = this.getInputStream();
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        for(int i = is.read(); i != -1; i=is.read()) {
+            baos.write(i);
+            if ((baos.size() / MB) + 1 > memoryLimit) {
+                //Entry exceeds memory limit
+                return -1L;
+            }
+        }
+        // end of stream reached without exceeding memory limit
+        return (long) Math.ceil((double) baos.size() / MB);
+    }
+
+    @Override
+    public void close() throws IOException {
+        zip.close();
+        closed = true;
+    }
+}

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/ErroneousEntry.java

@@ -8,8 +8,8 @@ public class ErroneousEntry extends CityGmlZipEntry {
 
     private ZipEntryErrorType errorType = null;
 
-    public ErroneousEntry(ZipEntry entry, ZipEntryErrorType errorType){
-        super(entry, true);
+    public ErroneousEntry(String filename, ZipEntryErrorType errorType){
+        super(filename, true);
         this.errorType = errorType;
     }