Feat: CityGml-validation of CityGmlZipEntry-Stream

Adds supporting functions for stream-reading of CityGmlZipEntry's
compressed CityGml-file, to allow validation of zipped files without
mounting them into the memory.

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/parser/CityGmlParser.java

@@ -292,10 +292,15 @@ public class CityGmlParser {
 
     }
 
-	public static void streamCityGml(CityGmlZipInputStream cgis, ParserConfiguration config, CityGmlConsumer cityObjectConsumer,
-									 String outputFile) throws CityGmlParseException {
-		return;
-	}
+	public static void streamCityGml(CityGmlZipEntry entry, ParserConfiguration config, CityGmlConsumer cityObjectConsumer,
+									 String outputFile) throws CityGmlParseException, IOException {
+		try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(entry)) {
+			parseEpsgCodeFromStream(cgis.getInputStream(), config);
+			startReadingCityGmlZipEntry(entry, config, null,cityObjectConsumer, outputFile);
+		} catch (ParserConfigurationException | SAXException | IOException e) {
+			throw new CityGmlParseException("Failed to read CityGML stream", e);
+		}
+    }
 
 	public static void streamCityGml(String file, ParserConfiguration config, CityGmlConsumer cityObjectConsumer,
 			String outputFile) throws CityGmlParseException {
@@ -311,7 +316,7 @@ public class CityGmlParser {
 	public static void streamCityGml(Path file, ParserConfiguration config, ProgressListener l,
 			CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGmlParseException {
 		parseEpsgCodeFromFile(file, config);
-		startReadingCityGml(file, config, l, cityObjectConsumer, outputFile);
+		startReadingCityGmlFile(file, config, l, cityObjectConsumer, outputFile);
 	}
 
 	public static CityModel parseOnlyCityModel(File inputFile) throws CityGmlParseException {
@@ -333,8 +338,8 @@ public class CityGmlParser {
 		throw new CityGmlParseException("Did not find any CityModel in CityGML file");
 	}
 
-	private static void startReadingCityGml(Path file, ParserConfiguration config, ProgressListener l,
-			CityGmlConsumer cityObjectConsumer, String outputFile) {
+	private static void startReadingCityGmlFile(Path file, ParserConfiguration config, ProgressListener l,
+												CityGmlConsumer cityObjectConsumer, String outputFile) {
 		try (ObservedInputStream ois = new ObservedInputStream(file.toFile())) {
 			if (l != null) {
 				ois.addListener(l::updateProgress);
@@ -347,13 +352,40 @@ public class CityGmlParser {
 	}
 
 	private static void readAndDiscardFeatures(Path file, ParserConfiguration config, ObservedInputStream ois,
+											   CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGMLReadException {
+		Citygml3FeatureMapper mapper = new Citygml3FeatureMapper(config, file);
+		readAndDiscardModel(mapper, ois, cityObjectConsumer, outputFile);
+
+	}
+
+	private static void startReadingCityGmlZipEntry(CityGmlZipEntry entry, ParserConfiguration config, ProgressListener l,
+													CityGmlConsumer cityObjectConsumer, String outputFile) {
+		try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(entry);
+			 ObservedInputStream ois = new ObservedInputStream(cgis.getInputStream(), entry.getFileSize())){
+			if (l != null) {
+				ois.addListener(l::updateProgress);
+			}
+			streamAndDiscardFeatures(entry, config, ois, cityObjectConsumer, outputFile);
+		} catch (IOException | CityGMLReadException e) {
+			logger.error(Localization.getText("CityGmlParser.errorReadingGmlFile"), e.getMessage());
+			logger.catching(Level.ERROR, e);
+		}
+	}
+
+	private static void streamAndDiscardFeatures(CityGmlZipEntry entry, ParserConfiguration config, ObservedInputStream ois,
+							 CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGMLReadException {
+		Citygml3FeatureMapper mapper = new Citygml3FeatureMapper(config, entry);
+		readAndDiscardModel(mapper, ois, cityObjectConsumer, outputFile);
+	}
+
+	private static void readAndDiscardModel(Citygml3FeatureMapper mapper, ObservedInputStream ois,
 			CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGMLReadException {
 		getContext();
 		CityGMLInputFactory inputFactory = context.createCityGMLInputFactory()
 				.withChunking(ChunkOptions.chunkByProperties(chunkProperties).skipCityModel(false));
 		CityGMLChunkWriter writer = null;
 		try (CityGMLReader reader = inputFactory.createCityGMLReader(ois)) {
-			Citygml3FeatureMapper mapper = new Citygml3FeatureMapper(config, file);
+
 			CityDoctorModel model = mapper.getModel();
 			boolean isInitialized = false;
 			while (reader.hasNext()) {

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipEntry.java

@@ -72,10 +72,10 @@ public class CityGmlZipEntry implements Serializable {
             return memoryLimit > fileSize;
         }
         try (CityGmlZipInputStream cgis = new CityGmlZipInputStream(this)){
-            long filesize = cgis.getFileSize(memoryLimit);
+            long filesize = cgis.getFileSize();
             if (filesize != -1){
                 this.fileSize = filesize;
-                return true;
+                return memoryLimit > fileSize;
             } else {
                 return false;
             }
@@ -113,4 +113,12 @@ public class CityGmlZipEntry implements Serializable {
         return model;
     }
 
+    public long getFileSize() throws IOException {
+        if (fileSize == -1L){
+            try(CityGmlZipInputStream cgis = new CityGmlZipInputStream(this)){
+                fileSize = cgis.getFileSize();
+            }
+        }
+        return fileSize;
+    }
 }

CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/zip/CityGmlZipInputStream.java

@@ -26,24 +26,19 @@ public class CityGmlZipInputStream implements AutoCloseable {
         return zip.getInputStream(zipEntry);
     }
 
-    public long getFileSize(long memoryLimit) throws IOException {
+    public long getFileSize() throws IOException {
         if (closed){
             throw new IOException("Stream closed");
         }
         if (zipEntry.getSize() != -1){
             return (long) Math.ceil((double) zipEntry.getSize() / MB);
         }
+        long bytes = 0;
         InputStream is = this.getInputStream();
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
         for(int i = is.read(); i != -1; i=is.read()) {
-            baos.write(i);
-            if ((baos.size() / MB) + 1 > memoryLimit) {
-                //Entry exceeds memory limit
-                return -1L;
-            }
+            bytes++;
         }
-        // end of stream reached without exceeding memory limit
-        return (long) Math.ceil((double) baos.size() / MB);
+        return (long) Math.ceil((double) bytes / MB);
     }
 
     @Override

CityDoctorParent/CityDoctorValidation/src/main/java/de/hft/stuttgart/citydoctor2/check/Checker.java

@@ -759,7 +759,7 @@ public class Checker {
 
 				CityGmlConsumer con = new StreamCityGmlConsumer(c, xmlReporter, pdfReporter, handler, config, null);
 				// parse and validate
-				CityGmlParser.streamCityGml(cgis, config.getParserConfiguration(), con, outputFile);
+				CityGmlParser.streamCityGml(entry, config.getParserConfiguration(), con, outputFile);
 
 				// write reports if available
 				writeReport(xmlReporter);