Merge branch 'dev_embedded_geodb' into 'dev'

!Geodatabase implementation

See merge request !36

CityDoctorParent/CityDoctorValidation/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>de.hft.stuttgart</groupId>
 		<artifactId>CityDoctorParent</artifactId>
-		<version>3.17.3</version>
+		<version>3.18.0</version>
 	</parent>
 	<artifactId>CityDoctorValidation</artifactId>
 	<name>CityDoctorValidation</name>

CityDoctorParent/CityDoctorValidation/src/main/java/de/hft/stuttgart/citydoctor2/CityDoctorValidation.java

@@ -21,13 +21,11 @@ package de.hft.stuttgart.citydoctor2;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.List;
 
-import de.hft.stuttgart.citydoctor2.zip.CityGmlZipArchive;
+import de.hft.stuttgart.citydoctor2.utils.Localization;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.citygml4j.core.ade.ADEException;
 
 import de.hft.stuttgart.citydoctor2.check.Checker;
 import de.hft.stuttgart.citydoctor2.check.ValidationConfiguration;
@@ -83,12 +81,12 @@ public class CityDoctorValidation {
 			if (optional) {
 				return null;
 			}
-			logger.error("No output file specified. (-out [FILE])");
+			logger.error(Localization.getText("CityDoctorValidation.noOutputFile"));
 			System.exit(11);
 		}
 		List<String> outFiles = argParser.getValues("out");
 		if (outFiles.size() != 1) {
-			logger.error("Specify exactly one file as output.");
+			logger.error(Localization.getText("CityDoctorValidation.notExactlyOneOutputFile"));
 			System.exit(12);
 		}
 		outputFile = outFiles.get(0);
@@ -157,15 +155,13 @@ public class CityDoctorValidation {
 											  ValidationConfiguration config, String outputFile) throws CityDoctorWriteException, CityGmlParseException, IOException, InvalidGmlFileException {
 		File inputFile = new File(input);
 		if (!inputFile.exists()) {
-			logger.error("Input file '{}' does not exist.", input);
+			logger.error(Localization.getText("CityDoctorValidation.inputFileNotExisting"), input);
 			System.exit(1);
 		}
 		if (inputFile.getName().endsWith(".gml") || inputFile.getName().endsWith(".xml")) {
 			startFileValidation(inputFile, xmlOutput, pdfOutput, config, outputFile);
-		} else if (inputFile.getName().endsWith(".zip")) {
-			startZipValidation(input, xmlOutput, pdfOutput, config, outputFile);
 		} else {
-			logger.error("Input file '{}' does not end with .gml or .xml and is not a zip archive.", input);
+			logger.error(Localization.getText("CityDoctorValidation.wrongInputFileFormat"), input);
 			System.exit(1);
 		}
 	}
@@ -179,10 +175,11 @@ public class CityDoctorValidation {
 	 * @param pdfOutput  the output path for the pdf report (optional)
 	 * @param config     the configuration path for the validation plan
 	 * @param outputFile storing the validated gml file with quality ade
-	 * @throws CityDoctorWriteException   if something goes wrong while writing the gml
-	 *                                 file
-	 * @throws CityGMLBuilderException
-	 * @throws ADEException 
+	 *
+	 * @throws IOException             When an exception while writing the report files happens.
+	 * @throws CityGmlParseException   When an error while parsing the cityGML file happens.
+	 * @throws InvalidGmlFileException If the cityGML file is not valid according to the cityGML schema.
+	 * @throws CityDoctorWriteException If something goes wrong while writing the output gml file
 	 */
 	public static void startFileValidation(File inputFile, String xmlOutput, String pdfOutput,
 			ValidationConfiguration config, String outputFile) throws IOException, CityGmlParseException,
@@ -201,34 +198,11 @@ public class CityDoctorValidation {
 		}
 	}
 
-	public static void startZipValidation(String inputZipFile, String xmlOutput, String pdfOutput,
-										  ValidationConfiguration config, String outputFile) throws CityGmlParseException, IOException {
-		xmlOutput = toDirectoryPath(xmlOutput);
-		pdfOutput = toDirectoryPath(pdfOutput);
-		outputFile = toDirectoryPath(outputFile);
-		if (xmlOutput == null && pdfOutput == null && outputFile == null) {
-			logger.warn("No output locations specified, results of validation will not be saved");
-		}
-		CityGmlZipArchive archive = CityGmlZipArchive.register(inputZipFile);
-		Checker.streamCheck(archive, xmlOutput, pdfOutput, config, outputFile);
-	}
-
-	public static String toDirectoryPath(String input) {
-		if (input == null) {
-			return null;
-		}
-		String parentDirectory = Path.of(input).getParent().toString() + File.separator;
-		String name = Path.of(input).getFileName().toString().replaceFirst("\\..+", "") + File.separator;
-		File f = new File(parentDirectory + name);
-		f.mkdir();
-		return f.getPath() + File.separator;
-	}
-
 	public static String getPdfOutput(ArgumentParser argParser) {
 		if (argParser.containsOption("pdfreport")) {
 			List<String> reportFiles = argParser.getValues("pdfreport");
 			if (reportFiles.size() != 1) {
-				logger.error("Specify exactly one file as pdfReport");
+				logger.error(Localization.getText("CityDoctorValidation.notExactlyOnePDFOutput"));
 				System.exit(6);
 			}
 			return reportFiles.get(0);
@@ -240,7 +214,7 @@ public class CityDoctorValidation {
 		if (argParser.containsOption("xmlreport")) {
 			List<String> reportFiles = argParser.getValues("xmlreport");
 			if (reportFiles.size() != 1) {
-				logger.error("Specify exactly one file as xmlReport");
+				logger.error(Localization.getText("CityDoctorValidation.notExactlyOneXMLOutput"));
 				System.exit(5);
 			}
 			return reportFiles.get(0);
@@ -257,16 +231,16 @@ public class CityDoctorValidation {
 		if (argParser.containsOption("config")) {
 			List<String> configFiles = argParser.getValues("config");
 			if (configFiles.size() != 1) {
-				logger.error("Specify exactly one file as configuration");
+				logger.error(Localization.getText("CityDoctorValidation.notExactlyOneConfigFile"));
 				System.exit(4);
 			}
 			return ValidationConfiguration.loadValidationConfig(configFiles.get(0));
 		} else {
 			if (optional) {
-				logger.warn("No configuration file given, loading default configuration");
+				logger.warn(Localization.getText("CityDoctorValidation.loadingDefaultConfig"));
 				return ValidationConfiguration.loadStandardValidationConfig();
 			} else {
-				logger.error("No configuration file specified with -config [FILE]");
+				logger.error(Localization.getText("CityDoctorValidation.noConfig"));
 				System.exit(7);
 				return null;
 			}
@@ -278,7 +252,7 @@ public class CityDoctorValidation {
 		if (argParser.containsOption("out")) {
 			List<String> outFiles = argParser.getValues("out");
 			if (outFiles.size() != 1) {
-				logger.error("Specify exactly one file as output.");
+				logger.error(Localization.getText("CityDoctorValidation.multipleOutputFiles"));
 				System.exit(3);
 			}
 			return outFiles.get(0);
@@ -297,12 +271,12 @@ public class CityDoctorValidation {
 			if (optional) {
 				return null;
 			}
-			logger.error("No input file specified. (-in [FILE])");
+			logger.error(Localization.getText("CityDoctorValidation.noInputFile"));
 			System.exit(1);
 		}
 		List<String> inFiles = argParser.getValues("in");
 		if (inFiles.size() != 1) {
-			logger.error("Specify exactly one file as input.");
+			logger.error(Localization.getText("CityDoctorValidation.notExactlyOneInputFile"));
 			System.exit(2);
 		}
 		inputFile = inFiles.get(0);

CityDoctorParent/CityDoctorValidation/src/main/java/de/hft/stuttgart/citydoctor2/CityDoctorValidationCLI.java

 package de.hft.stuttgart.citydoctor2;
 
-import java.util.concurrent.Callable;
 
+import de.hft.stuttgart.citydoctor2.check.ValidationConfiguration;
+import de.hft.stuttgart.citydoctor2.exceptions.CityDoctorWriteException;
+import de.hft.stuttgart.citydoctor2.parser.CityGmlParseException;
+import de.hft.stuttgart.citydoctor2.parser.InvalidGmlFileException;
+import picocli.CommandLine;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
 
+import java.io.File;
+import java.io.IOException;
+
 @Command(name =  "citydoctor2", mixinStandardHelpOptions = true, description = "Validates CityGML files according to a provided validation-plan.")
-public class CityDoctorValidationCLI implements Callable<Integer> {
+public class CityDoctorValidationCLI implements Runnable {
 
     @Option(names = {"-i" , "--in" , "--input"},
             description = "Filepath of the CityGML file.",
@@ -15,23 +22,41 @@ public class CityDoctorValidationCLI implements Callable<Integer> {
 
     @Option(names = {"-c" , "--config" , "--configuration", "--plan"},
             description = "Filepath of the validation configuration .yml",
-            required = true)
-    private String config;
+            defaultValue = Option.NULL_VALUE)
+    private File config;
 
     @Option(names = {"-x" , "--xml" , "--xmlReport"},
-            description = "Create a XML validation report at target location")
+            description = "Create a XML validation report at target location",
+            defaultValue = Option.NULL_VALUE)
     private String xmlReport;
 
     @Option(names = {"-p" , "--pdf" , "--pdfReport"},
-            description = "Create a PDF validation report at target location")
+            description = "Create a PDF validation report at target location",
+            defaultValue = Option.NULL_VALUE)
     private String pdfReport;
 
     @Option(names = {"-o" , "--out" , "--output"},
-            description = "Create a validated copy of the CityGML input with QualityADE at target location.")
+            description = "Create a validated copy of the CityGML input with QualityADE at target location.",
+            defaultValue = Option.NULL_VALUE)
     private String output;
 
     @Override
-    public Integer call() throws Exception {
-        return 0;
+    public void run(){
+        try {
+            ValidationConfiguration valConfig;
+            if(config == null){
+                valConfig=ValidationConfiguration.loadStandardValidationConfig();
+            } else{
+                valConfig=ValidationConfiguration.loadValidationConfig(config.getAbsolutePath());
+            }
+            CityDoctorValidation.startValidationProcess(input, xmlReport, pdfReport, valConfig,output);
+        } catch (CityDoctorWriteException | CityGmlParseException | IOException | InvalidGmlFileException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static void main(String[] args) {
+        int exitCode = new CommandLine(new CityDoctorValidationCLI()).execute(args);
+        System.exit(exitCode);
     }
 }

CityDoctorParent/CityDoctorValidation/src/main/java/de/hft/stuttgart/citydoctor2/check/Checker.java

@@ -28,9 +28,14 @@ import de.hft.stuttgart.citydoctor2.checks.CheckPrototype;
 import de.hft.stuttgart.citydoctor2.checks.Checks;
 import de.hft.stuttgart.citydoctor2.checks.SvrlContentHandler;
 import de.hft.stuttgart.citydoctor2.checks.util.FeatureCheckedListener;
+import de.hft.stuttgart.citydoctor2.database.CityObjectCache;
+import de.hft.stuttgart.citydoctor2.database.CityObjectCache;
 import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel;
 import de.hft.stuttgart.citydoctor2.datastructure.CityObject;
 import de.hft.stuttgart.citydoctor2.datastructure.FeatureType;
+import de.hft.stuttgart.citydoctor2.datastructure.GmlId;
+import de.hft.stuttgart.citydoctor2.datastructure.ImplicitGeometryHolder;
+import de.hft.stuttgart.citydoctor2.datastructure.GmlId;
 import de.hft.stuttgart.citydoctor2.parser.CityGmlConsumer;
 import de.hft.stuttgart.citydoctor2.parser.CityGmlParseException;
 import de.hft.stuttgart.citydoctor2.parser.CityGmlParser;
@@ -42,11 +47,8 @@ import de.hft.stuttgart.citydoctor2.reporting.XmlStreamReporter;
 import de.hft.stuttgart.citydoctor2.reporting.XmlValidationReporter;
 import de.hft.stuttgart.citydoctor2.reporting.pdf.PdfReporter;
 import de.hft.stuttgart.citydoctor2.reporting.pdf.PdfStreamReporter;
-import de.hft.stuttgart.citydoctor2.utils.ArchivePacker;
 import de.hft.stuttgart.citydoctor2.utils.Localization;
-import de.hft.stuttgart.citydoctor2.zip.CityGmlZipArchive;
-import de.hft.stuttgart.citydoctor2.zip.CityGmlZipEntry;
-import de.hft.stuttgart.citydoctor2.zip.CityGmlZipEntryFile;
+import de.hft.stuttgart.citydoctor2.utils.TimeFormatter;
 import de.hft.stuttgart.quality.model.enums.RequirementId;
 import de.hft.stuttgart.quality.model.enums.TopLevelFeatureType;
 import de.hft.stuttgart.quality.model.properties.CheckingProperty;
@@ -57,7 +59,6 @@ import de.hft.stuttgart.quality.model.properties.RequirementProperty;
 import de.hft.stuttgart.quality.model.types.Checking;
 import de.hft.stuttgart.quality.model.types.Parameter;
 import de.hft.stuttgart.quality.model.types.ValidationPlan;
-import org.apache.commons.io.FilenameUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -79,6 +80,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -89,7 +91,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-import java.util.stream.Stream;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * The main container class for checking. It contains the logic for validation,
@@ -191,18 +194,46 @@ public class Checker {
         if (config == null) {
             config = ValidationConfiguration.loadStandardValidationConfig();
         }
-        checkCityModel(model, l);
-        if (logger.isInfoEnabled()) {
-            logger.info(Localization.getText("Checker.checksFinished"));
+        AutoCloseable checkingContext = model.getCache().startCheckingMode();
+        try(checkingContext){
+            Callable<SvrlContentHandler> schematronThread = () -> executeSchematronValidationIfAvailable(config, model.getFile());
+            ExecutorService executor = Executors.newSingleThreadExecutor();
+            Future<SvrlContentHandler> handlerFuture =  executor.submit(schematronThread);
+            executor.shutdown();
+
+            checkCityModel(model, l);
+            getSchematronResults(handlerFuture, l);
+
+            model.setValidated(createValidationPlan());
+        } catch (Exception e){
+            logger.error("Validation failed due to encountering error", e);
         }
-        SvrlContentHandler handler = executeSchematronValidationIfAvailable(config, model.getFile());
-        if (handler != null) {
-            handleSchematronResults(handler);
+    }
+
+    private void getSchematronResults(Future<SvrlContentHandler> handlerFuture, ProgressListener l) {
+        if (!handlerFuture.isDone()) {
+            if (logger.isInfoEnabled()) {
+                logger.info(Localization.getText("Checker.schematronStillRunning"));
+            }
+            if(l !=null){
+                l.updateProgress(-0.5f);
+            }
+        }
+        try {
+            SvrlContentHandler handler = handlerFuture.get();
+            if (handler != null) {
+                handleSchematronResults(handler);
+            }
+        } catch (ExecutionException e) {
+            logger.error(e);
+        } catch (InterruptedException e) {
+            logger.error(Localization.getText("Checker.schematronInterrupted"), e);
+            Thread.currentThread().interrupt();
         }
-        model.setValidated(createValidationPlan());
     }
 
     private void handleSchematronResults(SvrlContentHandler handler) {
+        CityObjectCache cache = model.getCache();
     	handleSchematronErrorsGlobal(handler.getGeneralErrors());
         Map<String, CityObject> featureMap = new HashMap<>();
         boolean onlySchematron = execLayers.isEmpty();
@@ -222,7 +253,7 @@ public class Checker {
         }
         handler.getFeatureErrors().forEach((k, v) -> {
         	String trimmedId = k.trim();
-            CityObject co = featureMap.get(trimmedId);
+            CityObject co = cache.get(new GmlId(k));
             if (co == null) {
                 // gml id reported by schematron was not found, add to general errors
                 handleSchematronErrorsGlobal(v);
@@ -468,6 +499,10 @@ public class Checker {
             if (logger.isInfoEnabled()) {
                 logger.info(Localization.getText("Checker.schematronValidation"));
             }
+            if (!Files.exists(Path.of(config.getSchematronFilePath()))) {
+                logger.warn(Localization.getText("Checker.schematronFileNotFound"), config.getSchematronFilePath());
+                return null;
+            }
 
             try {
                 TransformerFactory transformerFactory = TransformerFactory.newInstance("net.sf.saxon.TransformerFactoryImpl", Checker.class.getClassLoader());
@@ -585,7 +620,7 @@ public class Checker {
         for (Entry<String, RequirementConfiguration> e : config.getRequirements().entrySet()) {
             de.hft.stuttgart.citydoctor2.check.Requirement req = Checks.getAvailableRequirements().get(e.getKey());
             if (req == null) {
-                logger.warn("Could not find any check that satisfies requirement {}, it will not be checked",
+                logger.warn(Localization.getText("Checker.noCheckSatisfiesRequirement"),
                         e.getKey());
             } else {
                 if (e.getValue().isEnabled()) {
@@ -652,31 +687,110 @@ public class Checker {
         }
     }
 
-    private void checkCityModel(CityDoctorModel model, ProgressListener l) {
-        Stream<CityObject> features = model.createFeatureStream();
-        float featureSum = model.getNumberOfFeatures();
+    private void checkCityModel(CityDoctorModel model, ProgressListener l){
+        List<GmlId> features = model.getFeatureIds();
+
+        AtomicInteger checkedCount = new AtomicInteger(0);
         // clear global errors
         model.getGlobalErrors().clear();
-        
-        // stupid lamda with final variable restrictions
-        int[] currentFeature = new int[1];
-        features.forEach(co -> {
-            if (config.getParserConfiguration().useLowMemoryConsumption()) {
-                // no edges have been created yet, create them
-                co.prepareForChecking();
+        CityObjectCache cache = model.getCache();
+        logger.trace("Setting up ThreadPool");
+        int threadCount = Runtime.getRuntime().availableProcessors() * 5;
+        ExecutorService exec =  new ThreadPoolExecutor
+                (threadCount, threadCount, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
+        try{
+            long startTime = System.nanoTime();
+            ImplicitGeometryHolder.preparePrototypeGeometriesForChecking();
+
+            List<Future<GmlId>> futures = runChecksOnFeatures(exec, cache, features, checkedCount, l);
+            List<GmlId> missedFeatures = getMissedFeatures(futures);
+
+            ImplicitGeometryHolder.clearMetaDataOfPrototypeGeometries();
+            if (!missedFeatures.isEmpty()){
+                logger.error(Localization.getText("Checker.dbUnresponsive"));
+                if (logger.isDebugEnabled()){
+                    logger.debug(missedFeatures.toString());
+                }
             }
-            // check every feature
-            executeChecksForCityObject(co);
+            long endTime = System.nanoTime();
+            if (logger.isInfoEnabled()){
+                long totalTime = (endTime - startTime) / 1_000_000; // Convert ns to ms
+                String messageString = Localization.getText("Checker.checksFinished");
+                String timeString = TimeFormatter.format(totalTime);
+                messageString = messageString.replace("{0}", timeString);
+                logger.info(messageString);
+            }
+        } catch (InterruptedException e){
+            // No interrupts implemented, so this block should actually never be entered
+            logger.error(Localization.getText("Checker.validationInterrupted"), e);
+            Thread.currentThread().interrupt();
+
+        } finally {
+            exec.shutdown();
+            try {
+                if(!exec.awaitTermination(1, TimeUnit.MINUTES)){
+                    exec.shutdownNow();
+                }
 
-            if (config.getParserConfiguration().useLowMemoryConsumption()) {
-                // low memory consumption, remove edges again
-                co.clearMetaInformation();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                logger.error(Localization.getText("Checker.shutdownInterrupted"));
             }
-            if (l != null) {
-                currentFeature[0]++;
-                l.updateProgress(currentFeature[0] / featureSum);
+        }
+    }
+
+    private List<Future<GmlId>> runChecksOnFeatures(ExecutorService exec, CityObjectCache cache, List<GmlId> ids, AtomicInteger checkedCount,
+                                            ProgressListener l)
+            throws InterruptedException {
+        float featureSum = ids.size() + (float) checkedCount.get();
+
+        logger.trace("Queueing up Checker tasks");
+        List<Callable<GmlId>> tasks = new ArrayList<>();
+        for (GmlId id : ids) {
+            tasks.add(()->{
+                CityObject co = cache.get(id);
+                if (co == null) {
+                    return id;
+                }
+                if(Thread.interrupted()){
+                    Thread.currentThread().interrupt();
+                    return null;
+                }
+                co.prepareForChecking();
+                executeChecksForCityObject(co);
+                co.clearMetaInformation();
+                cache.put(co);
+                checkedCount.incrementAndGet();
+                if (l!=null){
+                    l.updateProgress((checkedCount.get()) / featureSum);
+                }
+                return null;
+            });
+        }
+        logger.trace("Queueing up finished, invoking all tasks");
+        return exec.invokeAll(tasks);
+    }
+
+    private List<GmlId> getMissedFeatures(List<Future<GmlId>> futures) throws InterruptedException{
+        List<GmlId> missedList = new ArrayList<>();
+        Set<ExecutionException> errors = new HashSet<>();
+        for (Future<GmlId> future : futures) {
+            try{
+                GmlId gmlId = future.get();
+                if (gmlId != null) {
+                    missedList.add(gmlId);
+                }
+            } catch (ExecutionException e){
+                logger.debug("A Task failed due to an unexpected exception", e);
+                logger.debug(e.getCause());
+                errors.add(e);
             }
-        });
+        }
+        if (!errors.isEmpty()) {
+            logger.error(Localization.getText("Checker.futuresCheckFailures"));
+            logger.error(errors.toString());
+        }
+        return missedList;
     }
 
     private boolean filterObject(CityObject co) {
@@ -782,73 +896,6 @@ public class Checker {
         return hasUnusedDependency;
     }
 
-    public static void streamCheck(CityGmlZipArchive archive, String xmlDirectory, String pdfDirectory, ValidationConfiguration config,
-                                   String outputDirectory) throws IOException, CityGmlParseException {
-        streamCheck(archive, xmlDirectory, pdfDirectory, config, null, outputDirectory);
-    }
-
-    public static void streamCheck(CityGmlZipArchive archive, String xmlDirectory, String pdfDirectory, ValidationConfiguration config,
-                                   FeatureCheckedListener l, String outputDirectory) throws IOException, CityGmlParseException {
-
-
-        for (CityGmlZipEntry entry : archive.getEntries()) {
-            String entryName = FilenameUtils.removeExtension(entry.getDisplayName());
-            String xmlOutput = null;
-            String pdfOutput = null;
-            String outputFile = null;
-            if (xmlDirectory != null) {
-                xmlOutput = Path.of(xmlDirectory).resolve(entryName + "_report.xml").toString();
-            }
-            if (pdfDirectory != null) {
-                pdfOutput = Path.of(pdfDirectory).resolve(entryName + "_report.pdf").toString();
-            }
-            if (outputDirectory != null) {
-                outputFile = Path.of(outputDirectory).resolve(entryName + "_validated.gml").toString();
-            }
-
-            streamCheck(entry, xmlOutput, pdfOutput, config, l, outputFile);
-
-
-        }
-        if (outputDirectory != null) {
-            ArchivePacker.packAndDeleteDirectory(outputDirectory);
-        }
-    }
-
-
-    public static void streamCheck(CityGmlZipEntry entry, String xmlOutput, String pdfOutput, ValidationConfiguration config,
-                                   String outputFile) throws IOException, CityGmlParseException {
-        streamCheck(entry, xmlOutput, pdfOutput, config, null, outputFile);
-    }
-
-    public static void streamCheck(CityGmlZipEntry entry, String xmlOutput, String pdfOutput, ValidationConfiguration config,
-                                   FeatureCheckedListener l, String outputFile) throws IOException, CityGmlParseException {
-        try (BufferedOutputStream xmlBos = getXmlOutputMaybe(xmlOutput);
-             BufferedOutputStream pdfBos = getPdfOutputMaybe(pdfOutput)) {
-            Checker c = new Checker(config, null);
-            String fileName = entry.getEntrySubPath();
-
-            // create reporter if available
-            XmlStreamReporter xmlReporter = getXmlReporter(config, xmlBos, fileName);
-            PdfStreamReporter pdfReporter = getPdfReporter(config, pdfBos, fileName);
-
-            // execute schematron first
-            try (CityGmlZipEntryFile entryFile = new CityGmlZipEntryFile(entry)) {
-                SvrlContentHandler handler = executeSchematronValidationIfAvailable(config, entryFile.getInputStream());
-
-                CityGmlConsumer con = new StreamCityGmlConsumer(c, xmlReporter, pdfReporter, handler, config, l);
-                // parse and validate
-                CityGmlParser.streamCityGml(entry, config.getParserConfiguration(), con, outputFile);
-
-                // write reports if available
-                writeReport(xmlReporter);
-                writeReport(pdfReporter);
-            }
-        } catch (CheckReportWriteException e) {
-            logger.error(Localization.getText("Checker.failReports"), e);
-        }
-    }
-
     public static void streamCheck(File inputFile, String xmlOutput, String pdfOutput, ValidationConfiguration config,
                                    String outputFile) throws IOException, CityGmlParseException {
         streamCheck(inputFile, xmlOutput, pdfOutput, config, null, outputFile);

CityDoctorParent/CityDoctorValidation/src/main/java/de/hft/stuttgart/citydoctor2/check/StreamCityGmlConsumer.java

@@ -134,7 +134,7 @@ public class StreamCityGmlConsumer implements CityGmlConsumer {
 			pdfReporter.report(co);
 		}
 
-		// TODO: Update QualityADE to be compatible with CityGML 3.0 or deprecate it, unhandled top-level feature types are just skipped for now
+		// TODO: Update QualityADE to be compatible with CityGML 3.0, unhandled top-level feature types are just skipped for now
 		if (!(co instanceof AbstractTunnel || co instanceof CityFurniture || co instanceof GenericCityObject)) {
 			// store quality ade datastructures in cityobject
 			QualityADEUtils.writeQualityAde(co, val);

CityDoctorParent/CityDoctorValidation/src/main/java/de/hft/stuttgart/citydoctor2/checks/CheckContainer.java

@@ -230,6 +230,24 @@ public class CheckContainer extends Check {
 		}
 	}
 
+	@Override
+	public void check(TinObject to) {
+		try {
+			check.check(to);
+		}catch (Exception e) {
+			handleException(e, to);
+		}
+	}
+
+	@Override
+	public void check(ReliefObject ro) {
+		try {
+			check.check(ro);
+		}catch (Exception e) {
+			handleException(e, ro);
+		}
+	}
+
 	@Override
 	public void check(CityFurniture cf){
 		try {
@@ -248,6 +266,15 @@ public class CheckContainer extends Check {
 		}
 	}
 
+	@Override
+	public void check(OtherConstructionObject oco) {
+		try {
+			check.check(oco);
+		}catch (Exception e) {
+			handleException(e, oco);
+		}
+	}
+
 	@Override
 	public RequirementType getType() {
 		return check.getType();

CityDoctorParent/CityDoctorValidation/src/main/java/de/hft/stuttgart/citydoctor2/reporting/XmlValidationReporter.java

@@ -45,33 +45,8 @@ public class XmlValidationReporter implements Reporter {
 	@Override
 	public void writeReport(Checks checks, OutputStream filePath, CityDoctorModel model, ValidationConfiguration config) throws CheckReportWriteException {
 		StreamReporter streamReporter = new XmlStreamReporter(filePath, model.getFileName(), config);
-		for (BridgeObject bo : model.getBridges()) {
-			streamReporter.report(bo);
-		}
-		for (Building b : model.getBuildings()) {
-			streamReporter.report(b);
-		}
-		for (TransportationObject to : model.getTransportation()) {
-			streamReporter.report(to);
-		}
-		for (CityObject lo : model.getLand()) {
-			streamReporter.report(lo);
-		}
-		for (Vegetation v : model.getVegetation()) {
-			streamReporter.report(v);
-		}
-		for (WaterObject wo : model.getWater()) {
-			streamReporter.report(wo);
-		}
-		for (Tunnel tunnel : model.getTunnels()) {
-			streamReporter.report(tunnel);
-		}
-		for (CityFurniture cf : model.getCityFurniture()) {
-			streamReporter.report(cf);
-		}
-		for (GenericCityObject gco : model.getGenericCityObjects()) {
-			streamReporter.report(gco);
-		}
+		model.createFeatureStream().forEach(streamReporter::report);
+
 		for (CheckError err : model.getGlobalErrors()) {
 			streamReporter.reportGlobalError(err);
 		}

CityDoctorParent/CityDoctorValidation/src/main/resources/log4j2.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE Configuration>
+<Configuration status="INFO">
+    <Appenders>
+        <Console name="console" target="SYSTEM_OUT">
+            <PatternLayout
+                    pattern="[%-5level] %d{HH:mm:ss.SSS} %c{1} - %msg%n" />
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Root level="debug" additivity="false">
+            <AppenderRef ref="console" />
+        </Root>
+        <Logger name="com.zaxxer.hikari" level="INFO" additivity="false">
+            <AppenderRef ref="console"/>
+        </Logger>
+    </Loggers>
+</Configuration>
\ No newline at end of file

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/check/CheckerTest.java

@@ -24,7 +24,6 @@ import de.hft.stuttgart.citydoctor2.checkresult.GlobalErrorStatistics;
 import de.hft.stuttgart.citydoctor2.checkresult.GlobalStatistics;
 import de.hft.stuttgart.citydoctor2.checkresult.ModelStatistics;
 import de.hft.stuttgart.citydoctor2.checkresult.utility.CheckReportParseException;
-import de.hft.stuttgart.citydoctor2.checks.util.FeatureCheckedListener;
 import de.hft.stuttgart.citydoctor2.datastructure.Building;
 import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel;
 import de.hft.stuttgart.citydoctor2.exceptions.CityDoctorWriteException;
@@ -32,8 +31,6 @@ import de.hft.stuttgart.citydoctor2.parser.CityGmlParseException;
 import de.hft.stuttgart.citydoctor2.parser.CityGmlParser;
 import de.hft.stuttgart.citydoctor2.parser.InvalidGmlFileException;
 import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
-import de.hft.stuttgart.citydoctor2.zip.CityGmlZipArchive;
-import de.hft.stuttgart.citydoctor2.zip.CityGmlZipEntry;
 import org.apache.commons.io.FileUtils;
 import org.junit.Rule;
 import org.junit.Test;
@@ -68,7 +65,7 @@ public class CheckerTest {
                 "src/test/resources/SimpleSolid_SrefBS_SchematronTest.gml", config.getParserConfiguration());
         Checker checker = new Checker(config, model);
         checker.runChecks();
-        for (Building b : model.getBuildings()) {
+        for (Building b : model.getBuildings().toList()) {
             if (b.getGmlId().getGmlString().equals("_Simple_BD.1")) {
                 assertTrue(b.containsAnyError());
             } else {
@@ -128,76 +125,6 @@ public class CheckerTest {
         }
     }
 
-    @Test
-    public void testCliZipChecking() throws IOException, CityDoctorWriteException, CityGmlParseException, InvalidGmlFileException {
-        String output = folder.getRoot().toPath() + File.separator + "test";
-        String pdfOutput = folder.getRoot().toPath() + File.separator + "pdf" + File.separator;
-        String xmlOutput = folder.getRoot().toPath() + File.separator + "xml" + File.separator;
-        try {
-            String[] args = new String[10];
-            args[0] = "-in";
-            args[1] = "src/test/resources/zipArchive.zip";
-            args[2] = "-config";
-            args[3] = "src/test/resources/testConfigWithStreaming.yml";
-            args[4] = "-pdfReport";
-            args[5] = pdfOutput;
-            args[6] = "-xmlReport";
-            args[7] = xmlOutput;
-            args[8] = "-out";
-            args[9] = output + File.separator;
-            CityDoctorValidation.main(args);
-            File zipFile = new File(output + ".zip");
-            File pdfDir = new File(pdfOutput);
-            File xmlDir = new File(xmlOutput);
-            assertTrue(zipFile.exists());
-            assertTrue(pdfDir.exists());
-            assertTrue(xmlDir.exists());
-
-            assertTrue(pdfDir.isDirectory());
-            assertTrue(xmlDir.isDirectory());
-            assertEquals(5, pdfDir.listFiles().length);
-            assertEquals(5, xmlDir.listFiles().length);
-
-            CityGmlZipArchive cgmlArch = CityGmlZipArchive.register(output + ".zip");
-            assertNotNull(cgmlArch);
-            cgmlArch.mountArchive(new ParserConfiguration(8, true));
-            assertEquals(5, cgmlArch.getEntries().size());
-            for (CityGmlZipEntry entry : cgmlArch.getEntries()) {
-                assertNotNull(entry);
-                assertNull(entry.getErrorType());
-            }
-
-        } catch (CityGmlParseException | IOException | InvalidGmlFileException | CityDoctorWriteException e) {
-            throw e;
-        }
-    }
-
-    @Test
-    public void testZipEntryChecking() throws CityGmlParseException, IOException {
-        CityGmlZipArchive cgmlArch = CityGmlZipArchive.register("src/test/resources/zipArchive.zip");
-        assertNotNull(cgmlArch);
-        cgmlArch.mountArchive(new ParserConfiguration(8, false));
-        ValidationConfiguration config = ValidationConfiguration.loadStandardValidationConfig();
-        FeatureCheckedListener l = co -> {
-            assertTrue("CityObject should be validated", co.isValidated());
-        };
-        for (CityGmlZipEntry entry : cgmlArch.getEntries()) {
-            Checker.streamCheck(entry, null, null, config, l, null);
-        }
-    }
-
-    @Test
-    public void testZipArchiveChecking() throws CityGmlParseException, IOException {
-        CityGmlZipArchive cgmlArch = CityGmlZipArchive.register("src/test/resources/zipArchive.zip");
-        assertNotNull(cgmlArch);
-        cgmlArch.mountArchive(new ParserConfiguration(8, false));
-        ValidationConfiguration config = ValidationConfiguration.loadStandardValidationConfig();
-        FeatureCheckedListener l = co -> {
-            assertTrue(co.isValidated());
-        };
-        Checker.streamCheck(cgmlArch, null, null, config, l, null);
-    }
-
     @Test
     public void testReportCreation() throws IOException, CityGmlParseException, InvalidGmlFileException {
         Path tmpDir = null;
@@ -207,7 +134,7 @@ public class CheckerTest {
             Path xmlPath = tmpDir.resolve("report.xml");
             Path pdfPath = tmpDir.resolve("report.pdf");
             File features = new File("src/test/resources/feature_types.gml");
-            CityDoctorModel model = CityGmlParser.parseCityGmlFileSilently(
+            CityDoctorModel model = CityGmlParser.nestedCityGmlFileParse(
                     features.toString(), new ParserConfiguration(8, false));
             assertNotNull(model);
             Checker checker = new Checker(model);

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/check/FilterTest.java

@@ -45,12 +45,12 @@ public class FilterTest {
 		Checker c = new Checker(config, model);
 		c.runChecks();
 
-		Building b = model.getBuildings().get(0);
+		Building b = model.getBuildings().toList().get(0);
 		Geometry geom = b.getGeometries().get(0);
 
 		assertFalse(geom.hasCheckResults());
 
-		TransportationObject to = model.getTransportation().get(0);
+		TransportationObject to = model.getTransportation().toList().get(0);
 		Geometry geom2 = to.getGeometries().get(0);
 
 		assertTrue(geom2.getPolygons().get(0).hasCheckResults());
@@ -65,12 +65,12 @@ public class FilterTest {
 		Checker c = new Checker(config, model);
 		c.runChecks();
 
-		Building b = model.getBuildings().get(0);
+		Building b = model.getBuildings().toList().get(0);
 		Geometry geom = b.getGeometries().get(0);
 
 		assertFalse(geom.hasCheckResults());
 
-		TransportationObject to = model.getTransportation().get(0);
+		TransportationObject to = model.getTransportation().toList().get(0);
 		Geometry geom2 = to.getGeometries().get(0);
 
 		assertTrue(geom2.getPolygons().get(0).hasCheckResults());

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/checks/geometry/DegeneratedPolygonCheckTest.java

@@ -26,6 +26,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import de.hft.stuttgart.citydoctor2.database.UnconnectedCache;
 import org.junit.Test;
 
 import de.hft.stuttgart.citydoctor2.check.CheckError;
@@ -44,6 +45,7 @@ import de.hft.stuttgart.citydoctor2.datastructure.Vertex;
 import de.hft.stuttgart.citydoctor2.datastructure.Geometry.Orientation;
 import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
 import de.hft.stuttgart.quality.model.enums.RequirementId;
+import org.mockito.Mockito;
 
 public class DegeneratedPolygonCheckTest {
 
@@ -68,9 +70,10 @@ public class DegeneratedPolygonCheckTest {
 
 		Building b = new Building();
 		b.addGeometry(geom);
-
+        File file = Mockito.mock(File.class);
+        Mockito.when(file.getName()).thenReturn("building");
 		ParserConfiguration config = new ParserConfiguration(8, false);
-		CityDoctorModel model = new CityDoctorModel(config, new File(""));
+		CityDoctorModel model = new CityDoctorModel(config, file, new UnconnectedCache());
 		model.addBuilding(b);
 
 		ValidationConfiguration valConfig = ValidationConfiguration.loadStandardValidationConfig();

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/checks/geometry/SolidSelfIntCheckFalsePositiveBigMeshTest.java

@@ -35,7 +35,7 @@ public class SolidSelfIntCheckFalsePositiveBigMeshTest {
 		CityDoctorModel m = CityGmlParser.parseCityGmlFile(gml_filepath, config.getParserConfiguration());
 		Checker c = new Checker(config, m);
 		c.runChecks();
-		Building building = m.getBuildings().get(0);
+		Building building = m.getBuildings().toList().get(0);
 		/*
 		 *  The examples have no actual self-intersections, but can contain other actual model defects.
 		 *  If an error is detected, it is thus required to check if the

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/checks/geometry/SolidSelfIntCheckTest.java

@@ -95,7 +95,7 @@ public class SolidSelfIntCheckTest {
 		CityDoctorModel m = CityGmlParser.parseCityGmlFile("src/test/resources/SolidSelfIntTest1.gml", config.getParserConfiguration());
 		Checker c = new Checker(config, m);
 		c.runChecks();
-		assertFalse(m.getBuildings().get(0).containsAnyError());
+		assertFalse(m.getBuildings().toList().get(0).containsAnyError());
 	}
 	
 	
@@ -105,7 +105,7 @@ public class SolidSelfIntCheckTest {
 		CityDoctorModel m = CityGmlParser.parseCityGmlFile(gml_filepath, config.getParserConfiguration());
 		Checker c = new Checker(config, m);
 		c.runChecks();
-		Building building = m.getBuildings().get(0);
+		Building building = m.getBuildings().toList().get(0);
 		/*
 		 *  The examples have no actual self-intersections, but can contain other actual model defects.
 		 *  If an error is detected, it is thus required to check if the

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/systemtest/AllPolygonWrongOrientedTest.java

@@ -48,7 +48,7 @@ public class AllPolygonWrongOrientedTest {
 		Checker c = new Checker(config, model);
 		c.runChecks();
 		
-		Building b = model.getBuildings().get(0);
+		Building b = model.getBuildings().toList().get(0);
 		
 		assertFalse(b.containsError(CheckId.C_GE_S_ALL_POLYGONS_WRONG_ORIENTATION));
 	}

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/systemtest/ConCompCheckTest.java

@@ -51,7 +51,7 @@ public class ConCompCheckTest extends TestCleanUp {
 		Checker c = new Checker(m);
 		c.runChecks();
 		
-		Geometry geom = m.getBuildings().get(0).getGeometries().get(0);
+		Geometry geom = m.getBuildings().toList().get(0).getGeometries().get(0);
 		CheckResult cr = geom.getCheckResult(CheckId.C_GE_S_MULTIPLE_CONNECTED_COMPONENTS);
 		assertNotNull(cr);
 		assertEquals(ResultStatus.ERROR, cr.getResultStatus());
@@ -65,7 +65,7 @@ public class ConCompCheckTest extends TestCleanUp {
 		Checker c = new Checker(m);
 		c.runChecks();
 
-		Geometry geom = m.getBuildings().get(0).getGeometries().get(0);
+		Geometry geom = m.getBuildings().toList().get(0).getGeometries().get(0);
 		CheckResult cr = geom.getCheckResult(CheckId.C_GE_S_MULTIPLE_CONNECTED_COMPONENTS);
 		assertNotNull(cr);
 		assertEquals(ResultStatus.ERROR, cr.getResultStatus());

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/systemtest/DupPointsCheckTest.java

@@ -79,7 +79,7 @@ public class DupPointsCheckTest {
 	@Test
 	public void testDupPoints3() throws CityGmlParseException, IOException, InvalidGmlFileException {
 		CityDoctorModel m = TestUtil.loadAndCheckCityModel("src/test/resources/SimpleSolid_SrefBS.gml");
-		for (Polygon p : m.getBuildings().get(0).getGeometries().get(0).getPolygons()) {
+		for (Polygon p : m.getBuildings().toList().get(0).getGeometries().get(0).getPolygons()) {
 			LinearRing r = p.getExteriorRing();
 			CheckResult cr = r.getCheckResult(CheckId.C_GE_R_DUPLICATE_POINT);
 			assertEquals(ResultStatus.OK, cr.getResultStatus());

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/systemtest/HoleOutsideSystemTest.java

@@ -43,7 +43,7 @@ public class HoleOutsideSystemTest {
 	@Test
 	public void testHoleOutside1() throws CityGmlParseException, IOException, InvalidGmlFileException {
 		CityDoctorModel c = TestUtil.loadAndCheckCityModel("src/test/resources/SimpleSolid_SrefBS-GE-gml-PO-0001-T0002.gml");
-		for (Polygon p : c.getBuildings().get(0).getGeometries().get(0).getPolygons()) {
+		for (Polygon p : c.getBuildings().toList().get(0).getGeometries().get(0).getPolygons()) {
 			CheckResult cr = p.getCheckResult(CheckId.C_GE_P_HOLE_OUTSIDE);
 			assertEquals(ResultStatus.OK, cr.getResultStatus());
 		}

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/systemtest/InnerRingsNestedSystemTest.java

@@ -43,7 +43,7 @@ public class InnerRingsNestedSystemTest {
 	@Test
 	public void innerRingsNested1() throws CityGmlParseException, IOException, InvalidGmlFileException {
 		CityDoctorModel c = TestUtil.loadAndCheckCityModel("src/test/resources/SimpleSolid_SrefBS-GE-gml-PO-0001-T0002.gml");
-		for (Polygon p : c.getBuildings().get(0).getGeometries().get(0).getPolygons()) {
+		for (Polygon p : c.getBuildings().toList().get(0).getGeometries().get(0).getPolygons()) {
 			CheckResult cr = p.getCheckResult(CheckId.C_GE_P_INNER_RINGS_NESTED);
 			assertEquals(ResultStatus.OK, cr.getResultStatus());
 		}

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/systemtest/InteriorDisconnectedSystemTest.java

@@ -43,7 +43,7 @@ public class InteriorDisconnectedSystemTest {
 	@Test
 	public void testInteriorDisconnected1() throws CityGmlParseException, IOException, InvalidGmlFileException {
 		CityDoctorModel c = TestUtil.loadAndCheckCityModel("src/test/resources/SimpleSolid_SrefBS-GE-gml-PO-0001-T0002.gml");
-		for (Polygon p : c.getBuildings().get(0).getGeometries().get(0).getPolygons()) {
+		for (Polygon p : c.getBuildings().toList().get(0).getGeometries().get(0).getPolygons()) {
 			CheckResult cr = p.getCheckResult(CheckId.C_GE_P_INTERIOR_DISCONNECTED);
 			assertEquals(ResultStatus.OK, cr.getResultStatus());
 		}

CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/systemtest/MultipleComponentTest.java

@@ -43,7 +43,7 @@ public class MultipleComponentTest {
 	@Test
 	public void testNoMultipleComponents() throws CityGmlParseException, IOException, InvalidGmlFileException {
 		CityDoctorModel c = TestUtil.loadAndCheckCityModel("src/test/resources/SimpleSolid_SrefBS.gml");
-		Geometry g = c.getBuildings().get(0).getGeometries().get(0);
+		Geometry g = c.getBuildings().toList().get(0).getGeometries().get(0);
 		CheckResult cr = g.getCheckResult(CheckId.C_GE_S_MULTIPLE_CONNECTED_COMPONENTS);
 		assertEquals(ResultStatus.OK, cr.getResultStatus());
 	}
@@ -51,7 +51,7 @@ public class MultipleComponentTest {
 	@Test
 	public void testMultipleComponents() throws CityGmlParseException, IOException, InvalidGmlFileException {
 		CityDoctorModel c = TestUtil.loadAndCheckCityModel("src/test/resources/SimpleSolid_SrefBS-GE-gml-SO-0005-T0001.gml");
-		Geometry g = c.getBuildings().get(0).getGeometries().get(0);
+		Geometry g = c.getBuildings().toList().get(0).getGeometries().get(0);
 		CheckResult cr = g.getCheckResult(CheckId.C_GE_S_MULTIPLE_CONNECTED_COMPONENTS);
 		assertEquals(ResultStatus.ERROR, cr.getResultStatus());
 		assertEquals(ErrorId.GE_S_MULTIPLE_CONNECTED_COMPONENTS, cr.getError().getErrorId());