Refactor: Implement multithreaded checking

CityDoctorParent/CityDoctorValidation/src/main/java/de/hft/stuttgart/citydoctor2/check/Checker.java

@@ -28,9 +28,11 @@ import de.hft.stuttgart.citydoctor2.checks.CheckPrototype;
 import de.hft.stuttgart.citydoctor2.checks.Checks;
 import de.hft.stuttgart.citydoctor2.checks.SvrlContentHandler;
 import de.hft.stuttgart.citydoctor2.checks.util.FeatureCheckedListener;
+import de.hft.stuttgart.citydoctor2.database.CityObjectCache;
 import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel;
 import de.hft.stuttgart.citydoctor2.datastructure.CityObject;
 import de.hft.stuttgart.citydoctor2.datastructure.FeatureType;
+import de.hft.stuttgart.citydoctor2.datastructure.GmlId;
 import de.hft.stuttgart.citydoctor2.parser.CityGmlConsumer;
 import de.hft.stuttgart.citydoctor2.parser.CityGmlParseException;
 import de.hft.stuttgart.citydoctor2.parser.CityGmlParser;
@@ -79,6 +81,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -90,6 +93,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
 
 /**
@@ -196,9 +200,6 @@ public class Checker {
         if (logger.isInfoEnabled()) {
             logger.info(Localization.getText("Checker.checksFinished"));
         }
-        if (l != null) {
-            l.updateProgress(-1);
-        }
         SvrlContentHandler handler = executeSchematronValidationIfAvailable(config, model.getFile());
         if (handler != null) {
             handleSchematronResults(handler);
@@ -446,6 +447,10 @@ public class Checker {
             if (logger.isInfoEnabled()) {
                 logger.info(Localization.getText("Checker.schematronValidation"));
             }
+            if (!Files.exists(Path.of(config.getSchematronFilePath()))) {
+                logger.warn("Schematron file {} could not be found", config.getSchematronFilePath());
+                return null;
+            }
 
             try {
                 TransformerFactory transformerFactory = TransformerFactory.newInstance("net.sf.saxon.TransformerFactoryImpl", Checker.class.getClassLoader());
@@ -630,70 +635,93 @@ public class Checker {
         }
     }
 
-    private void checkCityModel(CityDoctorModel model, ProgressListener l) {
-        Stream<CityObject> features = model.createFeatureStream();
+    private void checkCityModel(CityDoctorModel model, ProgressListener l){
+        List<GmlId> features = model.getFeatureIds();
+        CityObjectCache cache = model.getCache();
         float featureSum = model.getNumberOfFeatures();
-        boolean lowMemoryMode = config.getParserConfiguration().useLowMemoryConsumption();
+        AtomicInteger checkedCount = new AtomicInteger(0);
+        AtomicInteger missingCount = new AtomicInteger(0);
         // clear global errors
         model.getGlobalErrors().clear();
         CheckerThreadsSynchronizer sync = new CheckerThreadsSynchronizer(model, l);
 
+        int threadCount = Runtime.getRuntime().availableProcessors() * 5;
         ExecutorService exec =  new ThreadPoolExecutor
-                (10, 10, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
-
-        Runnable run = new Runnable() {
-            @Override
-            public void run() {
-                while (sync.hasNext()) {
-                    CityObject co = sync.getNext();
-//                    if (lowMemoryMode) {
-                        co.prepareForChecking();
-//                    }
-                    executeChecksForCityObject(co);
-//                    if (lowMemoryMode) {
-                        co.clearMetaInformation();
-//                    }
-                    sync.updateFeature(co);
-                    sync.updateCheckedCount();
-                }
-            }
-        };
-        long startTime = System.nanoTime();
-        for (int i = 0; i < 10; i++) {
-            exec.execute(run);
-        }
-//        // stupid lamda with final variable restrictions
-//        int[] currentFeature = new int[1];
-//        features.forEach(co -> {
-//            if (lowMemoryMode) {
-//                // no edges have been created yet, create them
-//                co.prepareForChecking();
-//            }
-//            // check every feature
-//            executeChecksForCityObject(co);
-//
-//            if (lowMemoryMode) {
-//                // low memory consumption, remove edges again
-//                co.clearMetaInformation();
-//            }
-//            if (l != null) {
-//                currentFeature[0]++;
-//                l.updateProgress(currentFeature[0] / featureSum);
-//            }
-//        });
-
+                (threadCount, threadCount, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
 
+        long startTime = System.nanoTime();
+        for (GmlId id : features) {
+            exec.execute(()->{
+                CityObject co = cache.get(id);
+                if (co == null) {
+                    missingCount.incrementAndGet();
+                    if (l!=null){
+                        l.updateProgress((checkedCount.get() + missingCount.get()) / featureSum);
+                    }
+                    return;
+                }
+                co.prepareForChecking();
+                executeChecksForCityObject(co);
+                co.clearMetaInformation();
+                cache.put(co);
+                checkedCount.incrementAndGet();
+                if (l!=null){
+                    l.updateProgress((checkedCount.get() + missingCount.get()) / featureSum);
+                }
+            });
+        }
         exec.shutdown();
         try {
             exec.awaitTermination(30, TimeUnit.DAYS);
             long endTime = System.nanoTime();
-            logger.info("Time: " + (endTime -startTime) / 1_000_000 + " ms");
-            model.preCacheShownFeatures();
+            logger.info("Checks finished in " + (endTime -startTime) / 1_000_000 + " ms");
+            if (missingCount.get() > 0) {
+                int missing = missingCount.get();
+                String msg = String.format("Checker could not load %d/%.0f features (%.2f%%)", missing, featureSum, (missing/featureSum) * 100);
+                logger.warn(msg);
+            }
+//            model.preCacheShownFeatures();
         } catch (InterruptedException e) {
             throw new RuntimeException(e);
         }
     }
 
+    private void checkCityModelOLD(CityDoctorModel model, ProgressListener l) {
+        Stream<CityObject> features = model.createFeatureStream();
+        float featureSum = model.getNumberOfFeatures();
+        boolean lowMemoryMode = config.getParserConfiguration().useLowMemoryConsumption();
+
+        // clear global errors
+        model.getGlobalErrors().clear();
+
+        long startTime = System.nanoTime();
+
+        // stupid lamda with final variable restrictions
+        int[] currentFeature = new int[1];
+        features.forEach(co -> {
+            if (lowMemoryMode) {
+                // no edges have been created yet, create them
+                co.prepareForChecking();
+            }
+            // check every feature
+            executeChecksForCityObject(co);
+
+            if (lowMemoryMode) {
+                // low memory consumption, remove edges again
+                co.clearMetaInformation();
+            }
+            if (l != null) {
+                currentFeature[0]++;
+                l.updateProgress(currentFeature[0] / featureSum);
+            }
+        });
+
+        long endTime = System.nanoTime();
+        model.preCacheShownFeatures();
+        logger.info("Checks finished in " + (endTime -startTime) / 1_000_000 + " ms");
+
+    }
+
     private boolean filterObject(CityObject co) {
         return isObjectIncluded(co, includeFilters, excludeFilters);
     }

CityDoctorParent/Extensions/CityDoctorGUI/src/main/java/de/hft/stuttgart/citydoctor2/gui/CityDoctorController.java

@@ -1197,6 +1197,7 @@ public class CityDoctorController {
 
             Platform.runLater(() -> {
                 // apply check results to tree views
+                buildTrees();
                 updateFeatureTrees();
                 updateTree(mainWindow.getPolygonsView().getRoot());
                 for (CheckError e : model.getGlobalErrors()) {
@@ -1250,6 +1251,7 @@ public class CityDoctorController {
             }
             Platform.runLater(() -> {
                 // apply check results to tree views
+                buildTrees();
                 updateFeatureTrees();
                 updateTree(mainWindow.getPolygonsView().getRoot());
                 for (CheckError e : model.getGlobalErrors()) {