/*-
* Copyright 2020 Beuth Hochschule für Technik Berlin, Hochschule für Technik Stuttgart
*
* This file is part of CityDoctor2.
*
* CityDoctor2 is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* CityDoctor2 is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with CityDoctor2. If not, see .
*/
package de.hft.stuttgart.citydoctor2.check;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Stream;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Source;
import javax.xml.transform.TransformerException;
import javax.xml.transform.URIResolver;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamSource;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.w3c.dom.Document;
import de.hft.stuttgart.citydoctor2.check.error.SchematronError;
import de.hft.stuttgart.citydoctor2.checkresult.utility.CheckReportWriteException;
import de.hft.stuttgart.citydoctor2.checks.Checks;
import de.hft.stuttgart.citydoctor2.checks.SvrlContentHandler;
import de.hft.stuttgart.citydoctor2.checks.util.FeatureCheckedListener;
import de.hft.stuttgart.citydoctor2.datastructure.Building;
import de.hft.stuttgart.citydoctor2.datastructure.BuildingPart;
import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel;
import de.hft.stuttgart.citydoctor2.datastructure.CityObject;
import de.hft.stuttgart.citydoctor2.datastructure.FeatureType;
import de.hft.stuttgart.citydoctor2.parser.FeatureStream;
import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
import de.hft.stuttgart.citydoctor2.parser.ProgressListener;
import de.hft.stuttgart.citydoctor2.reporting.Reporter;
import de.hft.stuttgart.citydoctor2.reporting.StreamReporter;
import de.hft.stuttgart.citydoctor2.reporting.XmlStreamReporter;
import de.hft.stuttgart.citydoctor2.reporting.XmlValidationReporter;
import de.hft.stuttgart.citydoctor2.reporting.pdf.PdfReporter;
import de.hft.stuttgart.citydoctor2.reporting.pdf.PdfStreamReporter;
import de.hft.stuttgart.citydoctor2.utils.Localization;
import net.sf.saxon.s9api.DOMDestination;
import net.sf.saxon.s9api.Destination;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.SAXDestination;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XsltCompiler;
import net.sf.saxon.s9api.XsltExecutable;
import net.sf.saxon.s9api.XsltTransformer;
/**
* The main container class for checking. It contains the logic for validation,
* as well as contains the state of the checks performed.
*
* @author Matthias Betz
*
*/
public class Checker {
private static final Logger logger = LogManager.getLogger(Checker.class);
private ValidationConfiguration config;
private List> execLayers;
private List includeFilters;
private List excludeFilters;
private boolean isValidated = false;
private Checks checkConfig;
private CityDoctorModel model;
public Checker(CityDoctorModel model) {
this(ValidationConfiguration.loadStandardValidationConfig(), model);
}
public Checker(ValidationConfiguration config, CityDoctorModel model) {
this.model = model;
checkConfig = new Checks();
setValidationConfig(config);
}
public Checks getChecks() {
return checkConfig;
}
public CityDoctorModel getModel() {
return model;
}
/**
* Write the xml report for the given CityDoctorModel. If no report location is
* given or this checker has not validated anything, nothing is done.
*
* @param xmlOutput the output file location for the XML report. Can be null.
* @param model the model for which the report is written.
*/
public void writeXmlReport(String xmlOutput) {
if (!isValidated || xmlOutput == null) {
return;
}
File xmlFile = new File(xmlOutput);
if (xmlFile.getParentFile() != null) {
xmlFile.getParentFile().mkdirs();
}
Reporter reporter = new XmlValidationReporter();
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(xmlFile.getAbsolutePath()))) {
reporter.writeReport(checkConfig, bos, model, config);
} catch (CheckReportWriteException | IOException e) {
logger.error(Localization.getText("Checker.failXml"), e);
}
}
public void writePdfReport(String pdfOutput) {
if (!isValidated || pdfOutput == null) {
return;
}
File pdfFile = new File(pdfOutput);
if (pdfFile.getParentFile() != null) {
pdfFile.getParentFile().mkdirs();
}
Reporter reporter = new PdfReporter("assets/Logo.png");
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(pdfFile.getAbsolutePath()))) {
reporter.writeReport(checkConfig, bos, model, config);
} catch (IOException | CheckReportWriteException e) {
logger.error(Localization.getText("Checker.failPdf"), e);
}
}
public void runChecks() {
runChecks((ProgressListener) null);
}
public void runChecks(String xmlOutput) {
runChecks();
writeXmlReport(xmlOutput);
}
public void runChecks(String xmlOutput, String pdfOutput) {
runChecks();
writeXmlReport(xmlOutput);
writePdfReport(pdfOutput);
}
public void runChecks(String xmlOutput, String pdfOutput, ProgressListener l) {
runChecks(l);
writeXmlReport(xmlOutput);
writePdfReport(pdfOutput);
}
public void runChecks(ProgressListener l) {
if (config == null) {
config = ValidationConfiguration.loadStandardValidationConfig();
}
checkCityModel(model, l);
if (logger.isInfoEnabled()) {
logger.info(Localization.getText("Checker.checksFinished"));
}
SvrlContentHandler handler = executeSchematronValidationIfAvailable(config, model.getFile());
if (handler != null) {
model.addGlobalErrors(handler.getGeneralErrors());
Map featureMap = new HashMap<>();
model.createFeatureStream().forEach(f -> featureMap.put(f.getGmlId().getGmlString(), f));
for (Building b : model.getBuildings()) {
for (BuildingPart bp : b.getBuildingParts()) {
featureMap.put(bp.getGmlId().getGmlString(), bp);
}
}
handler.getFeatureErrors().forEach((k, v) -> {
if (k.trim().isEmpty()) {
// missing gml id, ignore?
return;
}
Checkable checkable = featureMap.get(k);
if (checkable == null) {
// gml id reported by schematron was not found, add to general errors
model.addGlobalError(v);
} else {
checkable.addCheckResult(new CheckResult(CheckId.C_SEM_SCHEMATRON, ResultStatus.ERROR, v));
}
});
}
isValidated = true;
}
private static SvrlContentHandler executeSchematronValidationIfAvailable(ValidationConfiguration config,
File file) {
if (config.getSchematronFilePath() != null && !config.getSchematronFilePath().isEmpty()) {
if (logger.isInfoEnabled()) {
logger.info(Localization.getText("Checker.schematronValidation"));
}
Processor processor = new Processor(false);
XsltCompiler xsltCompiler = processor.newXsltCompiler();
xsltCompiler.setURIResolver(new URIResolver() {
@Override
public Source resolve(String href, String base) throws TransformerException {
return new StreamSource(Checker.class.getResourceAsStream(href));
}
});
try {
XsltExecutable includeExecutable = xsltCompiler
.compile(new StreamSource(Checker.class.getResourceAsStream("iso_dsdl_include.xsl")));
XsltTransformer includeTransformer = includeExecutable.load();
includeTransformer.setSource(new StreamSource(new File(config.getSchematronFilePath())));
XsltExecutable expandExecutable = xsltCompiler
.compile(new StreamSource(Checker.class.getResourceAsStream("iso_abstract_expand.xsl")));
XsltTransformer expandTransformer = expandExecutable.load();
includeTransformer.setDestination(expandTransformer);
XsltExecutable xslt2Executable = xsltCompiler
.compile(new StreamSource(Checker.class.getResourceAsStream("iso_svrl_for_xslt2.xsl")));
XsltTransformer xslt2Transformer = xslt2Executable.load();
expandTransformer.setDestination(xslt2Transformer);
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
Document doc = factory.newDocumentBuilder().newDocument();
DOMDestination domDestination = new DOMDestination(doc);
xslt2Transformer.setDestination(domDestination);
includeTransformer.transform();
XsltExecutable schematronExecutable = xsltCompiler.compile(new DOMSource(doc));
XsltTransformer schematronTransformer = schematronExecutable.load();
schematronTransformer.setSource(new StreamSource(file));
SvrlContentHandler handler = new SvrlContentHandler();
Destination dest = new SAXDestination(handler);
schematronTransformer.setDestination(dest);
schematronTransformer.transform();
if (logger.isInfoEnabled()) {
logger.info(Localization.getText("Checker.finishedSchematron"));
}
return handler;
} catch (SaxonApiException | ParserConfigurationException e) {
logger.catching(e);
}
}
return null;
}
private void buildFilters() {
FilterConfiguration filterConfig = config.getFilter();
if (filterConfig == null) {
includeFilters = Collections.emptyList();
excludeFilters = Collections.emptyList();
return;
}
excludeFilters = buildExcludeFilters(filterConfig);
includeFilters = buildIncludeFilters(filterConfig);
}
private List buildExcludeFilters(FilterConfiguration filterConfig) {
if (filterConfig == null) {
return Collections.emptyList();
}
ExcludeFilterConfiguration excludeConfig = filterConfig.getExclude();
if (excludeConfig == null) {
return Collections.emptyList();
} else {
List filters = new ArrayList<>();
if (excludeConfig.getTypes() != null) {
for (FeatureType excludeType : excludeConfig.getTypes()) {
filters.add(new TypeFilter(excludeType));
}
}
if (excludeConfig.getIds() != null) {
for (String excludePattern : excludeConfig.getIds()) {
Filter f = new EqualsIgnoreCaseFilter(excludePattern);
filters.add(f);
}
}
return filters;
}
}
private List buildIncludeFilters(FilterConfiguration filterConfig) {
if (filterConfig == null) {
return Collections.emptyList();
}
IncludeFilterConfiguration includeConfig = filterConfig.getInclude();
if (includeConfig == null) {
return Collections.emptyList();
} else {
List filters = new ArrayList<>();
if (includeConfig.getTypes() != null) {
for (FeatureType includeType : includeConfig.getTypes()) {
filters.add(new TypeFilter(includeType));
}
}
if (includeConfig.getIds() != null) {
for (String includePattern : includeConfig.getIds()) {
Filter f = new EqualsIgnoreCaseFilter(includePattern);
filters.add(f);
}
}
return filters;
}
}
private void setValidationConfig(ValidationConfiguration config) {
if (config == null) {
throw new IllegalArgumentException("Validation configuration may not be null");
}
this.config = config;
buildFilters();
ParserConfiguration parserConfig = config.getParserConfiguration();
List checks = collectEnabledChecksAndInit(parserConfig, config);
execLayers = buildExecutionLayers(checks);
}
private List collectEnabledChecksAndInit(ParserConfiguration parserConfig, ValidationConfiguration config) {
List checks = new ArrayList<>();
for (Entry e : config.getChecks().entrySet()) {
if (e.getValue().isEnabled()) {
Check c = checkConfig.getCheckForId(e.getKey());
Map parameters = new HashMap<>();
parameters.putAll(e.getValue().getParameters());
parameters.put("numberOfRoundedPlaces", "" + config.getNumberOfRoundingPlaces());
parameters.put("minVertexDistance", "" + config.getMinVertexDistance());
// initialize checks with parameters
c.init(e.getValue().getParameters(), parserConfig);
checks.add(c);
}
}
return checks;
}
private void checkCityModel(CityDoctorModel model, ProgressListener l) {
Stream features = model.createFeatureStream();
float featureSum = model.getNumberOfFeatures();
// stupid lamda with final variable restrictions
int[] currentFeature = new int[1];
features.forEach(co -> {
if (config.getParserConfiguration().useLowMemoryConsumption()) {
// no edges have been created yet, create them
co.prepareForChecking();
}
// check every feature
executeChecksForCityObject(co);
if (config.getParserConfiguration().useLowMemoryConsumption()) {
// low memory consumption, remove edges again
co.clearMetaInformation();
}
if (l != null) {
currentFeature[0]++;
l.updateProgress(currentFeature[0] / featureSum);
}
});
}
private boolean filterObject(CityObject co) {
return isObjectIncluded(co, includeFilters, excludeFilters);
}
private boolean isObjectIncluded(CityObject co, List includeFilters, List excludeFilters) {
if (!includeFilters.isEmpty()) {
boolean include = false;
for (Filter f : includeFilters) {
if (f.matches(co)) {
include = true;
break;
}
}
if (!include) {
// not included, ignore
return false;
}
}
// check if object is excluded
for (Filter f : excludeFilters) {
if (f.matches(co)) {
// exclude object
return false;
}
}
return true;
}
/**
* Checks the city object if it has not been removed by the filters. The check
* result are stored into the city object itself.
*
* @param co the city object that is going to be checked
*/
private void executeChecksForCityObject(CityObject co) {
if (!filterObject(co)) {
return;
}
executeChecksForCheckable(co);
}
/**
* Executes all checks for the checkable. This will bypass the filters.
*
* @param co the checkable.
*/
public void executeChecksForCheckable(Checkable co) {
// throw away old results
co.clearAllContainedCheckResults();
if (logger.isDebugEnabled()) {
logger.debug(Localization.getText("Checker.checkFeature"), co);
}
for (int i = 0; i < execLayers.size(); i++) {
for (Check check : execLayers.get(i)) {
if (logger.isTraceEnabled()) {
logger.trace(Localization.getText("Checker.executeCheck"), check.getCheckId());
}
co.accept(check);
}
}
}
public static List> buildExecutionLayers(List checks) {
List> result = new ArrayList<>();
Set availableChecks = new HashSet<>(checks);
Set usedChecks = new HashSet<>();
while (!availableChecks.isEmpty()) {
List layer = new ArrayList<>();
Iterator iterator = availableChecks.iterator();
while (iterator.hasNext()) {
Check c = iterator.next();
boolean hasUnusedDependency = searchForUnusedDependency(usedChecks, c);
if (!hasUnusedDependency) {
iterator.remove();
layer.add(c);
}
}
if (layer.isEmpty()) {
throw new IllegalStateException("There are checks that have dependencies that are not executed or are unknown");
}
result.add(layer);
for (Check c : layer) {
usedChecks.add(c.getCheckId());
}
}
return result;
}
private static boolean searchForUnusedDependency(Set usedChecks, Check c) {
boolean hasUnusedDependency = false;
for (CheckId id : c.getDependencies()) {
if (!usedChecks.contains(id)) {
hasUnusedDependency = true;
break;
}
}
return hasUnusedDependency;
}
public static void streamCheck(FeatureStream stream, String xmlOutput, String pdfOutput,
ValidationConfiguration config) throws InterruptedException, IOException {
streamCheck(stream, xmlOutput, pdfOutput, config, "assets/Logo.png", null);
}
public static void streamCheck(FeatureStream stream, String xmlOutput, String pdfOutput,
ValidationConfiguration config, String logoLocation, FeatureCheckedListener l)
throws InterruptedException, IOException {
Checker c = new Checker(config, null);
try (BufferedOutputStream xmlBos = getXmlOutputMaybe(xmlOutput);
BufferedOutputStream pdfBos = getPdfOutputMaybe(pdfOutput)) {
XmlStreamReporter xmlReporter = null;
if (xmlOutput != null) {
xmlReporter = new XmlStreamReporter(xmlBos, stream.getFileName(), config);
}
PdfStreamReporter pdfReporter = null;
if (pdfOutput != null) {
pdfReporter = new PdfStreamReporter(pdfBos, stream.getFileName(), config, logoLocation);
}
CityObject co = null;
while ((co = stream.next()) != null) {
c.checkFeature(xmlReporter, pdfReporter, co);
if (l != null) {
l.featureChecked(co);
}
}
SvrlContentHandler handler = executeSchematronValidationIfAvailable(config, stream.getFile());
writeReport(xmlReporter, handler);
writeReport(pdfReporter, handler);
} catch (CheckReportWriteException e) {
logger.error(Localization.getText("Checker.failReports"), e);
}
}
private static void writeReport(StreamReporter reporter, SvrlContentHandler handler)
throws CheckReportWriteException {
if (reporter != null) {
if (handler != null) {
for (SchematronError err : handler.getGeneralErrors()) {
reporter.reportGlobalError(err);
}
for (Entry e : handler.getFeatureErrors().entrySet()) {
reporter.addError(e.getKey(), e.getValue());
}
}
reporter.finishReport();
}
}
private static BufferedOutputStream getPdfOutputMaybe(String pdfOutput) throws FileNotFoundException {
return pdfOutput != null ? new BufferedOutputStream(new FileOutputStream(pdfOutput)) : null;
}
private static BufferedOutputStream getXmlOutputMaybe(String xmlOutput) throws FileNotFoundException {
return xmlOutput != null ? new BufferedOutputStream(new FileOutputStream(xmlOutput)) : null;
}
private void checkFeature(XmlStreamReporter xmlReporter, PdfStreamReporter pdfReporter, CityObject co) {
if (logger.isDebugEnabled()) {
logger.debug(Localization.getText("Checker.checkFeature"), co);
}
executeChecksForCityObject(co);
if (xmlReporter != null) {
xmlReporter.report(co);
}
if (pdfReporter != null) {
pdfReporter.report(co);
}
}
}