Checker.java

/*-
 *  Copyright 2020 Beuth Hochschule für Technik Berlin, Hochschule für Technik Stuttgart
 * 
 *  This file is part of CityDoctor2.
 *
 *  CityDoctor2 is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  CityDoctor2 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with CityDoctor2.  If not, see <https://www.gnu.org/licenses/>.
 */
package de.hft.stuttgart.citydoctor2.check;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Stream;

import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Source;
import javax.xml.transform.TransformerException;
import javax.xml.transform.URIResolver;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamSource;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.w3c.dom.Document;

import de.hft.stuttgart.citydoctor2.check.error.SchematronError;
import de.hft.stuttgart.citydoctor2.checkresult.utility.CheckReportWriteException;
import de.hft.stuttgart.citydoctor2.checks.Checks;
import de.hft.stuttgart.citydoctor2.checks.SvrlContentHandler;
import de.hft.stuttgart.citydoctor2.checks.util.FeatureCheckedListener;
import de.hft.stuttgart.citydoctor2.datastructure.Building;
import de.hft.stuttgart.citydoctor2.datastructure.BuildingPart;
import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel;
import de.hft.stuttgart.citydoctor2.datastructure.CityObject;
import de.hft.stuttgart.citydoctor2.datastructure.FeatureType;
import de.hft.stuttgart.citydoctor2.parser.FeatureStream;
import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
import de.hft.stuttgart.citydoctor2.parser.ProgressListener;
import de.hft.stuttgart.citydoctor2.reporting.Reporter;
import de.hft.stuttgart.citydoctor2.reporting.StreamReporter;
import de.hft.stuttgart.citydoctor2.reporting.XmlStreamReporter;
import de.hft.stuttgart.citydoctor2.reporting.XmlValidationReporter;
import de.hft.stuttgart.citydoctor2.reporting.pdf.PdfReporter;
import de.hft.stuttgart.citydoctor2.reporting.pdf.PdfStreamReporter;
import de.hft.stuttgart.citydoctor2.utils.Localization;
import net.sf.saxon.s9api.DOMDestination;
import net.sf.saxon.s9api.Destination;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.SAXDestination;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XsltCompiler;
import net.sf.saxon.s9api.XsltExecutable;
import net.sf.saxon.s9api.XsltTransformer;

/**
 * The main container class for checking. It contains the logic for validation,
 * as well as contains the state of the checks performed.
 * 
 * @author Matthias Betz
 *
 */
public class Checker {

	private static final Logger logger = LogManager.getLogger(Checker.class);

	private ValidationConfiguration config;
	private List<List<Check>> execLayers;

	private List<Filter> includeFilters;
	private List<Filter> excludeFilters;

	private boolean isValidated = false;

	private Checks checkConfig;
	private CityDoctorModel model;

	public Checker(CityDoctorModel model) {
		this(ValidationConfiguration.loadStandardValidationConfig(), model);
	}

	public Checker(ValidationConfiguration config, CityDoctorModel model) {
		this.model = model;
		checkConfig = new Checks();
		setValidationConfig(config);
	}

	public Checks getChecks() {
		return checkConfig;
	}
	
	public CityDoctorModel getModel() {
		return model;
	}

	/**
	 * Write the xml report for the given CityDoctorModel. If no report location is
	 * given or this checker has not validated anything, nothing is done.
	 * 
	 * @param xmlOutput the output file location for the XML report. Can be null.
	 * @param model     the model for which the report is written.
	 */
	public void writeXmlReport(String xmlOutput) {
		if (!isValidated || xmlOutput == null) {
			return;
		}
		File xmlFile = new File(xmlOutput);
		if (xmlFile.getParentFile() != null) {
			xmlFile.getParentFile().mkdirs();
		}
		Reporter reporter = new XmlValidationReporter();
		try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(xmlFile.getAbsolutePath()))) {
			reporter.writeReport(checkConfig, bos, model, config);
		} catch (CheckReportWriteException | IOException e) {
			logger.error(Localization.getText("Checker.failXml"), e);
		}
	}

	public void writePdfReport(String pdfOutput) {
		if (!isValidated || pdfOutput == null) {
			return;
		}
		File pdfFile = new File(pdfOutput);
		if (pdfFile.getParentFile() != null) {
			pdfFile.getParentFile().mkdirs();
		}
		Reporter reporter = new PdfReporter("assets/Logo.png");
		try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(pdfFile.getAbsolutePath()))) {
			reporter.writeReport(checkConfig, bos, model, config);
		} catch (IOException | CheckReportWriteException e) {
			logger.error(Localization.getText("Checker.failPdf"), e);
		}
	}

	public void runChecks() {
		runChecks((ProgressListener) null);
	}

	public void runChecks(String xmlOutput) {
		runChecks();
		writeXmlReport(xmlOutput);
	}

	public void runChecks(String xmlOutput, String pdfOutput) {
		runChecks();
		writeXmlReport(xmlOutput);
		writePdfReport(pdfOutput);
	}

	public void runChecks(String xmlOutput, String pdfOutput, ProgressListener l) {
		runChecks(l);
		writeXmlReport(xmlOutput);
		writePdfReport(pdfOutput);
	}

	public void runChecks(ProgressListener l) {
		if (config == null) {
			config = ValidationConfiguration.loadStandardValidationConfig();
		}
		checkCityModel(model, l);
		if (logger.isInfoEnabled()) {
			logger.info(Localization.getText("Checker.checksFinished"));
		}
		SvrlContentHandler handler = executeSchematronValidationIfAvailable(config, model.getFile());
		if (handler != null) {
			model.addGlobalErrors(handler.getGeneralErrors());
			Map<String, Checkable> featureMap = new HashMap<>();
			model.createFeatureStream().forEach(f -> featureMap.put(f.getGmlId().getGmlString(), f));
			for (Building b : model.getBuildings()) {
				for (BuildingPart bp : b.getBuildingParts()) {
					featureMap.put(bp.getGmlId().getGmlString(), bp);
				}
			}
			handler.getFeatureErrors().forEach((k, v) -> {
				if (k.trim().isEmpty()) {
					// missing gml id, ignore?
					return;
				}
				Checkable checkable = featureMap.get(k);
				if (checkable == null) {
					// gml id reported by schematron was not found, add to general errors
					model.addGlobalError(v);
				} else {
					checkable.addCheckResult(new CheckResult(CheckId.C_SEM_SCHEMATRON, ResultStatus.ERROR, v));
				}
			});
		}
		isValidated = true;
	}
	
	public ValidationConfiguration getConfig() {
		return config;
	}

	public static SvrlContentHandler executeSchematronValidationIfAvailable(ValidationConfiguration config,
			File file) {
		if (config.getSchematronFilePath() != null && !config.getSchematronFilePath().isEmpty()) {
			if (logger.isInfoEnabled()) {
				logger.info(Localization.getText("Checker.schematronValidation"));
			}
			Processor processor = new Processor(false);
			XsltCompiler xsltCompiler = processor.newXsltCompiler();
			xsltCompiler.setURIResolver(new URIResolver() {

				@Override
				public Source resolve(String href, String base) throws TransformerException {
					return new StreamSource(Checker.class.getResourceAsStream(href));
				}

			});
			try {
				XsltExecutable includeExecutable = xsltCompiler
						.compile(new StreamSource(Checker.class.getResourceAsStream("iso_dsdl_include.xsl")));
				XsltTransformer includeTransformer = includeExecutable.load();
				includeTransformer.setSource(new StreamSource(new File(config.getSchematronFilePath())));

				XsltExecutable expandExecutable = xsltCompiler
						.compile(new StreamSource(Checker.class.getResourceAsStream("iso_abstract_expand.xsl")));
				XsltTransformer expandTransformer = expandExecutable.load();
				includeTransformer.setDestination(expandTransformer);

				XsltExecutable xslt2Executable = xsltCompiler
						.compile(new StreamSource(Checker.class.getResourceAsStream("iso_svrl_for_xslt2.xsl")));

				XsltTransformer xslt2Transformer = xslt2Executable.load();
				expandTransformer.setDestination(xslt2Transformer);

				DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
				factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
				Document doc = factory.newDocumentBuilder().newDocument();
				DOMDestination domDestination = new DOMDestination(doc);
				xslt2Transformer.setDestination(domDestination);
				includeTransformer.transform();

				XsltExecutable schematronExecutable = xsltCompiler.compile(new DOMSource(doc));
				XsltTransformer schematronTransformer = schematronExecutable.load();
				schematronTransformer.setSource(new StreamSource(file));
				SvrlContentHandler handler = new SvrlContentHandler();
				Destination dest = new SAXDestination(handler);
				schematronTransformer.setDestination(dest);
				schematronTransformer.transform();
				if (logger.isInfoEnabled()) {
					logger.info(Localization.getText("Checker.finishedSchematron"));
				}
				return handler;
			} catch (SaxonApiException | ParserConfigurationException e) {
				logger.catching(e);
			}
		}
		return null;
	}

	private void buildFilters() {
		FilterConfiguration filterConfig = config.getFilter();
		if (filterConfig == null) {
			includeFilters = Collections.emptyList();
			excludeFilters = Collections.emptyList();
			return;
		}
		excludeFilters = buildExcludeFilters(filterConfig);
		includeFilters = buildIncludeFilters(filterConfig);
	}

	private List<Filter> buildExcludeFilters(FilterConfiguration filterConfig) {
		if (filterConfig == null) {
			return Collections.emptyList();
		}
		ExcludeFilterConfiguration excludeConfig = filterConfig.getExclude();
		if (excludeConfig == null) {
			return Collections.emptyList();
		} else {
			List<Filter> filters = new ArrayList<>();
			if (excludeConfig.getTypes() != null) {
				for (FeatureType excludeType : excludeConfig.getTypes()) {
					filters.add(new TypeFilter(excludeType));
				}
			}
			if (excludeConfig.getIds() != null) {
				for (String excludePattern : excludeConfig.getIds()) {
					Filter f = new EqualsIgnoreCaseFilter(excludePattern);
					filters.add(f);
				}
			}
			return filters;
		}
	}

	private List<Filter> buildIncludeFilters(FilterConfiguration filterConfig) {
		if (filterConfig == null) {
			return Collections.emptyList();
		}
		IncludeFilterConfiguration includeConfig = filterConfig.getInclude();
		if (includeConfig == null) {
			return Collections.emptyList();
		} else {
			List<Filter> filters = new ArrayList<>();
			if (includeConfig.getTypes() != null) {
				for (FeatureType includeType : includeConfig.getTypes()) {
					filters.add(new TypeFilter(includeType));
				}
			}
			if (includeConfig.getIds() != null) {
				for (String includePattern : includeConfig.getIds()) {
					Filter f = new EqualsIgnoreCaseFilter(includePattern);
					filters.add(f);
				}
			}
			return filters;
		}
	}

	private void setValidationConfig(ValidationConfiguration config) {
		if (config == null) {
			throw new IllegalArgumentException("Validation configuration may not be null");
		}
		this.config = config;
		buildFilters();
		ParserConfiguration parserConfig = config.getParserConfiguration();
		List<Check> checks = collectEnabledChecksAndInit(parserConfig, config);
		execLayers = buildExecutionLayers(checks);
	}

	private List<Check> collectEnabledChecksAndInit(ParserConfiguration parserConfig, ValidationConfiguration config) {
		List<Check> checks = new ArrayList<>();
		for (Entry<CheckId, CheckConfiguration> e : config.getChecks().entrySet()) {
			if (e.getValue().isEnabled()) {
				Check c = checkConfig.getCheckForId(e.getKey());
				Map<String, String> parameters = new HashMap<>();
				parameters.putAll(e.getValue().getParameters());
				parameters.put("numberOfRoundingPlaces", "" + config.getNumberOfRoundingPlaces());
				parameters.put("minVertexDistance", "" + config.getMinVertexDistance());
				// initialize checks with parameters
				c.init(parameters, parserConfig);
				checks.add(c);
			}
		}
		return checks;
	}

	private void checkCityModel(CityDoctorModel model, ProgressListener l) {
		Stream<CityObject> features = model.createFeatureStream();
		float featureSum = model.getNumberOfFeatures();
		// stupid lamda with final variable restrictions
		int[] currentFeature = new int[1];
		features.forEach(co -> {
			if (config.getParserConfiguration().useLowMemoryConsumption()) {
				// no edges have been created yet, create them
				co.prepareForChecking();
			}
			// check every feature
			executeChecksForCityObject(co);
			
			if (config.getParserConfiguration().useLowMemoryConsumption()) {
				// low memory consumption, remove edges again
				co.clearMetaInformation();
			}
			if (l != null) {
				currentFeature[0]++;
				l.updateProgress(currentFeature[0] / featureSum);
			}
		});
	}

	private boolean filterObject(CityObject co) {
		return isObjectIncluded(co, includeFilters, excludeFilters);
	}

	private boolean isObjectIncluded(CityObject co, List<Filter> includeFilters, List<Filter> excludeFilters) {
		if (!includeFilters.isEmpty()) {
			boolean include = false;
			for (Filter f : includeFilters) {
				if (f.matches(co)) {
					include = true;
					break;
				}
			}
			if (!include) {
				// not included, ignore
				return false;
			}
		}
		// check if object is excluded
		for (Filter f : excludeFilters) {
			if (f.matches(co)) {
				// exclude object
				return false;
			}
		}
		return true;
	}

	/**
	 * Checks the city object if it has not been removed by the filters. The check
	 * result are stored into the city object itself.
	 * 
	 * @param co the city object that is going to be checked
	 */
	private void executeChecksForCityObject(CityObject co) {
		if (!filterObject(co)) {
			return;
		}
		executeChecksForCheckable(co);
	}

	/**
	 * Executes all checks for the checkable. This will bypass the filters.
	 * 
	 * @param co the checkable.
	 */
	public void executeChecksForCheckable(Checkable co) {
		// throw away old results
		co.clearAllContainedCheckResults();
		if (logger.isDebugEnabled()) {
			logger.debug(Localization.getText("Checker.checkFeature"), co);
		}
		for (int i = 0; i < execLayers.size(); i++) {
			for (Check check : execLayers.get(i)) {
				if (logger.isTraceEnabled()) {
					logger.trace(Localization.getText("Checker.executeCheck"), check.getCheckId());
				}
				co.accept(check);
			}
		}
	}

	public static List<List<Check>> buildExecutionLayers(List<Check> checks) {
		List<List<Check>> result = new ArrayList<>();
		
		Set<Check> availableChecks = new HashSet<>(checks);
		Set<CheckId> usedChecks = new HashSet<>();
		
		while (!availableChecks.isEmpty()) {
			List<Check> layer = new ArrayList<>();
			Iterator<Check> iterator = availableChecks.iterator();
			while (iterator.hasNext()) {
				Check c = iterator.next();
				boolean hasUnusedDependency = searchForUnusedDependency(usedChecks, c);
				if (!hasUnusedDependency) {
					iterator.remove();
					layer.add(c);
				}
			}
			if (layer.isEmpty()) {
				throw new IllegalStateException("There are checks that have dependencies that are not executed or are unknown");
			}
			result.add(layer);
			for (Check c : layer) {
				usedChecks.add(c.getCheckId());
			}
		}
		return result;
	}

	private static boolean searchForUnusedDependency(Set<CheckId> usedChecks, Check c) {
		boolean hasUnusedDependency = false;
		for (CheckId id : c.getDependencies()) {
			if (!usedChecks.contains(id)) {
				hasUnusedDependency = true;
				break;
			}
		}
		return hasUnusedDependency;
	}

	public static void streamCheck(FeatureStream stream, String xmlOutput, String pdfOutput,
			ValidationConfiguration config) throws InterruptedException, IOException {
		streamCheck(stream, xmlOutput, pdfOutput, config, "assets/Logo.png", null);
	}

	public static void streamCheck(FeatureStream stream, String xmlOutput, String pdfOutput,
			ValidationConfiguration config, String logoLocation, FeatureCheckedListener l)
			throws InterruptedException, IOException {
		Checker c = new Checker(config, null);
		try (BufferedOutputStream xmlBos = getXmlOutputMaybe(xmlOutput);
				BufferedOutputStream pdfBos = getPdfOutputMaybe(pdfOutput)) {
			XmlStreamReporter xmlReporter = null;
			if (xmlBos != null) {
				xmlReporter = new XmlStreamReporter(xmlBos, stream.getFileName(), config);
			}
			PdfStreamReporter pdfReporter = null;
			if (pdfBos != null) {
				pdfReporter = new PdfStreamReporter(pdfBos, stream.getFileName(), config, logoLocation);
			}
			CityObject co = null;
			while ((co = stream.next()) != null) {
				c.checkFeature(xmlReporter, pdfReporter, co);
				if (l != null) {
					l.featureChecked(co);
				}
			}
			SvrlContentHandler handler = executeSchematronValidationIfAvailable(config, stream.getFile());
			writeReport(xmlReporter, handler);
			writeReport(pdfReporter, handler);
		} catch (CheckReportWriteException e) {
			logger.error(Localization.getText("Checker.failReports"), e);
		}
	}

	public static void writeReport(StreamReporter reporter, SvrlContentHandler handler)
			throws CheckReportWriteException {
		if (reporter != null) {
			if (handler != null) {
				for (SchematronError err : handler.getGeneralErrors()) {
					reporter.reportGlobalError(err);
				}
				for (Entry<String, SchematronError> e : handler.getFeatureErrors().entrySet()) {
					reporter.addError(e.getKey(), e.getValue());
				}
			}
			reporter.finishReport();
		}
	}

	public static BufferedOutputStream getPdfOutputMaybe(String pdfOutput) throws FileNotFoundException {
		return pdfOutput != null ? new BufferedOutputStream(new FileOutputStream(pdfOutput)) : null;
	}

	public static BufferedOutputStream getXmlOutputMaybe(String xmlOutput) throws FileNotFoundException {
		return xmlOutput != null ? new BufferedOutputStream(new FileOutputStream(xmlOutput)) : null;
	}

	public void checkFeature(XmlStreamReporter xmlReporter, PdfStreamReporter pdfReporter, CityObject co) {
		if (logger.isDebugEnabled()) {
			logger.debug(Localization.getText("Checker.checkFeature"), co);
		}
		executeChecksForCityObject(co);
		if (xmlReporter != null) {
			xmlReporter.report(co);
		}
		if (pdfReporter != null) {
			pdfReporter.report(co);
		}
	}
}