CityGmlParser.java

/*-
 *  Copyright 2022 Beuth Hochschule für Technik Berlin, Hochschule für Technik Stuttgart
 * 
 *  This file is part of CityDoctor2.
 *
 *  CityDoctor2 is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  CityDoctor2 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with CityDoctor2.  If not, see <https://www.gnu.org/licenses/>.
 */
package de.hft.stuttgart.citydoctor2.parser;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.XMLConstants;
import javax.xml.namespace.QName;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.Source;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator;

import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.citygml4j.core.ade.ADEException;
import org.citygml4j.core.ade.ADERegistry;
import org.citygml4j.core.model.CityGMLVersion;
import org.citygml4j.core.model.ade.ADEProperty;
import org.citygml4j.core.model.core.AbstractCityObject;
import org.citygml4j.core.model.core.AbstractCityObjectProperty;
import org.citygml4j.core.model.core.AbstractFeature;
import org.citygml4j.core.model.core.CityModel;
import org.citygml4j.core.util.CityGMLConstants;
import org.citygml4j.xml.CityGMLContext;
import org.citygml4j.xml.CityGMLContextException;
import org.citygml4j.xml.module.citygml.CityGMLModules;
import org.citygml4j.xml.reader.ChunkOptions;
import org.citygml4j.xml.reader.CityGMLInputFactory;
import org.citygml4j.xml.reader.CityGMLReadException;
import org.citygml4j.xml.reader.CityGMLReader;
import org.citygml4j.xml.reader.FeatureInfo;
import org.citygml4j.xml.schema.CityGMLSchemaHandler;
import org.citygml4j.xml.writer.CityGMLChunkWriter;
import org.citygml4j.xml.writer.CityGMLOutputFactory;
import org.citygml4j.xml.writer.CityGMLWriteException;
import org.locationtech.proj4j.BasicCoordinateTransform;
import org.locationtech.proj4j.CRSFactory;
import org.locationtech.proj4j.CoordinateReferenceSystem;
import org.locationtech.proj4j.ProjCoordinate;
import org.locationtech.proj4j.proj.Projection;
import org.locationtech.proj4j.units.Units;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXNotRecognizedException;
import org.xml.sax.SAXNotSupportedException;
import org.xmlobjects.schema.SchemaHandlerException;

import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel;
import de.hft.stuttgart.citydoctor2.datastructure.CityObject;
import de.hft.stuttgart.citydoctor2.mapper.citygml3.Citygml3FeatureMapper;
import de.hft.stuttgart.citydoctor2.mapper.citygml3.GMLValidationHandler;
import de.hft.stuttgart.citydoctor2.math.Vector3d;
import de.hft.stuttgart.citydoctor2.utils.Localization;
import de.hft.stuttgart.quality.QualityADEContext;
import de.hft.stuttgart.quality.QualityADEModule;

/**
 * Utility class to parse CityGML files.
 * 
 * @author Matthias Betz
 *
 */
public class CityGmlParser {

	private static final String CITY_OBJECT_MEMBER = "cityObjectMember";

	private static final String WGS_84 = "EPSG:4326";

	private static final Logger logger = LogManager.getLogger(CityGmlParser.class);

	private static final CRSFactory CRS_FACTORY = new CRSFactory();
	// EPSG:31467
	private static final Pattern P_EPSG = Pattern.compile("^(EPSG:\\d+)$");
	// urn:ogc:def:crs,crs:EPSG:6.12:31467,crs:EPSG:6.12:5783
	// or
	// urn:ogc:def:crs,crs:EPSG::28992
	private static final Pattern P_OGC = Pattern.compile("urn:ogc:def:crs,crs:EPSG:[\\d\\.]*:([\\d]+)\\D*");

	private static final Pattern P_OGC2 = Pattern.compile("urn:ogc:def:crs:EPSG:[\\d\\.]*:([\\d]+)\\D*");

	// urn:adv:crs:DE_DHDN_3GK3*DE_DHHN92_NH
	// urn:adv:crs:ETRS89_UTM32*DE_DHHN92_NH
	private static final Pattern P_URN = Pattern.compile("urn:adv:crs:([^\\*]+)");

	private static final SAXParserFactory FACTORY;

	private static CityGMLContext context;
	private static List<QName> chunkProperties = new ArrayList<>();

	static {
		System.setProperty("javax.xml.transform.TransformerFactory", "com.sun.org.apache.xalan.internal.xsltc.trax.TransformerFactoryImpl");
		FACTORY = SAXParserFactory.newInstance();
		try {
			FACTORY.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
		} catch (SAXNotRecognizedException | SAXNotSupportedException | ParserConfigurationException e) {
			logger.catching(e);
		}

		chunkProperties.add(new QName(CityGMLConstants.CITYGML_1_0_CORE_NAMESPACE, CITY_OBJECT_MEMBER));
		chunkProperties.add(new QName(CityGMLConstants.CITYGML_2_0_CORE_NAMESPACE, CITY_OBJECT_MEMBER));
		chunkProperties.add(new QName(CityGMLConstants.CITYGML_3_0_CORE_NAMESPACE, CITY_OBJECT_MEMBER));
	}

	private CityGmlParser() {
	}

	public static synchronized CityGMLContext getContext() {
		if (context == null) {
			try {
				context = CityGMLContext.newInstance(CityGmlParser.class.getClassLoader());
				// also setup ades
				ADERegistry adeRegistry = ADERegistry.getInstance();
				adeRegistry.loadADE(new QualityADEContext());
			} catch (CityGMLContextException e) {
				logger.fatal("Unable to create citygml4j context", e);
				throw new IllegalStateException("Unable to create citygml4j context");
			} catch (ADEException e) {
				logger.fatal("Unable to add ADE plugins to citygml4j", e);
				throw new IllegalStateException("Unable to add ADE plugins to citygml4j");
			}
		}
		return context;
	}

	public static CityDoctorModel parseCityGmlFile(String file, ParserConfiguration config)
			throws CityGmlParseException, InvalidGmlFileException {
		return parseCityGmlFile(file, config, null, null);
	}

	public static CityDoctorModel parseCityGmlFile(String file, ParserConfiguration config, ProgressListener l)
			throws CityGmlParseException, InvalidGmlFileException {
		return parseCityGmlFile(file, config, l, null);
	}

	public static CityDoctorModel parseCityGmlFile(String filePath, ParserConfiguration config, ProgressListener l,
			GMLValidationHandler handler) throws CityGmlParseException, InvalidGmlFileException {
		CityGMLContext context = getContext();
		Path file = Paths.get(filePath);
		if (config.getValidate()) {
			List<String> messages = validateFile(context, handler, file);
			if (!messages.isEmpty()) {
				throw new InvalidGmlFileException("Invalid GML File. First error: \n" + messages.get(0));
			}
		}

		try {
			parseEpsgCodeFromFile(file, config);
			CityGMLInputFactory in = context.createCityGMLInputFactory()
					.withChunking(ChunkOptions.chunkByProperties(chunkProperties).skipCityModel(false));
			try (ObservedInputStream ois = new ObservedInputStream(file.toFile())) {
				if (l != null) {
					ois.addListener(l::updateProgress);
				}
				return readAndKeepFeatures(config, file, in, ois);
			}
		} catch (CityGMLReadException | IOException e) {
			throw new CityGmlParseException("Failed to read CityGML file", e);
		}
	}

	public static void streamCityGml(String file, ParserConfiguration config, CityGmlConsumer cityObjectConsumer,
			String outputFile) throws CityGmlParseException {
		Path f = Paths.get(file);
		streamCityGml(f, config, null, cityObjectConsumer, outputFile);
	}

	public static void streamCityGml(File file, ParserConfiguration parserConfig, CityGmlConsumer cityObjectConsumer,
			String outputFile) throws CityGmlParseException {
		streamCityGml(file.toPath(), parserConfig, null, cityObjectConsumer, outputFile);
	}

	public static void streamCityGml(Path file, ParserConfiguration config, ProgressListener l,
			CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGmlParseException {
		parseEpsgCodeFromFile(file, config);
		startReadingCityGml(file, config, l, cityObjectConsumer, outputFile);
	}

	public static CityModel parseOnlyCityModel(File inputFile) throws CityGmlParseException {
		try {
			CityGMLInputFactory inputFactory = context.createCityGMLInputFactory()
					.withChunking(ChunkOptions.chunkByProperties(chunkProperties).skipCityModel(false));
			try (CityGMLReader reader = inputFactory.createCityGMLReader(inputFile)) {
				while (reader.hasNext()) {
					AbstractFeature chunk = reader.next();
					if (chunk instanceof CityModel cModel) {
						cModel.setCityObjectMembers(null);
						return cModel;
					}
				}
			}
		} catch (CityGMLReadException e) {
			throw new CityGmlParseException(e);
		}
		throw new CityGmlParseException("Did not find any CityModel in CityGML file");
	}

	private static void startReadingCityGml(Path file, ParserConfiguration config, ProgressListener l,
			CityGmlConsumer cityObjectConsumer, String outputFile) {
		try (ObservedInputStream ois = new ObservedInputStream(file.toFile())) {
			if (l != null) {
				ois.addListener(l::updateProgress);
			}
			readAndDiscardFeatures(file, config, ois, cityObjectConsumer, outputFile);
		} catch (IOException | CityGMLReadException e) {
			logger.error(Localization.getText("CityGmlParser.errorReadingGmlFile"), e.getMessage());
			logger.catching(Level.ERROR, e);
		}
	}

	private static void readAndDiscardFeatures(Path file, ParserConfiguration config, ObservedInputStream ois,
			CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGMLReadException {
		getContext();
		CityGMLInputFactory inputFactory = context.createCityGMLInputFactory()
				.withChunking(ChunkOptions.chunkByProperties(chunkProperties).skipCityModel(false));
		CityGMLChunkWriter writer = null;
		try (CityGMLReader reader = inputFactory.createCityGMLReader(ois)) {
			Citygml3FeatureMapper mapper = new Citygml3FeatureMapper(config, file);
			CityDoctorModel model = mapper.getModel();
			boolean isInitialized = false;
			while (reader.hasNext()) {
				AbstractFeature chunk = reader.next();
				if (writer == null) {
					writer = createCityModelWriter(outputFile, reader);
				}
				if (!isInitialized && writer != null && reader.getParentInfo() != null
						&& reader.getParentInfo().getTypeName().getLocalPart().equals("CityModel")) {
					FeatureInfo parentInfo = reader.getParentInfo();
					writer.withCityModelInfo(parentInfo);
					isInitialized = true;
				}
				if (chunk instanceof AbstractCityObject ag) {
					ag.accept(mapper);
					drainCityModel(model, cityObjectConsumer);
					writeAbstractCityObject(writer, ag);
				} else if (chunk instanceof CityModel cModel) {
					cModel.setCityObjectMembers(null);
					mapper.setCityModel(cModel);
					cityObjectConsumer.accept(cModel);
					writeCityModel(writer, cModel);
				} else if (writer != null) {
					writer.writeMember(chunk);
				}
			}
			// end of stream
			logger.debug("End of gml file stream");
		} catch (CityGMLReadException e) {
			logger.error(Localization.getText("CityGmlParser.errorReadingGmlFile"), e.getMessage(), e);
		} catch (CityGMLWriteException e) {
			logger.error(Localization.getText("CityGmlParser.errorWritingGmlFile"), e.getMessage(), e);
		} finally {
			if (writer != null) {
				try {
					writer.close();
				} catch (CityGMLWriteException e) {
					// ignore
				}
			}
		}
	}

	private static void writeCityModel(CityGMLChunkWriter writer, CityModel cModel) {
		if (writer != null) {
			for (ADEProperty genEle : cModel.getADEProperties()) {
				writer.getCityModelInfo().addADEProperty(genEle);
			}
		}
	}

	private static void writeAbstractCityObject(CityGMLChunkWriter writer, AbstractCityObject ag)
			throws CityGMLWriteException {
		if (writer != null) {
			writer.writeMember(ag);
		}
	}

	private static CityGMLChunkWriter createCityModelWriter(String outputFile, CityGMLReader reader)
			throws CityGMLWriteException {
		if (outputFile == null) {
			return null;
		}
		CityGMLContext gmlContext = CityGmlParser.getContext();
		CityGMLVersion version = CityGMLModules.getCityGMLVersion(reader.getName().getNamespaceURI());
		CityGMLOutputFactory factory = gmlContext.createCityGMLOutputFactory(version);
		CityGMLChunkWriter writer = factory.createCityGMLChunkWriter(new File(outputFile),
				StandardCharsets.UTF_8.name());
		writer.withPrefix("qual", QualityADEModule.NAMESPACE_URI);
		writer.withSchemaLocation(QualityADEModule.NAMESPACE_URI, QualityADEModule.NAMESPACE_URI + "/qualityAde.xsd");
		writer.withIndent("  ");
		writer.withDefaultPrefixes();
		writer.withDefaultSchemaLocations();
		return writer;
	}

	private static CityDoctorModel readAndKeepFeatures(ParserConfiguration config, Path file,
			CityGMLInputFactory inputFactory, ObservedInputStream ois) throws CityGMLReadException {
		try (CityGMLReader reader = inputFactory.createCityGMLReader(ois)) {
			Citygml3FeatureMapper mapper = new Citygml3FeatureMapper(config, file);
			// model is read in chunked mode
			// object members are replaced by href in model
			// need to remove the refs and re-add unparsed objects
			List<AbstractCityObject> acos = new ArrayList<>();
			while (reader.hasNext()) {
				AbstractFeature chunk = reader.next();
				if (chunk instanceof CityModel cModel) {
					cModel.setCityObjectMembers(null);
					mapper.setCityModel(cModel);
					CityGMLVersion version = CityGMLModules.getCityGMLVersion(reader.getName().getNamespaceURI());
					mapper.setCityGMLVersion(version);
				} else if (chunk instanceof AbstractCityObject aco) {
					acos.add(aco);
					aco.accept(mapper);
				}
			}

			if (mapper.getModel().getCityModel() == null) {
				// file does not contain a city model?
				// create it for now
				mapper.setCityModel(new CityModel());
			}
			CityModel cModel = mapper.getModel().getCityModel();

			// remove those that should have been parsed
			List<AbstractCityObject> parsedCityObjects = mapper.getModel().createFeatureStream()
					.map(CityObject::getGmlObject).toList();
			acos.removeAll(parsedCityObjects);
			// re-add all not parsed objects
			for (AbstractCityObject aco : acos) {
				cModel.getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
			}
			if (logger.isInfoEnabled()) {
				logger.info(Localization.getText("CityGmlParser.parsedObjects"),
						mapper.getModel().getNumberOfFeatures());
			}
			return mapper.getModel();
		}
	}

	private static void parseEpsgCodeFromFile(Path file, ParserConfiguration config) throws CityGmlParseException {
		try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file.toFile()))) {
			parseEpsgCodeFromStream(bis, config);
		} catch (ParserConfigurationException | SAXException | IOException e) {
			throw new CityGmlParseException("Failed to read CityGML file", e);
		}
	}

	private static void parseEpsgCodeFromStream(InputStream is, ParserConfiguration config)
			throws ParserConfigurationException, SAXException {
		SAXParser parser = FACTORY.newSAXParser();
		CityGmlHandler handler = new CityGmlHandler();
		try {
			parser.parse(new InputSource(is), handler);
		} catch (EnvelopeFoundException e) {
			try {
				parseCoordinateSystem(config, handler);
			} catch (Exception e2) {
				logger.debug("Exception while parsing for EPSG code", e2);
				if (logger.isWarnEnabled()) {
					logger.warn(Localization.getText("CityGmlParser.noEPSG"));
				}
			}
		} catch (Exception e) {
			logger.debug("Exception while parsing for EPSG code", e);
			if (logger.isWarnEnabled()) {
				logger.warn(Localization.getText("CityGmlParser.noEPSG"));
			}
		}
	}

	private static void parseCoordinateSystem(ParserConfiguration config, CityGmlHandler handler) {
		if (handler.getEpsg() == null) {
			return;
		}
		CoordinateReferenceSystem crs = crsFromSrsName(handler.getEpsg());
		if (crs == null) {
			// could not find a coordinate system for srsName
			// assuming metric system
			return;
		}
		if (crs.getProjection().getUnits() == Units.METRES) {
			// coordinate system is in meters, do not convert
			if (logger.isInfoEnabled()) {
				logger.info(Localization.getText("CityGmlParser.noConversionNeeded"));
			}
			return;
		}
		parseMeterConversion(config, crs);
		Vector3d low = handler.getLowerCorner();
		Vector3d up = handler.getUpperCorner();
		double centerLong = low.getX() + ((up.getX() - low.getX()) / 2);
		double centerLat = low.getY() + ((up.getY() - low.getY()) / 2);
		if (!crs.getName().equals(WGS_84)) {
			// need to convert coordinates first to WGS84, then find UTM Zone
			CoordinateReferenceSystem wgs84 = crsFromSrsName(WGS_84);
			ProjCoordinate p1 = new ProjCoordinate();
			p1.setValue(centerLong, centerLat);
			ProjCoordinate p2 = new ProjCoordinate();
			BasicCoordinateTransform bct = new BasicCoordinateTransform(crs, wgs84);
			bct.transform(p1, p2);
			centerLong = p2.x;
			centerLat = p2.y;
		}
		int zone = (int) (31 + Math.round(centerLong / 6));
		CoordinateReferenceSystem utm;
		if (centerLat < 0) {
			// south
			logger.info("Converting coordiante system to UTM zone {}S", zone);
			utm = CRS_FACTORY.createFromParameters("UTM", "+proj=utm +ellps=WGS84 +units=m +zone=" + zone + " +south");
		} else {
			// north
			logger.info("Converting coordiante system to UTM zone {}N", zone);
			utm = CRS_FACTORY.createFromParameters("UTM", "+proj=utm +ellps=WGS84 +units=m +zone=" + zone);
		}
		config.setCoordinateSystem(crs, utm);
	}

	private static void parseMeterConversion(ParserConfiguration config, CoordinateReferenceSystem crs) {
		Projection projection = crs.getProjection();
		double fromMetres = projection.getFromMetres();
		if (fromMetres > 0) {
			// also transform height information
			config.setFromMetres(fromMetres);
		} else {
			config.setFromMetres(1.0);
		}
	}

	/**
	 * The srsName (The name by which this reference system is identified) inside
	 * the CityGML file can have multiple formats. This method tries to parse the
	 * string and detect the corresponding reference system. If it is found, it
	 * returns a proj4j.CoordinateReferenceSystem. It throws an
	 * IllegalArgumentException otherwise.
	 * 
	 * This method should be able to parse any EPSG id : e.g. "EPSG:1234". German
	 * Citygmls might also have "DE_DHDN_3GK3" or "ETRS89_UTM32" as srsName, so
	 * those are also included. It isn't guaranteed that those formats are correctly
	 * parsed, though.
	 * 
	 * The EPSG ids and parameters are defined in resources ('nad/epsg') inside
	 * proj4j-0.1.0.jar. Some EPSG ids are missing though, e.g. 7415
	 * 
	 * @param srsName
	 * @return CoordinateReferenceSystem
	 */
	private static CoordinateReferenceSystem crsFromSrsName(String srsName) {
		srsName = srsName.trim();
		Matcher mEPSG = P_EPSG.matcher(srsName);
		if (mEPSG.find()) {
			if ("EPSG:4979".contentEquals(srsName)) {
				srsName = "EPSG:4236";
			} else if ("EPSG:7415".contentEquals(srsName)) {
				return CRS_FACTORY.createFromParameters("EPSG:7415",
						"+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +towgs84=565.417,50.3319,465.552,-0.398957,0.343988,-1.8774,4.0725 +units=m +no_defs");
			}
			return CRS_FACTORY.createFromName(srsName);
		}

		Matcher mOGC = P_OGC.matcher(srsName);
		if (mOGC.find()) {
			return CRS_FACTORY.createFromName("EPSG:" + mOGC.group(1));
		}
		Matcher mOGC2 = P_OGC2.matcher(srsName);
		if (mOGC2.find()) {
			return CRS_FACTORY.createFromName("EPSG:" + mOGC2.group(1));
		}
		Matcher mURN = P_URN.matcher(srsName);
		// NOTE: Could use a HashMap if the switch/case becomes too long.
		if (mURN.find()) {
			switch (mURN.group(1)) {
			case "DE_DHDN_3GK2":
				return CRS_FACTORY.createFromName("EPSG:31466");
			case "DE_DHDN_3GK3":
				return CRS_FACTORY.createFromName("EPSG:31467");
			case "DE_DHDN_3GK4":
				return CRS_FACTORY.createFromName("EPSG:31468");
			case "DE_DHDN_3GK5":
				return CRS_FACTORY.createFromName("EPSG:31469");
			case "ETRS89_UTM32":
				return CRS_FACTORY.createFromName("EPSG:25832");
			default:
				return null;
			}
		}
		if (srsName.equals("http://www.opengis.net/def/crs/EPSG/0/6697")) {
			return CRS_FACTORY.createFromParameters("EPSG:6697", "+proj=longlat +ellps=GRS80 +no_defs +axis=neu");
		}
		return null;
	}

	private static List<String> validateFile(CityGMLContext context, GMLValidationHandler handler, Path file)
			throws CityGmlParseException {
		if (handler == null) {
			handler = new GMLValidationHandler();
		}
		try {
			CityGMLSchemaHandler schemaHandler = context.getDefaultSchemaHandler();
			Source[] schemas = schemaHandler.getSchemas();
			SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
			schemaFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
			Schema schema = schemaFactory.newSchema(schemas);
			Validator validator = schema.newValidator();
			validator.setErrorHandler(handler);
			validator.validate(new StreamSource(file.toFile()));
			return handler.getMessages();
		} catch (SchemaHandlerException | SAXException | IOException e) {
			throw new CityGmlParseException("Failed to validate CityGML file", e);
		}
	}

	private static void drainCityModel(CityDoctorModel model, CityGmlConsumer cityObjectConsumer) {
		drainCityObjectList(model.getBuildings(), cityObjectConsumer);
		drainCityObjectList(model.getBridges(), cityObjectConsumer);
		drainCityObjectList(model.getVegetation(), cityObjectConsumer);
		drainCityObjectList(model.getLand(), cityObjectConsumer);
		drainCityObjectList(model.getTransportation(), cityObjectConsumer);
		drainCityObjectList(model.getWater(), cityObjectConsumer);
	}

	private static void drainCityObjectList(List<? extends CityObject> objects, CityGmlConsumer cityObjectConsumer) {
		for (CityObject co : objects) {
			cityObjectConsumer.accept(co);
		}
		objects.clear();
	}

}