/*- * Copyright 2020 Beuth Hochschule für Technik Berlin, Hochschule für Technik Stuttgart * * This file is part of CityDoctor2. * * CityDoctor2 is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * CityDoctor2 is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with CityDoctor2. If not, see . */ package de.hft.stuttgart.citydoctor2.parser; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; import java.util.Enumeration; import java.util.List; import java.util.ServiceLoader; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import javax.xml.XMLConstants; import javax.xml.bind.ValidationEventHandler; import javax.xml.namespace.QName; import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.citygml4j.CityGMLContext; import org.citygml4j.builder.jaxb.CityGMLBuilder; import org.citygml4j.builder.jaxb.CityGMLBuilderException; import org.citygml4j.model.citygml.CityGML; import org.citygml4j.model.citygml.ade.ADEComponent; import org.citygml4j.model.citygml.ade.ADEException; import org.citygml4j.model.citygml.ade.binding.ADEContext; import org.citygml4j.model.citygml.ade.generic.ADEGenericElement; import org.citygml4j.model.citygml.core.AbstractCityObject; import org.citygml4j.model.citygml.core.CityModel; import org.citygml4j.model.citygml.core.CityObjectMember; import org.citygml4j.model.gml.feature.AbstractFeature; import org.citygml4j.model.module.citygml.CityGMLVersion; import org.citygml4j.xml.io.CityGMLInputFactory; import org.citygml4j.xml.io.CityGMLOutputFactory; import org.citygml4j.xml.io.reader.CityGMLReadException; import org.citygml4j.xml.io.reader.CityGMLReader; import org.citygml4j.xml.io.reader.FeatureReadMode; import org.citygml4j.xml.io.reader.ParentInfo; import org.citygml4j.xml.io.writer.CityGMLWriteException; import org.citygml4j.xml.io.writer.CityModelInfo; import org.citygml4j.xml.io.writer.CityModelWriter; import org.osgeo.proj4j.BasicCoordinateTransform; import org.osgeo.proj4j.CRSFactory; import org.osgeo.proj4j.CoordinateReferenceSystem; import org.osgeo.proj4j.ProjCoordinate; import org.osgeo.proj4j.proj.Projection; import org.osgeo.proj4j.units.Units; import org.w3c.dom.DOMException; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.SAXNotRecognizedException; import org.xml.sax.SAXNotSupportedException; import org.xml.sax.SAXParseException; import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel; import de.hft.stuttgart.citydoctor2.datastructure.CityObject; import de.hft.stuttgart.citydoctor2.mapper.FeatureMapper; import de.hft.stuttgart.citydoctor2.math.Vector3d; import de.hft.stuttgart.citydoctor2.utils.Localization; import de.hft.stuttgart.quality.QualityADEModule; /** * Utility class to parse CityGML files. * * @author Matthias Betz * */ public class CityGmlParser { private static final Logger logger = LogManager.getLogger(CityGmlParser.class); private static final CRSFactory CRS_FACTORY = new CRSFactory(); // EPSG:31467 private static final Pattern P_EPSG = Pattern.compile("^(EPSG:\\d+)$"); // urn:ogc:def:crs,crs:EPSG:6.12:31467,crs:EPSG:6.12:5783 // or // urn:ogc:def:crs,crs:EPSG::28992 private static final Pattern P_OGC = Pattern.compile("urn:ogc:def:crs,crs:EPSG:[\\d\\.]*:([\\d]+)\\D*"); private static final Pattern P_OGC2 = Pattern.compile("urn:ogc:def:crs:EPSG:[\\d\\.]*:([\\d]+)\\D*"); // urn:adv:crs:DE_DHDN_3GK3*DE_DHHN92_NH // urn:adv:crs:ETRS89_UTM32*DE_DHHN92_NH private static final Pattern P_URN = Pattern.compile("urn:adv:crs:([^\\*]+)"); private static final SAXParserFactory FACTORY; static { FACTORY = SAXParserFactory.newInstance(); try { FACTORY.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); } catch (SAXNotRecognizedException | SAXNotSupportedException | ParserConfigurationException e) { logger.catching(e); } } private CityGmlParser() { // only static access } public static CityDoctorModel parseCityGmlFile(String file, ParserConfiguration config) throws CityGmlParseException, InvalidGmlFileException { return parseCityGmlFile(file, config, null, null); } public static CityDoctorModel parseCityGmlFile(String file, ParserConfiguration config, ProgressListener l) throws CityGmlParseException, InvalidGmlFileException { return parseCityGmlFile(file, config, l, null); } public static CityDoctorModel parseCityGmlFile(String filePath, ParserConfiguration config, ProgressListener l, ValidationEventHandler handler) throws CityGmlParseException, InvalidGmlFileException { File file = new File(filePath); try { parseEpsgCodeFromFile(file, config); CityGMLBuilder builder = setupCityGmlBuilder(); CityGMLInputFactory inputFactory = setupGmlReader(builder, config); if (handler != null) { inputFactory.setValidationEventHandler(handler); } // try with resources for automatic closing try (ObservedInputStream ois = new ObservedInputStream(file)) { if (l != null) { ois.addListener(l::updateProgress); } return readAndKeepFeatures(config, file, inputFactory, ois); } } catch (CityGMLReadException e) { if (e.getCause() instanceof SAXParseException) { throw new InvalidGmlFileException( Localization.getText("CityGmlParser.notValidGmlFile") + e.getCause().getMessage(), e); } throw new CityGmlParseException(e); } catch (IOException | CityGMLBuilderException | ParserConfigurationException | SAXException | ADEException e) { throw new CityGmlParseException(e); } catch (DOMException e) { // the citygml4j split per feature tries to insert an element // this sometimes fails because of namespace mismatch // fallback solution is to parse the whole file at once // problems with memory consumption may arise try { if (logger.isWarnEnabled()) { logger.warn(Localization.getText("CityGmlParser.chunkReadFailed"), e); } return parseCityGmlFileComplete(file, config, l); } catch (CityGMLBuilderException | CityGMLReadException | IOException e1) { throw new CityGmlParseException(e1); } } } private static CityDoctorModel readAndKeepFeatures(ParserConfiguration config, File file, CityGMLInputFactory inputFactory, ObservedInputStream ois) throws CityGMLReadException { try (CityGMLReader reader = inputFactory.createCityGMLReader(file.getAbsolutePath(), ois)) { FeatureMapper mapper = new FeatureMapper(config, file); while (reader.hasNext()) { CityGML chunk = reader.nextFeature(); if (chunk instanceof AbstractCityObject) { AbstractCityObject ag = (AbstractCityObject) chunk; ag.accept(mapper); } else if (chunk instanceof CityModel) { CityModel cModel = (CityModel) chunk; cModel.unsetCityObjectMember(); mapper.setCityModel(cModel); } } if (logger.isInfoEnabled()) { logger.info(Localization.getText("CityGmlParser.parsedObjects"), mapper.getModel().getNumberOfFeatures()); } return mapper.getModel(); } } private static CityDoctorModel parseCityGmlFileComplete(File file, ParserConfiguration config, ProgressListener l) throws CityGMLBuilderException, IOException, CityGMLReadException, CityGmlParseException { CityGMLContext context = CityGMLContext.getInstance(); CityGMLBuilder builder = context.createCityGMLBuilder(); CityGMLInputFactory inputFactory = builder.createCityGMLInputFactory(); inputFactory.setProperty(CityGMLInputFactory.FAIL_ON_MISSING_ADE_SCHEMA, false); inputFactory.setProperty(CityGMLInputFactory.USE_VALIDATION, config.getValidate()); try (ObservedInputStream ois = new ObservedInputStream(file)) { if (l != null) { ois.addListener(l::updateProgress); } try (CityGMLReader reader = inputFactory.createCityGMLReader(file.getAbsolutePath(), ois)) { FeatureMapper mapper = new FeatureMapper(config, file); CityGML chunk = reader.nextFeature(); if (!(chunk instanceof CityModel)) { throw new CityGmlParseException("Did not read CityModel as first element of gml file."); } CityModel model = (CityModel) chunk; mapper.setCityModel(model); if (model.isSetCityObjectMember()) { for (CityObjectMember com : model.getCityObjectMember()) { if (com.isSetCityObject()) { com.getCityObject().accept(mapper); } } } model.unsetCityObjectMember(); return mapper.getModel(); } } } private static CityGMLInputFactory setupGmlReader(CityGMLBuilder builder, ParserConfiguration config) throws CityGMLBuilderException, ADEException { CityGMLInputFactory inputFactory = builder.createCityGMLInputFactory(); inputFactory.setProperty(CityGMLInputFactory.FEATURE_READ_MODE, FeatureReadMode.SPLIT_PER_FEATURE); if (config != null) { inputFactory.setProperty(CityGMLInputFactory.USE_VALIDATION, config.getValidate()); } inputFactory.setProperty(CityGMLInputFactory.FAIL_ON_MISSING_ADE_SCHEMA, false); inputFactory.setProperty(CityGMLInputFactory.EXCLUDE_FROM_SPLITTING, new QName[] { new QName("WallSurface"), new QName("RoofSurface"), new QName("GroundSurface"), new QName("CeilingSurface"), new QName("ClosureSurface"), new QName("FloorSurface"), new QName("InteriorWallSurface"), new QName("OuterCeilingSurface"), new QName("OuterFloorSurface"), new QName("BuildingInstallation"), new QName("BuildingPart"), new QName("Door"), new QName("Window") }); return inputFactory; } private static CityGMLBuilder setupCityGmlBuilder() throws ADEException, CityGMLBuilderException { CityGMLContext context = CityGMLContext.getInstance(); // setup energy ade stuff, so the parser doesn't crash on encountering this if (!context.hasADEContexts()) { for (ADEContext adeContext : ServiceLoader.load(ADEContext.class)) { context.registerADEContext(adeContext); } } CityGMLBuilder builder = context.createCityGMLBuilder(CityGmlParser.class.getClassLoader()); return builder; } public static void streamCityGml(String file, ParserConfiguration config, CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGmlParseException { File f = new File(file); streamCityGml(f, config, null, cityObjectConsumer, outputFile); } public static void streamCityGml(File file, ParserConfiguration config, ProgressListener l, CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGmlParseException { try { if (getExtension(file.getName()).equals("zip")) { streamZipFile(file, config); } parseEpsgCodeFromFile(file, config); CityGMLBuilder builder = setupCityGmlBuilder(); startReadingCityGml(file, config, l, cityObjectConsumer, builder, outputFile); } catch (CityGMLBuilderException | IOException | ParserConfigurationException | SAXException | ADEException e) { throw new CityGmlParseException(e); } } private static void startReadingCityGml(File file, ParserConfiguration config, ProgressListener l, CityGmlConsumer cityObjectConsumer, CityGMLBuilder builder, String outputFile) throws CityGMLBuilderException, ADEException { try (ObservedInputStream ois = new ObservedInputStream(file)) { if (l != null) { ois.addListener(l::updateProgress); } readAndDiscardFeatures(file, config, builder, ois, cityObjectConsumer, outputFile); } catch (IOException e) { logger.error(Localization.getText("CityGmlParser.errorReadingGmlFile"), e.getMessage()); logger.catching(Level.ERROR, e); } } private static void streamZipFile(File file, ParserConfiguration config) throws CityGmlParseException { try (ZipFile zip = new ZipFile(file)) { Enumeration entries = zip.entries(); while (entries.hasMoreElements()) { ZipEntry ze = entries.nextElement(); BufferedInputStream bis = new BufferedInputStream(zip.getInputStream(ze)); parseEpsgCodeFromStream(bis, config); } } catch (IOException e) { throw new UncheckedIOException(e); } catch (ParserConfigurationException | SAXException e) { throw new CityGmlParseException(e); } } public static String getExtension(String fileName) { char ch; int len; if (fileName == null || (len = fileName.length()) == 0 || (ch = fileName.charAt(len - 1)) == '/' || ch == '\\' || ch == '.') { return ""; } int dotInd = fileName.lastIndexOf('.'); int sepInd = Math.max(fileName.lastIndexOf('/'), fileName.lastIndexOf('\\')); if (dotInd <= sepInd) return ""; else return fileName.substring(dotInd + 1).toLowerCase(); } public static void streamCityGml(File file, ParserConfiguration parserConfig, CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGmlParseException { streamCityGml(file, parserConfig, null, cityObjectConsumer, outputFile); } /** * The srsName (The name by which this reference system is identified) inside * the CityGML file can have multiple formats. This method tries to parse the * string and detect the corresponding reference system. If it is found, it * returns a proj4j.CoordinateReferenceSystem. It throws an * IllegalArgumentException otherwise. * * This method should be able to parse any EPSG id : e.g. "EPSG:1234". German * Citygmls might also have "DE_DHDN_3GK3" or "ETRS89_UTM32" as srsName, so * those are also included. It isn't guaranteed that those formats are correctly * parsed, though. * * The EPSG ids and parameters are defined in resources ('nad/epsg') inside * proj4j-0.1.0.jar. Some EPSG ids are missing though, e.g. 7415 * * @param srsName * @return CoordinateReferenceSystem */ private static CoordinateReferenceSystem crsFromSrsName(String srsName) { srsName = srsName.trim(); Matcher mEPSG = P_EPSG.matcher(srsName); if (mEPSG.find()) { if ("EPSG:4979".contentEquals(srsName)) { srsName = "EPSG:4236"; } else if ("EPSG:7415".contentEquals(srsName)) { return CRS_FACTORY.createFromParameters("EPSG:7415", "+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +towgs84=565.417,50.3319,465.552,-0.398957,0.343988,-1.8774,4.0725 +units=m +no_defs"); } return CRS_FACTORY.createFromName(srsName); } Matcher mOGC = P_OGC.matcher(srsName); if (mOGC.find()) { return CRS_FACTORY.createFromName("EPSG:" + mOGC.group(1)); } Matcher mOGC2 = P_OGC2.matcher(srsName); if (mOGC2.find()) { return CRS_FACTORY.createFromName("EPSG:" + mOGC2.group(1)); } Matcher mURN = P_URN.matcher(srsName); // NOTE: Could use a HashMap if the switch/case becomes too long. if (mURN.find()) { switch (mURN.group(1)) { case "DE_DHDN_3GK2": return CRS_FACTORY.createFromName("EPSG:31466"); case "DE_DHDN_3GK3": return CRS_FACTORY.createFromName("EPSG:31467"); case "DE_DHDN_3GK4": return CRS_FACTORY.createFromName("EPSG:31468"); case "DE_DHDN_3GK5": return CRS_FACTORY.createFromName("EPSG:31469"); case "ETRS89_UTM32": return CRS_FACTORY.createFromName("EPSG:25832"); default: return null; } } if (srsName.equals("http://www.opengis.net/def/crs/EPSG/0/6697")) { return CRS_FACTORY.createFromParameters("EPSG:6697", "+proj=longlat +ellps=GRS80 +no_defs "); } return null; } public static CityModel parseOnlyCityModel(File inputFile) throws CityGmlParseException { try { CityGMLBuilder setupCityGmlBuilder = setupCityGmlBuilder(); CityGMLInputFactory inputFactory = setupGmlReader(setupCityGmlBuilder, null); try (CityGMLReader reader = inputFactory.createCityGMLReader(inputFile)) { while (reader.hasNext()) { CityGML chunk = reader.nextFeature(); if (chunk instanceof CityModel) { CityModel cModel = (CityModel) chunk; cModel.unsetCityObjectMember(); return cModel; } } } } catch (CityGMLBuilderException | CityGMLReadException | ADEException e) { throw new CityGmlParseException(e); } throw new CityGmlParseException("Did not find any CityModel in CityGML file"); } private static void readAndDiscardFeatures(File file, ParserConfiguration config, CityGMLBuilder builder, ObservedInputStream ois, CityGmlConsumer cityObjectConsumer, String outputFile) throws CityGMLBuilderException, ADEException { CityGMLInputFactory inputFactory = setupGmlReader(builder, config); try (CityGMLReader reader = inputFactory.createCityGMLReader(file.getAbsolutePath(), ois); CityModelWriter writer = createCityModelWriter(builder, outputFile)) { FeatureMapper mapper = new FeatureMapper(config, file); CityDoctorModel model = mapper.getModel(); boolean isInitialized = false; CityModelInfo cityModelInfo = null; while (reader.hasNext()) { CityGML chunk = reader.nextFeature(); if (!isInitialized && writer != null) { ParentInfo parentInfo = reader.getParentInfo(); cityModelInfo = new CityModelInfo(parentInfo); writer.setCityModelInfo(cityModelInfo); writer.writeStartDocument(); isInitialized = true; } if (chunk instanceof AbstractCityObject) { AbstractCityObject ag = (AbstractCityObject) chunk; ag.accept(mapper); drainCityModel(model, cityObjectConsumer); writeAbstractCityObject(writer, ag); } else if (chunk instanceof CityModel) { CityModel cModel = (CityModel) chunk; cModel.unsetCityObjectMember(); mapper.setCityModel(cModel); cityObjectConsumer.accept(cModel); writeCityModel(writer, cityModelInfo, cModel); } else if (chunk instanceof AbstractFeature && writer != null) { writer.writeFeatureMember((AbstractFeature) chunk); } } // end of stream logger.debug("End of gml file stream"); } catch (CityGMLReadException e) { logger.error(Localization.getText("CityGmlParser.errorReadingGmlFile"), e.getMessage(), e); } catch (CityGMLWriteException e) { logger.error(Localization.getText("CityGmlParser.errorWritingGmlFile"), e.getMessage(), e); } } private static void writeAbstractCityObject(CityModelWriter writer, AbstractCityObject ag) throws CityGMLWriteException { if (writer != null) { writer.writeFeatureMember(ag); } } private static void writeCityModel(CityModelWriter writer, CityModelInfo cityModelInfo, CityModel cModel) throws CityGMLWriteException { if (writer != null) { for (ADEGenericElement genEle : cModel.getGenericADEElement()) { cityModelInfo.addGenericADEElement(genEle); } for (ADEComponent adeComp : cModel.getGenericApplicationPropertyOfCityModel()) { cityModelInfo.addGenericApplicationPropertyOfCityModel(adeComp); } writer.writeEndDocument(); } } private static CityModelWriter createCityModelWriter(CityGMLBuilder builder, String outputFile) throws CityGMLWriteException { if (outputFile == null) { return null; } CityGMLOutputFactory factory = builder.createCityGMLOutputFactory(); CityModelWriter writer = factory.createCityModelWriter(new File(outputFile)); writer.setPrefix("qual", QualityADEModule.NAMESPACE_URI); writer.setSchemaLocation(QualityADEModule.NAMESPACE_URI, "https://transfer.hft-stuttgart.de/pages/qualityade/0.1/qualityAde.xsd"); writer.setIndentString(" "); writer.setPrefixes(CityGMLVersion.DEFAULT); writer.setSchemaLocations(CityGMLVersion.DEFAULT); return writer; } private static void parseEpsgCodeFromFile(File file, ParserConfiguration config) throws IOException, ParserConfigurationException, SAXException { try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) { parseEpsgCodeFromStream(bis, config); } } private static void parseEpsgCodeFromStream(InputStream is, ParserConfiguration config) throws ParserConfigurationException, SAXException { SAXParser parser = FACTORY.newSAXParser(); CityGmlHandler handler = new CityGmlHandler(); try { parser.parse(new InputSource(is), handler); } catch (EnvelopeFoundException e) { try { parseCoordinateSystem(config, handler); } catch (Exception e2) { logger.debug("Exception while parsing for EPSG code", e2); if (logger.isWarnEnabled()) { logger.warn(Localization.getText("CityGmlParser.noEPSG")); } } } catch (Exception e) { logger.debug("Exception while parsing for EPSG code", e); if (logger.isWarnEnabled()) { logger.warn(Localization.getText("CityGmlParser.noEPSG")); } } } private static void parseCoordinateSystem(ParserConfiguration config, CityGmlHandler handler) { if (handler.getEpsg() == null) { return; } CoordinateReferenceSystem crs = crsFromSrsName(handler.getEpsg()); if (crs == null) { // could not find a coordinate system for srsName // assuming metric system return; } ProjectionUnitExtractor extractor = new ProjectionUnitExtractor(crs.getProjection()); if (extractor.getUnit() == Units.METRES) { // coordinate system is in meters, do not convert if (logger.isInfoEnabled()) { logger.info(Localization.getText("CityGmlParser.noConversionNeeded")); } return; } parseMeterConversion(config, crs); Vector3d low = handler.getLowerCorner(); Vector3d up = handler.getUpperCorner(); double centerLong = low.getX() + ((up.getX() - low.getX()) / 2); double centerLat = low.getY() + ((up.getY() - low.getY()) / 2); if (!crs.getName().equals("EPSG:4326")) { // need to convert coordinates first to WGS84, then find UTM Zone CoordinateReferenceSystem wgs84 = crsFromSrsName("EPSG:4326"); ProjCoordinate p1 = new ProjCoordinate(); p1.setValue(centerLong, centerLat); ProjCoordinate p2 = new ProjCoordinate(); BasicCoordinateTransform bct = new BasicCoordinateTransform(crs, wgs84); bct.transform(p1, p2); centerLong = p2.y; centerLat = p2.x; } int zone = (int) (31 + Math.round(centerLong / 6)); CoordinateReferenceSystem utm; if (centerLat < 0) { // south logger.info("Converting coordiante system to UTM zone {}S", zone); utm = CRS_FACTORY.createFromParameters("UTM", "+proj=utm +ellps=WGS84 +units=m +zone=" + zone + " +south"); } else { // north logger.info("Converting coordiante system to UTM zone {}N", zone); utm = CRS_FACTORY.createFromParameters("UTM", "+proj=utm +ellps=WGS84 +units=m +zone=" + zone); } config.setCoordinateSystem(crs, utm); } private static void parseMeterConversion(ParserConfiguration config, CoordinateReferenceSystem crs) { Projection projection = crs.getProjection(); double fromMetres = projection.getFromMetres(); if (fromMetres > 0) { // also transform height information config.setFromMetres(fromMetres); } else { config.setFromMetres(1.0); } } private static void drainCityModel(CityDoctorModel model, CityGmlConsumer cityObjectConsumer) { drainCityObjectList(model.getBuildings(), cityObjectConsumer); drainCityObjectList(model.getBridges(), cityObjectConsumer); drainCityObjectList(model.getVegetation(), cityObjectConsumer); drainCityObjectList(model.getLand(), cityObjectConsumer); drainCityObjectList(model.getTransportation(), cityObjectConsumer); drainCityObjectList(model.getWater(), cityObjectConsumer); } private static void drainCityObjectList(List objects, CityGmlConsumer cityObjectConsumer) { for (CityObject co : objects) { cityObjectConsumer.accept(co); } objects.clear(); } }