/*- * Copyright 2020 Beuth Hochschule für Technik Berlin, Hochschule für Technik Stuttgart * * This file is part of CityDoctor2. * * CityDoctor2 is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * CityDoctor2 is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with CityDoctor2. If not, see . */ package de.hft.stuttgart.citydoctor2.parser; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; import java.util.Enumeration; import java.util.List; import java.util.ServiceLoader; import java.util.concurrent.ArrayBlockingQueue; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import javax.xml.XMLConstants; import javax.xml.namespace.QName; import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.citygml4j.CityGMLContext; import org.citygml4j.builder.jaxb.CityGMLBuilder; import org.citygml4j.builder.jaxb.CityGMLBuilderException; import org.citygml4j.model.citygml.CityGML; import org.citygml4j.model.citygml.ade.ADEException; import org.citygml4j.model.citygml.ade.binding.ADEContext; import org.citygml4j.model.citygml.core.AbstractCityObject; import org.citygml4j.model.citygml.core.CityModel; import org.citygml4j.model.citygml.core.CityObjectMember; import org.citygml4j.xml.io.CityGMLInputFactory; import org.citygml4j.xml.io.reader.CityGMLReadException; import org.citygml4j.xml.io.reader.CityGMLReader; import org.citygml4j.xml.io.reader.FeatureReadMode; import org.osgeo.proj4j.BasicCoordinateTransform; import org.osgeo.proj4j.CRSFactory; import org.osgeo.proj4j.CoordinateReferenceSystem; import org.osgeo.proj4j.ProjCoordinate; import org.osgeo.proj4j.proj.Projection; import org.osgeo.proj4j.units.Units; import org.w3c.dom.DOMException; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.SAXNotRecognizedException; import org.xml.sax.SAXNotSupportedException; import org.xml.sax.SAXParseException; import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel; import de.hft.stuttgart.citydoctor2.datastructure.CityObject; import de.hft.stuttgart.citydoctor2.mapper.FeatureMapper; import de.hft.stuttgart.citydoctor2.math.Vector3d; import de.hft.stuttgart.citydoctor2.utils.Localization; /** * Utility class to parse CityGML files. * * @author Matthias Betz * */ public class CityGmlParser { private static final Logger logger = LogManager.getLogger(CityGmlParser.class); private static final CRSFactory CRS_FACTORY = new CRSFactory(); // EPSG:31467 private static final Pattern P_EPSG = Pattern.compile("^(EPSG:\\d+)$"); // urn:ogc:def:crs,crs:EPSG:6.12:31467,crs:EPSG:6.12:5783 // or // urn:ogc:def:crs,crs:EPSG::28992 private static final Pattern P_OGC = Pattern.compile("urn:ogc:def:crs,crs:EPSG:[\\d\\.]*:([\\d]+)\\D*"); private static final Pattern P_OGC2 = Pattern.compile("urn:ogc:def:crs:EPSG:[\\d\\.]*:([\\d]+)\\D*"); // urn:adv:crs:DE_DHDN_3GK3*DE_DHHN92_NH // urn:adv:crs:ETRS89_UTM32*DE_DHHN92_NH private static final Pattern P_URN = Pattern.compile("urn:adv:crs:([^\\*]+)"); private static final SAXParserFactory FACTORY; static { FACTORY = SAXParserFactory.newInstance(); try { FACTORY.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); } catch (SAXNotRecognizedException | SAXNotSupportedException | ParserConfigurationException e) { logger.catching(e); } } private CityGmlParser() { // only static access } public static CityDoctorModel parseCityGmlFile(String file, ParserConfiguration config) throws CityGmlParseException, InvalidGmlFileException { return parseCityGmlFile(file, config, null); } public static CityDoctorModel parseCityGmlFile(String filePath, ParserConfiguration config, ProgressListener l) throws CityGmlParseException, InvalidGmlFileException { File file = new File(filePath); try { parseEpsgCodeFromFile(file, config); CityGMLInputFactory inputFactory = setupGmlReader(config); // try with resources for automatic closing try (ObservedInputStream ois = new ObservedInputStream(file)) { if (l != null) { ois.addListener(l::updateProgress); } try (CityGMLReader reader = inputFactory.createCityGMLReader(file.getAbsolutePath(), ois)) { FeatureMapper mapper = new FeatureMapper(config, file); while (reader.hasNext()) { CityGML chunk = reader.nextFeature(); if (chunk instanceof AbstractCityObject) { AbstractCityObject ag = (AbstractCityObject) chunk; ag.accept(mapper); } else if (chunk instanceof CityModel) { CityModel cModel = (CityModel) chunk; cModel.unsetCityObjectMember(); mapper.setCityModel(cModel); } } if (logger.isInfoEnabled()) { logger.info(Localization.getText("CityGmlParser.parsedObjects"), mapper.getModel().getNumberOfFeatures()); } return mapper.getModel(); } } } catch (CityGMLReadException e) { if (e.getCause() instanceof SAXParseException) { throw new InvalidGmlFileException(Localization.getText("CityGmlParser.notValidGmlFile") + e.getCause().getMessage(), e); } throw new CityGmlParseException(e); } catch (IOException | CityGMLBuilderException | ParserConfigurationException | SAXException | ADEException e) { throw new CityGmlParseException(e); } catch (DOMException e) { // the citygml4j split per feature tries to insert an element // this sometimes fails because of namespace mismatch // fallback solution is to parse the whole file at once // problems with memory consumption may arise try { if (logger.isWarnEnabled()) { logger.warn(Localization.getText("CityGmlParser.chunkReadFailed"), e); } return parseCityGmlFileComplete(file, config, l); } catch (CityGMLBuilderException | CityGMLReadException | IOException e1) { throw new CityGmlParseException(e1); } } } private static CityDoctorModel parseCityGmlFileComplete(File file, ParserConfiguration config, ProgressListener l) throws CityGMLBuilderException, IOException, CityGMLReadException, CityGmlParseException { CityGMLContext context = CityGMLContext.getInstance(); CityGMLBuilder builder = context.createCityGMLBuilder(); CityGMLInputFactory inputFactory = builder.createCityGMLInputFactory(); inputFactory.setProperty(CityGMLInputFactory.FAIL_ON_MISSING_ADE_SCHEMA, false); inputFactory.setProperty(CityGMLInputFactory.USE_VALIDATION, config.getValidate()); try (ObservedInputStream ois = new ObservedInputStream(file)) { if (l != null) { ois.addListener(l::updateProgress); } try (CityGMLReader reader = inputFactory.createCityGMLReader(file.getAbsolutePath(), ois)) { FeatureMapper mapper = new FeatureMapper(config, file); CityGML chunk = reader.nextFeature(); if (!(chunk instanceof CityModel)) { throw new CityGmlParseException("Did not read CityModel as first element of gml file."); } CityModel model = (CityModel) chunk; mapper.setCityModel(model); if (model.isSetCityObjectMember()) { for (CityObjectMember com : model.getCityObjectMember()) { if (com.isSetCityObject()) { com.getCityObject().accept(mapper); } } } model.unsetCityObjectMember(); return mapper.getModel(); } } } private static CityGMLInputFactory setupGmlReader(ParserConfiguration config) throws CityGMLBuilderException, ADEException { CityGMLContext context = CityGMLContext.getInstance(); // setup energy ade stuff, so the parser doesn't crash on encountering this if (!context.hasADEContexts()) { for (ADEContext adeContext : ServiceLoader.load(ADEContext.class)) { context.registerADEContext(adeContext); } } CityGMLBuilder builder = context.createCityGMLBuilder(CityGmlParser.class.getClassLoader()); CityGMLInputFactory inputFactory = builder.createCityGMLInputFactory(); inputFactory.setProperty(CityGMLInputFactory.FEATURE_READ_MODE, FeatureReadMode.SPLIT_PER_FEATURE); inputFactory.setProperty(CityGMLInputFactory.USE_VALIDATION, config.getValidate()); inputFactory.setProperty(CityGMLInputFactory.FAIL_ON_MISSING_ADE_SCHEMA, false); inputFactory.setProperty(CityGMLInputFactory.EXCLUDE_FROM_SPLITTING, new QName[] { new QName("WallSurface"), new QName("RoofSurface"), new QName("GroundSurface"), new QName("CeilingSurface"), new QName("ClosureSurface"), new QName("FloorSurface"), new QName("InteriorWallSurface"), new QName("OuterCeilingSurface"), new QName("OuterFloorSurface"), new QName("BuildingInstallation"), new QName("BuildingPart"), new QName("Door"), new QName("Window") }); return inputFactory; } public static FeatureStream streamCityGml(String file, ParserConfiguration config) throws CityGmlParseException { File f = new File(file); return streamCityGml(f, config, null, f.getName()); } public static FeatureStream streamCityGml(File file, ParserConfiguration config, ProgressListener l, String fileName) throws CityGmlParseException { try { if (getExtension(file.getName()).equals("zip")) { streamZipFile(file, config); } parseEpsgCodeFromFile(file, config); CityGMLInputFactory inputFactory = setupGmlReader(config); ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1); FeatureStream stream = new FeatureStream(queue, fileName, file); Thread readThread = new Thread(() -> { // try with resources for automatic closing try (ObservedInputStream ois = new ObservedInputStream(file)) { if (l != null) { ois.addListener(l::updateProgress); } readFeatures(file, config, inputFactory, queue, ois, stream); } catch (IOException e) { logger.error(Localization.getText("CityGmlParser.errorReadingGmlFile"), e.getMessage()); logger.catching(Level.ERROR, e); } }); stream.setThread(readThread); readThread.start(); return stream; } catch (CityGMLBuilderException | IOException | ParserConfigurationException | SAXException | ADEException e) { throw new CityGmlParseException(e); } } private static void streamZipFile(File file, ParserConfiguration config) throws CityGmlParseException { try (ZipFile zip = new ZipFile(file)) { Enumeration entries = zip.entries(); while (entries.hasMoreElements()) { ZipEntry ze = entries.nextElement(); BufferedInputStream bis = new BufferedInputStream(zip.getInputStream(ze)); parseEpsgCodeFromStream(bis, config); } } catch (IOException e) { throw new UncheckedIOException(e); } catch (ParserConfigurationException | SAXException e) { throw new CityGmlParseException(e); } } public static String getExtension(String fileName) { char ch; int len; if (fileName == null || (len = fileName.length()) == 0 || (ch = fileName.charAt(len - 1)) == '/' || ch == '\\' || ch == '.') { return ""; } int dotInd = fileName.lastIndexOf('.'); int sepInd = Math.max(fileName.lastIndexOf('/'), fileName.lastIndexOf('\\')); if (dotInd <= sepInd) return ""; else return fileName.substring(dotInd + 1).toLowerCase(); } public static FeatureStream streamCityGml(File file, ParserConfiguration parserConfig, String fileName) throws CityGmlParseException { return streamCityGml(file, parserConfig, null, fileName); } /** * The srsName (The name by which this reference system is identified) inside * the CityGML file can have multiple formats. This method tries to parse the * string and detect the corresponding reference system. If it is found, it * returns a proj4j.CoordinateReferenceSystem. It throws an * IllegalArgumentException otherwise. * * This method should be able to parse any EPSG id : e.g. "EPSG:1234". German * Citygmls might also have "DE_DHDN_3GK3" or "ETRS89_UTM32" as srsName, so * those are also included. It isn't guaranteed that those formats are correctly * parsed, though. * * The EPSG ids and parameters are defined in resources ('nad/epsg') inside * proj4j-0.1.0.jar. Some EPSG ids are missing though, e.g. 7415 * * @param srsName * @return CoordinateReferenceSystem */ private static CoordinateReferenceSystem crsFromSrsName(String srsName) { Matcher mEPSG = P_EPSG.matcher(srsName); if (mEPSG.find()) { if ("EPSG:4979".contentEquals(srsName)) { srsName = "EPSG:4236"; } else if ("EPSG:7415".contentEquals(srsName)) { return CRS_FACTORY.createFromParameters("EPSG:7415", "+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +towgs84=565.417,50.3319,465.552,-0.398957,0.343988,-1.8774,4.0725 +units=m +no_defs"); } return CRS_FACTORY.createFromName(srsName); } Matcher mOGC = P_OGC.matcher(srsName); if (mOGC.find()) { return CRS_FACTORY.createFromName("EPSG:" + mOGC.group(1)); } Matcher mOGC2 = P_OGC2.matcher(srsName); if (mOGC2.find()) { return CRS_FACTORY.createFromName("EPSG:" + mOGC2.group(1)); } Matcher mURN = P_URN.matcher(srsName); // NOTE: Could use a HashMap if the switch/case becomes too long. if (mURN.find()) { switch (mURN.group(1)) { case "DE_DHDN_3GK2": return CRS_FACTORY.createFromName("EPSG:31466"); case "DE_DHDN_3GK3": return CRS_FACTORY.createFromName("EPSG:31467"); case "DE_DHDN_3GK4": return CRS_FACTORY.createFromName("EPSG:31468"); case "DE_DHDN_3GK5": return CRS_FACTORY.createFromName("EPSG:31469"); case "ETRS89_UTM32": return CRS_FACTORY.createFromName("EPSG:25832"); default: return null; } } return null; } private static void readFeatures(File file, ParserConfiguration config, CityGMLInputFactory inputFactory, ArrayBlockingQueue queue, ObservedInputStream ois, FeatureStream stream) { try (CityGMLReader reader = inputFactory.createCityGMLReader(file.getAbsolutePath(), ois)) { FeatureMapper mapper = new FeatureMapper(config, file); CityDoctorModel model = mapper.getModel(); while (reader.hasNext() && !stream.isClosed()) { CityGML chunk = reader.nextFeature(); if (chunk instanceof AbstractCityObject) { AbstractCityObject ag = (AbstractCityObject) chunk; ag.accept(mapper); } else if (chunk instanceof CityModel) { CityModel cModel = (CityModel) chunk; cModel.unsetCityObjectMember(); mapper.setCityModel(cModel); stream.setCityDoctorModel(model); } drainCityModel(model, queue); } // end of stream queue.put(FeatureStream.POISON); logger.debug("End of gml file stream"); } catch (CityGMLReadException e) { logger.error(Localization.getText("CityGmlParser.errorReadingGmlFile"), e.getMessage(), e); } catch (InterruptedException e) { logger.warn("Interrupted while streaming gml file"); Thread.currentThread().interrupt(); } } private static void parseEpsgCodeFromFile(File file, ParserConfiguration config) throws IOException, ParserConfigurationException, SAXException { try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) { parseEpsgCodeFromStream(bis, config); } } private static void parseEpsgCodeFromStream(InputStream is, ParserConfiguration config) throws ParserConfigurationException, SAXException { SAXParser parser = FACTORY.newSAXParser(); CityGmlHandler handler = new CityGmlHandler(); try { parser.parse(new InputSource(is), handler); } catch (EnvelopeFoundException e) { try { parseCoordinateSystem(config, handler); } catch (Exception e2) { logger.debug("Exception while parsing for EPSG code", e2); if (logger.isWarnEnabled()) { logger.warn(Localization.getText("CityGmlParser.noEPSG")); } } } catch (Exception e) { logger.debug("Exception while parsing for EPSG code", e); if (logger.isWarnEnabled()) { logger.warn(Localization.getText("CityGmlParser.noEPSG")); } } } private static void parseCoordinateSystem(ParserConfiguration config, CityGmlHandler handler) { if (handler.getEpsg() == null) { return; } CoordinateReferenceSystem crs = crsFromSrsName(handler.getEpsg()); if (crs == null) { // could not find a coordinate system for srsName // assuming metric system return; } ProjectionUnitExtractor extractor = new ProjectionUnitExtractor(crs.getProjection()); if (extractor.getUnit() == Units.METRES) { // coordinate system is in meters, do not convert if (logger.isInfoEnabled()) { logger.info(Localization.getText("CityGmlParser.noConversionNeeded")); } return; } parseMeterConversion(config, crs); Vector3d low = handler.getLowerCorner(); Vector3d up = handler.getUpperCorner(); double centerLong = low.getX() + ((up.getX() - low.getX()) / 2); double centerLat = low.getY() + ((up.getY() - low.getY()) / 2); if (!crs.getName().equals("EPSG:4326")) { // need to convert coordinates first to WGS84, then find UTM Zone CoordinateReferenceSystem wgs84 = crsFromSrsName("EPSG:4326"); ProjCoordinate p1 = new ProjCoordinate(); p1.setValue(centerLong, centerLat); ProjCoordinate p2 = new ProjCoordinate(); BasicCoordinateTransform bct = new BasicCoordinateTransform(crs, wgs84); bct.transform(p1, p2); centerLong = p2.x; centerLat = p2.y; } int zone = (int) (31 + Math.round(centerLong / 6)); CoordinateReferenceSystem utm; if (centerLat < 0) { // south utm = CRS_FACTORY.createFromParameters("UTM", "+proj=utm +zone=" + zone + " +south"); } else { // north utm = CRS_FACTORY.createFromParameters("UTM", "+proj=utm +zone=" + zone); } config.setCoordinateSystem(crs, utm); } private static void parseMeterConversion(ParserConfiguration config, CoordinateReferenceSystem crs) { Projection projection = crs.getProjection(); double fromMetres = projection.getFromMetres(); if (fromMetres > 0) { // also transform height information config.setFromMetres(fromMetres); } else { config.setFromMetres(1.0); } } private static void drainCityModel(CityDoctorModel model, ArrayBlockingQueue queue) throws InterruptedException { drainCityObjectList(model.getBuildings(), queue); drainCityObjectList(model.getBridges(), queue); drainCityObjectList(model.getVegetation(), queue); drainCityObjectList(model.getLand(), queue); drainCityObjectList(model.getTransportation(), queue); drainCityObjectList(model.getWater(), queue); model.getBuildings().clear(); model.getBridges().clear(); model.getVegetation().clear(); model.getLand().clear(); model.getTransportation().clear(); model.getWater().clear(); } private static void drainCityObjectList(List buildings, ArrayBlockingQueue queue) throws InterruptedException { for (CityObject co : buildings) { queue.put(co); } } }