/*-
* Copyright 2020 Beuth Hochschule für Technik Berlin, Hochschule für Technik Stuttgart
*
* This file is part of CityDoctor2.
*
* CityDoctor2 is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* CityDoctor2 is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with CityDoctor2. If not, see .
*/
package de.hft.stuttgart.citydoctor2.parser;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.util.Enumeration;
import java.util.List;
import java.util.ServiceLoader;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.xml.XMLConstants;
import javax.xml.namespace.QName;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.citygml4j.CityGMLContext;
import org.citygml4j.builder.jaxb.CityGMLBuilder;
import org.citygml4j.builder.jaxb.CityGMLBuilderException;
import org.citygml4j.model.citygml.CityGML;
import org.citygml4j.model.citygml.ade.ADEException;
import org.citygml4j.model.citygml.ade.binding.ADEContext;
import org.citygml4j.model.citygml.core.AbstractCityObject;
import org.citygml4j.model.citygml.core.CityModel;
import org.citygml4j.model.citygml.core.CityObjectMember;
import org.citygml4j.xml.io.CityGMLInputFactory;
import org.citygml4j.xml.io.reader.CityGMLReadException;
import org.citygml4j.xml.io.reader.CityGMLReader;
import org.citygml4j.xml.io.reader.FeatureReadMode;
import org.osgeo.proj4j.BasicCoordinateTransform;
import org.osgeo.proj4j.CRSFactory;
import org.osgeo.proj4j.CoordinateReferenceSystem;
import org.osgeo.proj4j.ProjCoordinate;
import org.osgeo.proj4j.proj.Projection;
import org.osgeo.proj4j.units.Units;
import org.w3c.dom.DOMException;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXNotRecognizedException;
import org.xml.sax.SAXNotSupportedException;
import org.xml.sax.SAXParseException;
import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel;
import de.hft.stuttgart.citydoctor2.datastructure.CityObject;
import de.hft.stuttgart.citydoctor2.mapper.FeatureMapper;
import de.hft.stuttgart.citydoctor2.math.Vector3d;
import de.hft.stuttgart.citydoctor2.utils.Localization;
/**
* Utility class to parse CityGML files.
*
* @author Matthias Betz
*
*/
public class CityGmlParser {
private static final Logger logger = LogManager.getLogger(CityGmlParser.class);
private static final CRSFactory CRS_FACTORY = new CRSFactory();
// EPSG:31467
private static final Pattern P_EPSG = Pattern.compile("^(EPSG:\\d+)$");
// urn:ogc:def:crs,crs:EPSG:6.12:31467,crs:EPSG:6.12:5783
// or
// urn:ogc:def:crs,crs:EPSG::28992
private static final Pattern P_OGC = Pattern.compile("urn:ogc:def:crs,crs:EPSG:[\\d\\.]*:([\\d]+)\\D*");
private static final Pattern P_OGC2 = Pattern.compile("urn:ogc:def:crs:EPSG:[\\d\\.]*:([\\d]+)\\D*");
// urn:adv:crs:DE_DHDN_3GK3*DE_DHHN92_NH
// urn:adv:crs:ETRS89_UTM32*DE_DHHN92_NH
private static final Pattern P_URN = Pattern.compile("urn:adv:crs:([^\\*]+)");
private static final SAXParserFactory FACTORY;
static {
FACTORY = SAXParserFactory.newInstance();
try {
FACTORY.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
} catch (SAXNotRecognizedException | SAXNotSupportedException | ParserConfigurationException e) {
logger.catching(e);
}
}
private CityGmlParser() {
// only static access
}
public static CityDoctorModel parseCityGmlFile(String file, ParserConfiguration config)
throws CityGmlParseException, InvalidGmlFileException {
return parseCityGmlFile(file, config, null);
}
public static CityDoctorModel parseCityGmlFile(String filePath, ParserConfiguration config, ProgressListener l)
throws CityGmlParseException, InvalidGmlFileException {
File file = new File(filePath);
try {
parseEpsgCodeFromFile(file, config);
CityGMLInputFactory inputFactory = setupGmlReader(config);
// try with resources for automatic closing
try (ObservedInputStream ois = new ObservedInputStream(file)) {
if (l != null) {
ois.addListener(l::updateProgress);
}
try (CityGMLReader reader = inputFactory.createCityGMLReader(file.getAbsolutePath(), ois)) {
FeatureMapper mapper = new FeatureMapper(config, file);
while (reader.hasNext()) {
CityGML chunk = reader.nextFeature();
if (chunk instanceof AbstractCityObject) {
AbstractCityObject ag = (AbstractCityObject) chunk;
ag.accept(mapper);
} else if (chunk instanceof CityModel) {
CityModel cModel = (CityModel) chunk;
cModel.unsetCityObjectMember();
mapper.setCityModel(cModel);
}
}
if (logger.isInfoEnabled()) {
logger.info(Localization.getText("CityGmlParser.parsedObjects"), mapper.getModel().getNumberOfFeatures());
}
return mapper.getModel();
}
}
} catch (CityGMLReadException e) {
if (e.getCause() instanceof SAXParseException) {
throw new InvalidGmlFileException(Localization.getText("CityGmlParser.notValidGmlFile") + e.getCause().getMessage(), e);
}
throw new CityGmlParseException(e);
} catch (IOException | CityGMLBuilderException | ParserConfigurationException | SAXException | ADEException e) {
throw new CityGmlParseException(e);
} catch (DOMException e) {
// the citygml4j split per feature tries to insert an element
// this sometimes fails because of namespace mismatch
// fallback solution is to parse the whole file at once
// problems with memory consumption may arise
try {
if (logger.isWarnEnabled()) {
logger.warn(Localization.getText("CityGmlParser.chunkReadFailed"), e);
}
return parseCityGmlFileComplete(file, config, l);
} catch (CityGMLBuilderException | CityGMLReadException | IOException e1) {
throw new CityGmlParseException(e1);
}
}
}
private static CityDoctorModel parseCityGmlFileComplete(File file, ParserConfiguration config, ProgressListener l)
throws CityGMLBuilderException, IOException, CityGMLReadException, CityGmlParseException {
CityGMLContext context = CityGMLContext.getInstance();
CityGMLBuilder builder = context.createCityGMLBuilder();
CityGMLInputFactory inputFactory = builder.createCityGMLInputFactory();
inputFactory.setProperty(CityGMLInputFactory.FAIL_ON_MISSING_ADE_SCHEMA, false);
inputFactory.setProperty(CityGMLInputFactory.USE_VALIDATION, config.getValidate());
try (ObservedInputStream ois = new ObservedInputStream(file)) {
if (l != null) {
ois.addListener(l::updateProgress);
}
try (CityGMLReader reader = inputFactory.createCityGMLReader(file.getAbsolutePath(), ois)) {
FeatureMapper mapper = new FeatureMapper(config, file);
CityGML chunk = reader.nextFeature();
if (!(chunk instanceof CityModel)) {
throw new CityGmlParseException("Did not read CityModel as first element of gml file.");
}
CityModel model = (CityModel) chunk;
mapper.setCityModel(model);
if (model.isSetCityObjectMember()) {
for (CityObjectMember com : model.getCityObjectMember()) {
if (com.isSetCityObject()) {
com.getCityObject().accept(mapper);
}
}
}
model.unsetCityObjectMember();
return mapper.getModel();
}
}
}
private static CityGMLInputFactory setupGmlReader(ParserConfiguration config)
throws CityGMLBuilderException, ADEException {
CityGMLContext context = CityGMLContext.getInstance();
// setup energy ade stuff, so the parser doesn't crash on encountering this
if (!context.hasADEContexts()) {
for (ADEContext adeContext : ServiceLoader.load(ADEContext.class)) {
context.registerADEContext(adeContext);
}
}
CityGMLBuilder builder = context.createCityGMLBuilder(CityGmlParser.class.getClassLoader());
CityGMLInputFactory inputFactory = builder.createCityGMLInputFactory();
inputFactory.setProperty(CityGMLInputFactory.FEATURE_READ_MODE, FeatureReadMode.SPLIT_PER_FEATURE);
inputFactory.setProperty(CityGMLInputFactory.USE_VALIDATION, config.getValidate());
inputFactory.setProperty(CityGMLInputFactory.FAIL_ON_MISSING_ADE_SCHEMA, false);
inputFactory.setProperty(CityGMLInputFactory.EXCLUDE_FROM_SPLITTING,
new QName[] { new QName("WallSurface"), new QName("RoofSurface"), new QName("GroundSurface"),
new QName("CeilingSurface"), new QName("ClosureSurface"), new QName("FloorSurface"),
new QName("InteriorWallSurface"), new QName("OuterCeilingSurface"),
new QName("OuterFloorSurface"), new QName("BuildingInstallation"), new QName("BuildingPart"),
new QName("Door"), new QName("Window") });
return inputFactory;
}
public static FeatureStream streamCityGml(String file, ParserConfiguration config) throws CityGmlParseException {
File f = new File(file);
return streamCityGml(f, config, null, f.getName());
}
public static FeatureStream streamCityGml(File file, ParserConfiguration config, ProgressListener l,
String fileName) throws CityGmlParseException {
try {
if (getExtension(file.getName()).equals("zip")) {
streamZipFile(file, config);
}
parseEpsgCodeFromFile(file, config);
CityGMLInputFactory inputFactory = setupGmlReader(config);
ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1);
FeatureStream stream = new FeatureStream(queue, fileName, file);
Thread readThread = new Thread(() -> {
// try with resources for automatic closing
try (ObservedInputStream ois = new ObservedInputStream(file)) {
if (l != null) {
ois.addListener(l::updateProgress);
}
readFeatures(file, config, inputFactory, queue, ois, stream);
} catch (IOException e) {
logger.error(Localization.getText("CityGmlParser.errorReadingGmlFile"), e.getMessage());
logger.catching(Level.ERROR, e);
}
});
stream.setThread(readThread);
readThread.start();
return stream;
} catch (CityGMLBuilderException | IOException | ParserConfigurationException | SAXException | ADEException e) {
throw new CityGmlParseException(e);
}
}
private static void streamZipFile(File file, ParserConfiguration config) throws CityGmlParseException {
try (ZipFile zip = new ZipFile(file)) {
Enumeration extends ZipEntry> entries = zip.entries();
while (entries.hasMoreElements()) {
ZipEntry ze = entries.nextElement();
BufferedInputStream bis = new BufferedInputStream(zip.getInputStream(ze));
parseEpsgCodeFromStream(bis, config);
}
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (ParserConfigurationException | SAXException e) {
throw new CityGmlParseException(e);
}
}
public static String getExtension(String fileName) {
char ch;
int len;
if (fileName == null || (len = fileName.length()) == 0 || (ch = fileName.charAt(len - 1)) == '/' || ch == '\\'
|| ch == '.') {
return "";
}
int dotInd = fileName.lastIndexOf('.');
int sepInd = Math.max(fileName.lastIndexOf('/'), fileName.lastIndexOf('\\'));
if (dotInd <= sepInd)
return "";
else
return fileName.substring(dotInd + 1).toLowerCase();
}
public static FeatureStream streamCityGml(File file, ParserConfiguration parserConfig, String fileName)
throws CityGmlParseException {
return streamCityGml(file, parserConfig, null, fileName);
}
/**
* The srsName (The name by which this reference system is identified) inside
* the CityGML file can have multiple formats. This method tries to parse the
* string and detect the corresponding reference system. If it is found, it
* returns a proj4j.CoordinateReferenceSystem. It throws an
* IllegalArgumentException otherwise.
*
* This method should be able to parse any EPSG id : e.g. "EPSG:1234". German
* Citygmls might also have "DE_DHDN_3GK3" or "ETRS89_UTM32" as srsName, so
* those are also included. It isn't guaranteed that those formats are correctly
* parsed, though.
*
* The EPSG ids and parameters are defined in resources ('nad/epsg') inside
* proj4j-0.1.0.jar. Some EPSG ids are missing though, e.g. 7415
*
* @param srsName
* @return CoordinateReferenceSystem
*/
private static CoordinateReferenceSystem crsFromSrsName(String srsName) {
Matcher mEPSG = P_EPSG.matcher(srsName);
if (mEPSG.find()) {
if ("EPSG:4979".contentEquals(srsName)) {
srsName = "EPSG:4236";
} else if ("EPSG:7415".contentEquals(srsName)) {
return CRS_FACTORY.createFromParameters("EPSG:7415",
"+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +towgs84=565.417,50.3319,465.552,-0.398957,0.343988,-1.8774,4.0725 +units=m +no_defs");
}
return CRS_FACTORY.createFromName(srsName);
}
Matcher mOGC = P_OGC.matcher(srsName);
if (mOGC.find()) {
return CRS_FACTORY.createFromName("EPSG:" + mOGC.group(1));
}
Matcher mOGC2 = P_OGC2.matcher(srsName);
if (mOGC2.find()) {
return CRS_FACTORY.createFromName("EPSG:" + mOGC2.group(1));
}
Matcher mURN = P_URN.matcher(srsName);
// NOTE: Could use a HashMap if the switch/case becomes too long.
if (mURN.find()) {
switch (mURN.group(1)) {
case "DE_DHDN_3GK2":
return CRS_FACTORY.createFromName("EPSG:31466");
case "DE_DHDN_3GK3":
return CRS_FACTORY.createFromName("EPSG:31467");
case "DE_DHDN_3GK4":
return CRS_FACTORY.createFromName("EPSG:31468");
case "DE_DHDN_3GK5":
return CRS_FACTORY.createFromName("EPSG:31469");
case "ETRS89_UTM32":
return CRS_FACTORY.createFromName("EPSG:25832");
default:
return null;
}
}
return null;
}
private static void readFeatures(File file, ParserConfiguration config, CityGMLInputFactory inputFactory,
ArrayBlockingQueue queue, ObservedInputStream ois, FeatureStream stream) {
try (CityGMLReader reader = inputFactory.createCityGMLReader(file.getAbsolutePath(), ois)) {
FeatureMapper mapper = new FeatureMapper(config, file);
CityDoctorModel model = mapper.getModel();
while (reader.hasNext() && !stream.isClosed()) {
CityGML chunk = reader.nextFeature();
if (chunk instanceof AbstractCityObject) {
AbstractCityObject ag = (AbstractCityObject) chunk;
ag.accept(mapper);
} else if (chunk instanceof CityModel) {
CityModel cModel = (CityModel) chunk;
cModel.unsetCityObjectMember();
mapper.setCityModel(cModel);
stream.setCityDoctorModel(model);
}
drainCityModel(model, queue);
}
// end of stream
queue.put(FeatureStream.POISON);
logger.debug("End of gml file stream");
} catch (CityGMLReadException e) {
logger.error(Localization.getText("CityGmlParser.errorReadingGmlFile"), e.getMessage(), e);
} catch (InterruptedException e) {
logger.warn("Interrupted while streaming gml file");
Thread.currentThread().interrupt();
}
}
private static void parseEpsgCodeFromFile(File file, ParserConfiguration config)
throws IOException, ParserConfigurationException, SAXException {
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) {
parseEpsgCodeFromStream(bis, config);
}
}
private static void parseEpsgCodeFromStream(InputStream is, ParserConfiguration config)
throws ParserConfigurationException, SAXException {
SAXParser parser = FACTORY.newSAXParser();
CityGmlHandler handler = new CityGmlHandler();
try {
parser.parse(new InputSource(is), handler);
} catch (EnvelopeFoundException e) {
try {
parseCoordinateSystem(config, handler);
} catch (Exception e2) {
logger.debug("Exception while parsing for EPSG code", e2);
if (logger.isWarnEnabled()) {
logger.warn(Localization.getText("CityGmlParser.noEPSG"));
}
}
} catch (Exception e) {
logger.debug("Exception while parsing for EPSG code", e);
if (logger.isWarnEnabled()) {
logger.warn(Localization.getText("CityGmlParser.noEPSG"));
}
}
}
private static void parseCoordinateSystem(ParserConfiguration config, CityGmlHandler handler) {
if (handler.getEpsg() == null) {
return;
}
CoordinateReferenceSystem crs = crsFromSrsName(handler.getEpsg());
if (crs == null) {
// could not find a coordinate system for srsName
// assuming metric system
return;
}
ProjectionUnitExtractor extractor = new ProjectionUnitExtractor(crs.getProjection());
if (extractor.getUnit() == Units.METRES) {
// coordinate system is in meters, do not convert
if (logger.isInfoEnabled()) {
logger.info(Localization.getText("CityGmlParser.noConversionNeeded"));
}
return;
}
parseMeterConversion(config, crs);
Vector3d low = handler.getLowerCorner();
Vector3d up = handler.getUpperCorner();
double centerLong = low.getX() + ((up.getX() - low.getX()) / 2);
double centerLat = low.getY() + ((up.getY() - low.getY()) / 2);
if (!crs.getName().equals("EPSG:4326")) {
// need to convert coordinates first to WGS84, then find UTM Zone
CoordinateReferenceSystem wgs84 = crsFromSrsName("EPSG:4326");
ProjCoordinate p1 = new ProjCoordinate();
p1.setValue(centerLong, centerLat);
ProjCoordinate p2 = new ProjCoordinate();
BasicCoordinateTransform bct = new BasicCoordinateTransform(crs, wgs84);
bct.transform(p1, p2);
centerLong = p2.x;
centerLat = p2.y;
}
int zone = (int) (31 + Math.round(centerLong / 6));
CoordinateReferenceSystem utm;
if (centerLat < 0) {
// south
utm = CRS_FACTORY.createFromParameters("UTM", "+proj=utm +zone=" + zone + " +south");
} else {
// north
utm = CRS_FACTORY.createFromParameters("UTM", "+proj=utm +zone=" + zone);
}
config.setCoordinateSystem(crs, utm);
}
private static void parseMeterConversion(ParserConfiguration config, CoordinateReferenceSystem crs) {
Projection projection = crs.getProjection();
double fromMetres = projection.getFromMetres();
if (fromMetres > 0) {
// also transform height information
config.setFromMetres(fromMetres);
} else {
config.setFromMetres(1.0);
}
}
private static void drainCityModel(CityDoctorModel model, ArrayBlockingQueue queue)
throws InterruptedException {
drainCityObjectList(model.getBuildings(), queue);
drainCityObjectList(model.getBridges(), queue);
drainCityObjectList(model.getVegetation(), queue);
drainCityObjectList(model.getLand(), queue);
drainCityObjectList(model.getTransportation(), queue);
drainCityObjectList(model.getWater(), queue);
model.getBuildings().clear();
model.getBridges().clear();
model.getVegetation().clear();
model.getLand().clear();
model.getTransportation().clear();
model.getWater().clear();
}
private static void drainCityObjectList(List extends CityObject> buildings, ArrayBlockingQueue queue)
throws InterruptedException {
for (CityObject co : buildings) {
queue.put(co);
}
}
}