RegionChooser: Citygmls are now parsed with VTD-XML. It should be faster and...

RegionChooser: Citygmls are now parsed with VTD-XML. It should be faster and use less memory than the previous method. 

.classpath

@@ -6,5 +6,6 @@
 	<classpathentry kind="con" path="org.eclipse.fx.ide.jdt.core.JAVAFX_CONTAINER"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
 	<classpathentry combineaccessrules="false" kind="src" path="/GeoLibs"/>
+	<classpathentry kind="lib" path="lib/vtd-xml_2_13_1.jar"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

src/eu/simstadt/regionchooser/RegionChooserBrowser.java

@@ -18,6 +18,9 @@
 import com.vividsolutions.jts.geom.Geometry;
 import com.vividsolutions.jts.io.ParseException;
 import com.vividsolutions.jts.io.WKTReader;
+import com.ximpleware.NavException;
+import com.ximpleware.XPathEvalException;
+import com.ximpleware.XPathParseException;
 import eu.simstadt.nf4j.ExportJobFromJavaFXRegionChooser;
 import javafx.beans.value.ObservableValue;
 import javafx.concurrent.Task;
@@ -94,7 +97,8 @@ public void extractZIPtoGML(String zipFilename) throws IOException {
 
 		public void downloadRegionFromCityGML(String wktPolygon, String project, String citygml, String srsName)
 				throws IOException,
-				ParseException, SAXParseException, XMLStreamException {
+				ParseException, SAXParseException, XMLStreamException, NumberFormatException, XPathParseException,
+				NavException, XPathEvalException {
 			StringBuffer sb = RegionExtractor.selectRegionDirectlyFromCityGML(citygmlPath(project, citygml), wktPolygon,
 					srsName);
 

src/eu/simstadt/regionchooser/RegionExtractor.java

 package eu.simstadt.regionchooser;
 
 import java.io.IOException;
-import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import java.util.logging.Logger;
 import javax.xml.stream.XMLStreamException;
 import org.xml.sax.SAXParseException;
 import com.vividsolutions.jts.geom.Coordinate;
@@ -14,95 +12,90 @@
 import com.vividsolutions.jts.geom.Point;
 import com.vividsolutions.jts.io.ParseException;
 import com.vividsolutions.jts.io.WKTReader;
-import de.hft.stuttgart.citydoctor.writer.CityDoctorModel;
-import eu.simstadt.geo.GeoCoordinatesAccessor;
+import com.ximpleware.AutoPilot;
+import com.ximpleware.NavException;
+import com.ximpleware.VTDGen;
+import com.ximpleware.VTDNav;
+import com.ximpleware.XPathEvalException;
+import com.ximpleware.XPathParseException;
+import sun.misc.Launcher;
 
 
 public class RegionExtractor
 {
 	static private WKTReader wktReader = new WKTReader();
 
+	private static final Logger LOGGER = Logger.getLogger(Launcher.class.getName());
+
 	static public StringBuffer selectRegionDirectlyFromCityGML(Path citygmlPath, String wktPolygon, String srsName)
-			throws IOException, ParseException, SAXParseException, XMLStreamException {
+			throws SAXParseException, XMLStreamException, ParseException, XPathParseException, NavException,
+			NumberFormatException, XPathEvalException, IOException {
 
 		Geometry poly = wktReader.read(wktPolygon);
-
 		final GeometryFactory gf = new GeometryFactory();
-		//TODO: Don't read all the file. Not possible for 15GB gml files
-		String s = new String(Files.readAllBytes(citygmlPath), Charset.defaultCharset());
+		StringBuffer sb = new StringBuffer();
 
-		Pattern cityObjectPattern = Pattern
-				.compile("(?s)<(core:)?cityObjectMember[^>]*>.*?<\\/(core:)?cityObjectMember>\\s*");
+		int foundBuildingsCount = 0;
+		int buildingsCount = 0;
+		long offsetAndLength;
+		int buildingOffset = 0;
+		int buildingLength = 0;
+		int coordinatesOffset = 0;
+		int coordinatesLength = 0;
+		VTDGen parser = new VTDGen();
+		if (parser.parseFile(citygmlPath.toString(), false)) {
 
-		Pattern coordinatesPattern = null;
-		switch (srsName) {
-		case "EPSG:31467":
-			coordinatesPattern = Pattern
-					.compile("(?<![\\d\\.])(3\\d\\d\\d\\d\\d\\d[\\.\\d]*) (5\\d\\d\\d\\d\\d\\d[\\.\\d]*)");
-			break;
-		case "EPSG:25832":
-			coordinatesPattern = Pattern
-					.compile("(?<![\\d\\.])([2-6]\\d{5}[\\.\\d]*) ([567]\\d{6}[\\.\\d]*)"); //NOTE: ETRS89 / UTM zone 32N. Used for Essen
-			break;
-		case "EPSG:32118":
-			coordinatesPattern = Pattern
-					.compile("(?<![\\d\\.])([23]\\d\\d\\d\\d\\d[\\.\\d]*) ([4-8]\\d\\d\\d\\d[\\.\\d]*)"); //NOTE: NYC
-			break;
-		case "EPSG:2263":
-			coordinatesPattern = Pattern
-					.compile("(?<![\\d\\.])([19]\\d{5,6}[\\.\\d]*) ([1-4]\\d{5}[\\.\\d]*)"); //NOTE: NYC feet
-			break;
-		case "EPSG:3068":
-			coordinatesPattern = Pattern
-					.compile("(?<![\\d\\.])(2\\d\\d\\d\\d[\\.\\d]*) (2\\d\\d\\d\\d[\\.\\d]*)"); //NOTE: Berlin
-			break;
-		case "EPSG:32632":
-			coordinatesPattern = Pattern
-					.compile("(?<![\\d\\.])(69\\d\\d\\d\\d[\\.\\d]*) (533\\d\\d\\d\\d[\\.\\d]*)"); //NOTE: Munich
-			break;
-		default:
-			System.out.println(srsName + " not supported yet.");
-			CityDoctorModel model = CityDoctorModel.loadWithEnergyADEAndNoSchemaValidation(citygmlPath.toFile());
-			GeoCoordinatesAccessor coordinates = GeoCoordinatesAccessor.coordinatesComputedFromBuildingsIfNeeded(model);
-			System.out.println(coordinates.getLowerCorner());
-			System.out.println(coordinates.getUpperCorner());
+			VTDNav navigator = parser.getNav();
+			AutoPilot buildingsFinder = new AutoPilot(navigator);
+			buildingsFinder.selectXPath("//cityObjectMember");
+			while ((buildingsFinder.evalXPath()) != -1) {
+				AutoPilot coordinatesFinder = new AutoPilot(navigator);
+				coordinatesFinder.selectXPath(".//posList");
+				offsetAndLength = navigator.getElementFragment();
+				buildingOffset = (int) offsetAndLength;
+				buildingLength = (int) (offsetAndLength >> 32);
+				String cityObject = navigator.toRawString(buildingOffset, buildingLength);
+				// Add header
+				if (buildingsCount == 0) {
+					sb.append(navigator.toRawString(0, buildingOffset));
+				}
+				buildingsCount += 1;
+				if (buildingsCount % 1000 == 0) {
+					LOGGER.info("1000 buildings parsed");
+				}
 
-			throw new IllegalArgumentException("Sorry. " + srsName + " is not supported yet.");
-		}
+				int coordinatesCount = 0;
+				double xTotal = 0;
+				double yTotal = 0;
 
-		Matcher cityObjectMatcher = cityObjectPattern.matcher(s);
-		StringBuffer sb = new StringBuffer();
-		int i = 0;
-		while (cityObjectMatcher.find())
+				while (coordinatesFinder.evalXPath() != -1) {
+					offsetAndLength = navigator.getContentFragment();
+					coordinatesOffset = (int) offsetAndLength;
+					coordinatesLength = (int) (offsetAndLength >> 32);
+					String posList = navigator.toRawString(coordinatesOffset, coordinatesLength);
+					String[] coordinates = posList.split(" ");
+					for (int k = 0; k < coordinates.length; k = k + 3) {
+						coordinatesCount++;
+						xTotal += Double.valueOf(coordinates[k]);
+						yTotal += Double.valueOf(coordinates[k + 1]);
+					}
+				}
 
-		{
-			cityObjectMatcher.appendReplacement(sb, "");
-			String cityObject = cityObjectMatcher.group();
-			Matcher coordinatesMatcher = coordinatesPattern.matcher(cityObject);
-			int coordinatesCount = 0;
-			double xTotal = 0;
-			double yTotal = 0;
-			while (coordinatesMatcher.find()) {
-				coordinatesCount++;
-				xTotal += Double.valueOf(coordinatesMatcher.group(1));
-				yTotal += Double.valueOf(coordinatesMatcher.group(2));
-			}
-			double x = xTotal / coordinatesCount;
-			double y = yTotal / coordinatesCount;
-			Coordinate coord = new Coordinate(x, y);
-			Point point = gf.createPoint(coord);
-			if (point.within(poly)) {
-				i++;
-				sb.append(cityObject);
+				double x = xTotal / coordinatesCount;
+				double y = yTotal / coordinatesCount;
+				Coordinate coord = new Coordinate(x, y);
+				Point point = gf.createPoint(coord);
+				if (point.within(poly)) {
+					foundBuildingsCount++;
+					sb.append(cityObject);
+				}
 			}
+			LOGGER.info("Buildings found in selected region " + foundBuildingsCount);
+			int footerOffset = buildingOffset + buildingLength;
+			int footerLength = (int) (Files.size(citygmlPath) - footerOffset);
+			sb.append(navigator.toRawString(footerOffset, footerLength));
 		}
-		System.out.println("Buildings found in selected region " + i);
-		if (i > 0)
 
-		{
-			cityObjectMatcher.appendTail(sb);
-		}
 		return sb;
 	}
-
 }

src/eu/simstadt/regionchooser/website/script/simstadt_openlayers.js

@@ -310,7 +310,7 @@ function downloadRegionFromNovaFACTORY(i) {
 }
 
 function sketchAsWKT(srsName) {
-	console.log(srsName);
+	console.log(srsName); //TODO: Check if proj4 knows this coordinate system
 	srsName = (typeof srsName === 'undefined') ? 'EPSG:4326' : srsName;
 	var wktFormat = new ol.format.WKT();
 	return wktFormat.writeFeature(sketch, {

test/eu/simstadt/regionchooser/test/RegionExtractorTests.java

@@ -35,6 +35,8 @@ public void testExtract3BuildingsFromGSK3Model() throws Throwable {
 		assertTrue(churchGMLString.contains("DEBW_LOD2_203056"));
 		assertTrue(churchGMLString.contains("DEBW_LOD2_2869"));
 		assertTrue(churchGMLString.contains("DEBW_LOD2_2909"));
+		assertTrue(churchGMLString.contains("<core:CityModel")); // Header
+		assertTrue(churchGMLString.contains("</core:CityModel")); // Footer
 	}
 
 	@Test
@@ -49,6 +51,8 @@ public void testExtract3BuildingsFromNAD83Model() throws Throwable {
 		assertTrue(archGMLString.contains("WASHINGTON SQUARE"));
 		assertTrue(archGMLString.contains("uuid_c0980a6e-05ea-4d09-bc83-efab226945a1"));
 		assertTrue(archGMLString.contains("uuid_0985cebb-922d-4b3e-95e5-15dc6089cd28"));
+		assertTrue(archGMLString.contains("<CityModel")); // Header
+		assertTrue(archGMLString.contains("</CityModel")); // Footer
 	}
 
 	@Test
@@ -60,5 +64,7 @@ public void testExtract0BuildingsWithWrongCoordinates() throws Throwable {
 		String emptyGMLString = RegionExtractor.selectRegionDirectlyFromCityGML(citygmlPath, wktPolygon, "EPSG:32118")
 				.toString();
 		assertEquals(countRegexMatches(emptyGMLString, "<(core:)?cityObjectMember"), 0);
+		assertTrue(emptyGMLString.contains("<CityModel")); // Header
+		assertTrue(emptyGMLString.contains("</CityModel")); // Footer
 	}
 }