Skip to content
GitLab
Commit 97349c76 authored by Luna Riegel's avatar Luna Riegel
Browse files

Refactor: Batch sql commits during parsing

parent b9f844bd
Pipeline #12022 passed with stage
in 2 minutes and 40 seconds
Hide whitespace changes
Inline Side-by-side
CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/database/CacheManager.java
View file @ 97349c76
......@@ -32,6 +32,12 @@ public class CacheManager {
return entry.getModel().getCache().retrieveLibraryObject(entry.getDisplayName());
}
public static void finishParsing(){
if (cache != null){
cache.finishParsing();
}
}
public static UnconnectedCache getUnconnectedCache(){
return new UnconnectedCache();
}
......
CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/database/CityObjectCache.java
View file @ 97349c76
......@@ -18,7 +18,7 @@ public interface CityObjectCache {
Stream<CityObject> getStream(List<GmlId> ids);
int getFeatureCount();
int getTotalFeatureCount();
void resetCache();
......
CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/database/DatabaseHandler.java
View file @ 97349c76
......@@ -23,27 +23,24 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class DatabaseHandler {
private static final Logger logger = LogManager.getLogger(DatabaseHandler.class);
private static final String DB_NAME = "cd_db";
private final String fileName;
// TODO: Change handler back to using only a single featuretable
static{
// Creates the Database at initialization
try {
// Cleanup hook Runtime.getRuntime().addShutdownHook(new Thread(() -> {}));
H2GISDBFactory.createSpatialDataBase(DB_NAME,
true, ";FILE_LOCK=SOCKET");
} catch (Exception e) {
logger.fatal("FATAL ERROR: Creation of embedded database failed!");
logger.fatal(e.getMessage());
Alert alert = new Alert(Alert.AlertType.ERROR);
alert.setTitle("FATAL ERROR");
alert.setHeaderText("FATAL ERROR");
alert.setContentText("Creation of embedded database failed!");
alert.showAndWait();
}
}
......@@ -60,7 +57,7 @@ public class DatabaseHandler {
}
//TODO: Refactor this to remove filename
public String getFileName() {
return fileName;
}
......@@ -73,12 +70,15 @@ public class DatabaseHandler {
logger.debug("Dropped existing features table");
}
}
// TODO: Remove type and add a row with an atomic integer model id
// TODO: Refactor this to only use a single feature table
try (PreparedStatement createPs = con.prepareStatement(String.format("CREATE TABLE %s (gmlid VARCHAR(255)" +
" PRIMARY KEY, bbox GEOMETRY, data BLOB, type VARCHAR(255) NOT NULL)", fileName))) {
" PRIMARY KEY, bbox GEOMETRY, data BLOB)", fileName))) {
createPs.executeUpdate();
if (logger.isDebugEnabled()) {
logger.debug("Created features table");
}
//CREATE SPATIAL INDEX GEO_TABLE_SPATIAL_INDEX ON GEO_TABLE(THE_GEOM); <- Create spatial id
}
} catch (SQLException e) {
......@@ -93,14 +93,13 @@ public class DatabaseHandler {
*/
public void marshallCityObject(CityObject co) {
try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) {
try (PreparedStatement ps = con.prepareStatement(String.format("MERGE INTO %s VALUES (?, ?, ?, ?)", fileName))) {
try (PreparedStatement ps = con.prepareStatement(String.format("MERGE INTO %s VALUES (?, ?, ?)", fileName))) {
ps.setString(1, co.getGmlId().toString());
String bbox = BoundingBox.of(co).to2DWkt();
ps.setString(2, bbox);
if (logger.isDebugEnabled()) {
logger.debug("Attempting to serialize CityObject: {} ", co);
}
ps.setString(4, co.getClass().getName());
try {
byte[] bytes = SerializationUtils.serialize(co);
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
......@@ -118,6 +117,47 @@ public class DatabaseHandler {
}
}
public boolean tryMarshallCityObjectMap(Map<GmlId, CityObject> map) {
// Id String for error logging
String id = "";
try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) {
try (PreparedStatement ps = con.prepareStatement(String.format("MERGE INTO %s VALUES (?, ?, ?)", fileName))) {
con.setAutoCommit(false);
for (Map.Entry<GmlId, CityObject> entry : map.entrySet()) {
CityObject co = entry.getValue();
id = entry.getKey().toString();
ps.setString(1, id);
String bbox = BoundingBox.of(co).to2DWkt();
ps.setString(2, bbox);
if (logger.isTraceEnabled()) {
logger.trace("Attempting to serialize CityObject: {} ", co);
}
try {
byte[] bytes = SerializationUtils.serialize(co);
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
ps.setBinaryStream(3, bis, bytes.length);
ps.addBatch();
} catch (SerializationException e) {
logger.error("Serialization of CityObject {} failed",co.getGmlId());
logger.error(e.getMessage());
}
}
ps.executeBatch();
con.commit();
} catch (SQLException e) {
con.rollback();
logger.error("Batch-commit aborted: Saving of CityObject {} to embedded database critically failed!", id);
logger.error(e.getMessage());
return false;
}
} catch (SQLException e){
logger.error("Connection to embedded database could not be opened!");
logger.error(e.getMessage());
return false;
}
return true;
}
/**
* Retrieves a CityObject from the embedded database.
* @param id the gmlId of the requested Feature
......@@ -126,7 +166,7 @@ public class DatabaseHandler {
public CityObject unmarshallCityObject(GmlId id) {
try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) {
try (PreparedStatement ps = con.prepareStatement(String.format(
"SELECT gmlid, bbox, data, type FROM %s WHERE gmlid = ?", fileName))) {
"SELECT gmlid, bbox, data FROM %s WHERE gmlid = ?", fileName))) {
ps.setString(1, id.toString());
ResultSet rs = ps.executeQuery();
if (rs.next()) {
......@@ -156,7 +196,7 @@ public class DatabaseHandler {
public List<CityObject> getAllCityObjects() {
List<CityObject> cos = new ArrayList<>();
try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) {
try (PreparedStatement ps = con.prepareStatement(String.format("SELECT gmlid, bbox, data, type FROM %s", fileName))) {
try (PreparedStatement ps = con.prepareStatement(String.format("SELECT gmlid, bbox, data FROM %s", fileName))) {
ResultSet rs = ps.executeQuery();
cos.addAll(retrieveCityObjects(rs));
}
......
CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/database/FeatureCache.java
View file @ 97349c76
......@@ -6,10 +6,13 @@ import com.github.benmanes.caffeine.cache.RemovalCause;
import de.hft.stuttgart.citydoctor2.datastructure.CityObject;
import de.hft.stuttgart.citydoctor2.datastructure.GmlId;
import de.hft.stuttgart.citydoctor2.parser.InvalidGmlFileException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
......@@ -17,9 +20,15 @@ import java.util.stream.Stream;
* Cache for CityObjects
*/
public class FeatureCache implements CityObjectCache{
private static final Logger logger = LogManager.getLogger(FeatureCache.class);
private final LoadingCache<GmlId, CityObject> cache;
private DatabaseHandler handler;
private final Map<String, DatabaseHandler> handlerList = new HashMap<>();
private AtomicInteger putsCounter = new AtomicInteger(0);
private boolean parsingFinished = false;
FeatureCache(String fileName) {
handler = new DatabaseHandler(fileName);
......@@ -33,13 +42,56 @@ public class FeatureCache implements CityObjectCache{
@Override
public void put(CityObject cityObject) {
handler.marshallCityObject(cityObject);
cache.put(cityObject.getGmlId(), cityObject);
/* Batch marshall during parsing to improve performance
*/
if (!parsingFinished){
int count = putsCounter.incrementAndGet();
if (count >= 2500) {
batchMarshall();
}
}
}
private void batchMarshall(){
logger.trace("Setting up batch Marshaller");
Map<GmlId, CityObject> cacheContent= new HashMap<>(cache.asMap());
cache.invalidateAll();
putsCounter.set(0);
Thread batcher = new Thread(() -> {
logger.trace("Entered batch Marshaller");
boolean success = handler.tryMarshallCityObjectMap(cacheContent);
if (!success) {
logger.trace("Batch marshalling failed, falling back to one-by-one marshalling");
// Batch marshalling failed, marshall them one-by-one
for (Map.Entry<GmlId, CityObject> entry : cacheContent.entrySet()) {
handler.marshallCityObject(entry.getValue());
}
}
logger.trace("Batch marshalling done");
});
batcher.start();
logger.trace("Batch Marshaller started");
}
@Override
public CityObject get(GmlId id) {
return cache.get(id);
CityObject result = cache.get(id);
if (result == null){
logger.trace("Cache-value missing for {}", id);
//Key has null value, try to restore from Database
result = handler.unmarshallCityObject(id);
if (result == null){
logger.error("CityObject with id {} is null in cache and database", id);
} else {
cache.put(id, result);
}
}
return result;
}
public void changeFileName(String fileName){
......@@ -62,20 +114,23 @@ public class FeatureCache implements CityObjectCache{
}
@Override
public int getFeatureCount() {
public int getTotalFeatureCount() {
return handler.getFeatureCount();
}
@Override
public void resetCache(){
logger.trace("Resetting cache");
cache.invalidateAll();
putsCounter.set(0);
parsingFinished = false;
}
@Override
public List<CityObject> retrieveLibraryObject(String fileName) throws InvalidGmlFileException {
String prevHandler = handler.getFileName();
this.changeFileName(fileName);
int featureCount = getFeatureCount();
int featureCount = getTotalFeatureCount();
if (featureCount != 1){
if (featureCount == 0){
throw new InvalidGmlFileException(String.format("LibraryObject ZIP-entry %s contains no Feature", fileName));
......@@ -88,5 +143,20 @@ public class FeatureCache implements CityObjectCache{
return objects;
}
public void finishParsing() {
if (!parsingFinished) {
HashMap<GmlId, CityObject> cacheContent= new HashMap<>(cache.asMap());
boolean success = handler.tryMarshallCityObjectMap(cacheContent);
if (!success) {
logger.trace("Batch marshalling failed, falling back to one-by-one marshalling");
// Batch marshalling failed, marshall them one-by-one
for (Map.Entry<GmlId, CityObject> entry : cacheContent.entrySet()) {
handler.marshallCityObject(entry.getValue());
}
}
parsingFinished = true;
}
}
}
CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/database/UnconnectedCache.java
View file @ 97349c76
......@@ -41,7 +41,7 @@ public class UnconnectedCache implements CityObjectCache{
}
@Override
public int getFeatureCount() {
public int getTotalFeatureCount() {
return cache.size();
}
......
CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/datastructure/CityDoctorModel.java
View file @ 97349c76
......@@ -27,7 +27,6 @@ import de.hft.stuttgart.citydoctor2.exceptions.CityDoctorWriteException;
import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
import de.hft.stuttgart.citydoctor2.writer.CityGMLWriterUtils;
import de.hft.stuttgart.quality.model.types.ValidationPlan;
import org.apache.commons.io.FilenameUtils;
import org.citygml4j.core.model.CityGMLVersion;
import org.citygml4j.core.model.core.CityModel;
......@@ -131,6 +130,53 @@ public class CityDoctorModel {
getTunnels(), getWater(), getCityFurniture(), getGenericCityObjects()).flatMap(co -> co);
}
public List<GmlId> getFeatureIds() {
List<GmlId> featureIds = new ArrayList<>();
featureIds.addAll(buildings);
featureIds.addAll(vegetation);
featureIds.addAll(bridges);
featureIds.addAll(land);
featureIds.addAll(roads);
featureIds.addAll(tunnels);
featureIds.addAll(water);
featureIds.addAll(cityfurniture);
featureIds.addAll(genericObjects);
return featureIds;
}
public boolean containsGmlId(String id) {
if (id == null) {
return false;
}
GmlId gmlId = new GmlId(id);
if (buildings.contains(gmlId)) {
return true;
}
if (vegetation.contains(gmlId)) {
return true;
}
if (bridges.contains(gmlId)) {
return true;
}
if (land.contains(gmlId)) {
return true;
}
if (roads.contains(gmlId)) {
return true;
}
if (tunnels.contains(gmlId)) {
return true;
}
if (water.contains(gmlId)) {
return true;
}
if (cityfurniture.contains(gmlId)) {
return true;
}
return genericObjects.contains(gmlId);
}
public void accept(CheckableVisitor visitor) {
createFeatureStream().forEach(cityObject -> cityObject.accept(visitor));
}
......@@ -260,6 +306,10 @@ public class CityDoctorModel {
return buildings.stream().map(co -> (Building) cache.get(co));
}
public List<GmlId> getBuildingIds() {
return buildings;
}
public int getBuildingCount(){
return buildings.size();
}
......@@ -273,6 +323,10 @@ public class CityDoctorModel {
return bridges.stream().map(co -> (BridgeObject) cache.get(co));
}
public List<GmlId> getBridgeIds() {
return bridges;
}
public int getBridgeCount(){
return bridges.size();
}
......@@ -286,6 +340,10 @@ public class CityDoctorModel {
return cityfurniture.stream().map(co -> (CityFurniture) cache.get(co));
}
public List<GmlId> getCityFurnitureIds() {
return cityfurniture;
}
public int getCityFurnitureCount(){
return cityfurniture.size();
}
......@@ -299,6 +357,10 @@ public class CityDoctorModel {
return genericObjects.stream().map(co -> (GenericCityObject) cache.get(co));
}
public List<GmlId> getGenericCityObjectIds() {
return genericObjects;
}
public int getGenericCityObjectCount(){
return genericObjects.size();
}
......@@ -312,6 +374,10 @@ public class CityDoctorModel {
return roads.stream().map(co -> (TransportationObject) cache.get(co));
}
public List<GmlId> getTransportationIds() {
return roads;
}
public int getTransportationCount(){
return roads.size();
}
......@@ -325,6 +391,10 @@ public class CityDoctorModel {
return tunnels.stream().map(co -> (Tunnel) cache.get(co));
}
public List<GmlId> getTunnelIds() {
return tunnels;
}
public int getTunnelCount(){
return tunnels.size();
}
......@@ -338,6 +408,10 @@ public class CityDoctorModel {
return water.stream().map(co -> (WaterObject) cache.get(co));
}
public List<GmlId> getWaterIds() {
return water;
}
public int getWaterCount(){
return water.size();
}
......@@ -361,6 +435,10 @@ public class CityDoctorModel {
return land.stream().map(cache::get);
}
public List<GmlId> getLandIds() {
return land;
}
public int getLandCount(){
return land.size();
}
......@@ -374,6 +452,10 @@ public class CityDoctorModel {
return vegetation.stream().map(co -> (Vegetation) cache.get(co));
}
public List<GmlId> getVegetationIds() {
return vegetation;
}
public int getVegetationCount(){
return vegetation.size();
}
......
CityDoctorParent/CityDoctorModel/src/main/java/de/hft/stuttgart/citydoctor2/parser/CityGmlParser.java
View file @ 97349c76
......@@ -30,7 +30,6 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
......@@ -45,6 +44,7 @@ import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator;
import de.hft.stuttgart.citydoctor2.database.CacheManager;
import de.hft.stuttgart.citydoctor2.datastructure.GmlId;
import de.hft.stuttgart.citydoctor2.datastructure.ImplicitGeometryHolder;
import org.apache.logging.log4j.Level;
......@@ -572,6 +572,9 @@ public class CityGmlParser {
// object members are replaced by href in model
// need to remove the refs and re-add unparsed objects
List<AbstractCityObject> acos = new ArrayList<>();
CityModel cgmlModel = null;
CityDoctorModel cdModel = mapper.getModel();
List<AbstractCityObject> queuedObjects = new ArrayList<>();
while (reader.hasNext()) {
AbstractFeature chunk = reader.next();
version = CityGMLModules.getCityGMLVersion(reader.getName().getNamespaceURI());
......@@ -579,39 +582,65 @@ public class CityGmlParser {
cModel.setCityObjectMembers(null);
mapper.setCityModel(cModel);
mapper.setCityGMLVersion(version);
cgmlModel = cModel;
if (!queuedObjects.isEmpty()) {
for (AbstractCityObject aco : queuedObjects) {
if (!cdModel.containsGmlId(aco.getId())) {
cgmlModel.getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
}
}
}
} else if (chunk instanceof AbstractCityObject aco) {
acos.add(aco);
acos.add(aco); // getid
String id = aco.getId();
aco.accept(mapper);
if (cgmlModel == null) {
// Add object to queue till the CityModel has been parsed
queuedObjects.add(aco);
}else if (!cdModel.containsGmlId(id)){
cgmlModel.getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
}
// TODO: add check if aco was parsed successfully, add aco to cmodel if it wasnt
}
}
if (mapper.getModel().getCityModel() == null) {
// file does not contain a city model?
// create it for now
logger.warn("File does not contain a CityModel definition! Temporary CityModel has been created.");
mapper.setCityModel(new CityModel());
for (AbstractCityObject aco : queuedObjects) {
if (!cdModel.containsGmlId(aco.getId())) {
mapper.getModel().getCityModel().getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
}
}
}
CityModel cModel = mapper.getModel().getCityModel();
// remove those that should have been parsed
logger.debug("Getting parsed object from CityDoctorModel");
List<AbstractCityObject> parsedCityObjects = mapper.getModel().createFeatureStream().filter(Objects::nonNull)
.map(CityObject::getGmlObject).toList();
logger.debug("Removing all parsed city objects from acos");
acos.removeAll(parsedCityObjects);
// re-add all not parsed objects
if (logger.isInfoEnabled() && verbose){
logger.info("Resolving HRefs");
}
for (AbstractCityObject aco : acos) {
cModel.getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
}
// cgmlModel = mapper.getModel().getCityModel();
//
// // remove those that should have been parsed
// if (logger.isInfoEnabled() && verbose){
// logger.info("Resolving HRefs");
// }
//// List<AbstractCityObject> parsedCityObjects = mapper.getModel().createFeatureStream().filter(Objects::nonNull)
//// .map(CityObject::getGmlObject).toList();
//// mapper.getModel().createFeatureStream().filter(Objects::nonNull)
//// .map(CityObject::getGmlObject).forEach(acos::remove);
// mapper.getModel().getFeatureIds().forEach(id -> {acos.removeIf(aco -> aco.getId().equals(id.getGmlString()));});
//// acos.removeAll(parsedCityObjects);
// // re-add all not parsed objects
// for (AbstractCityObject aco : acos) {
// cgmlModel.getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
// }
// After parsing is complete, try to resolve deferred RelativeGeometries
ImplicitGeometryHolder.resolveDeferredObjects(mapper.getModel().getCache());
if (logger.isInfoEnabled() && verbose) {
logger.info(Localization.getText("CityGmlParser.parsedObjects"),
mapper.getModel().getNumberOfFeatures());
}
// After parsing is complete, try to resolve deferred RelativeGeometries
ImplicitGeometryHolder.resolveDeferredObjects(mapper.getModel().getCache());
mapper.setCityGMLVersion(version);
CacheManager.finishParsing();
return mapper.getModel();
}
}
......
CityDoctorParent/CityDoctorModel/src/test/java/de/hft/stuttgart/citydoctor2/database/FeatureCacheTest.java
View file @ 97349c76
package de.hft.stuttgart.citydoctor2.database;
import de.hft.stuttgart.citydoctor2.datastructure.Building;
import de.hft.stuttgart.citydoctor2.datastructure.GeometryTestUtils;
import de.hft.stuttgart.citydoctor2.datastructure.GeometryType;
import de.hft.stuttgart.citydoctor2.datastructure.Lod;
import de.hft.stuttgart.citydoctor2.datastructure.*;
import org.citygml4j.core.model.building.AbstractBuilding;
import org.junit.Test;
......@@ -19,11 +16,26 @@ public class FeatureCacheTest {
FeatureCache cache = new FeatureCache("test");
Building b = new Building();
b.setGmlObject(mock(AbstractBuilding.class));
b.setGmlId(new GmlId("test"));
b.addGeometry(GeometryTestUtils.createDummyGeometry(GeometryType.SOLID, Lod.LOD0));
assertEquals(0, cache.getFeatureCount());
assertEquals(0, cache.getTotalFeatureCount());
cache.put(b);
assertEquals(1, cache.getFeatureCount());
cache.finishParsing();
assertEquals(1, cache.getTotalFeatureCount());
}
@Test
public void testGet(){
FeatureCache cache = new FeatureCache("test");
DatabaseHandler handler = new DatabaseHandler("test");
Building b = new Building();
b.setGmlId(new GmlId("building"));
b.addGeometry(GeometryTestUtils.createDummyGeometry(GeometryType.SOLID, Lod.LOD0));
handler.marshallCityObject(b);
CityObject b2 = cache.get(new GmlId("building"));
assertEquals(b.getGmlId(), b2.getGmlId());
}
......
CityDoctorParent/CityDoctorValidation/src/test/java/de/hft/stuttgart/citydoctor2/checks/geometry/DegeneratedPolygonCheckTest.java
View file @ 97349c76
......@@ -44,6 +44,7 @@ import de.hft.stuttgart.citydoctor2.datastructure.Vertex;
import de.hft.stuttgart.citydoctor2.datastructure.Geometry.Orientation;
import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
import de.hft.stuttgart.quality.model.enums.RequirementId;
import org.mockito.Mockito;
public class DegeneratedPolygonCheckTest {
......@@ -68,9 +69,10 @@ public class DegeneratedPolygonCheckTest {
Building b = new Building();
b.addGeometry(geom);
File file = Mockito.mock(File.class);
Mockito.when(file.getName()).thenReturn("building");
ParserConfiguration config = new ParserConfiguration(8, false);
CityDoctorModel model = new CityDoctorModel(config, new File(""));
CityDoctorModel model = new CityDoctorModel(config, file);
model.addBuilding(b);
ValidationConfiguration valConfig = ValidationConfiguration.loadStandardValidationConfig();
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal