Commit 97349c76 authored by Luna Riegel's avatar Luna Riegel
Browse files

Refactor: Batch sql commits during parsing

parent b9f844bd
Pipeline #12022 passed with stage
in 2 minutes and 40 seconds
...@@ -32,6 +32,12 @@ public class CacheManager { ...@@ -32,6 +32,12 @@ public class CacheManager {
return entry.getModel().getCache().retrieveLibraryObject(entry.getDisplayName()); return entry.getModel().getCache().retrieveLibraryObject(entry.getDisplayName());
} }
public static void finishParsing(){
if (cache != null){
cache.finishParsing();
}
}
public static UnconnectedCache getUnconnectedCache(){ public static UnconnectedCache getUnconnectedCache(){
return new UnconnectedCache(); return new UnconnectedCache();
} }
......
...@@ -18,7 +18,7 @@ public interface CityObjectCache { ...@@ -18,7 +18,7 @@ public interface CityObjectCache {
Stream<CityObject> getStream(List<GmlId> ids); Stream<CityObject> getStream(List<GmlId> ids);
int getFeatureCount(); int getTotalFeatureCount();
void resetCache(); void resetCache();
......
...@@ -23,27 +23,24 @@ import java.sql.ResultSet; ...@@ -23,27 +23,24 @@ import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
public class DatabaseHandler { public class DatabaseHandler {
private static final Logger logger = LogManager.getLogger(DatabaseHandler.class); private static final Logger logger = LogManager.getLogger(DatabaseHandler.class);
private static final String DB_NAME = "cd_db"; private static final String DB_NAME = "cd_db";
private final String fileName; private final String fileName;
// TODO: Change handler back to using only a single featuretable
static{ static{
// Creates the Database at initialization // Creates the Database at initialization
try { try {
// Cleanup hook Runtime.getRuntime().addShutdownHook(new Thread(() -> {}));
H2GISDBFactory.createSpatialDataBase(DB_NAME, H2GISDBFactory.createSpatialDataBase(DB_NAME,
true, ";FILE_LOCK=SOCKET"); true, ";FILE_LOCK=SOCKET");
} catch (Exception e) { } catch (Exception e) {
logger.fatal("FATAL ERROR: Creation of embedded database failed!"); logger.fatal("FATAL ERROR: Creation of embedded database failed!");
logger.fatal(e.getMessage()); logger.fatal(e.getMessage());
Alert alert = new Alert(Alert.AlertType.ERROR);
alert.setTitle("FATAL ERROR");
alert.setHeaderText("FATAL ERROR");
alert.setContentText("Creation of embedded database failed!");
alert.showAndWait();
} }
} }
...@@ -60,7 +57,7 @@ public class DatabaseHandler { ...@@ -60,7 +57,7 @@ public class DatabaseHandler {
} }
//TODO: Refactor this to remove filename
public String getFileName() { public String getFileName() {
return fileName; return fileName;
} }
...@@ -73,12 +70,15 @@ public class DatabaseHandler { ...@@ -73,12 +70,15 @@ public class DatabaseHandler {
logger.debug("Dropped existing features table"); logger.debug("Dropped existing features table");
} }
} }
// TODO: Remove type and add a row with an atomic integer model id
// TODO: Refactor this to only use a single feature table
try (PreparedStatement createPs = con.prepareStatement(String.format("CREATE TABLE %s (gmlid VARCHAR(255)" + try (PreparedStatement createPs = con.prepareStatement(String.format("CREATE TABLE %s (gmlid VARCHAR(255)" +
" PRIMARY KEY, bbox GEOMETRY, data BLOB, type VARCHAR(255) NOT NULL)", fileName))) { " PRIMARY KEY, bbox GEOMETRY, data BLOB)", fileName))) {
createPs.executeUpdate(); createPs.executeUpdate();
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("Created features table"); logger.debug("Created features table");
} }
//CREATE SPATIAL INDEX GEO_TABLE_SPATIAL_INDEX ON GEO_TABLE(THE_GEOM); <- Create spatial id
} }
} catch (SQLException e) { } catch (SQLException e) {
...@@ -93,14 +93,13 @@ public class DatabaseHandler { ...@@ -93,14 +93,13 @@ public class DatabaseHandler {
*/ */
public void marshallCityObject(CityObject co) { public void marshallCityObject(CityObject co) {
try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) { try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) {
try (PreparedStatement ps = con.prepareStatement(String.format("MERGE INTO %s VALUES (?, ?, ?, ?)", fileName))) { try (PreparedStatement ps = con.prepareStatement(String.format("MERGE INTO %s VALUES (?, ?, ?)", fileName))) {
ps.setString(1, co.getGmlId().toString()); ps.setString(1, co.getGmlId().toString());
String bbox = BoundingBox.of(co).to2DWkt(); String bbox = BoundingBox.of(co).to2DWkt();
ps.setString(2, bbox); ps.setString(2, bbox);
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("Attempting to serialize CityObject: {} ", co); logger.debug("Attempting to serialize CityObject: {} ", co);
} }
ps.setString(4, co.getClass().getName());
try { try {
byte[] bytes = SerializationUtils.serialize(co); byte[] bytes = SerializationUtils.serialize(co);
ByteArrayInputStream bis = new ByteArrayInputStream(bytes); ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
...@@ -118,6 +117,47 @@ public class DatabaseHandler { ...@@ -118,6 +117,47 @@ public class DatabaseHandler {
} }
} }
public boolean tryMarshallCityObjectMap(Map<GmlId, CityObject> map) {
// Id String for error logging
String id = "";
try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) {
try (PreparedStatement ps = con.prepareStatement(String.format("MERGE INTO %s VALUES (?, ?, ?)", fileName))) {
con.setAutoCommit(false);
for (Map.Entry<GmlId, CityObject> entry : map.entrySet()) {
CityObject co = entry.getValue();
id = entry.getKey().toString();
ps.setString(1, id);
String bbox = BoundingBox.of(co).to2DWkt();
ps.setString(2, bbox);
if (logger.isTraceEnabled()) {
logger.trace("Attempting to serialize CityObject: {} ", co);
}
try {
byte[] bytes = SerializationUtils.serialize(co);
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
ps.setBinaryStream(3, bis, bytes.length);
ps.addBatch();
} catch (SerializationException e) {
logger.error("Serialization of CityObject {} failed",co.getGmlId());
logger.error(e.getMessage());
}
}
ps.executeBatch();
con.commit();
} catch (SQLException e) {
con.rollback();
logger.error("Batch-commit aborted: Saving of CityObject {} to embedded database critically failed!", id);
logger.error(e.getMessage());
return false;
}
} catch (SQLException e){
logger.error("Connection to embedded database could not be opened!");
logger.error(e.getMessage());
return false;
}
return true;
}
/** /**
* Retrieves a CityObject from the embedded database. * Retrieves a CityObject from the embedded database.
* @param id the gmlId of the requested Feature * @param id the gmlId of the requested Feature
...@@ -126,7 +166,7 @@ public class DatabaseHandler { ...@@ -126,7 +166,7 @@ public class DatabaseHandler {
public CityObject unmarshallCityObject(GmlId id) { public CityObject unmarshallCityObject(GmlId id) {
try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) { try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) {
try (PreparedStatement ps = con.prepareStatement(String.format( try (PreparedStatement ps = con.prepareStatement(String.format(
"SELECT gmlid, bbox, data, type FROM %s WHERE gmlid = ?", fileName))) { "SELECT gmlid, bbox, data FROM %s WHERE gmlid = ?", fileName))) {
ps.setString(1, id.toString()); ps.setString(1, id.toString());
ResultSet rs = ps.executeQuery(); ResultSet rs = ps.executeQuery();
if (rs.next()) { if (rs.next()) {
...@@ -156,7 +196,7 @@ public class DatabaseHandler { ...@@ -156,7 +196,7 @@ public class DatabaseHandler {
public List<CityObject> getAllCityObjects() { public List<CityObject> getAllCityObjects() {
List<CityObject> cos = new ArrayList<>(); List<CityObject> cos = new ArrayList<>();
try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) { try (Connection con = new ConnectionWrapper(H2GISDBFactory.openSpatialDataBase(DB_NAME))) {
try (PreparedStatement ps = con.prepareStatement(String.format("SELECT gmlid, bbox, data, type FROM %s", fileName))) { try (PreparedStatement ps = con.prepareStatement(String.format("SELECT gmlid, bbox, data FROM %s", fileName))) {
ResultSet rs = ps.executeQuery(); ResultSet rs = ps.executeQuery();
cos.addAll(retrieveCityObjects(rs)); cos.addAll(retrieveCityObjects(rs));
} }
......
...@@ -6,10 +6,13 @@ import com.github.benmanes.caffeine.cache.RemovalCause; ...@@ -6,10 +6,13 @@ import com.github.benmanes.caffeine.cache.RemovalCause;
import de.hft.stuttgart.citydoctor2.datastructure.CityObject; import de.hft.stuttgart.citydoctor2.datastructure.CityObject;
import de.hft.stuttgart.citydoctor2.datastructure.GmlId; import de.hft.stuttgart.citydoctor2.datastructure.GmlId;
import de.hft.stuttgart.citydoctor2.parser.InvalidGmlFileException; import de.hft.stuttgart.citydoctor2.parser.InvalidGmlFileException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream; import java.util.stream.Stream;
...@@ -17,9 +20,15 @@ import java.util.stream.Stream; ...@@ -17,9 +20,15 @@ import java.util.stream.Stream;
* Cache for CityObjects * Cache for CityObjects
*/ */
public class FeatureCache implements CityObjectCache{ public class FeatureCache implements CityObjectCache{
private static final Logger logger = LogManager.getLogger(FeatureCache.class);
private final LoadingCache<GmlId, CityObject> cache; private final LoadingCache<GmlId, CityObject> cache;
private DatabaseHandler handler; private DatabaseHandler handler;
private final Map<String, DatabaseHandler> handlerList = new HashMap<>(); private final Map<String, DatabaseHandler> handlerList = new HashMap<>();
private AtomicInteger putsCounter = new AtomicInteger(0);
private boolean parsingFinished = false;
FeatureCache(String fileName) { FeatureCache(String fileName) {
handler = new DatabaseHandler(fileName); handler = new DatabaseHandler(fileName);
...@@ -33,13 +42,56 @@ public class FeatureCache implements CityObjectCache{ ...@@ -33,13 +42,56 @@ public class FeatureCache implements CityObjectCache{
@Override @Override
public void put(CityObject cityObject) { public void put(CityObject cityObject) {
handler.marshallCityObject(cityObject);
cache.put(cityObject.getGmlId(), cityObject); cache.put(cityObject.getGmlId(), cityObject);
/* Batch marshall during parsing to improve performance
*/
if (!parsingFinished){
int count = putsCounter.incrementAndGet();
if (count >= 2500) {
batchMarshall();
}
}
}
private void batchMarshall(){
logger.trace("Setting up batch Marshaller");
Map<GmlId, CityObject> cacheContent= new HashMap<>(cache.asMap());
cache.invalidateAll();
putsCounter.set(0);
Thread batcher = new Thread(() -> {
logger.trace("Entered batch Marshaller");
boolean success = handler.tryMarshallCityObjectMap(cacheContent);
if (!success) {
logger.trace("Batch marshalling failed, falling back to one-by-one marshalling");
// Batch marshalling failed, marshall them one-by-one
for (Map.Entry<GmlId, CityObject> entry : cacheContent.entrySet()) {
handler.marshallCityObject(entry.getValue());
}
}
logger.trace("Batch marshalling done");
});
batcher.start();
logger.trace("Batch Marshaller started");
} }
@Override @Override
public CityObject get(GmlId id) { public CityObject get(GmlId id) {
return cache.get(id); CityObject result = cache.get(id);
if (result == null){
logger.trace("Cache-value missing for {}", id);
//Key has null value, try to restore from Database
result = handler.unmarshallCityObject(id);
if (result == null){
logger.error("CityObject with id {} is null in cache and database", id);
} else {
cache.put(id, result);
}
}
return result;
} }
public void changeFileName(String fileName){ public void changeFileName(String fileName){
...@@ -62,20 +114,23 @@ public class FeatureCache implements CityObjectCache{ ...@@ -62,20 +114,23 @@ public class FeatureCache implements CityObjectCache{
} }
@Override @Override
public int getFeatureCount() { public int getTotalFeatureCount() {
return handler.getFeatureCount(); return handler.getFeatureCount();
} }
@Override @Override
public void resetCache(){ public void resetCache(){
logger.trace("Resetting cache");
cache.invalidateAll(); cache.invalidateAll();
putsCounter.set(0);
parsingFinished = false;
} }
@Override @Override
public List<CityObject> retrieveLibraryObject(String fileName) throws InvalidGmlFileException { public List<CityObject> retrieveLibraryObject(String fileName) throws InvalidGmlFileException {
String prevHandler = handler.getFileName(); String prevHandler = handler.getFileName();
this.changeFileName(fileName); this.changeFileName(fileName);
int featureCount = getFeatureCount(); int featureCount = getTotalFeatureCount();
if (featureCount != 1){ if (featureCount != 1){
if (featureCount == 0){ if (featureCount == 0){
throw new InvalidGmlFileException(String.format("LibraryObject ZIP-entry %s contains no Feature", fileName)); throw new InvalidGmlFileException(String.format("LibraryObject ZIP-entry %s contains no Feature", fileName));
...@@ -88,5 +143,20 @@ public class FeatureCache implements CityObjectCache{ ...@@ -88,5 +143,20 @@ public class FeatureCache implements CityObjectCache{
return objects; return objects;
} }
public void finishParsing() {
if (!parsingFinished) {
HashMap<GmlId, CityObject> cacheContent= new HashMap<>(cache.asMap());
boolean success = handler.tryMarshallCityObjectMap(cacheContent);
if (!success) {
logger.trace("Batch marshalling failed, falling back to one-by-one marshalling");
// Batch marshalling failed, marshall them one-by-one
for (Map.Entry<GmlId, CityObject> entry : cacheContent.entrySet()) {
handler.marshallCityObject(entry.getValue());
}
}
parsingFinished = true;
}
}
} }
...@@ -41,7 +41,7 @@ public class UnconnectedCache implements CityObjectCache{ ...@@ -41,7 +41,7 @@ public class UnconnectedCache implements CityObjectCache{
} }
@Override @Override
public int getFeatureCount() { public int getTotalFeatureCount() {
return cache.size(); return cache.size();
} }
......
...@@ -27,7 +27,6 @@ import de.hft.stuttgart.citydoctor2.exceptions.CityDoctorWriteException; ...@@ -27,7 +27,6 @@ import de.hft.stuttgart.citydoctor2.exceptions.CityDoctorWriteException;
import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration; import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
import de.hft.stuttgart.citydoctor2.writer.CityGMLWriterUtils; import de.hft.stuttgart.citydoctor2.writer.CityGMLWriterUtils;
import de.hft.stuttgart.quality.model.types.ValidationPlan; import de.hft.stuttgart.quality.model.types.ValidationPlan;
import org.apache.commons.io.FilenameUtils;
import org.citygml4j.core.model.CityGMLVersion; import org.citygml4j.core.model.CityGMLVersion;
import org.citygml4j.core.model.core.CityModel; import org.citygml4j.core.model.core.CityModel;
...@@ -131,6 +130,53 @@ public class CityDoctorModel { ...@@ -131,6 +130,53 @@ public class CityDoctorModel {
getTunnels(), getWater(), getCityFurniture(), getGenericCityObjects()).flatMap(co -> co); getTunnels(), getWater(), getCityFurniture(), getGenericCityObjects()).flatMap(co -> co);
} }
public List<GmlId> getFeatureIds() {
List<GmlId> featureIds = new ArrayList<>();
featureIds.addAll(buildings);
featureIds.addAll(vegetation);
featureIds.addAll(bridges);
featureIds.addAll(land);
featureIds.addAll(roads);
featureIds.addAll(tunnels);
featureIds.addAll(water);
featureIds.addAll(cityfurniture);
featureIds.addAll(genericObjects);
return featureIds;
}
public boolean containsGmlId(String id) {
if (id == null) {
return false;
}
GmlId gmlId = new GmlId(id);
if (buildings.contains(gmlId)) {
return true;
}
if (vegetation.contains(gmlId)) {
return true;
}
if (bridges.contains(gmlId)) {
return true;
}
if (land.contains(gmlId)) {
return true;
}
if (roads.contains(gmlId)) {
return true;
}
if (tunnels.contains(gmlId)) {
return true;
}
if (water.contains(gmlId)) {
return true;
}
if (cityfurniture.contains(gmlId)) {
return true;
}
return genericObjects.contains(gmlId);
}
public void accept(CheckableVisitor visitor) { public void accept(CheckableVisitor visitor) {
createFeatureStream().forEach(cityObject -> cityObject.accept(visitor)); createFeatureStream().forEach(cityObject -> cityObject.accept(visitor));
} }
...@@ -260,6 +306,10 @@ public class CityDoctorModel { ...@@ -260,6 +306,10 @@ public class CityDoctorModel {
return buildings.stream().map(co -> (Building) cache.get(co)); return buildings.stream().map(co -> (Building) cache.get(co));
} }
public List<GmlId> getBuildingIds() {
return buildings;
}
public int getBuildingCount(){ public int getBuildingCount(){
return buildings.size(); return buildings.size();
} }
...@@ -273,6 +323,10 @@ public class CityDoctorModel { ...@@ -273,6 +323,10 @@ public class CityDoctorModel {
return bridges.stream().map(co -> (BridgeObject) cache.get(co)); return bridges.stream().map(co -> (BridgeObject) cache.get(co));
} }
public List<GmlId> getBridgeIds() {
return bridges;
}
public int getBridgeCount(){ public int getBridgeCount(){
return bridges.size(); return bridges.size();
} }
...@@ -286,6 +340,10 @@ public class CityDoctorModel { ...@@ -286,6 +340,10 @@ public class CityDoctorModel {
return cityfurniture.stream().map(co -> (CityFurniture) cache.get(co)); return cityfurniture.stream().map(co -> (CityFurniture) cache.get(co));
} }
public List<GmlId> getCityFurnitureIds() {
return cityfurniture;
}
public int getCityFurnitureCount(){ public int getCityFurnitureCount(){
return cityfurniture.size(); return cityfurniture.size();
} }
...@@ -299,6 +357,10 @@ public class CityDoctorModel { ...@@ -299,6 +357,10 @@ public class CityDoctorModel {
return genericObjects.stream().map(co -> (GenericCityObject) cache.get(co)); return genericObjects.stream().map(co -> (GenericCityObject) cache.get(co));
} }
public List<GmlId> getGenericCityObjectIds() {
return genericObjects;
}
public int getGenericCityObjectCount(){ public int getGenericCityObjectCount(){
return genericObjects.size(); return genericObjects.size();
} }
...@@ -312,6 +374,10 @@ public class CityDoctorModel { ...@@ -312,6 +374,10 @@ public class CityDoctorModel {
return roads.stream().map(co -> (TransportationObject) cache.get(co)); return roads.stream().map(co -> (TransportationObject) cache.get(co));
} }
public List<GmlId> getTransportationIds() {
return roads;
}
public int getTransportationCount(){ public int getTransportationCount(){
return roads.size(); return roads.size();
} }
...@@ -325,6 +391,10 @@ public class CityDoctorModel { ...@@ -325,6 +391,10 @@ public class CityDoctorModel {
return tunnels.stream().map(co -> (Tunnel) cache.get(co)); return tunnels.stream().map(co -> (Tunnel) cache.get(co));
} }
public List<GmlId> getTunnelIds() {
return tunnels;
}
public int getTunnelCount(){ public int getTunnelCount(){
return tunnels.size(); return tunnels.size();
} }
...@@ -338,6 +408,10 @@ public class CityDoctorModel { ...@@ -338,6 +408,10 @@ public class CityDoctorModel {
return water.stream().map(co -> (WaterObject) cache.get(co)); return water.stream().map(co -> (WaterObject) cache.get(co));
} }
public List<GmlId> getWaterIds() {
return water;
}
public int getWaterCount(){ public int getWaterCount(){
return water.size(); return water.size();
} }
...@@ -361,6 +435,10 @@ public class CityDoctorModel { ...@@ -361,6 +435,10 @@ public class CityDoctorModel {
return land.stream().map(cache::get); return land.stream().map(cache::get);
} }
public List<GmlId> getLandIds() {
return land;
}
public int getLandCount(){ public int getLandCount(){
return land.size(); return land.size();
} }
...@@ -374,6 +452,10 @@ public class CityDoctorModel { ...@@ -374,6 +452,10 @@ public class CityDoctorModel {
return vegetation.stream().map(co -> (Vegetation) cache.get(co)); return vegetation.stream().map(co -> (Vegetation) cache.get(co));
} }
public List<GmlId> getVegetationIds() {
return vegetation;
}
public int getVegetationCount(){ public int getVegetationCount(){
return vegetation.size(); return vegetation.size();
} }
......
...@@ -30,7 +30,6 @@ import java.nio.file.Paths; ...@@ -30,7 +30,6 @@ import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
...@@ -45,6 +44,7 @@ import javax.xml.validation.Schema; ...@@ -45,6 +44,7 @@ import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory; import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator; import javax.xml.validation.Validator;
import de.hft.stuttgart.citydoctor2.database.CacheManager;
import de.hft.stuttgart.citydoctor2.datastructure.GmlId; import de.hft.stuttgart.citydoctor2.datastructure.GmlId;
import de.hft.stuttgart.citydoctor2.datastructure.ImplicitGeometryHolder; import de.hft.stuttgart.citydoctor2.datastructure.ImplicitGeometryHolder;
import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Level;
...@@ -572,6 +572,9 @@ public class CityGmlParser { ...@@ -572,6 +572,9 @@ public class CityGmlParser {
// object members are replaced by href in model // object members are replaced by href in model
// need to remove the refs and re-add unparsed objects // need to remove the refs and re-add unparsed objects
List<AbstractCityObject> acos = new ArrayList<>(); List<AbstractCityObject> acos = new ArrayList<>();
CityModel cgmlModel = null;
CityDoctorModel cdModel = mapper.getModel();
List<AbstractCityObject> queuedObjects = new ArrayList<>();
while (reader.hasNext()) { while (reader.hasNext()) {
AbstractFeature chunk = reader.next(); AbstractFeature chunk = reader.next();
version = CityGMLModules.getCityGMLVersion(reader.getName().getNamespaceURI()); version = CityGMLModules.getCityGMLVersion(reader.getName().getNamespaceURI());
...@@ -579,39 +582,65 @@ public class CityGmlParser { ...@@ -579,39 +582,65 @@ public class CityGmlParser {
cModel.setCityObjectMembers(null); cModel.setCityObjectMembers(null);
mapper.setCityModel(cModel); mapper.setCityModel(cModel);
mapper.setCityGMLVersion(version); mapper.setCityGMLVersion(version);
cgmlModel = cModel;
if (!queuedObjects.isEmpty()) {
for (AbstractCityObject aco : queuedObjects) {
if (!cdModel.containsGmlId(aco.getId())) {
cgmlModel.getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
}
}
}
} else if (chunk instanceof AbstractCityObject aco) { } else if (chunk instanceof AbstractCityObject aco) {
acos.add(aco); acos.add(aco); // getid
String id = aco.getId();
aco.accept(mapper); aco.accept(mapper);
if (cgmlModel == null) {
// Add object to queue till the CityModel has been parsed
queuedObjects.add(aco);
}else if (!cdModel.containsGmlId(id)){
cgmlModel.getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
}
// TODO: add check if aco was parsed successfully, add aco to cmodel if it wasnt
} }
} }
if (mapper.getModel().getCityModel() == null) { if (mapper.getModel().getCityModel() == null) {
// file does not contain a city model? // file does not contain a city model?
// create it for now // create it for now
logger.warn("File does not contain a CityModel definition! Temporary CityModel has been created.");
mapper.setCityModel(new CityModel()); mapper.setCityModel(new CityModel());
for (AbstractCityObject aco : queuedObjects) {
if (!cdModel.containsGmlId(aco.getId())) {
mapper.getModel().getCityModel().getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
}
}
} }
CityModel cModel = mapper.getModel().getCityModel(); // cgmlModel = mapper.getModel().getCityModel();
//
// remove those that should have been parsed // // remove those that should have been parsed
logger.debug("Getting parsed object from CityDoctorModel"); // if (logger.isInfoEnabled() && verbose){
List<AbstractCityObject> parsedCityObjects = mapper.getModel().createFeatureStream().filter(Objects::nonNull) // logger.info("Resolving HRefs");
.map(CityObject::getGmlObject).toList(); // }
logger.debug("Removing all parsed city objects from acos"); //// List<AbstractCityObject> parsedCityObjects = mapper.getModel().createFeatureStream().filter(Objects::nonNull)
acos.removeAll(parsedCityObjects); //// .map(CityObject::getGmlObject).toList();
// re-add all not parsed objects //// mapper.getModel().createFeatureStream().filter(Objects::nonNull)
if (logger.isInfoEnabled() && verbose){ //// .map(CityObject::getGmlObject).forEach(acos::remove);
logger.info("Resolving HRefs"); // mapper.getModel().getFeatureIds().forEach(id -> {acos.removeIf(aco -> aco.getId().equals(id.getGmlString()));});
} //// acos.removeAll(parsedCityObjects);
for (AbstractCityObject aco : acos) { // // re-add all not parsed objects
cModel.getCityObjectMembers().add(new AbstractCityObjectProperty(aco)); // for (AbstractCityObject aco : acos) {
} // cgmlModel.getCityObjectMembers().add(new AbstractCityObjectProperty(aco));
// }
// After parsing is complete, try to resolve deferred RelativeGeometries
ImplicitGeometryHolder.resolveDeferredObjects(mapper.getModel().getCache());
if (logger.isInfoEnabled() && verbose) { if (logger.isInfoEnabled() && verbose) {
logger.info(Localization.getText("CityGmlParser.parsedObjects"), logger.info(Localization.getText("CityGmlParser.parsedObjects"),
mapper.getModel().getNumberOfFeatures()); mapper.getModel().getNumberOfFeatures());
} }
// After parsing is complete, try to resolve deferred RelativeGeometries
ImplicitGeometryHolder.resolveDeferredObjects(mapper.getModel().getCache());
mapper.setCityGMLVersion(version); mapper.setCityGMLVersion(version);
CacheManager.finishParsing();
return mapper.getModel(); return mapper.getModel();
} }
} }
......
package de.hft.stuttgart.citydoctor2.database; package de.hft.stuttgart.citydoctor2.database;
import de.hft.stuttgart.citydoctor2.datastructure.Building; import de.hft.stuttgart.citydoctor2.datastructure.*;
import de.hft.stuttgart.citydoctor2.datastructure.GeometryTestUtils;
import de.hft.stuttgart.citydoctor2.datastructure.GeometryType;
import de.hft.stuttgart.citydoctor2.datastructure.Lod;
import org.citygml4j.core.model.building.AbstractBuilding; import org.citygml4j.core.model.building.AbstractBuilding;
import org.junit.Test; import org.junit.Test;
...@@ -19,11 +16,26 @@ public class FeatureCacheTest { ...@@ -19,11 +16,26 @@ public class FeatureCacheTest {
FeatureCache cache = new FeatureCache("test"); FeatureCache cache = new FeatureCache("test");
Building b = new Building(); Building b = new Building();
b.setGmlObject(mock(AbstractBuilding.class)); b.setGmlObject(mock(AbstractBuilding.class));
b.setGmlId(new GmlId("test"));
b.addGeometry(GeometryTestUtils.createDummyGeometry(GeometryType.SOLID, Lod.LOD0)); b.addGeometry(GeometryTestUtils.createDummyGeometry(GeometryType.SOLID, Lod.LOD0));
assertEquals(0, cache.getFeatureCount()); assertEquals(0, cache.getTotalFeatureCount());
cache.put(b); cache.put(b);
assertEquals(1, cache.getFeatureCount()); cache.finishParsing();
assertEquals(1, cache.getTotalFeatureCount());
}
@Test
public void testGet(){
FeatureCache cache = new FeatureCache("test");
DatabaseHandler handler = new DatabaseHandler("test");
Building b = new Building();
b.setGmlId(new GmlId("building"));
b.addGeometry(GeometryTestUtils.createDummyGeometry(GeometryType.SOLID, Lod.LOD0));
handler.marshallCityObject(b);
CityObject b2 = cache.get(new GmlId("building"));
assertEquals(b.getGmlId(), b2.getGmlId());
} }
......
...@@ -44,6 +44,7 @@ import de.hft.stuttgart.citydoctor2.datastructure.Vertex; ...@@ -44,6 +44,7 @@ import de.hft.stuttgart.citydoctor2.datastructure.Vertex;
import de.hft.stuttgart.citydoctor2.datastructure.Geometry.Orientation; import de.hft.stuttgart.citydoctor2.datastructure.Geometry.Orientation;
import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration; import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
import de.hft.stuttgart.quality.model.enums.RequirementId; import de.hft.stuttgart.quality.model.enums.RequirementId;
import org.mockito.Mockito;
public class DegeneratedPolygonCheckTest { public class DegeneratedPolygonCheckTest {
...@@ -68,9 +69,10 @@ public class DegeneratedPolygonCheckTest { ...@@ -68,9 +69,10 @@ public class DegeneratedPolygonCheckTest {
Building b = new Building(); Building b = new Building();
b.addGeometry(geom); b.addGeometry(geom);
File file = Mockito.mock(File.class);
Mockito.when(file.getName()).thenReturn("building");
ParserConfiguration config = new ParserConfiguration(8, false); ParserConfiguration config = new ParserConfiguration(8, false);
CityDoctorModel model = new CityDoctorModel(config, new File("")); CityDoctorModel model = new CityDoctorModel(config, file);
model.addBuilding(b); model.addBuilding(b);
ValidationConfiguration valConfig = ValidationConfiguration.loadStandardValidationConfig(); ValidationConfiguration valConfig = ValidationConfiguration.loadStandardValidationConfig();
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment