Commit d1bfb680 authored by Riegel's avatar Riegel
Browse files

Refactor: Rework parsing to use Streams instead of tempDir

2 merge requests!28Version 3.17.0 Release,!26Add ZIP-archive support
Showing with 30 additions and 41 deletions
+30 -41
package de.hft.stuttgart.citydoctor2.ziploader; package de.hft.stuttgart.citydoctor2.ziploader;
import de.hft.stuttgart.citydoctor2.check.Checker; import de.hft.stuttgart.citydoctor2.check.Checker;
import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel;
import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration; import de.hft.stuttgart.citydoctor2.parser.ParserConfiguration;
import de.hft.stuttgart.citydoctor2.ziploader.utils.ArchivePacker; import de.hft.stuttgart.citydoctor2.ziploader.utils.ArchivePacker;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.Serial; import java.io.Serial;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipFile; import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
...@@ -32,6 +26,7 @@ public class CityGmlArchive { ...@@ -32,6 +26,7 @@ public class CityGmlArchive {
private static final long serialVersionUID = 2168389511043362615L; private static final long serialVersionUID = 2168389511043362615L;
private final List<CityGmlZipEntry> entries; private final List<CityGmlZipEntry> entries;
private final Path archivePath;
public static CityGmlArchive fromZipFile(String zipFile, ParserConfiguration config) { public static CityGmlArchive fromZipFile(String zipFile, ParserConfiguration config) {
ArrayList<CityGmlZipEntry> archiveEntries = new ArrayList<>(); ArrayList<CityGmlZipEntry> archiveEntries = new ArrayList<>();
...@@ -60,11 +55,13 @@ public class CityGmlArchive { ...@@ -60,11 +55,13 @@ public class CityGmlArchive {
} }
} }
logger.info("Zip-archive successfully uncompressed, {} CityGml files detected", archiveEntries.size()); logger.info("Zip-archive successfully uncompressed, {} CityGml files detected", archiveEntries.size());
return new CityGmlArchive(archiveEntries); return new CityGmlArchive(archiveEntries, Path.of(zipFile));
} }
private CityGmlArchive(List<CityGmlZipEntry> entries) { private CityGmlArchive(List<CityGmlZipEntry> entries, Path archivePath) {
this.entries = entries; this.entries = entries;
entries.forEach(e -> e.setArchive(this));
this.archivePath = archivePath;
} }
public void exportToZipFile(String path) { public void exportToZipFile(String path) {
...@@ -78,6 +75,10 @@ public class CityGmlArchive { ...@@ -78,6 +75,10 @@ public class CityGmlArchive {
} }
} }
public Path getArchivePath() {
return archivePath;
}
public List<CityGmlZipEntry> getEntries() { public List<CityGmlZipEntry> getEntries() {
return entries; return entries;
} }
......
...@@ -18,26 +18,22 @@ public class CityGmlZipEntry { ...@@ -18,26 +18,22 @@ public class CityGmlZipEntry {
private static final Logger logger = LogManager.getLogger(CityGmlZipEntry.class); private static final Logger logger = LogManager.getLogger(CityGmlZipEntry.class);
private String fileName; private final String fileName;
private String archivePath; private final CityDoctorModel model;
private CityDoctorModel model; private CityGmlArchive parentArchive;
private boolean erroneousEntry = false;
private boolean isLibraryObject = false;
private boolean validated = false; private boolean validated = false;
private ZipEntryErrorType errorType = null; private ZipEntryErrorType errorType = null;
public static CityGmlZipEntry of(ZipEntry entry, ParserConfiguration config, String archivePath, Path tempDir, InputStream is){
CityGmlZipEntry cgzEntry = new CityGmlZipEntry(); public static CityGmlZipEntry of(ZipEntry entry, ZipFile archive, ParserConfiguration config){
cgzEntry.fileName = entry.getName();
cgzEntry.archivePath = archivePath;
if ((double) entry.getSize() /1024/1024 > 20) { if ((double) entry.getSize() /1024/1024 > 20) {
return new ErroneousEntry(entry, ZipEntryErrorType.EXCESSIVE_FILESIZE); return new ErroneousEntry(entry, ZipEntryErrorType.EXCESSIVE_FILESIZE);
} else { } else {
try { try {
Files.copy(is, Path.of(tempDir.toString(), entry.getName())); ZipInputStream zis = new ZipInputStream(archive.getInputStream(entry));
CityGmlParser.gagLogger(true); CityGmlParser.gagLogger(true);
cgzEntry.model = CityGmlParser.parseCityGmlFile(tempDir.toString() + File.separator + entry.getName(), config); CityDoctorModel model = CityGmlParser.parseCityGmlStream(zis, config);
return new CityGmlZipEntry(entry.getName(), model);
} catch (CityGmlParseException | InvalidGmlFileException e) { } catch (CityGmlParseException | InvalidGmlFileException e) {
logger.error(e); logger.error(e);
return new ErroneousEntry(entry, ZipEntryErrorType.INVALID_CITY_GML_FILE); return new ErroneousEntry(entry, ZipEntryErrorType.INVALID_CITY_GML_FILE);
...@@ -46,7 +42,6 @@ public class CityGmlZipEntry { ...@@ -46,7 +42,6 @@ public class CityGmlZipEntry {
return new ErroneousEntry(entry, ZipEntryErrorType.IO_ERROR); return new ErroneousEntry(entry, ZipEntryErrorType.IO_ERROR);
} }
} }
return cgzEntry;
} }
private CityGmlZipEntry(String filename, CityDoctorModel model){ private CityGmlZipEntry(String filename, CityDoctorModel model){
...@@ -58,6 +53,15 @@ public class CityGmlZipEntry { ...@@ -58,6 +53,15 @@ public class CityGmlZipEntry {
this.fileName = entry.getName(); this.fileName = entry.getName();
this.model = null; this.model = null;
} }
public void setArchive(CityGmlArchive archive){
parentArchive = archive;
}
public CityGmlArchive getArchive(){
return parentArchive;
}
public void validateModel(){ public void validateModel(){
Checker checker = new Checker(this.getModel()); Checker checker = new Checker(this.getModel());
checker.runChecks(); checker.runChecks();
......
...@@ -3,6 +3,7 @@ package de.hft.stuttgart.citydoctor2.ziploader.utils; ...@@ -3,6 +3,7 @@ package de.hft.stuttgart.citydoctor2.ziploader.utils;
import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel; import de.hft.stuttgart.citydoctor2.datastructure.CityDoctorModel;
import de.hft.stuttgart.citydoctor2.ziploader.CityGmlArchive; import de.hft.stuttgart.citydoctor2.ziploader.CityGmlArchive;
import de.hft.stuttgart.citydoctor2.ziploader.CityGmlZipEntry; import de.hft.stuttgart.citydoctor2.ziploader.CityGmlZipEntry;
import de.hft.stuttgart.citydoctor2.ziploader.ErroneousEntry;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
...@@ -31,7 +32,7 @@ public class ArchivePacker { ...@@ -31,7 +32,7 @@ public class ArchivePacker {
tmpDir = Files.createTempDirectory("zipTmp"); tmpDir = Files.createTempDirectory("zipTmp");
tmpDir.toFile().deleteOnExit(); tmpDir.toFile().deleteOnExit();
for (CityGmlZipEntry entry : archive.getEntries()) { for (CityGmlZipEntry entry : archive.getEntries()) {
if (entry.isErroneousEntry()){ if (entry instanceof ErroneousEntry){
continue; continue;
} }
CityDoctorModel model = entry.getModel(); CityDoctorModel model = entry.getModel();
......
...@@ -5,13 +5,10 @@ import org.apache.commons.io.FileUtils; ...@@ -5,13 +5,10 @@ import org.apache.commons.io.FileUtils;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.mockito.Mockito;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.zip.ZipEntry;
import static org.junit.Assert.*; import static org.junit.Assert.*;
...@@ -42,20 +39,6 @@ public class ZipTest { ...@@ -42,20 +39,6 @@ public class ZipTest {
cgmlArch.exportToZipFile("src/test/resources/mock"); cgmlArch.exportToZipFile("src/test/resources/mock");
} }
@Test
public void testExcessiveFilesize(){
ZipEntry zipEntry = Mockito.mock(ZipEntry.class);
Path mockPath = Mockito.mock(Path.class);
InputStream mockIs = Mockito.mock(InputStream.class);
// Mock filesize of 30 Gb
Mockito.when(zipEntry.getSize()).thenReturn(1024L*1024L*30);
Mockito.when(zipEntry.getName()).thenReturn("mock.gml");
CityGmlZipEntry cgmlZE = CityGmlZipEntry.of(zipEntry, config, "test", mockPath, mockIs);
assertNotNull(cgmlZE);
assertTrue(cgmlZE.isErroneousEntry());
assertEquals(CityGmlZipEntry.ZipEntryErrorType.EXCESSIVE_FILESIZE, cgmlZE.getErrorType());
}
@Test @Test
public void testZipping() throws IOException { public void testZipping() throws IOException {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment