Merge branch 'feat-dtt-transition' into 'master'

Feat dtt transition

See merge request HFTSoftwareProject/MoDoCoT-Backend!3

Dockerfile

@@ -3,16 +3,15 @@
 #base image
 FROM openjdk:11-jre-slim
 
-ADD target/modocot-backend.jar app.jar
+ADD target/dtt-backend.jar app.jar
 
 # Prepare environment.
 # Create needed folders
-RUN mkdir /modocot && \
-    mkdir /modocot/data && \
-    mkdir /modocot/config
+RUN mkdir /data && \
+    mkdir /data/config
 
-VOLUME /modocot/data
+VOLUME /data
 
-env SPRING_CONFIG_ADDITIONAL_LOCATION "file:/home/modocot/config/"
+env SPRING_CONFIG_ADDITIONAL_LOCATION "file:/data/config/"
 
 ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"]

Jenkinsfile

@@ -4,7 +4,7 @@ def version = ""
 pipeline {
 
     environment {
-        registry = "hftstuttgart/modocot-backend"
+        registry = "hftstuttgart/dtt-backend"
         registryCredential = 'Dockerhub'
         dockerImage = ''
     }

pom.xml

@@ -3,11 +3,11 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>de.hftstuttgart</groupId>
-    <artifactId>modocot-backend</artifactId>
+    <artifactId>dtt-backend</artifactId>
     <version>${buildNumber}</version>
     <packaging>jar</packaging>
 
-    <name>Modocot-Backend</name>
+    <name>Dockerized-Testing-Toolkit-Backend</name>
 
     <parent>
         <groupId>org.springframework.boot</groupId>
@@ -59,7 +59,7 @@
         <dependency>
             <groupId>org.eclipse.jgit</groupId>
             <artifactId>org.eclipse.jgit</artifactId>
-            <version>5.7.0.202003110725-r</version>
+            <version>5.10.0.202012080955-r</version>
         </dependency>
         <dependency>
             <groupId>log4j</groupId>

src/main/java/de/hftstuttgart/ModocotBackendApplication.java → src/main/java/de/hftstuttgart/Application.java

@@ -4,10 +4,11 @@ import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 
 @SpringBootApplication
-public class ModocotBackendApplication {
+public class Application
+{
 
     public static void main(String[] args) {
-        SpringApplication.run(ModocotBackendApplication.class, args);
+        SpringApplication.run(Application.class, args);
     }
 
 }

src/main/java/de/hftstuttgart/models/LegacyMoodleResult.java

-package de.hftstuttgart.models;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class LegacyMoodleResult {
-
-    public List<LegacyMoodleTestResult> testResults;
-    public List<LegacyMoodleCompilationError> compilationErrors;
-
-    public static LegacyMoodleResult convertToModdleResult(ModocotResultSummary summary) {
-        LegacyMoodleTestResult tests = new LegacyMoodleTestResult();
-        tests.failureCount = summary.failureCount;
-        tests.testCount = summary.testCount;
-        tests.successfulTests = summary.successes.stream()
-            .map(s -> s.name)
-            .collect(Collectors.toList());
-        tests.testFailures = summary.failures.stream()
-            .map(f -> {
-                LegacyMoodleTestFailure failure = new LegacyMoodleTestFailure();
-                failure.testHeader = f.name;
-                failure.message = f.failureReason;
-                failure.trace = f.stacktrace;
-
-                return failure;
-            })
-            .collect(Collectors.toList());
-
-        List<LegacyMoodleCompilationError> compilationErrors;
-        compilationErrors = summary.compilationErrors.stream()
-            .map(f -> {
-                LegacyMoodleCompilationError cError = new LegacyMoodleCompilationError();
-                cError.javaFileName = f.name;
-                cError.message = f.failureReason;
-                cError.lineNumber = f.lineNumber;
-                cError.columnNumber = f.columnNumber;
-                cError.position = f.position;
-
-                return cError;
-            })
-            .collect(Collectors.toList());
-
-        LegacyMoodleResult result = new LegacyMoodleResult();
-        result.testResults = Collections.singletonList(tests);
-        result.compilationErrors = compilationErrors;
-
-        return result;
-    }
-
-    private static class LegacyMoodleTestResult {
-        public String testName = "UnitTests";
-        public int testCount;
-        public int failureCount;
-        public List<String> successfulTests;
-        public List<LegacyMoodleTestFailure> testFailures;
-    }
-
-    private static class LegacyMoodleTestFailure {
-        public String testHeader;
-        public String message;
-        public String trace;
-    }
-
-    private static class LegacyMoodleCompilationError {
-        public String message;
-        public String javaFileName;
-        public int lineNumber;
-        public int columnNumber;
-        public int position;
-    }
-}

src/main/java/de/hftstuttgart/models/ModocotResultSummary.java

-package de.hftstuttgart.models;
-
-import java.util.Set;
-
-public class ModocotResultSummary
-{
-    public long timestamp;
-    public int testCount;
-    public int failureCount;
-    public int successCount;
-    public String globalStacktrace;
-
-    public Set<ModocotResult> successes;
-    public Set<ModocotResult> failures;
-    public Set<ModocotResult> compilationErrors;
-}

src/main/java/de/hftstuttgart/models/ModocotResult.java → src/main/java/de/hftstuttgart/models/Result.java

 package de.hftstuttgart.models;
 
-public class ModocotResult
+public class Result
 {
     public String name;
     public int state;
@@ -15,8 +15,9 @@ public class ModocotResult
 
     public static enum State
     {
+        UNKNOWN,
         SUCCESS,
         FAILURE,
-        UNKNOWN
+        COMPILATIONERROR,
     }
 }

src/main/java/de/hftstuttgart/models/ResultSummary.java

+package de.hftstuttgart.models;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class ResultSummary
+{
+    public long timestamp               = System.currentTimeMillis() / 1000;
+    public String globalStacktrace      = null;
+    public Set<Result> results   = new HashSet<>();
+}

src/main/java/de/hftstuttgart/rest/v1/task/TaskUpload.java

 package de.hftstuttgart.rest.v1.task;
 
-import de.hftstuttgart.models.LegacyMoodleResult;
-import de.hftstuttgart.models.ModocotResultSummary;
+import de.hftstuttgart.models.ResultSummary;
 import de.hftstuttgart.utils.*;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -42,12 +41,12 @@ public class TaskUpload {
         this.executeTestUtil = executeTestUtil;
 
         // set path of temporary directory on host and inside our container
-        this.testTmpPathModocot = Paths.get(env.getProperty("modocot.tests.tmp.dir"));
+        this.testTmpPathModocot = Paths.get(env.getProperty("tests.tmp.dir"));
     }
 
     @RequestMapping(method = RequestMethod.POST)
-    public LegacyMoodleResult uploadAndTestFile(@RequestParam("taskFile") MultipartFile taskFileRef,
-                                    @RequestParam("assignmentId") String assignmentId
+    public ResultSummary uploadAndTestFile(@RequestParam("taskFile") MultipartFile taskFileRef,
+                                           @RequestParam("assignmentId") String assignmentId
     ) throws IOException, InterruptedException {
         LOG.info("submission for testing received");
 
@@ -82,11 +81,7 @@ public class TaskUpload {
 
         // run test
         LOG.debug("calling test execution");
-        ModocotResultSummary resultSummary = executeTestUtil.runTests(assignmentId, workDirectory);
-
-        // convert to moddle plugin readable format and return to moodle
-        LOG.debug("convert to moodle understandable format");
-        LegacyMoodleResult moodleResult = LegacyMoodleResult.convertToModdleResult(resultSummary);
+        ResultSummary resultSummary = executeTestUtil.runTests(assignmentId, workDirectory);
 
         if (mimeInfo.equals("text/plain")) {
             LOG.info("check for provided Ticketsystem information");
@@ -94,6 +89,6 @@ public class TaskUpload {
         }
 
         LOG.info("submission tested successfully");
-        return moodleResult;
+        return resultSummary;
     }
 }

src/main/java/de/hftstuttgart/rest/v1/unittest/UnitTestUpload.java

@@ -27,8 +27,8 @@ import java.util.regex.Pattern;
 public class UnitTestUpload {
 
     private static final Logger LOG = LogManager.getLogger(UnitTestUpload.class);
-    public final static String modocotTestConfigRegex = "^modocot::(.*)::(.*|none)::(.*|none)::(.*)$";
-    public final static String modocotDueConfigRegex = "^modocot::(.*)::(.*|none)::(.*|none)$";
+    public final static String TESTCONFIGREGEX = "^dtt::(.*)::(.*|none)::(.*|none)::(.*)$";
+    public final static String SUBMISSIONCONFIGREGEX = "^dtt::(.*)::(.*|none)::(.*|none)$";
 
     private final JGitUtil jGitUtil;
     private final String assignmentBasePath;
@@ -36,7 +36,7 @@ public class UnitTestUpload {
     public UnitTestUpload(Environment env, JGitUtil jGitUtil) {
         this.jGitUtil = jGitUtil;
 
-        Path p = Paths.get(env.getProperty("modocot.dir"), env.getProperty("modocot.dir.test.folder.name"));
+        Path p = Paths.get(env.getProperty("data.dir"), env.getProperty("data.dir.test.folder.name"));
         this.assignmentBasePath = p.toAbsolutePath().toString();
     }
 
@@ -64,7 +64,7 @@ public class UnitTestUpload {
         unitTestFileRef.transferTo(file);
         LOG.debug(String.format("saved config file to: %s", file.getAbsolutePath()));
 
-        Pattern pattern = Pattern.compile(this.modocotTestConfigRegex);
+        Pattern pattern = Pattern.compile(this.TESTCONFIGREGEX);
         Matcher config = null;
 
         LOG.debug("reading test configuration file");

src/main/java/de/hftstuttgart/utils/DockerUtil.java

@@ -2,6 +2,7 @@ package de.hftstuttgart.utils;
 
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.command.CreateContainerResponse;
+import com.github.dockerjava.api.exception.DockerException;
 import com.github.dockerjava.api.model.Bind;
 import com.github.dockerjava.api.model.HostConfig;
 import com.github.dockerjava.core.DefaultDockerClientConfig;
@@ -40,18 +41,33 @@ public class DockerUtil {
     public int runContainer(String image, Bind... binds) throws InterruptedException, IOException
     {
         LOG.debug(String.format("pull image: %s", image));
-        dockerClient.pullImageCmd(image)
-            .start()
-            .awaitCompletion()
-            .close();
+        try {
+            dockerClient.pullImageCmd(image)
+                .start()
+                .awaitCompletion()
+                .close();
+        } catch (DockerException e) {
+            LOG.error(String.format(
+                "pulling docker image %s failed with %s, trying with local image",
+                image,
+                e.getMessage()));
+        }
 
         LOG.debug("creating container");
-        CreateContainerResponse containerResponse = dockerClient.createContainerCmd("testcontainer")
-            .withImage(image)
-            .withHostConfig(
-                HostConfig.newHostConfig()
-                    .withBinds(binds))
-            .exec();
+        CreateContainerResponse containerResponse;
+        try {
+            containerResponse = dockerClient.createContainerCmd("testcontainer")
+                .withImage(image)
+                .withHostConfig(
+                    HostConfig.newHostConfig()
+                        .withBinds(binds))
+                .exec();
+        } catch (DockerException e) {
+            LOG.error(String.format(
+                "Creating Docker Testrunner container failed with %s", e.getMessage()));
+            throw e;
+        }
+
         LOG.debug(String.format("container created: %s", containerResponse.getId()));
 
         LOG.debug(String.format("starting container %s", containerResponse.getId()));

src/main/java/de/hftstuttgart/utils/ExecuteTestUtil.java

@@ -3,7 +3,7 @@ package de.hftstuttgart.utils;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.dockerjava.api.model.Bind;
 import com.github.dockerjava.api.model.Volume;
-import de.hftstuttgart.models.ModocotResultSummary;
+import de.hftstuttgart.models.ResultSummary;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 import org.springframework.core.env.Environment;
@@ -35,21 +35,21 @@ public class ExecuteTestUtil  {
 
         // set base path for assignments to be stored
         Path p = Paths.get(
-            env.getProperty("modocot.dir"),
-            env.getProperty("modocot.dir.test.folder.name"));
+            env.getProperty("data.dir"),
+            env.getProperty("data.dir.test.folder.name"));
         this.assignmentBasePath = p.toAbsolutePath().toString();
 
         // set path of temporary directory on host and inside our container
         this.testTmpPathHost = Paths.get(env.getProperty("host.tests.tmp.dir"));
-        this.testTmpPathModocot = Paths.get(env.getProperty("modocot.tests.tmp.dir"));
+        this.testTmpPathModocot = Paths.get(env.getProperty("tests.tmp.dir"));
     }
 
-    public ModocotResultSummary runTests(String assignmentId, Path workDirectory) throws IOException, InterruptedException {
+    public ResultSummary runTests(String assignmentId, Path workDirectory) throws IOException, InterruptedException {
 
         // define paths for the test, the submission and where the result is to be expected afterwards
-        Path testPath   = Paths.get(workDirectory.toAbsolutePath().toString(), "test");
-        Path srcPath    = Paths.get(workDirectory.toAbsolutePath().toString(), "src");
-        Path resultPath = Paths.get(workDirectory.toAbsolutePath().toString(), "result");
+        Path testPath   = Paths.get(workDirectory.toAbsolutePath().toString(), "/test");
+        Path srcPath    = Paths.get(workDirectory.toAbsolutePath().toString(), "/src");
+        Path resultPath = Paths.get(workDirectory.toAbsolutePath().toString(), "/result");
 
         // clone stored test to tmpdir
         LOG.debug("copying pre-downloaded unitttest repo");
@@ -99,9 +99,9 @@ public class ExecuteTestUtil  {
         // start test-container with professor given image and bind mounts for test, submission and result
         dockerUtil.runContainer(
             config.group(4),
-            new Bind(testPathHost.toAbsolutePath().toString(), new Volume("/modocot/test")),
-            new Bind(srcPathHost.toAbsolutePath().toString(), new Volume("/modocot/src")),
-            new Bind(resultPathHost.toAbsolutePath().toString(), new Volume("/modocot/result"))
+            new Bind(testPathHost.toAbsolutePath().toString(), new Volume("/data/test")),
+            new Bind(srcPathHost.toAbsolutePath().toString(), new Volume("/data/src")),
+            new Bind(resultPathHost.toAbsolutePath().toString(), new Volume("/data/result"))
         );
 
         // define expected result file
@@ -115,9 +115,9 @@ public class ExecuteTestUtil  {
 
         LOG.debug("parse results json");
         ObjectMapper objectMapper = new ObjectMapper();
-        ModocotResultSummary resultSummary = objectMapper.readValue(
+        ResultSummary resultSummary = objectMapper.readValue(
             resultFile.toURI().toURL(),
-            ModocotResultSummary.class);
+            ResultSummary.class);
 
         return resultSummary;
     }

src/main/java/de/hftstuttgart/utils/RegexUtil.java

@@ -32,11 +32,11 @@ public class RegexUtil {
         Pattern pattern;
         switch (configType) {
             case PROFESSOR:
-                pattern = Pattern.compile(UnitTestUpload.modocotTestConfigRegex);
+                pattern = Pattern.compile(UnitTestUpload.TESTCONFIGREGEX);
                 break;
 
             case STUDENT:
-                pattern = Pattern.compile(UnitTestUpload.modocotDueConfigRegex);
+                pattern = Pattern.compile(UnitTestUpload.SUBMISSIONCONFIGREGEX);
                 break;
 
             default:

src/main/java/de/hftstuttgart/utils/UnifiedTicketingUtil.java

 package de.hftstuttgart.utils;
 
-import de.hftstuttgart.models.ModocotResult;
-import de.hftstuttgart.models.ModocotResultSummary;
+import de.hftstuttgart.models.Result;
+import de.hftstuttgart.models.ResultSummary;
 import de.hftstuttgart.unifiedticketing.core.Filter;
 import de.hftstuttgart.unifiedticketing.core.Ticket;
 import de.hftstuttgart.unifiedticketing.core.TicketBuilder;
@@ -29,7 +29,7 @@ public class UnifiedTicketingUtil {
     private final static String MODOCOT_LABEL = "MoDoCoT created";
     private final static String MODOCOT_TITLE = " | " + MODOCOT_LABEL;
 
-    public static String createTicketDescriptionFromResult(TicketSystem ts, ModocotResult result, boolean compilationError) {
+    public static String createTicketDescriptionFromResult(TicketSystem ts, Result result, boolean compilationError) {
         StringBuilder sb = new StringBuilder();
 
         // heading
@@ -73,7 +73,7 @@ public class UnifiedTicketingUtil {
         return sb.toString();
     }
 
-    public static String createTicketTitleFromResult(TicketSystem ts, ModocotResult result, boolean compilationError) {
+    public static String createTicketTitleFromResult(TicketSystem ts, Result result, boolean compilationError) {
         StringBuilder sb = new StringBuilder();
         String separator = " | ";
 
@@ -97,7 +97,7 @@ public class UnifiedTicketingUtil {
         return sb.toString();
     }
 
-    public static Ticket createTicketFromResult(TicketSystem ts, ModocotResult result, boolean compilationError) {
+    public static Ticket createTicketFromResult(TicketSystem ts, Result result, boolean compilationError) {
         TicketBuilder tb = ts.createTicket()
             .title(createTicketTitleFromResult(ts, result, compilationError))
             .description(createTicketDescriptionFromResult(ts, result, compilationError));
@@ -142,7 +142,7 @@ public class UnifiedTicketingUtil {
         return ret;
     }
 
-    public static String getHashForFailure(ModocotResult result) {
+    public static String getHashForFailure(Result result) {
         MessageDigest digest;
 
         try {
@@ -166,7 +166,7 @@ public class UnifiedTicketingUtil {
         return hexString.substring(hexString.length() - 9, hexString.length() - 1);
     }
 
-    public static void processResult(TicketSystem ts, Set<Ticket> tickets, ModocotResult result, boolean compilationError) {
+    public static void processResult(TicketSystem ts, Set<Ticket> tickets, Result result, boolean compilationError) {
         LOG.debug(String.format("retrieving hash for %s", result.name));
         String hash = getHashForFailure(result);
 
@@ -195,7 +195,7 @@ public class UnifiedTicketingUtil {
      * @param meta student uploaded file
      * @param resultSummary summary from the testrunner container
      */
-    public static void reportResults(InputStream meta, ModocotResultSummary resultSummary) {
+    public static void reportResults(InputStream meta, ResultSummary resultSummary) {
         try {
             reportResults(meta, resultSummary, false);
         } catch (InterruptedException e) {
@@ -212,7 +212,7 @@ public class UnifiedTicketingUtil {
      * @param wait if we should block until the ticket creation has finished
      * @throws InterruptedException
      */
-    public static void reportResults(InputStream meta, ModocotResultSummary resultSummary, boolean wait) throws InterruptedException {
+    public static void reportResults(InputStream meta, ResultSummary resultSummary, boolean wait) throws InterruptedException {
 
         LOG.debug("preparing thread for ticket result submitting");
         Thread unifiedTicketingUtil = new Thread(() -> {
@@ -257,22 +257,26 @@ public class UnifiedTicketingUtil {
         }
     }
 
-    public static void reportToTicketsystem(TicketSystem ts, ModocotResultSummary resultSummary) {
+    public static void reportToTicketsystem(TicketSystem ts, ResultSummary resultSummary) {
         // tickets existing yet
         LOG.debug("fetching existing tickets");
         Set<Ticket> tickets = fetchExistingModocotTickets(ts);
 
         // for each fail or compile error
         LOG.debug("start failed tests reporting");
-        resultSummary.failures.forEach(f -> processResult(ts, tickets, f, false));
+        resultSummary.results.stream()
+            .filter(r -> r.state == Result.State.FAILURE.ordinal())
+            .forEach(f -> processResult(ts, tickets, f, false));
         LOG.debug("start compilation errors reporting");
-        resultSummary.compilationErrors.forEach(c -> processResult(ts, tickets, c, true));
+        resultSummary.results.stream()
+            .filter(r -> r.state == Result.State.COMPILATIONERROR.ordinal())
+            .forEach(c -> processResult(ts, tickets, c, true));
 
         LOG.debug("closing all remaining tickets, no longer appeared");
         tickets.forEach(ticket -> ticket.close().save());
     }
 
-    public static Ticket updateTicketFromResult(TicketSystem ts, Ticket ticket, ModocotResult result, boolean compilationError) {
+    public static Ticket updateTicketFromResult(TicketSystem ts, Ticket ticket, Result result, boolean compilationError) {
         ticket
             .open()
             .setTitle(createTicketTitleFromResult(ts, result, compilationError))

src/main/resources/application.properties

@@ -7,7 +7,7 @@ spring.http.multipart.max-file-size=5Mb
 ###############################################
 
 # Holds the uploaded Zip-Files
-modocot.tests.tmp.dir=/tmp/modocot-tests
-host.tests.tmp.dir=${modocot.tests.tmp.dir}
-modocot.dir=/modocot/data
-modocot.dir.test.folder.name=UnitTests
+tests.tmp.dir=/tmp/dtt-tests
+host.tests.tmp.dir=${tests.tmp.dir}
+data.dir=/data
+data.dir.test.folder.name=UnitTests