removed dtt remnants

acd8c826 · Lückemeyer · d1fab229 · acd8c826 · acd8c826 · acd8c826
Commit acd8c826 authored 1 year ago by Lückemeyer
Hide whitespace changes
Inline Side-by-side

Showing

with 500 additions and 500 deletions
+500 -500
--- a/src/main/java/de/hftstuttgart/dtabackend/rest/v1/task/TaskUpload.java
+++ b/src/main/java/de/hftstuttgart/dtabackend/rest/v1/task/TaskUpload.java
@@ -52,7 +52,7 @@ public class TaskUpload {
        LOG.info("submission for testing received");
        LOG.debug("creating new temporary directory");
-        Path workDirectory = Files.createTempDirectory(testTmpPath, "dtt");
+        Path workDirectory = Files.createTempDirectory(testTmpPath, "dta");
        LOG.debug(String.format("working dir for test is: %s", workDirectory.toAbsolutePath().toString()));
        // define paths for the test, the submission and where the result is to be expected afterwards
@@ -61,7 +61,7 @@ public class TaskUpload {
        String mimeInfo = new Tika().detect(taskFileRef.getInputStream());
        switch (mimeInfo) {
            case "text/plain":
-                LOG.debug("textfile uploaded, searching for dtt config");
+                LOG.debug("textfile uploaded, searching for dta config");
                // find URI in config file
                Matcher config = RegexUtil.findStudentConfig(taskFileRef.getInputStream());

--- a/src/main/java/de/hftstuttgart/dtabackend/rest/v1/unittest/UnitTestUpload.java
+++ b/src/main/java/de/hftstuttgart/dtabackend/rest/v1/unittest/UnitTestUpload.java
 package de.hftstuttgart.dtabackend.rest.v1.unittest;
 import de.hftstuttgart.dtabackend.utils.JGitUtil;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.core.env.Environment;
 import org.springframework.util.FileSystemUtils;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 import jakarta.servlet.annotation.MultipartConfig;
 import java.io.*;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 /**
 * Rest controller for anything related to the TEST files.
 */
 @RestController
 @RequestMapping("/v1/unittest")
 @MultipartConfig
 public class UnitTestUpload {
    private static final Logger LOG = LogManager.getLogger(UnitTestUpload.class);
    public final static String TESTCONFIGREGEX = "^dtt::(.*)::(.*|none)::(.*|none)::(.*)$";
    public final static String SUBMISSIONCONFIGREGEX = "^dtt::(.*)::(.*|none)::(.*|none)$";
    private final JGitUtil jGitUtil;
    private final String assignmentBasePath;
    public UnitTestUpload(Environment env, JGitUtil jGitUtil) {
        this.jGitUtil = jGitUtil;
        Path p = Paths.get(env.getProperty("data.dir"), env.getProperty("data.dir.test.folder.name"));
        this.assignmentBasePath = p.toAbsolutePath().toString();
    }
    /**
     * Create a subfolder for the specific assignment.
     * This is called when the teacher creates an assignment and uploads the JUnit test files
     *
     * @param unitTestFileRef The text file which contains the JUnit tests meta data
     * @param assignmentId    ID of the created assignment. Generated by Moodle
     */
    @RequestMapping(method = RequestMethod.POST)
    public void uploadUnitTestFile(
        @RequestParam("unitTestFile") MultipartFile unitTestFileRef,
        @RequestParam("assignmentId") String assignmentId
    ) throws IOException {
        LOG.info("received new assignment");
        File file = Paths.get(
                this.assignmentBasePath,
                assignmentId + ".txt")
            .toFile();
        file.mkdirs();
        // save assignment config
        unitTestFileRef.transferTo(file);
        LOG.debug(String.format("saved config file to: %s", file.getAbsolutePath()));
        Pattern pattern = Pattern.compile(TESTCONFIGREGEX);
        Matcher config = null;
        LOG.debug("reading test configuration file");
        // open saved config in a try-with
        try (BufferedReader br = new BufferedReader(
            new InputStreamReader(
                new FileInputStream(file)))) {
            String line;
            // search for a URI while none is found and there are lines left
            while (config == null && (line = br.readLine()) != null) {
                Matcher matcher = pattern.matcher(line);
                if (matcher.matches()) {
-                    LOG.debug(String.format("found dtt test line: %s", line));
+                    LOG.debug(String.format("found dta test line: %s", line));
                    config = matcher;
                }
            }
        } catch (IOException e) {
            LOG.error("Error while reading repo config", e);
        }
        finally {
            if (config == null) {
                throw new RuntimeException("couldn't find repo config for unittest clone");
            }
        }
        LOG.debug("calling test repo clone");
        // cloning assignment repo to persistent space
        jGitUtil.cloneRepository(
            config,
            Paths.get(this.assignmentBasePath, assignmentId).toAbsolutePath().toString());
        LOG.info(String.format("stored new assignment: %s", file.getAbsolutePath()));
    }
    /**
     * Delete the folder for the assignment.
     * Called when the teacher deletes the JUnitTest assignment
     * <p>
     * {{url}}:8080/v1/unittest?assignmentId=111
     *
     * @param assignmentId ID of the assignment to delete. Generated by Moodle
     */
    @RequestMapping(method = RequestMethod.DELETE)
    public void deleteUnitTestFiles(@RequestParam("assignmentId") String assignmentId) {
        LOG.info(String.format("received deletion order for assignment %s", assignmentId));
        // deleting config file
        File file = Paths.get(
            this.assignmentBasePath,
            assignmentId + ".txt")
            .toFile();
        file.delete();
        // deleting local copy of repository
        file = Paths.get(
            this.assignmentBasePath,
            assignmentId).toFile();
        FileSystemUtils.deleteRecursively(file);
        LOG.info(String.format("assignment %s deletion complete", assignmentId));
    }
 }
--- a/src/main/java/de/hftstuttgart/dtabackend/utils/RegexUtil.java
+++ b/src/main/java/de/hftstuttgart/dtabackend/utils/RegexUtil.java
 package de.hftstuttgart.dtabackend.utils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import de.hftstuttgart.dtabackend.rest.v1.unittest.UnitTestUpload;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 public class RegexUtil {
    public enum ConfigType {
        TEACHER,
        STUDENT,
    }
    private static final Logger LOG = LogManager.getLogger(RegexUtil.class);
    public static Matcher findStudentConfig(InputStream is) {
        return findConfig(is, ConfigType.STUDENT);
    }
    public static Matcher findProfessorConfig(InputStream is) {
        return findConfig(is, ConfigType.TEACHER);
    }
    public static Matcher findConfig(InputStream is, ConfigType configType) {
        Pattern pattern;
        switch (configType) {
            case TEACHER:
                pattern = Pattern.compile(UnitTestUpload.TESTCONFIGREGEX);
                break;
            case STUDENT:
                pattern = Pattern.compile(UnitTestUpload.SUBMISSIONCONFIGREGEX);
                break;
            default:
                String msg = String.format("unknown config type: %s", configType.name());
                LOG.error(msg);
                throw new RuntimeException(msg);
        }
        Matcher config = null;
        LOG.debug("reading config file");
        // open received file in a try-with
        try (BufferedReader br = new BufferedReader(
            new InputStreamReader(
                is))) {
            String line;
            // as long as we haven't found a configuration and have lines left, search
            while (config == null && (line = br.readLine()) != null) {
                Matcher matcher = pattern.matcher(line);
                if (matcher.matches()) {
-                    LOG.debug(String.format("found dtt line: %s", line));
+                    LOG.debug(String.format("found dta line: %s", line));
                    config = matcher;
                }
            }
        } catch (IOException e) {
            LOG.error("Error while reading repo config", e);
        }
        finally {
            if (config == null) {
                throw new RuntimeException("couldn't find repo config for clone");
            }
        }
        return config;
    }
 }
--- a/src/main/java/de/hftstuttgart/dtabackend/utils/UnifiedTicketingUtil.java
+++ b/src/main/java/de/hftstuttgart/dtabackend/utils/UnifiedTicketingUtil.java
 package de.hftstuttgart.dtabackend.utils;
 import de.hftstuttgart.dtabackend.models.Result;
 import de.hftstuttgart.dtabackend.models.ResultSummary;
 import de.hftstuttgart.unifiedticketing.core.Filter;
 import de.hftstuttgart.unifiedticketing.core.Ticket;
 import de.hftstuttgart.unifiedticketing.core.TicketBuilder;
 import de.hftstuttgart.unifiedticketing.core.TicketSystem;
 import de.hftstuttgart.unifiedticketing.exceptions.UnifiedticketingException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 public class UnifiedTicketingUtil {
    private final static Logger LOG = LogManager.getLogger(UnifiedTicketingUtil.class);
-    private final static String LABEL = "DTT created";
+    private final static String LABEL = "DTA created";
    private final static String TITLE = " | " + LABEL;
    public static String createTicketDescriptionFromResult(TicketSystem ts, Result result, boolean compilationError) {
        StringBuilder sb = new StringBuilder();
        // heading
        if (compilationError) {
            sb.append(String.format("# %s is not compilable", result.name));
        } else {
            sb.append(String.format("# Unittest for %s fails", result.name));
        }
        sb.append("\n\n");
        // meta data table
        sb.append("|||\n");
        sb.append("|:-|:-|\n");
        if (compilationError) {
            sb.append(String.format("|File|`%s`|\n", result.name));
        } else {
            sb.append(String.format("|Test|`%s`|\n", result.name));
        }
        sb.append(String.format("|Reason|`%s`|\n", result.failureReason));
        if (compilationError) {
            sb.append(String.format("|Line|`%s`|\n", result.lineNumber));
            sb.append(String.format("|Column|`%s`|\n", result.columnNumber));
            sb.append(String.format("|Position|`%s`|\n", result.position));
        } else {
            sb.append(String.format("|Type|`%s`|\n", result.failureType));
        }
        sb.append("\n\n");
        // stacktrace
        sb.append("<details>\n\n");
        sb.append("<summary>show stacktrace</summary>\n\n");
        sb.append("```");
        sb.append(result.stacktrace);
        sb.append("\n```");
        sb.append("\n\n</details>\n");
        return sb.toString();
    }
    public static String createTicketTitleFromResult(TicketSystem ts, Result result, boolean compilationError) {
        StringBuilder sb = new StringBuilder();
        String separator = " | ";
        if (compilationError) sb.append("compilation fails");
        else sb.append("test method fails");
        sb.append(separator);
        sb.append(result.name);
        if (!compilationError) {
            sb.append(separator);
            sb.append(result.failureReason);
        }
        // if label-support is not present, place global identifier into title
        if (!ts.hasLabelSupport()) sb.append(TITLE);
        sb.append(separator);
        sb.append(getHashForFailure(result));
        return sb.toString();
    }
    public static Ticket createTicketFromResult(TicketSystem ts, Result result, boolean compilationError) {
        TicketBuilder tb = ts.createTicket()
            .title(createTicketTitleFromResult(ts, result, compilationError))
            .description(createTicketDescriptionFromResult(ts, result, compilationError));
        if (ts.hasLabelSupport()) tb.labels(Collections.singleton(LABEL));
        return tb.create();
    }
    public static Set<Ticket> fetchExistingTickets(TicketSystem ts) {
        Set<Ticket> ret = new HashSet<>();
        Filter f = ts.find();
        // depending on label support, identify tickets by label or title containing string
        if (ts.hasLabelSupport()) {
            LOG.debug(String.format(
-                "ticketsystem has label support, using label %s to find dtt tickets", LABEL));
+                "ticketsystem has label support, using label %s to find dta tickets", LABEL));
            f.withLabel(LABEL);
        }
        else {
            LOG.debug(String.format(
                "ticketsystem without labels, searching for ticket titles containing %s", TITLE));
            f.withTitleContain(TITLE);
        }
        LOG.debug("prepare pagination cycling");
        // set first page and page size for pagination
        int page = 1;
        int pageSize = 10;
        f.setPageSize(pageSize);
        LOG.debug(String.format("using pagination with %s elements per page", pageSize));
        // declare list for received tickets
        List<Ticket> received;
        // go into do-while to evaluate after receiving if another round is needed
        do {
            LOG.debug(String.format("calling page %s", page));
            f.setPage(page++);
            received = f.get();
            ret.addAll(received);
        } while (f.getLastReceivedItemCount() >= pageSize);
        return ret;
    }
    public static String getHashForFailure(Result result) {
        MessageDigest digest;
        try {
            digest = MessageDigest.getInstance("SHA-512");
        } catch (NoSuchAlgorithmException e) {
            throw new UnifiedticketingException(e);
        }
        byte[] data = digest.digest(result.name.getBytes(StandardCharsets.UTF_8));
        StringBuilder hexString = new StringBuilder(2 * data.length);
        for (byte character: data)
        {
            String hex = Integer.toHexString(0xff & character);
            if (hex.length() == 1)
            {
                hexString.append('0');
            }
            hexString.append(hex);
        }
        return hexString.substring(hexString.length() - 9, hexString.length() - 1);
    }
    public static void processResult(TicketSystem ts, Set<Ticket> tickets, Result result, boolean compilationError) {
        LOG.debug(String.format("retrieving hash for %s", result.name));
        String hash = getHashForFailure(result);
        // check if corresponding ticket exists yet, otherwise create new one
        Ticket ticket = tickets.stream()
            .filter(t -> t.getTitle().endsWith(hash))
            .findFirst()
            .orElse(null);
        if (ticket != null) {
            // if yet existing, remove from found list
            LOG.debug("found ticket with matching hash, removing from collection");
            tickets.remove(ticket);
            LOG.debug("updating ticket with new result");
            updateTicketFromResult(ts, ticket, result, compilationError);
        } else {
            LOG.debug("no ticket found, creating new one");
            createTicketFromResult(ts, result, compilationError);
        }
    }
    /**
     * search file for unified-ticketing URI's and report to every set ticket system,
     * not waiting for it to finish and catching an eventually interrupted thread.
     *
     * @param meta student uploaded file
     * @param resultSummary summary from the testrunner container
     */
    public static void reportResults(InputStream meta, ResultSummary resultSummary) {
        try {
            reportResults(meta, resultSummary, false);
        } catch (InterruptedException e) {
            LOG.error(String.format("Unified-Ticketing got interrupted with: %s", e.getMessage()));
        }
    }
    /**
     * report all failures and compilation errors to all configured ticket systems.
     * You can optionally wait for the ticket creation to finish, which is done in a separate thread.
     *
     * @param meta student uploaded file
     * @param resultSummary summary from the testrunner container
     * @param wait if we should block until the ticket creation has finished
     * @throws InterruptedException
     */
    public static void reportResults(InputStream meta, ResultSummary resultSummary, boolean wait) throws InterruptedException {
        LOG.debug("preparing thread for ticket result submitting");
        Thread unifiedTicketingUtil = new Thread(() -> {
            // read student transmitted file
            try (BufferedReader br = new BufferedReader(new InputStreamReader(meta))) {
                String line = null;
                while ((line = br.readLine()) != null) {
                    TicketSystem ts = null;
                    // try each line as URI for a unified-ticketing instantiation
                    try {
                        ts = TicketSystem.fromUri(line);
                        LOG.info(String.format("ticket system for reporting found: %s", ts.baseUrl));
                    } catch (UnifiedticketingException e) {
                        // ignore if line didn't match a unified-ticketing URI
                    }
                    // if a ticketsystem got instantiated, start the submission.
                    // If errors occur, log it this time.
                    try {
                        if (ts != null) reportToTicketsystem(ts, resultSummary);
                    } catch (UnifiedticketingException e) {
                        LOG.warn(String.format(
                            "reporting fails to ticketsystem %s failed with: ",
                            ts.baseUrl,
                            e.getMessage()));
                    }
                }
            } catch (IOException e) {
                LOG.error(String.format("couldn't read config to find lines for ticket reporting: %s", e.getMessage()));
            }
        });
        LOG.debug("starting ticket submitting thread");
        unifiedTicketingUtil.start();
        if (wait) {
            LOG.debug("wait for ticket submission completion");
            unifiedTicketingUtil.join();
        } else {
            LOG.debug("tickets will be submitted in background");
        }
    }
    public static void reportToTicketsystem(TicketSystem ts, ResultSummary resultSummary) {
        // tickets existing yet
        LOG.debug("fetching existing tickets");
        Set<Ticket> tickets = fetchExistingTickets(ts);
        // for each fail or compile error
        LOG.debug("start failed tests reporting");
        resultSummary.results.stream()
            .filter(r -> r.state == Result.State.FAILURE.ordinal())
            .forEach(f -> processResult(ts, tickets, f, false));
        LOG.debug("start compilation errors reporting");
        resultSummary.results.stream()
            .filter(r -> r.state == Result.State.COMPILATIONERROR.ordinal())
            .forEach(c -> processResult(ts, tickets, c, true));
        LOG.debug("closing all remaining tickets, no longer appeared");
        tickets.forEach(ticket -> ticket.close().save());
    }
    public static Ticket updateTicketFromResult(TicketSystem ts, Ticket ticket, Result result, boolean compilationError) {
        ticket
            .open()
            .setTitle(createTicketTitleFromResult(ts, result, compilationError))
            .setDescription(createTicketDescriptionFromResult(ts, result, compilationError));
        if (ts.hasLabelSupport()) ticket.addLabel(LABEL);
        return ticket.save();
    }
 }
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -7,7 +7,7 @@ spring.http.multipart.max-file-size=5Mb
 ###############################################
 # Holds the uploaded Zip-Files
-tests.tmp.dir=/tmp/dta-tests
+tests.tmp.dir=~/dta-tests
 host.tests.tmp.dir=${tests.tmp.dir}
 data.dir=/data
 data.dir.test.folder.name=UnitTests