Skip to content

GitLab

Verified Commit db826d5d authored by Lukas Wiest's avatar Lukas Wiest 🚂
Browse files

refactor(rest): refactor TaskUpload to V2 

- creates tmp dir
- copies over the stored assignment repo
- clones student submission repo
- runs the image the prof specified in its assignment, mounting
    - the assignment repo
    - the subission repo
    - an empty folder expecting the result file to be in after container
        finished
- reads result file
- converts to moodle understandable instances
- returns result to moodle

BREAKING CHANGE: Gitea eliminated
BREAKING CHANGE: Jenkins eliminated
BREAKING CHANGE: expected file type changed from zip to text
BREAKING CHANGE: textfile has to conform new modocot URI
BREAKING CHANGE: no Jenkins pipe needed in submission repo
BREAKING CHANGE: no Dockerfile needed in submission repo
parent 3a59600d
Hide whitespace changes
Inline Side-by-side
src/main/java/de/hftstuttgart/rest/v1/task/TaskUpload.java
View file @ db826d5d
package de.hftstuttgart.rest.v1.task;
import de.hftstuttgart.config.ModocotProperties;
import de.hftstuttgart.rest.v1.jenkins.RestAPIController;
import de.hftstuttgart.utils.BackendUtil;
import de.hftstuttgart.utils.BuildState;
import de.hftstuttgart.utils.GitTeaUtil;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Volume;
import de.hftstuttgart.models.LegacyMoodleResult;
import de.hftstuttgart.models.ModocotResultSummary;
import de.hftstuttgart.rest.v1.unittest.UnitTestUpload;
import de.hftstuttgart.utils.DockerUtil;
import de.hftstuttgart.utils.FileUtil;
import de.hftstuttgart.utils.JGitUtil;
import de.hftstuttgart.utils.TaskUploadUtils;
import de.hftstuttgart.utils.UnzipUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.core.env.Environment;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
......@@ -17,10 +19,12 @@ import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.annotation.MultipartConfig;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Rest controller for everything related to the TASK files
......@@ -31,98 +35,173 @@ import java.util.UUID;
public class TaskUpload {
private static final Logger LOG = LogManager.getLogger(TaskUpload.class);
private final ModocotProperties modocotProperties;
private final TaskUploadUtils taskUploadUtils;
private final JGitUtil jGitUtil;
private final DockerUtil dockerUtil;
private final String assignmentBasePath;
private final GitTeaUtil gitTeaUtil;
private final Path testTmpPathHost;
private final Path testTmpPathModocot;
public TaskUpload(ModocotProperties modocotProperties,
TaskUploadUtils taskUploadUtils,
JGitUtil jGitUtil,
GitTeaUtil gitTeaUtil) {
this.modocotProperties = modocotProperties;
this.taskUploadUtils = taskUploadUtils;
public TaskUpload(
Environment env,
JGitUtil jGitUtil,
DockerUtil dockerUtil
) {
this.jGitUtil = jGitUtil;
this.gitTeaUtil = gitTeaUtil;
this.assignmentBasePath =
this.modocotProperties.getModocotParentDirectory()
+ File.separator
+ this.modocotProperties.getModocotAssignmentFolderPrefix();
}
@RequestMapping(method = RequestMethod.POST)
public String uploadAndTestFile(@RequestParam("taskFile") MultipartFile taskFileRef,
@RequestParam("assignmentId") String assignmentId) throws IOException, InterruptedException {
String jobId = assignmentId + "_" + UUID.randomUUID();
String subFolderPath = this.assignmentBasePath + assignmentId;
LOG.info("subFolderPath: " + subFolderPath);
File workDirectory = new File(subFolderPath);
workDirectory.mkdirs();
LOG.info("created new File");
File file = new File(subFolderPath, String.valueOf(UUID.randomUUID()));
this.dockerUtil = dockerUtil;
taskFileRef.transferTo(file);
List<File> unzippedFiles = UnzipUtil.unzip(file);
// set base path for assignments to be stored
Path p = Paths.get(
env.getProperty("modocot.dir"),
env.getProperty("modocot.dir.test.folder.name"));
this.assignmentBasePath = p.toAbsolutePath().toString();
if (file.exists()) file.delete();
startFileRead(assignmentId, subFolderPath, unzippedFiles);
BuildState buildState = this.taskUploadUtils.startTask(jobId, subFolderPath);
// set path of temporary directory on host and inside our container
this.testTmpPathHost = Paths.get(env.getProperty("host.tests.tmp.dir"));
this.testTmpPathModocot = Paths.get(env.getProperty("modocot.tests.tmp.dir"));
}
if (buildState != BuildState.BLUE) {
String jenkinsError = this.taskUploadUtils.getJenkinsConsoleOutput(jobId);
LOG.error(jenkinsError);
RestAPIController.JOB_MAP.remove(jobId);
this.gitTeaUtil.deleteRepository(jobId);
this.taskUploadUtils.deleteJenkinsJob(jobId);
return jenkinsError;
@RequestMapping(method = RequestMethod.POST)
public LegacyMoodleResult uploadAndTestFile(@RequestParam("taskFile") MultipartFile taskFileRef,
@RequestParam("assignmentId") String assignmentId
) throws IOException, InterruptedException {
LOG.info("submission for testing received");
LOG.debug("creating new temporary directory");
Path workDirectory = Files.createTempDirectory(testTmpPathModocot, "modocot");
LOG.debug(String.format("working dir for test is: %s", workDirectory.toAbsolutePath().toString()));
// define paths for the test, the submission and where the result is to be expected afterwards
Path testPath = Paths.get(workDirectory.toAbsolutePath().toString(), "test");
Path srcPath = Paths.get(workDirectory.toAbsolutePath().toString(), "src");
Path resultPath = Paths.get(workDirectory.toAbsolutePath().toString(), "result");
// clone stored test to tmpdir
LOG.debug("copying pre-downloaded unitttest repo");
FileUtil.copyFolder(
Paths.get(
assignmentBasePath,
assignmentId
),
testPath
);
LOG.debug("copy test config");
Files.copy(
Paths.get(
assignmentBasePath,
assignmentId + ".txt"
),
Paths.get(
workDirectory.toAbsolutePath().toString(),
"config.txt"
)
);
// clone student code to tmpdir
Pattern pattern = Pattern.compile(UnitTestUpload.modocotDueConfigRegex);
Matcher config = null;
LOG.debug("reading task file");
// open received file in a try-with
try (BufferedReader br = new BufferedReader(
new InputStreamReader(
taskFileRef.getInputStream()))) {
String line;
// as long as we haven't found a configuration and have lines left, search
while (config == null && (line = br.readLine()) != null) {
Matcher matcher = pattern.matcher(line);
if (matcher.matches()) {
LOG.debug(String.format("found modocot line: %s", line));
config = matcher;
}
}
} catch (IOException e) {
LOG.error("Error while reading repo config", e);
}
finally {
if (config == null) {
throw new RuntimeException("couldn't find repo config for unittest clone");
}
}
Thread.sleep(3000);
String userResult = RestAPIController.JOB_MAP.remove(jobId);
this.gitTeaUtil.deleteRepository(jobId);
this.taskUploadUtils.deleteJenkinsJob(jobId);
LOG.debug("calling repo clone");
jGitUtil.cloneRepository(config, srcPath.toAbsolutePath().toString());
Files.createDirectory(resultPath);
LOG.info("starting unittest");
pattern = Pattern.compile(UnitTestUpload.modocotTestConfigRegex);
config = null;
LOG.debug("reading test config");
// open test config and read it in again, important to know with which docker image the test should be run
try (BufferedReader br = new BufferedReader(
new InputStreamReader(
new FileInputStream(Paths.get(workDirectory.toAbsolutePath().toString(), "config.txt").toFile())))) {
String line;
while (config == null && (line = br.readLine()) != null) {
Matcher matcher = pattern.matcher(line);
if (matcher.matches()) {
LOG.debug(String.format("found modocot line: %s", line));
config = matcher;
}
}
} catch (IOException e) {
LOG.error("Error while reading test config", e);
}
finally {
if (config == null) {
throw new RuntimeException("couldn't find test config for unittest");
}
}
if (!this.taskUploadUtils.isValidJSON(userResult)) {
throw new IllegalArgumentException("userResult from JUnitTestLauncher is invalid: " + userResult);
// define the paths to mount as Binds from Host to the test-container
Path testPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
testPath.getName(testPath.getNameCount()-1).toString()
);
Path srcPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
srcPath.getName(srcPath.getNameCount()-1).toString()
);
Path resultPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
resultPath.getName(resultPath.getNameCount()-1).toString()
);
// start test-container with professor given image and bind mounts for test, submission and result
dockerUtil.runContainer(
config.group(4),
new Bind(testPathHost.toAbsolutePath().toString(), new Volume("/modocot/test")),
new Bind(srcPathHost.toAbsolutePath().toString(), new Volume("/modocot/src")),
new Bind(resultPathHost.toAbsolutePath().toString(), new Volume("/modocot/result"))
);
// define expected result file
File resultFile = Paths.get(resultPath.toAbsolutePath().toString(), "result.json").toFile();
// check if result file is there
if (!resultFile.exists() || !resultFile.isFile()) {
LOG.error(String.format("couln't find result file in %s", resultFile.getAbsolutePath()));
throw new RuntimeException("no resultfile found");
}
return userResult;
}
LOG.debug("parse results json");
ObjectMapper objectMapper = new ObjectMapper();
ModocotResultSummary resultSummary = objectMapper.readValue(
resultFile.toURI().toURL(),
ModocotResultSummary.class);
/**
* Extracting all lines from repo.txt and cloning extracted repository
*
* @param assignmentId String assignmentId
* @param subFolderPath working-directory
* @param unzippedFiles List of all Files from {@link MultipartFile}
*/
private void startFileRead(String assignmentId, String subFolderPath, List<File> unzippedFiles) {
boolean gotRepoFile = unzippedFiles.stream().anyMatch(zipFile -> zipFile.getName().equalsIgnoreCase("repo.txt"));
if (gotRepoFile) {
List<String> lines = BackendUtil.extractLinesFromRepoFile(unzippedFiles, "repo.txt");
String repoUrl = (lines.size() > 0 && !lines.get(0).equals("")) ?
lines.get(0) : "";
String credentials = (lines.size() > 1 && !lines.get(1).equals("")) ?
lines.get(1) : null;
File unit = new File(subFolderPath + "/src/UnitTests");
unit.mkdirs();
File task = new File(subFolderPath + "/src/Test");
task.mkdirs();
this.jGitUtil.cloneRepository("http://" + this.modocotProperties.getDockerHostIp()
+ ":3000/" + this.modocotProperties.getGitTeaUsername() + "/"
+ assignmentId + ".git",
unit, null, true, false);
this.jGitUtil.cloneRepository(repoUrl, task, credentials, credentials != null, true);
}
// convert to moddle plugin readable format and return to moodle
LOG.debug("convert to moodle understandable format");
LegacyMoodleResult moodleResult = LegacyMoodleResult.convertToModdleResult(resultSummary);
LOG.info("submission tested successfully");
return moodleResult;
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal