Skip to content

GitLab

Verified Commit db826d5d authored by Lukas Wiest's avatar Lukas Wiest 🚂
Browse files

refactor(rest): refactor TaskUpload to V2 

- creates tmp dir
- copies over the stored assignment repo
- clones student submission repo
- runs the image the prof specified in its assignment, mounting
    - the assignment repo
    - the subission repo
    - an empty folder expecting the result file to be in after container
        finished
- reads result file
- converts to moodle understandable instances
- returns result to moodle

BREAKING CHANGE: Gitea eliminated
BREAKING CHANGE: Jenkins eliminated
BREAKING CHANGE: expected file type changed from zip to text
BREAKING CHANGE: textfile has to conform new modocot URI
BREAKING CHANGE: no Jenkins pipe needed in submission repo
BREAKING CHANGE: no Dockerfile needed in submission repo
parent 3a59600d
Hide whitespace changes
Inline Side-by-side
src/main/java/de/hftstuttgart/rest/v1/task/TaskUpload.java
View file @ db826d5d
package de.hftstuttgart.rest.v1.task; package de.hftstuttgart.rest.v1.task;
import de.hftstuttgart.config.ModocotProperties; import com.fasterxml.jackson.databind.ObjectMapper;
import de.hftstuttgart.rest.v1.jenkins.RestAPIController; import com.github.dockerjava.api.model.Bind;
import de.hftstuttgart.utils.BackendUtil; import com.github.dockerjava.api.model.Volume;
import de.hftstuttgart.utils.BuildState; import de.hftstuttgart.models.LegacyMoodleResult;
import de.hftstuttgart.utils.GitTeaUtil; import de.hftstuttgart.models.ModocotResultSummary;
import de.hftstuttgart.rest.v1.unittest.UnitTestUpload;
import de.hftstuttgart.utils.DockerUtil;
import de.hftstuttgart.utils.FileUtil;
import de.hftstuttgart.utils.JGitUtil; import de.hftstuttgart.utils.JGitUtil;
import de.hftstuttgart.utils.TaskUploadUtils;
import de.hftstuttgart.utils.UnzipUtil;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.springframework.core.env.Environment;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
...@@ -17,10 +19,12 @@ import org.springframework.web.bind.annotation.RestController; ...@@ -17,10 +19,12 @@ import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import javax.servlet.annotation.MultipartConfig; import javax.servlet.annotation.MultipartConfig;
import java.io.File; import java.io.*;
import java.io.IOException; import java.nio.file.Files;
import java.util.List; import java.nio.file.Path;
import java.util.UUID; import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/** /**
* Rest controller for everything related to the TASK files * Rest controller for everything related to the TASK files
...@@ -31,98 +35,173 @@ import java.util.UUID; ...@@ -31,98 +35,173 @@ import java.util.UUID;
public class TaskUpload { public class TaskUpload {
private static final Logger LOG = LogManager.getLogger(TaskUpload.class); private static final Logger LOG = LogManager.getLogger(TaskUpload.class);
private final ModocotProperties modocotProperties;
private final TaskUploadUtils taskUploadUtils;
private final JGitUtil jGitUtil; private final JGitUtil jGitUtil;
private final DockerUtil dockerUtil;
private final String assignmentBasePath; private final String assignmentBasePath;
private final GitTeaUtil gitTeaUtil; private final Path testTmpPathHost;
private final Path testTmpPathModocot;
public TaskUpload(ModocotProperties modocotProperties, public TaskUpload(
TaskUploadUtils taskUploadUtils, Environment env,
JGitUtil jGitUtil, JGitUtil jGitUtil,
GitTeaUtil gitTeaUtil) { DockerUtil dockerUtil
this.modocotProperties = modocotProperties; ) {
this.taskUploadUtils = taskUploadUtils;
this.jGitUtil = jGitUtil; this.jGitUtil = jGitUtil;
this.gitTeaUtil = gitTeaUtil; this.dockerUtil = dockerUtil;
this.assignmentBasePath =
this.modocotProperties.getModocotParentDirectory()
+ File.separator
+ this.modocotProperties.getModocotAssignmentFolderPrefix();
}
@RequestMapping(method = RequestMethod.POST)
public String uploadAndTestFile(@RequestParam("taskFile") MultipartFile taskFileRef,
@RequestParam("assignmentId") String assignmentId) throws IOException, InterruptedException {
String jobId = assignmentId + "_" + UUID.randomUUID();
String subFolderPath = this.assignmentBasePath + assignmentId;
LOG.info("subFolderPath: " + subFolderPath);
File workDirectory = new File(subFolderPath);
workDirectory.mkdirs();
LOG.info("created new File");
File file = new File(subFolderPath, String.valueOf(UUID.randomUUID()));
taskFileRef.transferTo(file); // set base path for assignments to be stored
List<File> unzippedFiles = UnzipUtil.unzip(file); Path p = Paths.get(
env.getProperty("modocot.dir"),
env.getProperty("modocot.dir.test.folder.name"));
this.assignmentBasePath = p.toAbsolutePath().toString();
if (file.exists()) file.delete(); // set path of temporary directory on host and inside our container
this.testTmpPathHost = Paths.get(env.getProperty("host.tests.tmp.dir"));
startFileRead(assignmentId, subFolderPath, unzippedFiles); this.testTmpPathModocot = Paths.get(env.getProperty("modocot.tests.tmp.dir"));
}
BuildState buildState = this.taskUploadUtils.startTask(jobId, subFolderPath);
if (buildState != BuildState.BLUE) { @RequestMapping(method = RequestMethod.POST)
String jenkinsError = this.taskUploadUtils.getJenkinsConsoleOutput(jobId); public LegacyMoodleResult uploadAndTestFile(@RequestParam("taskFile") MultipartFile taskFileRef,
LOG.error(jenkinsError); @RequestParam("assignmentId") String assignmentId
RestAPIController.JOB_MAP.remove(jobId); ) throws IOException, InterruptedException {
this.gitTeaUtil.deleteRepository(jobId); LOG.info("submission for testing received");
this.taskUploadUtils.deleteJenkinsJob(jobId);
return jenkinsError; LOG.debug("creating new temporary directory");
Path workDirectory = Files.createTempDirectory(testTmpPathModocot, "modocot");
LOG.debug(String.format("working dir for test is: %s", workDirectory.toAbsolutePath().toString()));
// define paths for the test, the submission and where the result is to be expected afterwards
Path testPath = Paths.get(workDirectory.toAbsolutePath().toString(), "test");
Path srcPath = Paths.get(workDirectory.toAbsolutePath().toString(), "src");
Path resultPath = Paths.get(workDirectory.toAbsolutePath().toString(), "result");
// clone stored test to tmpdir
LOG.debug("copying pre-downloaded unitttest repo");
FileUtil.copyFolder(
Paths.get(
assignmentBasePath,
assignmentId
),
testPath
);
LOG.debug("copy test config");
Files.copy(
Paths.get(
assignmentBasePath,
assignmentId + ".txt"
),
Paths.get(
workDirectory.toAbsolutePath().toString(),
"config.txt"
)
);
// clone student code to tmpdir
Pattern pattern = Pattern.compile(UnitTestUpload.modocotDueConfigRegex);
Matcher config = null;
LOG.debug("reading task file");
// open received file in a try-with
try (BufferedReader br = new BufferedReader(
new InputStreamReader(
taskFileRef.getInputStream()))) {
String line;
// as long as we haven't found a configuration and have lines left, search
while (config == null && (line = br.readLine()) != null) {
Matcher matcher = pattern.matcher(line);
if (matcher.matches()) {
LOG.debug(String.format("found modocot line: %s", line));
config = matcher;
}
}
} catch (IOException e) {
LOG.error("Error while reading repo config", e);
}
finally {
if (config == null) {
throw new RuntimeException("couldn't find repo config for unittest clone");
}
} }
Thread.sleep(3000); LOG.debug("calling repo clone");
jGitUtil.cloneRepository(config, srcPath.toAbsolutePath().toString());
String userResult = RestAPIController.JOB_MAP.remove(jobId);
Files.createDirectory(resultPath);
this.gitTeaUtil.deleteRepository(jobId);
LOG.info("starting unittest");
this.taskUploadUtils.deleteJenkinsJob(jobId); pattern = Pattern.compile(UnitTestUpload.modocotTestConfigRegex);
config = null;
LOG.debug("reading test config");
// open test config and read it in again, important to know with which docker image the test should be run
try (BufferedReader br = new BufferedReader(
new InputStreamReader(
new FileInputStream(Paths.get(workDirectory.toAbsolutePath().toString(), "config.txt").toFile())))) {
String line;
while (config == null && (line = br.readLine()) != null) {
Matcher matcher = pattern.matcher(line);
if (matcher.matches()) {
LOG.debug(String.format("found modocot line: %s", line));
config = matcher;
}
}
} catch (IOException e) {
LOG.error("Error while reading test config", e);
}
finally {
if (config == null) {
throw new RuntimeException("couldn't find test config for unittest");
}
}
if (!this.taskUploadUtils.isValidJSON(userResult)) { // define the paths to mount as Binds from Host to the test-container
throw new IllegalArgumentException("userResult from JUnitTestLauncher is invalid: " + userResult); Path testPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
testPath.getName(testPath.getNameCount()-1).toString()
);
Path srcPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
srcPath.getName(srcPath.getNameCount()-1).toString()
);
Path resultPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
resultPath.getName(resultPath.getNameCount()-1).toString()
);
// start test-container with professor given image and bind mounts for test, submission and result
dockerUtil.runContainer(
config.group(4),
new Bind(testPathHost.toAbsolutePath().toString(), new Volume("/modocot/test")),
new Bind(srcPathHost.toAbsolutePath().toString(), new Volume("/modocot/src")),
new Bind(resultPathHost.toAbsolutePath().toString(), new Volume("/modocot/result"))
);
// define expected result file
File resultFile = Paths.get(resultPath.toAbsolutePath().toString(), "result.json").toFile();
// check if result file is there
if (!resultFile.exists() || !resultFile.isFile()) {
LOG.error(String.format("couln't find result file in %s", resultFile.getAbsolutePath()));
throw new RuntimeException("no resultfile found");
} }
return userResult; LOG.debug("parse results json");
} ObjectMapper objectMapper = new ObjectMapper();
ModocotResultSummary resultSummary = objectMapper.readValue(
resultFile.toURI().toURL(),
ModocotResultSummary.class);
/** // convert to moddle plugin readable format and return to moodle
* Extracting all lines from repo.txt and cloning extracted repository LOG.debug("convert to moodle understandable format");
* LegacyMoodleResult moodleResult = LegacyMoodleResult.convertToModdleResult(resultSummary);
* @param assignmentId String assignmentId
* @param subFolderPath working-directory LOG.info("submission tested successfully");
* @param unzippedFiles List of all Files from {@link MultipartFile} return moodleResult;
*/
private void startFileRead(String assignmentId, String subFolderPath, List<File> unzippedFiles) {
boolean gotRepoFile = unzippedFiles.stream().anyMatch(zipFile -> zipFile.getName().equalsIgnoreCase("repo.txt"));
if (gotRepoFile) {
List<String> lines = BackendUtil.extractLinesFromRepoFile(unzippedFiles, "repo.txt");
String repoUrl = (lines.size() > 0 && !lines.get(0).equals("")) ?
lines.get(0) : "";
String credentials = (lines.size() > 1 && !lines.get(1).equals("")) ?
lines.get(1) : null;
File unit = new File(subFolderPath + "/src/UnitTests");
unit.mkdirs();
File task = new File(subFolderPath + "/src/Test");
task.mkdirs();
this.jGitUtil.cloneRepository("http://" + this.modocotProperties.getDockerHostIp()
+ ":3000/" + this.modocotProperties.getGitTeaUsername() + "/"
+ assignmentId + ".git",
unit, null, true, false);
this.jGitUtil.cloneRepository(repoUrl, task, credentials, credentials != null, true);
}
} }
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal