Skip to content

GitLab

Verified Commit 7c10b571 authored by Lukas Wiest's avatar Lukas Wiest 🚂
Browse files

feat: major rework of MoDoCoT

parents 0e7ec0ac 19e0fb0c
Hide whitespace changes
Inline Side-by-side
src/main/java/de/hftstuttgart/dockertests/app/index.html deleted 100644 → 0
View file @ 0e7ec0ac
<!DOCTYPE html>
<html>
<head>
<title>Example</title>
</head>
<body>
<p>Hello World</p>
</body>
</html>
\ No newline at end of file
src/main/java/de/hftstuttgart/dockertests/filetest.txt deleted 100644 → 0
View file @ 0e7ec0ac
Bind mount persistance data through bind Mount
src/main/java/de/hftstuttgart/exceptions/CorruptedZipFileException.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.exceptions;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
/**
* Created by Marcel Bochtler on 28.11.16.
*/
@ResponseStatus(HttpStatus.BAD_REQUEST)
public class CorruptedZipFileException extends RuntimeException {
public CorruptedZipFileException(String s) {
super(s);
}
}
src/main/java/de/hftstuttgart/exceptions/NoZipFileException.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.exceptions;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
/**
* Created by Marcel Bochtler on 29.11.16.
*/
@ResponseStatus(HttpStatus.BAD_REQUEST)
public class NoZipFileException extends RuntimeException {
public NoZipFileException(String message) {
super(message);
}
}
src/main/java/de/hftstuttgart/models/JenkinsJobData.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.hftstuttgart.utils.BuildState;
import java.util.List;
@JsonInclude
public class JenkinsJobData {
public JenkinsJobData() {
}
public String getClassData() {
return classData;
}
public List<Job> getJobs() {
return jobs;
}
@JsonProperty("_class")
private String classData;
private List<Job> jobs;
@JsonProperty("jobs")
public void unpackingJobs(List<Job> jobs) {
this.jobs = jobs;
}
public static class Job {
@JsonProperty("_class")
private String jobClassData;
@JsonProperty("name")
private String name;
@JsonProperty("color")
private BuildState color;
public String getJobClassData() {
return jobClassData;
}
public void setJobClassData(String jobClassData) {
this.jobClassData = jobClassData;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public BuildState getColor() {
return color;
}
public void setColor(BuildState color) {
this.color = color;
}
}
}
src/main/java/de/hftstuttgart/models/LegacyMoodleResult.java 0 → 100644
View file @ 7c10b571
package de.hftstuttgart.models;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
public class LegacyMoodleResult {
public List<LegacyMoodleTestResult> testResults;
public List<LegacyMoodleCompilationError> compilationErrors;
public static LegacyMoodleResult convertToModdleResult(ModocotResultSummary summary) {
LegacyMoodleTestResult tests = new LegacyMoodleTestResult();
tests.failureCount = summary.failureCount;
tests.testCount = summary.testCount;
tests.successfulTests = summary.successes.stream()
.map(s -> s.name)
.collect(Collectors.toList());
tests.testFailures = summary.failures.stream()
.map(f -> {
LegacyMoodleTestFailure failure = new LegacyMoodleTestFailure();
failure.testHeader = f.name;
failure.message = f.failureReason;
failure.trace = f.stacktrace;
return failure;
})
.collect(Collectors.toList());
List<LegacyMoodleCompilationError> compilationErrors;
compilationErrors = summary.compilationErrors.stream()
.map(f -> {
LegacyMoodleCompilationError cError = new LegacyMoodleCompilationError();
cError.javaFileName = f.name;
cError.message = f.failureReason;
cError.lineNumber = f.lineNumber;
cError.columnNumber = f.columnNumber;
cError.position = f.position;
return cError;
})
.collect(Collectors.toList());
LegacyMoodleResult result = new LegacyMoodleResult();
result.testResults = Collections.singletonList(tests);
result.compilationErrors = compilationErrors;
return result;
}
private static class LegacyMoodleTestResult {
public String testName = "UnitTests";
public int testCount;
public int failureCount;
public List<String> successfulTests;
public List<LegacyMoodleTestFailure> testFailures;
}
private static class LegacyMoodleTestFailure {
public String testHeader;
public String message;
public String trace;
}
private static class LegacyMoodleCompilationError {
public String message;
public String javaFileName;
public int lineNumber;
public int columnNumber;
public int position;
}
}
src/main/java/de/hftstuttgart/models/ModocotResult.java 0 → 100644
View file @ 7c10b571
package de.hftstuttgart.models;
public class ModocotResult
{
public String name;
public int state;
public String failureType;
public String failureReason;
public String stacktrace;
public int columnNumber;
public int lineNumber;
public int position;
public static enum State
{
SUCCESS,
FAILURE,
UNKNOWN
}
}
src/main/java/de/hftstuttgart/models/ModocotResultSummary.java 0 → 100644
View file @ 7c10b571
package de.hftstuttgart.models;
import java.util.Set;
public class ModocotResultSummary
{
public long timestamp;
public int testCount;
public int failureCount;
public int successCount;
public String globalStacktrace;
public Set<ModocotResult> successes;
public Set<ModocotResult> failures;
public Set<ModocotResult> compilationErrors;
}
src/main/java/de/hftstuttgart/models/TestResult.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.models;
import org.junit.runner.notification.Failure;
import java.util.List;
public class TestResult {
private String testName;
private int testCount;
private int failureCount;
private List<String> successfulTests;
private List<Failure> testFailures;
public String getTestName() {
return testName;
}
public void setTestName(String testName) {
this.testName = testName;
}
public int getTestCount() {
return testCount;
}
public void setTestCount(int testCount) {
this.testCount = testCount;
}
public int getFailureCount() {
return failureCount;
}
public void setFailureCount(int failureCount) {
this.failureCount = failureCount;
}
public List<String> getSuccessfulTests() {
return successfulTests;
}
public void setSuccessfulTests(List<String> successfulTests) {
this.successfulTests = successfulTests;
}
public List<Failure> getTestFailures() {
return testFailures;
}
public void setTestFailures(List<Failure> testFailures) {
this.testFailures = testFailures;
}
@Override
public String toString() {
return "TestResult{" +
"testName='" + testName + '\'' +
", testCount=" + testCount +
", failureCount=" + failureCount +
", testFailures=" + testFailures +
'}';
}
}
src/main/java/de/hftstuttgart/models/UserResult.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.models;
import javax.tools.Diagnostic;
import java.util.List;
public class UserResult {
private List<TestResult> testResults;
private List<Diagnostic> compilationErrors;
public UserResult(List<TestResult> testResults) {
this.testResults = testResults;
}
public List<TestResult> getTestResults() {
return testResults;
}
public void setTestResults(List<TestResult> testResults) {
this.testResults = testResults;
}
public List<Diagnostic> getCompilationErrors() {
return compilationErrors;
}
public void setCompilationErrors(List<Diagnostic> compilationErrors) {
this.compilationErrors = compilationErrors;
}
}
src/main/java/de/hftstuttgart/rest/v1/jenkins/RestAPIController.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.rest.v1.jenkins;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap;
import java.util.Map;
@RestController
public class RestAPIController {
private static final Logger LOG = LogManager.getLogger(RestAPIController.class);
public static final Map<String, String> JOB_MAP = new HashMap<>();
@PostMapping("/v1/uploaduserresults")
public void uploadsResults(@RequestParam("jobID") String jobId, @RequestBody String userResult) {
LOG.info("result: " + userResult);
if (!JOB_MAP.containsKey(jobId)) {
String keys = String.join(", ", JOB_MAP.keySet());
throw new IllegalArgumentException(
String.format("Key %s does not exist in JOB_MAP, available Keys: [%s]", jobId, keys));
}
JOB_MAP.put(jobId, userResult);
}
}
\ No newline at end of file
src/main/java/de/hftstuttgart/rest/v1/task/TaskUpload.java
View file @ 7c10b571
package de.hftstuttgart.rest.v1.task; package de.hftstuttgart.rest.v1.task;
import de.hftstuttgart.config.ModocotProperties; import com.fasterxml.jackson.databind.ObjectMapper;
import de.hftstuttgart.rest.v1.jenkins.RestAPIController; import com.github.dockerjava.api.model.Bind;
import de.hftstuttgart.utils.BackendUtil; import com.github.dockerjava.api.model.Volume;
import de.hftstuttgart.utils.BuildState; import de.hftstuttgart.models.LegacyMoodleResult;
import de.hftstuttgart.utils.GitTeaUtil; import de.hftstuttgart.models.ModocotResultSummary;
import de.hftstuttgart.rest.v1.unittest.UnitTestUpload;
import de.hftstuttgart.utils.DockerUtil;
import de.hftstuttgart.utils.FileUtil;
import de.hftstuttgart.utils.JGitUtil; import de.hftstuttgart.utils.JGitUtil;
import de.hftstuttgart.utils.TaskUploadUtils;
import de.hftstuttgart.utils.UnzipUtil;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.springframework.core.env.Environment;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
...@@ -17,10 +19,12 @@ import org.springframework.web.bind.annotation.RestController; ...@@ -17,10 +19,12 @@ import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import javax.servlet.annotation.MultipartConfig; import javax.servlet.annotation.MultipartConfig;
import java.io.File; import java.io.*;
import java.io.IOException; import java.nio.file.Files;
import java.util.List; import java.nio.file.Path;
import java.util.UUID; import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/** /**
* Rest controller for everything related to the TASK files * Rest controller for everything related to the TASK files
...@@ -31,98 +35,173 @@ import java.util.UUID; ...@@ -31,98 +35,173 @@ import java.util.UUID;
public class TaskUpload { public class TaskUpload {
private static final Logger LOG = LogManager.getLogger(TaskUpload.class); private static final Logger LOG = LogManager.getLogger(TaskUpload.class);
private final ModocotProperties modocotProperties;
private final TaskUploadUtils taskUploadUtils;
private final JGitUtil jGitUtil; private final JGitUtil jGitUtil;
private final DockerUtil dockerUtil;
private final String assignmentBasePath; private final String assignmentBasePath;
private final GitTeaUtil gitTeaUtil; private final Path testTmpPathHost;
private final Path testTmpPathModocot;
public TaskUpload(ModocotProperties modocotProperties, public TaskUpload(
TaskUploadUtils taskUploadUtils, Environment env,
JGitUtil jGitUtil, JGitUtil jGitUtil,
GitTeaUtil gitTeaUtil) { DockerUtil dockerUtil
this.modocotProperties = modocotProperties; ) {
this.taskUploadUtils = taskUploadUtils;
this.jGitUtil = jGitUtil; this.jGitUtil = jGitUtil;
this.gitTeaUtil = gitTeaUtil; this.dockerUtil = dockerUtil;
this.assignmentBasePath =
this.modocotProperties.getModocotParentDirectory()
+ File.separator
+ this.modocotProperties.getModocotAssignmentFolderPrefix();
}
@RequestMapping(method = RequestMethod.POST)
public String uploadAndTestFile(@RequestParam("taskFile") MultipartFile taskFileRef,
@RequestParam("assignmentId") String assignmentId) throws IOException, InterruptedException {
String jobId = assignmentId + "_" + UUID.randomUUID();
String subFolderPath = this.assignmentBasePath + assignmentId;
LOG.info("subFolderPath: " + subFolderPath);
File workDirectory = new File(subFolderPath);
workDirectory.mkdirs();
LOG.info("created new File");
File file = new File(subFolderPath, String.valueOf(UUID.randomUUID()));
taskFileRef.transferTo(file); // set base path for assignments to be stored
List<File> unzippedFiles = UnzipUtil.unzip(file); Path p = Paths.get(
env.getProperty("modocot.dir"),
env.getProperty("modocot.dir.test.folder.name"));
this.assignmentBasePath = p.toAbsolutePath().toString();
if (file.exists()) file.delete(); // set path of temporary directory on host and inside our container
this.testTmpPathHost = Paths.get(env.getProperty("host.tests.tmp.dir"));
startFileRead(assignmentId, subFolderPath, unzippedFiles); this.testTmpPathModocot = Paths.get(env.getProperty("modocot.tests.tmp.dir"));
}
BuildState buildState = this.taskUploadUtils.startTask(jobId, subFolderPath);
if (buildState != BuildState.BLUE) { @RequestMapping(method = RequestMethod.POST)
String jenkinsError = this.taskUploadUtils.getJenkinsConsoleOutput(jobId); public LegacyMoodleResult uploadAndTestFile(@RequestParam("taskFile") MultipartFile taskFileRef,
LOG.error(jenkinsError); @RequestParam("assignmentId") String assignmentId
RestAPIController.JOB_MAP.remove(jobId); ) throws IOException, InterruptedException {
this.gitTeaUtil.deleteRepository(jobId); LOG.info("submission for testing received");
this.taskUploadUtils.deleteJenkinsJob(jobId);
return jenkinsError; LOG.debug("creating new temporary directory");
Path workDirectory = Files.createTempDirectory(testTmpPathModocot, "modocot");
LOG.debug(String.format("working dir for test is: %s", workDirectory.toAbsolutePath().toString()));
// define paths for the test, the submission and where the result is to be expected afterwards
Path testPath = Paths.get(workDirectory.toAbsolutePath().toString(), "test");
Path srcPath = Paths.get(workDirectory.toAbsolutePath().toString(), "src");
Path resultPath = Paths.get(workDirectory.toAbsolutePath().toString(), "result");
// clone stored test to tmpdir
LOG.debug("copying pre-downloaded unitttest repo");
FileUtil.copyFolder(
Paths.get(
assignmentBasePath,
assignmentId
),
testPath
);
LOG.debug("copy test config");
Files.copy(
Paths.get(
assignmentBasePath,
assignmentId + ".txt"
),
Paths.get(
workDirectory.toAbsolutePath().toString(),
"config.txt"
)
);
// clone student code to tmpdir
Pattern pattern = Pattern.compile(UnitTestUpload.modocotDueConfigRegex);
Matcher config = null;
LOG.debug("reading task file");
// open received file in a try-with
try (BufferedReader br = new BufferedReader(
new InputStreamReader(
taskFileRef.getInputStream()))) {
String line;
// as long as we haven't found a configuration and have lines left, search
while (config == null && (line = br.readLine()) != null) {
Matcher matcher = pattern.matcher(line);
if (matcher.matches()) {
LOG.debug(String.format("found modocot line: %s", line));
config = matcher;
}
}
} catch (IOException e) {
LOG.error("Error while reading repo config", e);
}
finally {
if (config == null) {
throw new RuntimeException("couldn't find repo config for unittest clone");
}
} }
Thread.sleep(3000); LOG.debug("calling repo clone");
jGitUtil.cloneRepository(config, srcPath.toAbsolutePath().toString());
String userResult = RestAPIController.JOB_MAP.remove(jobId);
Files.createDirectory(resultPath);
this.gitTeaUtil.deleteRepository(jobId);
LOG.info("starting unittest");
this.taskUploadUtils.deleteJenkinsJob(jobId); pattern = Pattern.compile(UnitTestUpload.modocotTestConfigRegex);
config = null;
LOG.debug("reading test config");
// open test config and read it in again, important to know with which docker image the test should be run
try (BufferedReader br = new BufferedReader(
new InputStreamReader(
new FileInputStream(Paths.get(workDirectory.toAbsolutePath().toString(), "config.txt").toFile())))) {
String line;
while (config == null && (line = br.readLine()) != null) {
Matcher matcher = pattern.matcher(line);
if (matcher.matches()) {
LOG.debug(String.format("found modocot line: %s", line));
config = matcher;
}
}
} catch (IOException e) {
LOG.error("Error while reading test config", e);
}
finally {
if (config == null) {
throw new RuntimeException("couldn't find test config for unittest");
}
}
if (!this.taskUploadUtils.isValidJSON(userResult)) { // define the paths to mount as Binds from Host to the test-container
throw new IllegalArgumentException("userResult from JUnitTestLauncher is invalid: " + userResult); Path testPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
testPath.getName(testPath.getNameCount()-1).toString()
);
Path srcPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
srcPath.getName(srcPath.getNameCount()-1).toString()
);
Path resultPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
resultPath.getName(resultPath.getNameCount()-1).toString()
);
// start test-container with professor given image and bind mounts for test, submission and result
dockerUtil.runContainer(
config.group(4),
new Bind(testPathHost.toAbsolutePath().toString(), new Volume("/modocot/test")),
new Bind(srcPathHost.toAbsolutePath().toString(), new Volume("/modocot/src")),
new Bind(resultPathHost.toAbsolutePath().toString(), new Volume("/modocot/result"))
);
// define expected result file
File resultFile = Paths.get(resultPath.toAbsolutePath().toString(), "result.json").toFile();
// check if result file is there
if (!resultFile.exists() || !resultFile.isFile()) {
LOG.error(String.format("couln't find result file in %s", resultFile.getAbsolutePath()));
throw new RuntimeException("no resultfile found");
} }
return userResult; LOG.debug("parse results json");
} ObjectMapper objectMapper = new ObjectMapper();
ModocotResultSummary resultSummary = objectMapper.readValue(
resultFile.toURI().toURL(),
ModocotResultSummary.class);
/** // convert to moddle plugin readable format and return to moodle
* Extracting all lines from repo.txt and cloning extracted repository LOG.debug("convert to moodle understandable format");
* LegacyMoodleResult moodleResult = LegacyMoodleResult.convertToModdleResult(resultSummary);
* @param assignmentId String assignmentId
* @param subFolderPath working-directory LOG.info("submission tested successfully");
* @param unzippedFiles List of all Files from {@link MultipartFile} return moodleResult;
*/
private void startFileRead(String assignmentId, String subFolderPath, List<File> unzippedFiles) {
boolean gotRepoFile = unzippedFiles.stream().anyMatch(zipFile -> zipFile.getName().equalsIgnoreCase("repo.txt"));
if (gotRepoFile) {
List<String> lines = BackendUtil.extractLinesFromRepoFile(unzippedFiles, "repo.txt");
String repoUrl = (lines.size() > 0 && !lines.get(0).equals("")) ?
lines.get(0) : "";
String credentials = (lines.size() > 1 && !lines.get(1).equals("")) ?
lines.get(1) : null;
File unit = new File(subFolderPath + "/src/UnitTests");
unit.mkdirs();
File task = new File(subFolderPath + "/src/Test");
task.mkdirs();
this.jGitUtil.cloneRepository("http://" + this.modocotProperties.getDockerHostIp()
+ ":3000/" + this.modocotProperties.getGitTeaUsername() + "/"
+ assignmentId + ".git",
unit, null, true, false);
this.jGitUtil.cloneRepository(repoUrl, task, credentials, credentials != null, true);
}
} }
} }
src/main/java/de/hftstuttgart/rest/v1/unittest/UnitTestUpload.java
View file @ 7c10b571
package de.hftstuttgart.rest.v1.unittest; package de.hftstuttgart.rest.v1.unittest;
import de.hftstuttgart.config.ModocotProperties;
import de.hftstuttgart.utils.BackendUtil;
import de.hftstuttgart.utils.FileUtil;
import de.hftstuttgart.utils.GitTeaUtil;
import de.hftstuttgart.utils.JGitUtil; import de.hftstuttgart.utils.JGitUtil;
import de.hftstuttgart.utils.UnzipUtil;
import io.gitea.model.Repository;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.eclipse.jgit.transport.PushResult; import org.springframework.core.env.Environment;
import org.springframework.util.FileSystemUtils;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
...@@ -17,13 +12,11 @@ import org.springframework.web.bind.annotation.RestController; ...@@ -17,13 +12,11 @@ import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import javax.servlet.annotation.MultipartConfig; import javax.servlet.annotation.MultipartConfig;
import java.io.File; import java.io.*;
import java.io.IOException; import java.nio.file.Path;
import java.util.List; import java.nio.file.Paths;
import java.util.Objects; import java.util.regex.Matcher;
import java.util.UUID; import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/** /**
* Rest controller for anything related to the TEST files. * Rest controller for anything related to the TEST files.
...@@ -34,93 +27,77 @@ import java.util.stream.Stream; ...@@ -34,93 +27,77 @@ import java.util.stream.Stream;
public class UnitTestUpload { public class UnitTestUpload {
private static final Logger LOG = LogManager.getLogger(UnitTestUpload.class); private static final Logger LOG = LogManager.getLogger(UnitTestUpload.class);
public final static String modocotTestConfigRegex = "^modocot::(.*)::(.*|none)::(.*|none)::(.*)$";
public final static String modocotDueConfigRegex = "^modocot::(.*)::(.*|none)::(.*|none)$";
private final ModocotProperties modocotProperties;
private final GitTeaUtil gitTeaUtil;
private final JGitUtil jGitUtil; private final JGitUtil jGitUtil;
private final String assignmentBasePath; private final String assignmentBasePath;
public UnitTestUpload(ModocotProperties modocotProperties, public UnitTestUpload(Environment env, JGitUtil jGitUtil) {
GitTeaUtil gitTeaUtil,
JGitUtil jGitUtil) {
this.modocotProperties = modocotProperties;
this.gitTeaUtil = gitTeaUtil;
this.jGitUtil = jGitUtil; this.jGitUtil = jGitUtil;
this.assignmentBasePath =
modocotProperties.getModocotParentDirectory() Path p = Paths.get(env.getProperty("modocot.dir"), env.getProperty("modocot.dir.test.folder.name"));
+ File.separator this.assignmentBasePath = p.toAbsolutePath().toString();
+ modocotProperties.getModocotAssignmentFolderPrefix();
} }
/** /**
* Create a subfolder for the specific assignment. * Create a subfolder for the specific assignment.
* This is called when the teacher creates an assignment and uploads the JUnit test files * This is called when the teacher creates an assignment and uploads the JUnit test files
* *
* @param unitTestFileRef The zip file which contains the JUnit tests * @param unitTestFileRef The text file which contains the JUnit tests meta data
* @param assignmentId ID of the created assignment. Generated by Moodle * @param assignmentId ID of the created assignment. Generated by Moodle
*/ */
@RequestMapping(method = RequestMethod.POST) @RequestMapping(method = RequestMethod.POST)
public void uploadUnitTestFile(@RequestParam("unitTestFile") MultipartFile unitTestFileRef, @RequestParam("assignmentId") String assignmentId) throws IOException { public void uploadUnitTestFile(
@RequestParam("unitTestFile") MultipartFile unitTestFileRef,
// path to the directory in which we will be working @RequestParam("assignmentId") String assignmentId
String subFolderPath = this.assignmentBasePath + assignmentId; ) throws IOException {
LOG.info("received new assignment");
LOG.info("work-directory: " + subFolderPath);
// creating the work-directory File file = Paths.get(
File workDirectory = new File(subFolderPath); this.assignmentBasePath,
workDirectory.mkdirs(); assignmentId + ".txt")
.toFile();
// creating the file which the unitTestFileRef will be transferred into file.mkdirs();
File file = new File(subFolderPath, String.valueOf(UUID.randomUUID()));
// save assignment config
// transferring MultipartFile into temporary file
unitTestFileRef.transferTo(file); unitTestFileRef.transferTo(file);
LOG.debug(String.format("saved config file to: %s", file.getAbsolutePath()));
// unzipping temporary file to work-directory
List<File> zipFiles = UnzipUtil.unzip(file); Pattern pattern = Pattern.compile(this.modocotTestConfigRegex);
Matcher config = null;
// unzipping temporary file (not needed anymore)
if (file.exists()) file.delete(); LOG.debug("reading test configuration file");
// open saved config in a try-with
// check if any extracted files are named repo.txt try (BufferedReader br = new BufferedReader(
if (zipFiles.stream().anyMatch(zipFile -> zipFile.getName().equalsIgnoreCase("repo.txt"))) { new InputStreamReader(
// reading all from the repo.txt new FileInputStream(file)))) {
List<String> lines = BackendUtil.extractLinesFromRepoFile(zipFiles, "repo.txt"); String line;
// either set the first line of repo.txt or ""
String repoUrl = (lines.size() > 0 && !lines.get(0).equals("")) ? // search for a modocot URI while none is found and there are lines left
lines.get(0) : ""; while (config == null && (line = br.readLine()) != null) {
LOG.info("repoUrl: " + repoUrl); Matcher matcher = pattern.matcher(line);
// either set the second line of repo.txt or null if (matcher.matches()) {
String credentials = ((lines.size() > 1) && !lines.get(1).equals("")) ? LOG.debug(String.format("found modocot test line: %s", line));
lines.get(1) : null; config = matcher;
LOG.info("credentials: " + credentials); }
// cloning repository repoUrl into work-directory, using credentials from second line in repo.txt }
this.jGitUtil.cloneRepository(repoUrl, workDirectory, credentials, credentials != null,true); } catch (IOException e) {
} LOG.error("Error while reading repo config", e);
Stream.of(Objects.requireNonNull(workDirectory.listFiles()))
.forEach(fi -> System.out.println(fi.getAbsolutePath()));
if(this.gitTeaUtil.repositoryExists(assignmentId)) {
this.gitTeaUtil.deleteRepository(assignmentId);
} }
// creating repository on internal giTea, name = assignmentId finally {
Repository repo = this.gitTeaUtil.createRepository(assignmentId); if (config == null) {
System.out.println(repo); throw new RuntimeException("couldn't find repo config for unittest clone");
// giTea runs in docker and returns CloneUrl as localhost, replacing localhost with docker-host ip
repo.setCloneUrl(repo.getCloneUrl().replace("localhost", modocotProperties.getDockerHostIp()));
// committing everything in work-directory and push to created repository
Iterable<PushResult> pushResults = this.jGitUtil.commitAllAndPush(workDirectory, repo, false);
if (pushResults != null) {
for (PushResult pushResult : pushResults) {
LOG.info("Push-Result: " + pushResult.getMessages());
} }
} }
// deleting created work-directory
FileUtil.deleteFolderRecursively(workDirectory);
LOG.info("Uploaded unit test file: " + workDirectory); LOG.debug("calling test repo clone");
// cloning assignment repo to persistent space
jGitUtil.cloneRepository(
config,
Paths.get(this.assignmentBasePath, assignmentId).toAbsolutePath().toString());
LOG.info(String.format("stored new assignment: %s", file.getAbsolutePath()));
} }
/** /**
...@@ -133,10 +110,21 @@ public class UnitTestUpload { ...@@ -133,10 +110,21 @@ public class UnitTestUpload {
*/ */
@RequestMapping(method = RequestMethod.DELETE) @RequestMapping(method = RequestMethod.DELETE)
public void deleteUnitTestFiles(@RequestParam("assignmentId") String assignmentId) { public void deleteUnitTestFiles(@RequestParam("assignmentId") String assignmentId) {
String path = this.assignmentBasePath + assignmentId; LOG.info(String.format("received deletion order for assignment %s", assignmentId));
File dir = new File(path);
if (dir.exists()) { // deleting config file
FileUtil.deleteFolderRecursively(dir); File file = Paths.get(
} this.assignmentBasePath,
assignmentId + ".txt")
.toFile();
file.delete();
// deleting local copy of repository
file = Paths.get(
this.assignmentBasePath,
assignmentId).toFile();
FileSystemUtils.deleteRecursively(file);
LOG.info(String.format("assignment %s deletion complete", assignmentId));
} }
} }
src/main/java/de/hftstuttgart/utils/BackendUtil.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.List;
import java.util.stream.Collectors;
public class BackendUtil {
/**
* Loop through all {@link File}s in {@code filesFromZipFile} and find {@link File} {@code fileNameToSearchFor}. <br>
* And return a {@link List} of Strings for each line in file {@code fileNameToSearchFor}
*
* @param filesFromZipFile all files which get extracted from previous zipFile
* @param fileNameToSearchFor search for specific name
* @return {@link List} of Strings for each line in file {@code fileNameToSearchFor}
*/
public static List<String> extractLinesFromRepoFile(List<File> filesFromZipFile, String fileNameToSearchFor) {
if(filesFromZipFile.size() < 1 && fileNameToSearchFor != null) {
throw new IllegalArgumentException();
}
List<String> lines = null;
for (File file : filesFromZipFile) {
if (file.getName().equalsIgnoreCase(fileNameToSearchFor)) {
try {
FileInputStream fis = new FileInputStream(file);
BufferedReader br = new BufferedReader(new InputStreamReader(fis));
lines = br.lines().collect(Collectors.toList());
br.close();
if (file.exists())
file.delete();
} catch (IOException ignored) { }
}
}
return lines;
}
}
src/main/java/de/hftstuttgart/utils/BuildState.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.utils;
import com.fasterxml.jackson.annotation.JsonProperty;
public enum BuildState {
@JsonProperty("blue") BLUE,
@JsonProperty("notbuilt") NOTBUILT,
@JsonProperty("notbuilt_anime") NOTBUILT_ANMIE,
@JsonProperty("red") RED,
@JsonProperty("yellow") YELLOW,
@JsonProperty("grey") GREY,
@JsonProperty("disabled") DISABLED,
@JsonProperty("aborted") ABORTED,
@JsonProperty("aborted_anime") ABORTED_ANIME,
@JsonProperty("grey_anime") GREY_ANIME
}
src/main/java/de/hftstuttgart/utils/DockerUtil.java 0 → 100644
View file @ 7c10b571
package de.hftstuttgart.utils;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.HostConfig;
import com.github.dockerjava.core.DefaultDockerClientConfig;
import com.github.dockerjava.core.DockerClientConfig;
import com.github.dockerjava.core.DockerClientImpl;
import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
import com.github.dockerjava.transport.DockerHttpClient;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
import java.io.IOException;
@Component
public class DockerUtil {
private static final Logger LOG = LogManager.getLogger(DockerUtil.class);
private DockerClient dockerClient;
public DockerUtil(Environment env) {
LOG.info("initializing Docker tools");
LOG.debug("create docker client config");
DockerClientConfig dockerClientConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build();
LOG.debug("create docker http client");
DockerHttpClient httpClient = new ApacheDockerHttpClient.Builder()
.dockerHost(dockerClientConfig.getDockerHost())
.sslConfig(dockerClientConfig.getSSLConfig())
.build();
LOG.debug("create docker client");
dockerClient = DockerClientImpl.getInstance(dockerClientConfig, httpClient);
}
public int runContainer(String image, Bind... binds) throws InterruptedException, IOException
{
LOG.debug(String.format("pull image: %s", image));
dockerClient.pullImageCmd(image)
.start()
.awaitCompletion()
.close();
LOG.debug("creating container");
CreateContainerResponse containerResponse = dockerClient.createContainerCmd("testcontainer")
.withImage(image)
.withHostConfig(
HostConfig.newHostConfig()
.withBinds(binds))
.exec();
LOG.debug(String.format("container created: %s", containerResponse.getId()));
LOG.debug(String.format("starting container %s", containerResponse.getId()));
dockerClient.startContainerCmd(containerResponse.getId()).exec();
LOG.debug(String.format("waiting for completion of container %s", containerResponse.getId()));
int ret = dockerClient
.waitContainerCmd(containerResponse.getId())
.start()
.awaitCompletion()
.awaitStatusCode();
LOG.debug(String.format("container completed with status %d", ret));
LOG.debug(String.format("deleting container %s", containerResponse.getId()));
dockerClient.removeContainerCmd(containerResponse.getId())
.withRemoveVolumes(true)
.exec();
return ret;
}
}
src/main/java/de/hftstuttgart/utils/FileUtil.java
View file @ 7c10b571
package de.hftstuttgart.utils; package de.hftstuttgart.utils;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
/** /**
* Helper Class for all file related tasks. * Helper Class for all file related tasks.
...@@ -27,4 +30,14 @@ public class FileUtil { ...@@ -27,4 +30,14 @@ public class FileUtil {
folder.delete(); folder.delete();
} }
public static void copyFolder(Path src, Path dst) throws IOException {
Files.walk(src)
.forEach(source -> {
try {
Files.copy(source, dst.resolve(src.relativize(source)));
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
}
});
}
} }
src/main/java/de/hftstuttgart/utils/GitTeaUtil.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.utils;
import de.hftstuttgart.config.ModocotProperties;
import io.gitea.ApiClient;
import io.gitea.ApiException;
import io.gitea.Configuration;
import io.gitea.api.RepositoryApi;
import io.gitea.api.UserApi;
import io.gitea.auth.HttpBasicAuth;
import io.gitea.model.CreateRepoOption;
import io.gitea.model.Repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;
@Component
public class GitTeaUtil {
private static final Logger LOG = LogManager.getLogger(GitTeaUtil.class);
private final ModocotProperties modocotProperties;
public GitTeaUtil(ModocotProperties modocotProperties) {
this.modocotProperties = modocotProperties;
}
/**
* Create Git-Repository with name {@code repositoryName} on internal Git-Server
*
* @param repositoryName name of new Repository
* @return newly created Repository
*/
public Repository createRepository(String repositoryName) {
setupAuth();
Repository repo = null;
try {
repo = new UserApi()
.createCurrentUserRepo(new CreateRepoOption().name(repositoryName));
} catch (ApiException e) {
LOG.error(String.format("Error while creating repository: %s", repositoryName), e);
}
if (repo != null) {
LOG.info("Created repository {} on {}", repositoryName, repo.getCloneUrl());
} else {
throw new IllegalStateException("Repository is null");
}
return repo;
}
/**
* Delete Git-Repository with name {@code repositoryName} on internal Git-Server
*
* @param repositoryName name of new Repository
*/
public void deleteRepository(String repositoryName) {
setupAuth();
try {
new RepositoryApi().repoDelete(this.modocotProperties.getGitTeaUsername(), repositoryName);
} catch (ApiException e) {
LOG.error("Error while deleting repository:" + e.getMessage());
}
}
/**
* Check if a Git-Repository with the given name exists.
*
* @param repositoryName name of the Repository
*/
public boolean repositoryExists(String repositoryName) {
setupAuth();
Repository repository = null;
try {
repository = new RepositoryApi().repoGet(this.modocotProperties.getGitTeaUsername(), repositoryName);
} catch (ApiException e) {
LOG.error("Error while deleting repository:" + e.getMessage());
}
return repository != null;
}
private void setupAuth() {
ApiClient defaultClient = Configuration.getDefaultApiClient();
defaultClient.setBasePath(this.modocotProperties.getGitTeaBasePath());
HttpBasicAuth basicAuth = (HttpBasicAuth) defaultClient.getAuthentication("BasicAuth");
basicAuth.setUsername(this.modocotProperties.getGitTeaUsername());
basicAuth.setPassword(this.modocotProperties.getGitTeaPassword());
}
}
src/main/java/de/hftstuttgart/utils/JGitUtil.java
View file @ 7c10b571
package de.hftstuttgart.utils; package de.hftstuttgart.utils;
import de.hftstuttgart.config.ModocotProperties;
import io.gitea.model.Repository;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.transport.PushResult;
import org.eclipse.jgit.transport.URIish;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.FileSystemUtils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.util.regex.Matcher;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.ProxySelector;
import java.net.SocketAddress;
import java.net.URI;
import java.util.Arrays;
import java.util.List;
@Component @Component
public class JGitUtil { public class JGitUtil {
private static final Logger LOG = LogManager.getLogger(JGitUtil.class); private static final Logger LOG = LogManager.getLogger(JGitUtil.class);
private final ModocotProperties modocotProperties; public JGitUtil() {}
public JGitUtil(ModocotProperties modocotProperties) { public void cloneRepository(Matcher config, String targetPath) {
this.modocotProperties = modocotProperties; LOG.debug(String.format("cloning repository: %s", config.group(1)));
}
/** File targetDirectory = new File(targetPath);
* Clone Repository from {@code uriToClone} into Directory {@code cloneDirectory}, and using token {@code token}.<br> if (targetDirectory.exists()) {
* {@code boolean proxy} determines, if a proxy should be used LOG.debug("clone target directory existing yet, deleting now");
* FileSystemUtils.deleteRecursively(targetDirectory);
* @param uriToClone Repository cloneUrl
* @param cloneDirectory {@link File} directory in which Repository should be cloned into
* @param token AuthToken, when cloning Repository. If {@code null} use {@code ModocotProperties}
* @param proxy boolean if a proxy should be used
*/
public void cloneRepository(String uriToClone, File cloneDirectory, String token, boolean useToken, boolean proxy) {
if (!cloneDirectory.isDirectory()) {
String error = String.format("%s is not a directory, cannot clone", cloneDirectory.getAbsolutePath());
LOG.error(error);
throw new IllegalArgumentException(error);
}
LOG.info(String.format("Cloning all files from %s into %s", uriToClone, cloneDirectory.getAbsolutePath()));
try {
deleteDotGitFolder(cloneDirectory);
UsernamePasswordCredentialsProvider credProvider = null;
LOG.info("using token: " + useToken);
if (useToken) {
LOG.info("token: " + token);
if (token != null) {
LOG.info("Using token: " + token);
String[] credentials = token.split(":");
credProvider = new UsernamePasswordCredentialsProvider(credentials[0], credentials[1]);
} else {
LOG.info("Using credentials: " + this.modocotProperties.getGitTeaUsername() + ", " + this.modocotProperties.getGitTeaPassword());
credProvider = new UsernamePasswordCredentialsProvider(
this.modocotProperties.getGitTeaUsername(),
this.modocotProperties.getGitTeaPassword());
}
}
LOG.info("Starting cloning, url: " + uriToClone);
setProxy(proxy);
if (useToken) {
Git.cloneRepository()
.setCredentialsProvider(credProvider)
.setURI(uriToClone)
.setDirectory(cloneDirectory)
.call()
.close();
} else {
Git.cloneRepository()
.setURI(uriToClone)
.setDirectory(cloneDirectory)
.call()
.close();
}
} catch (GitAPIException e) {
LOG.error(String.format("Error while cloning from %s", uriToClone), e);
} }
}
/**
* Commit {@code directory} and push all files to the given {@code repository}.
*
* @param directory {@link File} directory to commit
* @param repository {@link Repository} Repository to push to
* @param proxy boolean if a proxy should be used
* @return {@code Iterable<PushResult>}
*/
public Iterable<PushResult> commitAllAndPush(File directory, Repository repository, boolean proxy) {
if (!directory.isDirectory()) {
String error = String.format("%s is not a directory, cannot commit or push", directory.getAbsolutePath());
LOG.error(error);
throw new IllegalArgumentException(error);
}
LOG.info(String.format("Committing all files in: %s and pushing them to: %s", directory.getAbsolutePath(), repository.getCloneUrl()));
try { try {
deleteDotGitFolder(directory); LOG.debug("preparing clone");
// "git init" new repository CloneCommand cloneCommand = Git.cloneRepository()
Git.init() .setDirectory(targetDirectory)
.setDirectory(directory) .setURI(config.group(1));
.call();
if (!config.group(2).equals("none") && !config.group(3).equals("none")) {
// open new repository LOG.debug("setting credentials");
Git git = Git.open(directory); cloneCommand.setCredentialsProvider(
new UsernamePasswordCredentialsProvider(config.group(2), config.group(3)));
// "git add ." on repository }
git.add()
.addFilepattern(".")
.call();
// "git commit -m %gitTeaCommitMessage%" on repository
git.commit()
.setMessage(this.modocotProperties.getGitTeaDefaultCommitMessage())
.call();
// add new remote from repository
git.remoteAdd()
.setName(this.modocotProperties.getGitTeaOrigin())
.setUri(new URIish(repository.getCloneUrl()))
.call();
// "git push" to new origin
setProxy(proxy);
return git.push()
.setCredentialsProvider(new UsernamePasswordCredentialsProvider(
this.modocotProperties.getGitTeaUsername(),
this.modocotProperties.getGitTeaPassword()))
.call();
} catch (Exception e) {
LOG.error(String.format("Error while committing to repo: %s from file: %s", repository, directory.getAbsolutePath()), e);
}
return null;
}
private void deleteDotGitFolder(File directory) { LOG.debug("cloning...");
File temp = new File(directory.getPath() + "/src/UnitTests/.git"); cloneCommand.call()
if (temp.exists() && temp.isDirectory() && temp.canWrite()) { .close();
FileUtil.deleteFolderRecursively(temp);
} }
temp = new File(directory.getPath() + "/src/.git"); catch (GitAPIException e) {
if (temp.exists() && temp.isDirectory() && temp.canWrite()) { LOG.error(String.format("Error while cloning from %s", config.group(1)), e);
FileUtil.deleteFolderRecursively(temp);
} }
temp = new File(directory.getPath() + "/.git");
if (temp.exists() && temp.isDirectory() && temp.canWrite()) {
FileUtil.deleteFolderRecursively(temp);
}
}
private void setProxy(boolean proxy) {
if (proxy) {
ProxySelector.setDefault(new ProxySelector() {
final ProxySelector delegate = ProxySelector.getDefault();
@Override LOG.debug(String.format("cloned from %s to %s", config.group(1), targetDirectory));
public List<Proxy> select(URI uri) {
// Filter the URIs to be proxied
if (uri.toString().contains("https")) {
return Arrays.asList(new Proxy(Proxy.Type.HTTP, InetSocketAddress
.createUnresolved("proxy.hft-stuttgart.de", 80)));
}
if (uri.toString().contains("http")) {
return Arrays.asList(new Proxy(Proxy.Type.HTTP, InetSocketAddress
.createUnresolved("proxy.hft-stuttgart.de", 80)));
}
// revert to the default behaviour
return delegate == null ? Arrays.asList(Proxy.NO_PROXY)
: delegate.select(uri);
}
@Override
public void connectFailed(URI uri, SocketAddress sa, IOException ioe) {
if (uri == null || sa == null || ioe == null) {
throw new IllegalArgumentException(
"Arguments can't be null.");
}
}
});
}
} }
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal