Skip to content

GitLab

Verified Commit 7c10b571 authored by Lukas Wiest's avatar Lukas Wiest 🚂
Browse files

feat: major rework of MoDoCoT

parents 0e7ec0ac 19e0fb0c
Hide whitespace changes
Inline Side-by-side
src/main/java/de/hftstuttgart/utils/RestCall.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.utils;
import de.hftstuttgart.config.ModocotProperties;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.tomcat.util.codec.binary.Base64;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
@Component
public class RestCall {
private static final Logger LOG = LogManager.getLogger(RestCall.class);
private final ModocotProperties modocotProperties;
public RestCall(ModocotProperties modocotProperties) {
this.modocotProperties = modocotProperties;
}
public <T> ResponseEntity<T> exchange(String specificUrl, HttpMethod method, T body, Class<T> responseType, Object... uriVariables) {
RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_XML);
headers.add("Authorization", "Basic " + new String(Base64.encodeBase64((this.modocotProperties.getJenkinsApiToken()).getBytes())));
headers.add("user", "admin");
LOG.info(method.toString() + ", to: " + this.modocotProperties.getJenkinsBaseUrl() + specificUrl);
return restTemplate.exchange(this.modocotProperties.getJenkinsBaseUrl() + specificUrl, method, new HttpEntity<>(body, headers), responseType, uriVariables);
}
}
src/main/java/de/hftstuttgart/utils/TaskUploadUtils.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.utils;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.hftstuttgart.config.ModocotProperties;
import de.hftstuttgart.models.JenkinsJobData;
import de.hftstuttgart.rest.v1.jenkins.RestAPIController;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import freemarker.template.Version;
import io.gitea.model.Repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
@Component
public class TaskUploadUtils {
private static final Logger LOG = LogManager.getLogger(TaskUploadUtils.class);
private final GitTeaUtil gitTeaUtil;
private final ModocotProperties modocotProperties;
private final JGitUtil jGitUtil;
private final RestCall restCall;
public TaskUploadUtils(GitTeaUtil gitTeaUtil,
ModocotProperties modocotProperties,
JGitUtil jGitUtil,
RestCall restCall) {
this.gitTeaUtil = gitTeaUtil;
this.modocotProperties = modocotProperties;
this.jGitUtil = jGitUtil;
this.restCall = restCall;
}
/**
* Start jenkinsJob and return its BuildStatus after completion
*
* @param jobId String jenkinsJobId
* @param subFolderPath workingDirectory
* @return Jenkins BuildStatus
*/
public BuildState startTask(String jobId, String subFolderPath) throws IOException, InterruptedException {
// we clone the Student submission to /Test subfolder and then copy those files into the working directory
File f = new File(subFolderPath + "/src/Test");
for (File fi : f.listFiles()) {
Files.move(Paths.get(fi.getPath()), Paths.get(subFolderPath + "/src/" + fi.getName()));
}
if (f.exists()) f.delete();
File temp = new File(subFolderPath + "/src/UnitTests/.git");
if (temp.exists() && temp.isDirectory() && temp.canWrite()) {
FileUtil.deleteFolderRecursively(temp);
}
temp = new File(subFolderPath + "/src/.git");
if (temp.exists() && temp.isDirectory() && temp.canWrite()) {
FileUtil.deleteFolderRecursively(temp);
}
// creating the jobId.json containing the jobId
FileWriter fileWriter = new FileWriter(subFolderPath + "/src/jobId.json");
fileWriter.write(new ObjectMapper().readTree("{\"jobId\":\"" + jobId + "\"}").toString());
fileWriter.close();
// creating repository with name jobId
Repository repository = this.gitTeaUtil.createRepository(jobId);
repository.setCloneUrl(repository.getCloneUrl().replace("localhost", modocotProperties.getDockerHostIp()));
// committing work-directory and pushing all files to repository
this.jGitUtil.commitAllAndPush(new File(subFolderPath), repository, false);
FileUtil.deleteFolderRecursively(new File(subFolderPath));
// persisting the jobId
RestAPIController.JOB_MAP.put(jobId, null);
createJenkinsJob(jobId, repository.getCloneUrl());
buildJenkinsJob(jobId);
// waiting for jenkinsJob to finish then returning its BuildStatus
int timeout = 0;
BuildState buildState = getJenkinsBuildState(jobId);
while (buildState != BuildState.BLUE && buildState != BuildState.RED) {
buildState = getJenkinsBuildState(jobId);
if (timeout >= 100) break;
Thread.sleep(6000);
timeout++;
}
return buildState;
}
public void deleteJenkinsJob(String jobId) {
LOG.info("deleteJenkinsJob jobId: " + jobId);
ResponseEntity<String> response = this.restCall.exchange("job/" + jobId + "/doDelete", HttpMethod.POST, null, String.class);
}
private BuildState getJenkinsBuildState(String jenkinsJob) {
LOG.info("getJenkinsBuildState jenkinsJob: " + jenkinsJob);
ResponseEntity<JenkinsJobData> response = this.restCall.exchange("api/json?tree=jobs[name,color]", HttpMethod.GET, null, JenkinsJobData.class);
if (response.getBody() == null) {
throw new NullPointerException("Jenkins Response was null");
}
return response.getBody()
.getJobs()
.stream()
.filter(job -> job.getName().equals(jenkinsJob))
.findFirst()
.get()
.getColor();
}
public String getJenkinsConsoleOutput(String jenkinsJob) {
ResponseEntity<String> response = this.restCall.exchange("job/" + jenkinsJob + "/lastBuild/consoleText", HttpMethod.GET, null, String.class);
return response.getBody();
}
private void buildJenkinsJob(String user) {
LOG.info("buildJenkinsJob user: " + user);
ResponseEntity<String> response = this.restCall.exchange("job/" + user + "/build", HttpMethod.POST, null, String.class);
}
private void createJenkinsJob(String gitUser, String gitUrl) {
LOG.info("createJenkinsJob gitUser: " + gitUser + ", gitUrl: " + gitUrl);
ResponseEntity<String> response = this.restCall.exchange("createItem?name=" + gitUser, HttpMethod.POST, createXmlFile(gitUser, gitUrl), String.class);
}
private String createXmlFile(String gitUser, String gitUrl) {
// fill data map for template
Map<String, String> templateData = new HashMap<>();
templateData.put("gitUser", gitUser);
templateData.put("gitUrl", gitUrl);
// freemarker create config
Configuration cfg = new Configuration(new Version("2.3.30"));
cfg.setClassForTemplateLoading(this.getClass(), "/templates");
cfg.setDefaultEncoding("UTF-8");
// fuse config and data
StringWriter out = new StringWriter();
try {
Template template = cfg.getTemplate("JenkinsFile.ftl");
template.process(templateData, out);
LOG.info(String.format("Template created with gitUrl: %s and gitUser: %s", gitUrl, gitUser));
} catch (IOException | TemplateException e) {
e.printStackTrace();
}
return out.getBuffer().toString();
}
public boolean isValidJSON(final String json) {
boolean valid = false;
try {
final JsonParser parser = new ObjectMapper().getFactory().createParser(json);
while (parser.nextToken() != null) {
}
valid = true;
} catch (IOException e) {
LOG.error("Json is invalid", e);
}
return valid;
}
}
src/main/java/de/hftstuttgart/utils/UnzipUtil.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.utils;
import de.hftstuttgart.exceptions.CorruptedZipFileException;
import de.hftstuttgart.exceptions.NoZipFileException;
import org.apache.log4j.Logger;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipInputStream;
/**
* Created by Marcel Bochtler on 13.11.16.
* Based on: https://www.mkyong.com/java/how-to-decompress-files-from-a-zip-file/
*/
public class UnzipUtil {
private static final Logger LOG = Logger.getLogger(UnzipUtil.class);
/**
* Unzips files and saves them to the disk.
* Checks if the zip file is valid.
*/
public static List<File> unzip(File zipFile) throws IOException {
String outputFolder = zipFile.getParentFile().getAbsolutePath();
List<File> unzippedFiles = new ArrayList<>();
byte[] buffer = new byte[1024];
//create output directory is not exists
File folder = new File(zipFile.getAbsolutePath());
if (!folder.exists()) {
folder.mkdir();
}
try (ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(zipFile))) {
ZipEntry zipEntry = zipInputStream.getNextEntry();
if (zipEntry == null) {
String message = "The file " + zipFile.getAbsolutePath() + " does not seem be a zip file";
LOG.error(message);
throw new NoZipFileException(message);
}
while (zipEntry != null) {
String fileName = zipEntry.getName();
File unzippedFile = new File(outputFolder + File.separator + fileName);
LOG.info("Unzipped file: " + unzippedFile.getName());
// create all non exists folders
// else we will hit FileNotFoundException for compressed folder
new File(unzippedFile.getParent()).mkdirs();
FileOutputStream fos = new FileOutputStream(unzippedFile);
int length;
while ((length = zipInputStream.read(buffer)) > 0) {
fos.write(buffer, 0, length);
}
fos.close();
zipEntry = zipInputStream.getNextEntry();
unzippedFiles.add(unzippedFile);
}
if (zipFile.exists()) {
zipFile.delete();
}
return unzippedFiles;
} catch (ZipException ze) {
String msg = "Failed to unzip file " + zipFile;
LOG.error(msg);
throw new CorruptedZipFileException(msg);
}
}
}
src/main/resources/application-debug.properties 0 → 100644
View file @ 7c10b571
logging.level.de.hftstuttgart=TRACE
src/main/resources/application-server.properties deleted 100644 → 0
View file @ 0e7ec0ac
# Multipart settings
spring.http.multipart.enabled=true
spring.http.multipart.max-file-size=5Mb
server.port=8081
docker.hostIp=10.40.10.144
jenkins.api.token=dome:1194017120ee74c91ee314e6f796e7b2ae
jenkins.url=http://${docker.hostIp}:8080/
###############################################
# Modocot properties
###############################################
# Holds the uploaded Zip-Files
modocot.dir.parent=/home/modocot
modocot.dir.assignment.prefix=Assignment_
modocot.dir.test.folder.name=UnitTests
gitTea.basePath=http://${docker.hostIp}:3000/api/v1
gitTea.username=giteaUser
gitTea.password=giteaUser1!
gitTea.defaultCommitMessage=Commit all changes including additions
gitTea.defaultOrigin=origin
\ No newline at end of file
src/main/resources/application.properties
View file @ 7c10b571
......@@ -2,24 +2,12 @@
spring.http.multipart.enabled=true
spring.http.multipart.max-file-size=5Mb
server.port=8081
docker.hostIp=10.0.2.15
jenkins.api.token=token:114e49fcd53d583f81113697a64d0e7630
jenkins.url=http://${docker.hostIp}:8080/
###############################################
# Modocot properties
###############################################
# Holds the uploaded Zip-Files
modocot.dir.parent=/home/doom/modocot/
modocot.dir.assignment.prefix=Assignment_
modocot.tests.tmp.dir=/tmp/modocot-tests
host.tests.tmp.dir=${modocot.tests.tmp.dir}
modocot.dir=/modocot/data
modocot.dir.test.folder.name=UnitTests
gitTea.basePath=http://${docker.hostIp}:3000/api/v1
gitTea.username=giteaUser
gitTea.password=giteaUser1!
gitTea.defaultCommitMessage=Commit all changes including additions
gitTea.defaultOrigin=origin
src/main/resources/git-credentials.properties deleted 100644 → 0
View file @ 0e7ec0ac
modocot.git.username=username
modocot.git.password=password
\ No newline at end of file
src/main/resources/templates/JenkinsFile.ftl deleted 100644 → 0
View file @ 0e7ec0ac
<?xml version='1.1' encoding='UTF-8'?>
<flow-definition plugin="workflow-job@2.39">
<actions>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobAction plugin="pipeline-model-definition@1.6.0"/>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction plugin="pipeline-model-definition@1.6.0">
<jobProperties/>
<triggers/>
<parameters/>
<options/>
</org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction>
</actions>
<description>Pipeline for User: ${gitUser} </description>
<keepDependencies>false</keepDependencies>
<properties/>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.80">
<scm class="hudson.plugins.git.GitSCM" plugin="git@4.2.2">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>${gitUrl}</url>
<credentialsId>giteaUser</credentialsId>
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>*/master</name>
</hudson.plugins.git.BranchSpec>
</branches>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<submoduleCfg class="list"/>
<extensions/>
</scm>
<scriptPath>src/Jenkinsfile</scriptPath>
<lightweight>true</lightweight>
</definition>
<triggers/>
<disabled>false</disabled>
</flow-definition>
\ No newline at end of file
src/main/resources/templates/JenkinsLocalTestFile.ftl deleted 100644 → 0
View file @ 0e7ec0ac
<project>
<actions/>
<description>${gitUser} ${gitUrl}</description>
<keepDependencies>false</keepDependencies>
<properties>
<hudson.plugins.jira.JiraProjectProperty plugin="jira@3.0.17"/>
</properties>
<scm class="hudson.scm.NullSCM"/>
<canRoam>true</canRoam>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
\ No newline at end of file
src/test/java/de/hftstuttgart/rest/v1/unittest/IntegrationTest.java deleted 100644 → 0
View file @ 0e7ec0ac
package de.hftstuttgart.rest.v1.unittest;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
import java.io.File;
import java.io.FileInputStream;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class IntegrationTest {
@Autowired
private WebApplicationContext webApplicationContext;
MockMvc mockMvc;
@Before
public void setup() throws Exception {
mockMvc = MockMvcBuilders.webAppContextSetup(webApplicationContext).build();
}
@Test
@Ignore
public void validUnitTestFileTest() throws Exception {
// Upload tests
File unitTestFile = new File(Thread.currentThread().getContextClassLoader().getResource("tests.zip").getFile());
MockMultipartFile testFileMock = new MockMultipartFile("unitTestFile", new FileInputStream(unitTestFile));
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/v1/unittest")
.file(testFileMock)
.param("assignmentId", "111"))
.andExpect(status().is(200));
// Upload tasks
File taskFile = new File(Thread.currentThread().getContextClassLoader().getResource("tasks.zip").getFile());
MockMultipartFile taskFileMock = new MockMultipartFile("taskFile", new FileInputStream(taskFile));
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/v1/task")
.file(taskFileMock)
.param("assignmentId", "111"))
.andExpect(status().is(200))
.andExpect(content().string(("{\n" +
" \"testResults\" : [ {\n" +
" \"testName\" : \"CalculatorTest\",\n" +
" \"testCount\" : 5,\n" +
" \"failureCount\" : 0,\n" +
" \"successfulTests\" : [ \"add\", \"div\", \"sub\", \"sum\", \"mult\" ],\n" +
" \"testFailures\" : [ ]\n" +
" } ],\n" +
" \"compilationErrors\" : [ ]\n" +
"}")
.replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))));
}
@Test
@Ignore
public void corruptedZipTest() throws Exception {
File file = new File(Thread.currentThread().getContextClassLoader().getResource("corrupted.zip").getFile());
MockMultipartFile mockFile = new MockMultipartFile("unitTestFile", new FileInputStream(file));
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/v1/unittest")
.file(mockFile)
.param("assignmentId", "222"))
.andExpect(status().is(400));
}
@Test
@Ignore
public void renamedTxtFileTest() throws Exception {
File file = new File(Thread.currentThread().getContextClassLoader().getResource("textfile.zip").getFile());
MockMultipartFile mockFile = new MockMultipartFile("unitTestFile", new FileInputStream(file));
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/v1/unittest")
.file(mockFile)
.param("assignmentId", "333"))
.andExpect(status().is(400));
}
@Test
@Ignore
public void noAssignmentIdTest() throws Exception {
File file = new File(Thread.currentThread().getContextClassLoader().getResource("textfile.zip").getFile());
MockMultipartFile mockFile = new MockMultipartFile("unitTestFile", new FileInputStream(file));
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/v1/unittest")
.file(mockFile))
.andExpect(status().is(400));
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal