An error occurred while loading the file. Please try again.
ExecuteTestUtil.java 12.90 KiB
package de.hftstuttgart.dtabackend.utils;
import com.fasterxml.jackson.core.exc.StreamReadException;
import com.fasterxml.jackson.databind.DatabindException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Volume;
import de.hftstuttgart.dtabackend.models.ExerciseCompetencyProfile;
import de.hftstuttgart.dtabackend.models.ICompetencyProfile;
import de.hftstuttgart.dtabackend.models.Recommendation;
import de.hftstuttgart.dtabackend.models.ResultSummary;
import de.hftstuttgart.dtabackend.models.TestCompetencyProfile;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
import java.io.*;
import java.net.MalformedURLException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Component
public class ExecuteTestUtil  {
    private static final Logger LOG = LogManager.getLogger(ExecuteTestUtil.class);
    private final DockerUtil dockerUtil;
    private final String assignmentBasePath;
    private final Path testTmpPathHost;
    public ExecuteTestUtil(
        Environment env,
        DockerUtil dockerUtil
    ) {
        this.dockerUtil = dockerUtil;
        // set base path for assignments to be stored
        Path p = Paths.get(
            env.getProperty("data.dir"),
            env.getProperty("data.dir.test.folder.name"));
        this.assignmentBasePath = p.toAbsolutePath().toString();
        // set path of temporary directory on host _and_ inside our container, _must_ be identical 
        this.testTmpPathHost = Paths.get(env.getProperty("host.tests.tmp.dir"));
    public ResultSummary runTests(String assignmentId, Path workDirectory) throws IOException, InterruptedException {
        // define paths for the test, the submission and where the result is to be expected afterwards
        Path testPath   = Paths.get(workDirectory.toAbsolutePath().toString(), "/test");
        Path srcPath    = Paths.get(workDirectory.toAbsolutePath().toString(), "/src");
        Path resultPath = Paths.get(workDirectory.toAbsolutePath().toString(), "/result");
        // clone stored test to tmpdir
        LOG.debug("copying pre-downloaded unitttest repo");
        FileUtil.copyFolder(
            Paths.get(assignmentBasePath, assignmentId),
            testPath);
		LOG.debug("copying exercise manifest from: %s in testPath: %s",
assignmentBasePath + assignmentId + "_checkout", testPath.toString() ); Files.copy(Paths.get( assignmentBasePath, assignmentId + "_checkout", CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME), Paths.get( testPath.toString(), CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME)); LOG.debug("copy test config"); Files.copy( Paths.get(assignmentBasePath, assignmentId + ".txt"), Paths.get(workDirectory.toAbsolutePath().toString(), "config.txt")); Files.createDirectory(resultPath); LOG.info("reading test config"); Matcher config = RegexUtil.extractConfig( new FileInputStream(Paths.get(workDirectory.toAbsolutePath().toString(), "config.txt").toFile()), Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX)); String image=""; if(config==null) { config = RegexUtil.extractConfig( new FileInputStream(Paths.get(workDirectory.toAbsolutePath().toString(), "config.txt").toFile()), Pattern.compile(RegexUtil.TESTCONFIGREGEX)); if(config==null) { throw new RuntimeException("couldn't find repo config for unittest image extraction"); } image=config.group(4); } else { image=config.group(5); } // define the paths to mount as Binds from Host to the test-container Path testPathHost = Paths.get( testTmpPathHost.toAbsolutePath().toString(), workDirectory.getName(workDirectory.getNameCount()-1).toString(), testPath.getName(testPath.getNameCount()-1).toString() ); Path srcPathHost = Paths.get( testTmpPathHost.toAbsolutePath().toString(), workDirectory.getName(workDirectory.getNameCount()-1).toString(), srcPath.getName(srcPath.getNameCount()-1).toString() ); Path resultPathHost = Paths.get( testTmpPathHost.toAbsolutePath().toString(), workDirectory.getName(workDirectory.getNameCount()-1).toString(), resultPath.getName(resultPath.getNameCount()-1).toString() ); // start test-container with professor given image and bind mounts for test, submission and result dockerUtil.runContainer( image, new Bind(testPathHost.toAbsolutePath().toString(), new Volume("/data/test")), new Bind(srcPathHost.toAbsolutePath().toString(), new Volume("/data/src")), new Bind(resultPathHost.toAbsolutePath().toString(), new Volume("/data/result")) ); ResultSummary resultSummary = generateResult(assignmentId, resultPath, testPathHost); return resultSummary; } private ResultSummary generateResult(String assignmentId, Path resultPath, Path testPathHost) throws IOException, StreamReadException, DatabindException, MalformedURLException { // define expected result file File resultFile = Paths.get(resultPath.toAbsolutePath().toString(), "result.json").toFile(); // check if result file is there
if (!resultFile.exists() || !resultFile.isFile()) { LOG.error(String.format("Could not find result file in %s", resultFile.getAbsolutePath())); throw new RuntimeException("no resultfile found"); } LOG.debug("parse results json"); ObjectMapper objectMapper = new ObjectMapper(); ResultSummary resultSummary = objectMapper.readValue(resultFile.toURI().toURL(), ResultSummary.class); LOG.debug("result json returned time "+ resultSummary.timestamp + " with "+resultSummary.results.size()+ " test results."); LOG.info("Checking for optional test competency profile information for paedagogical agent functionality..."); List<TestCompetencyProfile> testCompetencyProfiles=CompetencyAssessmentUtil.readTestCompetencyProfiles(testPathHost, CompetencyAssessmentUtil.TEST_COMPETENCY_MANIFEST_FILE_NAME); LOG.debug(String.format( "Reading Test Competency Profiles: basePath=%s, fileName=%s", testPathHost, CompetencyAssessmentUtil.TEST_COMPETENCY_MANIFEST_FILE_NAME )); if(testCompetencyProfiles!=null) { LOG.info("Found optional test competency profiles, generating agent profile data..."); resultSummary.overallTestCompetencyProfile=CompetencyAssessmentUtil.packFloats(CompetencyAssessmentUtil.sumTestCompetencyProfiles(testCompetencyProfiles)); resultSummary.successfulTestCompetencyProfile=CompetencyAssessmentUtil.packFloats(CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, true)); LOG.info("Checking for optional exercise competency profile information for paedagogical agent exercise recommendation functionality..."); //testPathHost or assignmentBasePath Path exerciseManifestFile = Paths.get(testPathHost.toString(), CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME); LOG.debug(String.format( "Constructing Path for exercise manifest: testPath=%s, fileName=%s -> Resulting Path=%s", testPathHost.toString(), CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME, exerciseManifestFile.toString() )); if (Files.exists(exerciseManifestFile)) { LOG.info("Found optional exercise competency profiles, generating recommendations..."); resultSummary.recommendations = recommendNextExercises(assignmentId, testPathHost, testCompetencyProfiles, resultSummary); } } return resultSummary; } /* * exercise recommendation part */ public List<Recommendation> recommendNextExercises(String assignmentId, Path testPathHost, List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary) throws FileNotFoundException { // fetch repo url from original test upload Pattern pattern = Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX); File file = Paths.get(assignmentBasePath, assignmentId + ".txt").toFile(); FileInputStream configFileStream = new FileInputStream(file); Matcher config = RegexUtil.extractConfig(configFileStream, pattern); String testRepoURL = config.group(1); List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = CompetencyAssessmentUtil.readExerciseCompetencyProfiles(testPathHost, CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME); int currentTopicIndex=0; float currentDifficulty=0.0f; //build course topic order Map<String, Integer> topicOrder = new HashMap<>(); int order = 1; for (ExerciseCompetencyProfile e : exerciseCompetencyProfiles) { if (!topicOrder.containsKey(e.exerciseTopicName)) { topicOrder.put(e.exerciseTopicName, order++); } if (e.exerciseURL.equals(testRepoURL)) { currentTopicIndex = order; currentDifficulty = e.difficulty; } } //filter exercises according to success
float[] unsuccessful = CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, false); List<ExerciseCompetencyProfile> filteredExercises = filterExercisesByTopicsAndDifficulty( exerciseCompetencyProfiles, topicOrder, currentTopicIndex, testRepoURL, currentDifficulty, unsuccessful, resultSummary); //compute recommendations List<Recommendation> recommendedExercises = new ArrayList<>(); for (ExerciseCompetencyProfile exerciseProfile : filteredExercises) { Recommendation recommendation = new Recommendation(exerciseProfile.exerciseTopicName, exerciseProfile.exerciseURL, exerciseProfile.exerciseName, exerciseProfile.difficulty, calculateScore(exerciseProfile, unsuccessful, topicOrder, currentDifficulty)); recommendedExercises.add(recommendation); LOG.info("Recommending exercise "+recommendation.topic+"/"+recommendation.exerciseName+" with score "+recommendation.score); } //sort the recommendations for successful or resilient learners, otherwise reverse in display recommendedExercises.stream().sorted(Recommendation.COMPARE_BY_SCORE).collect(Collectors.toList()); return recommendedExercises; } public static List<ExerciseCompetencyProfile> filterExercisesByTopicsAndDifficulty(List<ExerciseCompetencyProfile> exerciseCompetencyProfiles, Map<String, Integer> topicOrder, int currentTopicIndex, String testRepoURL, float currentDifficulty, float[] unsuccessful, ResultSummary resultSummary) { //filter out all advanced topics in any case //option for later: include next topic if fullsuccess and current difficulty == max difficulty List<ExerciseCompetencyProfile> filteredExercises = exerciseCompetencyProfiles.stream() .filter(testProfile -> topicOrder.get(testProfile.exerciseTopicName) <= currentTopicIndex) .collect(Collectors.toList()); //filter by difficulty according to success if (isFullSuccess(unsuccessful)) { filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty >= currentDifficulty && !testRepoURL.equals(profile.exerciseURL)).collect(Collectors.toList()); } else { filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty <= currentDifficulty).collect(Collectors.toList()); } return filteredExercises; } public static boolean isFullSuccess(float[] unsuccessful) { for (float value : unsuccessful) { if (value != 0.0f) { return false; } } return true; } public static float calculateScore(ExerciseCompetencyProfile exerciseProfile, float[] unsuccessful, Map<String, Integer> topicOrder, float currentDifficulty) { //ensure factor 1 for full success not to blank out score, thus offset the base float score = 1.0f; //competency profile difference to not fully achieved competencies component for (int i = 0; i < exerciseProfile.competencyAssessments.length-1; i++) { score += exerciseProfile.competencyAssessments[i] * unsuccessful[i]; } //difficulty component score = score * (exerciseProfile.difficulty*(0.5f+Math.abs(currentDifficulty-exerciseProfile.difficulty))); //topic component score *= topicOrder.get(exerciseProfile.exerciseTopicName); score = Math.round(score * 10.0f) / 10.0f; return score; } }