Commit 11f401b1 authored by Lückemeyer's avatar Lückemeyer
Browse files

added recommendations

No related merge requests found
Showing with 296 additions and 95 deletions
+296 -95
......@@ -163,6 +163,14 @@
<version>4.11.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.tmatesoft.svnkit/svnkit -->
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.10.11</version>
</dependency>
</dependencies>
<build>
......
package de.hftstuttgart.dtabackend.models;
import java.util.Comparator;
import java.util.Objects;
public class ExerciseCompetencyProfile implements ICompetencyProfile {
public String exerciseTopicName;
public String exerciseName;
public String exerciseURL;
public float difficulty;
public float[] competencyAssessments=new float[MAX_COMPETENCY_DIMENSIONS];
@Override
public boolean equals(Object other) {
return other instanceof ExerciseCompetencyProfile && Objects.equals(exerciseURL, ((ExerciseCompetencyProfile) other).exerciseURL);
}
}
package de.hftstuttgart.dtabackend.models;
public interface ICompetencyProfile {
int MAX_COMPETENCY_DIMENSIONS = 16;
String COMPETENCY_SEPARATOR = ";";
public static float[] competencyProjection(float[] cp, float[] cp2) {
float z[] = new float[MAX_COMPETENCY_DIMENSIONS];
for (int i = 0; i < MAX_COMPETENCY_DIMENSIONS; i++) {
z[i] = cp[i] * cp2[i];
}
return z;
}
public static float[] competencyShare(float[] cp, float[] cpTotal) {
float z[] = new float[MAX_COMPETENCY_DIMENSIONS];
for (int i = 0; i < MAX_COMPETENCY_DIMENSIONS; i++) {
z[i] = cp[i] / cpTotal[i];
}
return z;
}
public static float[] competencySum(float[] cp, float[] cp2) {
float z[] = new float[MAX_COMPETENCY_DIMENSIONS];
for (int i = 0; i < MAX_COMPETENCY_DIMENSIONS; i++) {
z[i] = cp[i] + cp2[i];
}
return z;
}
}
package de.hftstuttgart.dtabackend.models;
import java.util.Comparator;
public class Recommendation {
public String topic;
public String exerciseName;
public String url;
public float difficulty;
public float score;
public static final Comparator<Recommendation> COMPARE_BY_SCORE = Comparator.comparingDouble(other -> other.score);
public Recommendation(String topic, String exerciseName, String url, float difficulty, float score) {
this.topic = topic;
this.exerciseName = exerciseName;
this.url = url;
this.difficulty = difficulty;
this.score = score;
}
}
package de.hftstuttgart.dtabackend.models;
import java.util.HashSet;
import java.util.Set;
public class ResultSummary
{
public long timestamp = System.currentTimeMillis() / 1000;
public String globalStacktrace = null;
public String successfulTestCompetencyProfile;
public String overallTestCompetencyProfile;
public Set<Result> results = new HashSet<>();
}
package de.hftstuttgart.dtabackend.models;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class ResultSummary
{
public long timestamp = System.currentTimeMillis() / 1000;
public String globalStacktrace = null;
public String successfulTestCompetencyProfile;
public String overallTestCompetencyProfile;
public Set<Result> results = new HashSet<>();
public List<Recommendation> recommendations = new ArrayList<>();
}
package de.hftstuttgart.dtabackend.models;
public class TestCompetencyProfile {
public static final int MAX_COMPETENCY_DIMENSIONS = 16;
public static final String COMPETENCY_SEPARATOR=";";
public String testPackageName;
public String testClassName;
public String testName;
public float[] competencyAssessments=new float[MAX_COMPETENCY_DIMENSIONS];
public static float[] competencyProjection(float[] cp, float[] cp2) {
float z[] = new float[MAX_COMPETENCY_DIMENSIONS];
for (int i = 0; i < MAX_COMPETENCY_DIMENSIONS; i++) {
z[i] = cp[i] * cp2[i];
}
return z;
}
public static float[] competencyShare(float[] cp, float[] cpTotal) {
float z[] = new float[MAX_COMPETENCY_DIMENSIONS];
for (int i = 0; i < MAX_COMPETENCY_DIMENSIONS; i++) {
z[i] = cp[i] / cpTotal[i];
}
return z;
}
public static float[] competencySum(float[] cp, float[] cp2) {
float z[] = new float[MAX_COMPETENCY_DIMENSIONS];
for (int i = 0; i < MAX_COMPETENCY_DIMENSIONS; i++) {
z[i] = cp[i] + cp2[i];
}
return z;
}
@Override
public boolean equals(Object other) {
return other instanceof TestCompetencyProfile &&
testPackageName.equals(((TestCompetencyProfile)other).testPackageName) &&
testClassName.equals(((TestCompetencyProfile)other).testClassName) &&
testName.equals(((TestCompetencyProfile)other).testName);
}
}
package de.hftstuttgart.dtabackend.models;
public class TestCompetencyProfile implements ICompetencyProfile {
public String testPackageName;
public String testClassName;
public String testName;
public float[] competencyAssessments=new float[MAX_COMPETENCY_DIMENSIONS];
@Override
public boolean equals(Object other) {
return other instanceof TestCompetencyProfile &&
testPackageName.equals(((TestCompetencyProfile)other).testPackageName) &&
testClassName.equals(((TestCompetencyProfile)other).testClassName) &&
testName.equals(((TestCompetencyProfile)other).testName);
}
}
......@@ -31,16 +31,16 @@ import jakarta.servlet.annotation.MultipartConfig;
public class TaskUpload {
private static final Logger LOG = LogManager.getLogger(TaskUpload.class);
private final RepoUtil jGitUtil;
private final RepoUtil repoUtil;
private final Path testTmpPath;
private final ExecuteTestUtil executeTestUtil;
public TaskUpload(
Environment env,
RepoUtil jGitUtil,
RepoUtil repoUtil,
ExecuteTestUtil executeTestUtil
) {
this.jGitUtil = jGitUtil;
this.repoUtil = repoUtil;
this.executeTestUtil = executeTestUtil;
// set path of temporary directory on host and inside our container
......@@ -78,7 +78,7 @@ public class TaskUpload {
subDir=config.group(4);
}
LOG.debug("calling repo clone");
jGitUtil.cloneRepository(config, srcPath.toAbsolutePath().toString(), subDir);
repoUtil.cloneRepository(config, srcPath.toAbsolutePath().toString(), subDir);
break;
case "application/zip":
......
......@@ -30,11 +30,11 @@ import java.util.regex.Pattern;
public class UnitTestUpload {
private static final Logger LOG = LogManager.getLogger(UnitTestUpload.class);
private final RepoUtil jGitUtil;
private final RepoUtil repoUtil;
private final String assignmentBasePath;
public UnitTestUpload(Environment env, RepoUtil jGitUtil) {
this.jGitUtil = jGitUtil;
public UnitTestUpload(Environment env, RepoUtil repoUtil) {
this.repoUtil = repoUtil;
Path p = Paths.get(env.getProperty("data.dir"), env.getProperty("data.dir.test.folder.name"));
this.assignmentBasePath = p.toAbsolutePath().toString();
......@@ -78,7 +78,7 @@ public class UnitTestUpload {
}
LOG.debug("calling test repo clone");
// cloning assignment repo to persistent space
jGitUtil.cloneRepository(config, Paths.get(this.assignmentBasePath, assignmentId).toAbsolutePath().toString(), subDir);
repoUtil.cloneRepository(config, Paths.get(this.assignmentBasePath, assignmentId).toAbsolutePath().toString(), subDir);
LOG.info(String.format("stored new assignment: %s", file.getAbsolutePath()));
}
......
......@@ -16,8 +16,9 @@ import org.apache.logging.log4j.Logger;
import com.fasterxml.jackson.core.exc.StreamReadException;
import com.fasterxml.jackson.databind.DatabindException;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.hftstuttgart.dtabackend.models.ExerciseCompetencyProfile;
import de.hftstuttgart.dtabackend.models.ICompetencyProfile;
import de.hftstuttgart.dtabackend.models.Result;
import de.hftstuttgart.dtabackend.models.ResultSummary;
import de.hftstuttgart.dtabackend.models.TestCompetencyProfile;
......@@ -28,31 +29,33 @@ public class CompetencyAssessmentUtil {
private static final Logger LOG = LogManager.getLogger(CompetencyAssessmentUtil.class);
public static String TEST_COMPETENCY_MANIFEST_FILE_NAME="competency-tests.mft";
public static String EXERCISE_COMPETENCY_MANIFEST_FILE_NAME="exercise-tests.mft";
public static void main(String[] args) throws StreamReadException, DatabindException, MalformedURLException, IOException {
ResultSummary summary=ExecuteTestUtil.generateResult(Path.of(args[0]), Path.of(args[1]));
/*public static void main(String[] args) throws StreamReadException, DatabindException, MalformedURLException, IOException {
ResultSummary summary=ExecuteTestUtil.generateResult("1", Path.of(args[0]), Path.of(args[1]));
System.out.println(summary.successfulTestCompetencyProfile);
}
*/
public static float[] sumTestCompetencyProfiles(List<TestCompetencyProfile> testCompetencyProfiles) {
float[] tcpTotalProfile=new float[TestCompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
float[] tcpTotalProfile=new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for(TestCompetencyProfile currentProfile: testCompetencyProfiles) {
tcpTotalProfile=TestCompetencyProfile.competencySum(tcpTotalProfile, currentProfile.competencyAssessments);
tcpTotalProfile=ICompetencyProfile.competencySum(tcpTotalProfile, currentProfile.competencyAssessments);
}
return tcpTotalProfile;
}
public static float[] sumSuccessfulCompetencyProfiles(List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary) {
float[] sumSuccessful=new float[TestCompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
public static float[] sumSuccessfulCompetencyProfiles(List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary, boolean success) {
float[] sumSuccessful=new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for(Result currentResult: resultSummary.results) {
if(currentResult.state==Result.State.SUCCESS.ordinal()) {
boolean isSuccess = Integer.valueOf(currentResult.state).equals(Result.State.SUCCESS.ordinal());
if (isSuccess == success) {
TestCompetencyProfile currentProfile=new TestCompetencyProfile();
currentProfile.testPackageName=(currentResult.packageName!=null)?currentResult.packageName:"";
currentProfile.testClassName=(currentResult.className!=null)?currentResult.className:"";
currentProfile.testName=(currentResult.name!=null)?currentResult.name:"";
int testIndex=testCompetencyProfiles.indexOf(currentProfile);
if(testIndex!=-1) {
sumSuccessful=TestCompetencyProfile.competencySum(sumSuccessful, testCompetencyProfiles.get(testIndex).competencyAssessments);
sumSuccessful=ICompetencyProfile.competencySum(sumSuccessful, testCompetencyProfiles.get(testIndex).competencyAssessments);
}
}
}
......@@ -66,12 +69,12 @@ public class CompetencyAssessmentUtil {
String testEntry=testCompetencyManifest.readLine();
while(testEntry!=null)
{
String[] testEntyComponents=testEntry.split(TestCompetencyProfile.COMPETENCY_SEPARATOR);
String[] testEntyComponents=testEntry.split(ICompetencyProfile.COMPETENCY_SEPARATOR);
TestCompetencyProfile currentProfile=new TestCompetencyProfile();
currentProfile.testPackageName=testEntyComponents[0];
currentProfile.testClassName=testEntyComponents[1];
currentProfile.testName=testEntyComponents[2];
for(int competencyIndex=0; competencyIndex<TestCompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
for(int competencyIndex=0; competencyIndex<ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex]=Float.valueOf(testEntyComponents[competencyIndex+3]);
}
testCompetencyProfiles.add(currentProfile);
......@@ -88,6 +91,41 @@ public class CompetencyAssessmentUtil {
}
return testCompetencyProfiles;
}
public static List<ExerciseCompetencyProfile> readExerciseCompetencyProfiles(Path exercisePath, String fileName) {
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = new ArrayList<>();
try (BufferedReader exerciseCompetencyManifest = new BufferedReader(new FileReader(new File(exercisePath.toFile(), fileName)))) {
String exerciseEntry = exerciseCompetencyManifest.readLine();
while (exerciseEntry != null) {
String[] exerciseEntyComponents = exerciseEntry.split(ExerciseCompetencyProfile.COMPETENCY_SEPARATOR);
ExerciseCompetencyProfile currentProfile = new ExerciseCompetencyProfile();
currentProfile.exerciseTopicName = exerciseEntyComponents[0];
currentProfile.exerciseName = exerciseEntyComponents[1];
currentProfile.exerciseURL = exerciseEntyComponents[2];
for (int competencyIndex = 0; competencyIndex < ExerciseCompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex] = Float.valueOf(exerciseEntyComponents[competencyIndex+3]);
}
currentProfile.difficulty = Float.parseFloat(exerciseEntyComponents[19]);
exerciseCompetencyProfiles.add(currentProfile);
exerciseEntry = exerciseCompetencyManifest.readLine();
}
exerciseCompetencyManifest.close();
LOG.info("Added " + exerciseCompetencyProfiles.size() + " test competency profiles from exercise competency manifest.");
} catch (FileNotFoundException e) {
LOG.info("Exercise competency manifest file not found.");
} catch (IOException e) {
LOG.info("Exercise competency manifest file unreadable.");
}
return exerciseCompetencyProfiles;
}
public static String packFloats(float[] array) {
return IntStream.range(0, array.length)
......
......@@ -5,6 +5,10 @@ import com.fasterxml.jackson.databind.DatabindException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Volume;
import de.hftstuttgart.dtabackend.models.ExerciseCompetencyProfile;
import de.hftstuttgart.dtabackend.models.ICompetencyProfile;
import de.hftstuttgart.dtabackend.models.Recommendation;
import de.hftstuttgart.dtabackend.models.ResultSummary;
import de.hftstuttgart.dtabackend.models.TestCompetencyProfile;
......@@ -18,26 +22,26 @@ import java.net.MalformedURLException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Component
public class ExecuteTestUtil {
private static final Logger LOG = LogManager.getLogger(ExecuteTestUtil.class);
private final RepoUtil jGitUtil;
private final DockerUtil dockerUtil;
private final String assignmentBasePath;
private final Path testTmpPathHost;
private final Path testTmpPath;
public ExecuteTestUtil(
Environment env,
RepoUtil jGitUtil,
DockerUtil dockerUtil
) {
this.jGitUtil = jGitUtil;
this.dockerUtil = dockerUtil;
// set base path for assignments to be stored
......@@ -46,9 +50,8 @@ public class ExecuteTestUtil {
env.getProperty("data.dir.test.folder.name"));
this.assignmentBasePath = p.toAbsolutePath().toString();
// set path of temporary directory on host and inside our container
// set path of temporary directory on host _and_ inside our container, _must_ be identical
this.testTmpPathHost = Paths.get(env.getProperty("host.tests.tmp.dir"));
this.testTmpPath = Paths.get(env.getProperty("tests.tmp.dir"));
}
public ResultSummary runTests(String assignmentId, Path workDirectory) throws IOException, InterruptedException {
......@@ -114,12 +117,12 @@ public class ExecuteTestUtil {
new Bind(resultPathHost.toAbsolutePath().toString(), new Volume("/data/result"))
);
ResultSummary resultSummary = generateResult(resultPath, testPathHost);
ResultSummary resultSummary = generateResult(assignmentId, resultPath, testPathHost);
return resultSummary;
}
static ResultSummary generateResult(Path resultPath, Path testPathHost)
private ResultSummary generateResult(String assignmentId, Path resultPath, Path testPathHost)
throws IOException, StreamReadException, DatabindException, MalformedURLException {
// define expected result file
File resultFile = Paths.get(resultPath.toAbsolutePath().toString(), "result.json").toFile();
......@@ -132,9 +135,7 @@ public class ExecuteTestUtil {
LOG.debug("parse results json");
ObjectMapper objectMapper = new ObjectMapper();
ResultSummary resultSummary = objectMapper.readValue(
resultFile.toURI().toURL(),
ResultSummary.class);
ResultSummary resultSummary = objectMapper.readValue(resultFile.toURI().toURL(), ResultSummary.class);
LOG.debug("result json returned time "+ resultSummary.timestamp + " with "+resultSummary.results.size()+ " test results.");
LOG.info("Checking for optional test competency profile information for paedagogical agent functionality...");
......@@ -142,8 +143,112 @@ public class ExecuteTestUtil {
if(testCompetencyProfiles!=null) {
LOG.info("Found optional test competency profiles, generating agent profile data...");
resultSummary.overallTestCompetencyProfile=CompetencyAssessmentUtil.packFloats(CompetencyAssessmentUtil.sumTestCompetencyProfiles(testCompetencyProfiles));
resultSummary.successfulTestCompetencyProfile=CompetencyAssessmentUtil.packFloats(CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary));
resultSummary.successfulTestCompetencyProfile=CompetencyAssessmentUtil.packFloats(CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, true));
LOG.info("Checking for optional exercise competency profile information for paedagogical agent exercise recommendation functionality...");
//testPathHost or assignmentBasePath
Path exerciseManifestFile = Paths.get(testPathHost.toAbsolutePath().toString(), CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME);
if (Files.exists(exerciseManifestFile)) {
LOG.info("Found optional exercise competency profiles, generating recommendations...");
resultSummary.recommendations = recommendNextExercises(assignmentId, testPathHost, testCompetencyProfiles, resultSummary);
}
}
return resultSummary;
}
}
/*
* exercise recommendation part
*/
public List<Recommendation> recommendNextExercises(String assignmentId, Path testPathHost, List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary)
throws FileNotFoundException {
// fetch repo url from original test upload
Pattern pattern = Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX);
File file = Paths.get(assignmentBasePath, assignmentId + ".txt").toFile();
FileInputStream configFileStream = new FileInputStream(file);
Matcher config = RegexUtil.extractConfig(configFileStream, pattern);
String testRepoURL = config.group(1);
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = CompetencyAssessmentUtil.readExerciseCompetencyProfiles(testPathHost, CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME);
int currentTopicIndex=0;
float currentDifficulty=0.0f;
//build course topic order
Map<String, Integer> topicOrder = new HashMap<>();
int order = 1;
for (ExerciseCompetencyProfile e : exerciseCompetencyProfiles) {
if (!topicOrder.containsKey(e.exerciseTopicName)) {
topicOrder.put(e.exerciseTopicName, order++);
}
if (e.exerciseURL.equals(testRepoURL)) {
currentTopicIndex = order;
currentDifficulty = e.difficulty;
}
}
//filter exercises according to success
float[] unsuccessful = CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, false);
List<ExerciseCompetencyProfile> filteredExercises = filterExercisesByTopicsAndDifficulty(
exerciseCompetencyProfiles, topicOrder, currentTopicIndex, testRepoURL,
currentDifficulty, unsuccessful, resultSummary);
//compute recommendations
List<Recommendation> recommendedExercises = new ArrayList<>();
for (ExerciseCompetencyProfile exerciseProfile : filteredExercises) {
Recommendation recommendation = new Recommendation(exerciseProfile.exerciseTopicName, exerciseProfile.exerciseURL, exerciseProfile.exerciseName,
exerciseProfile.difficulty, calculateScore(exerciseProfile, unsuccessful, topicOrder, currentDifficulty));
recommendedExercises.add(recommendation);
LOG.info("Recommending exercise "+recommendation.topic+"/"+recommendation.exerciseName+" with score "+recommendation.score);
}
//sort the recommendations for successful or resilient learners, otherwise reverse in display
recommendedExercises.stream().sorted(Recommendation.COMPARE_BY_SCORE).collect(Collectors.toList());
return recommendedExercises;
}
public static List<ExerciseCompetencyProfile> filterExercisesByTopicsAndDifficulty(List<ExerciseCompetencyProfile> exerciseCompetencyProfiles,
Map<String, Integer> topicOrder, int currentTopicIndex, String testRepoURL, float currentDifficulty, float[] unsuccessful,
ResultSummary resultSummary) {
//filter out all advanced topics in any case
//option for later: include next topic if fullsuccess and current difficulty == max difficulty
List<ExerciseCompetencyProfile> filteredExercises = exerciseCompetencyProfiles.stream()
.filter(testProfile -> topicOrder.get(testProfile.exerciseTopicName) <= currentTopicIndex)
.collect(Collectors.toList());
//filter by difficulty according to success
if (isFullSuccess(unsuccessful)) {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty >= currentDifficulty && !testRepoURL.equals(profile.exerciseURL)).collect(Collectors.toList());
} else {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty <= currentDifficulty).collect(Collectors.toList());
}
return filteredExercises;
}
public static boolean isFullSuccess(float[] unsuccessful) {
for (float value : unsuccessful) {
if (value != 0.0f) {
return false;
}
}
return true;
}
public static float calculateScore(ExerciseCompetencyProfile exerciseProfile, float[] unsuccessful, Map<String, Integer> topicOrder, float currentDifficulty) {
//ensure factor 1 for full success not to blank out score, thus offset the base
float score = 1.0f;
//competency profile difference to not fully achieved competencies component
for (int i = 0; i < exerciseProfile.competencyAssessments.length-1; i++) {
score += exerciseProfile.competencyAssessments[i] * unsuccessful[i];
}
//difficulty component
score = score * (exerciseProfile.difficulty*(0.5f+Math.abs(currentDifficulty-exerciseProfile.difficulty)));
//topic component
score *= topicOrder.get(exerciseProfile.exerciseTopicName);
score = Math.round(score * 10.0f) / 10.0f;
return score;
}
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment