Skip to content
GitLab
An error occurred while fetching the assigned milestone of the selected merge_request.

Fix: dev recommendations

Merged mamunozgil requested to merge amg-dev-recommendations into master
Compare
and
2 files
+ 241
- 170
src/main/java/…/…‎/dtabackend/utils‎
CompetencyAsse‎ssmentUtil.java‎ +98 -81
ExecuteTes‎tUtil.java‎ +143 -89
src/main/java/de/hftstuttgart/dtabackend/utils/CompetencyAssessmentUtil.java
+ 98
- 81
@@ -27,110 +27,127 @@ import java.io.FileNotFoundException;
public class CompetencyAssessmentUtil {
private static final Logger LOG = LogManager.getLogger(CompetencyAssessmentUtil.class);
public static String TEST_COMPETENCY_MANIFEST_FILE_NAME="competency-tests.mft";
public static String EXERCISE_COMPETENCY_MANIFEST_FILE_NAME="exercise-tests.mft";
/*public static void main(String[] args) throws StreamReadException, DatabindException, MalformedURLException, IOException {
ResultSummary summary=ExecuteTestUtil.generateResult("1", Path.of(args[0]), Path.of(args[1]));
System.out.println(summary.successfulTestCompetencyProfile);
}
*/
public static String TEST_COMPETENCY_MANIFEST_FILE_NAME = "competency-tests.mft";
public static String EXERCISE_COMPETENCY_MANIFEST_FILE_NAME = "exercise-tests.mft";
public static float[] sumTestCompetencyProfiles(List<TestCompetencyProfile> testCompetencyProfiles) {
float[] tcpTotalProfile=new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for(TestCompetencyProfile currentProfile: testCompetencyProfiles) {
tcpTotalProfile=ICompetencyProfile.competencySum(tcpTotalProfile, currentProfile.competencyAssessments);
LOG.debug("Starting sumTestCompetencyProfiles with {} test profiles.", testCompetencyProfiles.size());
float[] tcpTotalProfile = new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for (TestCompetencyProfile currentProfile : testCompetencyProfiles) {
LOG.debug("Adding competency assessments from profile: {}", currentProfile.testName);
tcpTotalProfile = ICompetencyProfile.competencySum(tcpTotalProfile, currentProfile.competencyAssessments);
}
LOG.debug("Completed summing test competency profiles. Total profile: {}", packFloats(tcpTotalProfile));
return tcpTotalProfile;
}
public static float[] sumSuccessfulCompetencyProfiles(List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary, boolean success) {
float[] sumSuccessful=new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for(Result currentResult: resultSummary.results) {
LOG.debug("Starting sumSuccessfulCompetencyProfiles. Success flag: {}", success);
float[] sumSuccessful = new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for (Result currentResult : resultSummary.results) {
boolean isSuccess = Integer.valueOf(currentResult.state).equals(Result.State.SUCCESS.ordinal());
if (isSuccess == success) {
TestCompetencyProfile currentProfile=new TestCompetencyProfile();
currentProfile.testPackageName=(currentResult.packageName!=null)?currentResult.packageName:"";
currentProfile.testClassName=(currentResult.className!=null)?currentResult.className:"";
currentProfile.testName=(currentResult.name!=null)?currentResult.name:"";
int testIndex=testCompetencyProfiles.indexOf(currentProfile);
if(testIndex!=-1) {
sumSuccessful=ICompetencyProfile.competencySum(sumSuccessful, testCompetencyProfiles.get(testIndex).competencyAssessments);
LOG.debug("Processing result: {} with success state: {}", currentResult.name, isSuccess);
if (isSuccess == success) {
TestCompetencyProfile currentProfile = new TestCompetencyProfile();
currentProfile.testPackageName = (currentResult.packageName != null) ? currentResult.packageName : "";
currentProfile.testClassName = (currentResult.className != null) ? currentResult.className : "";
currentProfile.testName = (currentResult.name != null) ? currentResult.name : "";
int testIndex = testCompetencyProfiles.indexOf(currentProfile);
if (testIndex != -1) {
LOG.debug("Found matching profile for result: {}. Adding competencies.", currentResult.name);
sumSuccessful = ICompetencyProfile.competencySum(sumSuccessful, testCompetencyProfiles.get(testIndex).competencyAssessments);
} else {
LOG.debug("No matching profile found for result: {}", currentResult.name);
}
}
}
LOG.debug("Completed summing successful competency profiles. Sum: {}", packFloats(sumSuccessful));
return sumSuccessful;
}
public static List<TestCompetencyProfile> readTestCompetencyProfiles(Path testPath, String fileName) {
List<TestCompetencyProfile> testCompetencyProfiles=new ArrayList<TestCompetencyProfile>();
try {
BufferedReader testCompetencyManifest=new BufferedReader(new FileReader(new File(testPath.toFile(), fileName)));
String testEntry=testCompetencyManifest.readLine();
while(testEntry!=null)
{
String[] testEntyComponents=testEntry.split(ICompetencyProfile.COMPETENCY_SEPARATOR);
TestCompetencyProfile currentProfile=new TestCompetencyProfile();
currentProfile.testPackageName=testEntyComponents[0];
currentProfile.testClassName=testEntyComponents[1];
currentProfile.testName=testEntyComponents[2];
for(int competencyIndex=0; competencyIndex<ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex]=Float.valueOf(testEntyComponents[competencyIndex+3]);
LOG.debug("Reading test competency profiles from path: {}, file: {}", testPath, fileName);
List<TestCompetencyProfile> testCompetencyProfiles = new ArrayList<>();
try (BufferedReader testCompetencyManifest = new BufferedReader(new FileReader(new File(testPath.toFile(), fileName)))) {
String testEntry = testCompetencyManifest.readLine();
while (testEntry != null) {
String[] testEntyComponents = testEntry.split(ICompetencyProfile.COMPETENCY_SEPARATOR);
TestCompetencyProfile currentProfile = new TestCompetencyProfile();
currentProfile.testPackageName = testEntyComponents[0];
currentProfile.testClassName = testEntyComponents[1];
currentProfile.testName = testEntyComponents[2];
for (int competencyIndex = 0; competencyIndex < ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex] = Float.valueOf(testEntyComponents[competencyIndex + 3]);
}
testCompetencyProfiles.add(currentProfile);
testEntry=testCompetencyManifest.readLine();
LOG.debug("Added test competency profile: {}", currentProfile.testName);
testEntry = testCompetencyManifest.readLine();
}
testCompetencyManifest.close();
LOG.info("Added "+testCompetencyProfiles.size()+" test competency profiles from test competency manifest. Optional agent functionality enabled.");
LOG.info("Added {} test competency profiles from manifest.", testCompetencyProfiles.size());
} catch (FileNotFoundException e) {
LOG.info("Test competency manifest file for agent feedback not found. Skipping optional functionality.");
testCompetencyProfiles=null;
LOG.info("Test competency manifest file not found. Skipping functionality.");
} catch (IOException e) {
LOG.info("Test competency manifest file for agent feedback unreadable. Skipping optional functionality.");
testCompetencyProfiles=null;
}
LOG.info("Test competency manifest file unreadable. Skipping functionality.");
}
return testCompetencyProfiles;
}
public static List<ExerciseCompetencyProfile> readExerciseCompetencyProfiles(Path exercisePath, String fileName) {
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = new ArrayList<>();
try (BufferedReader exerciseCompetencyManifest = new BufferedReader(new FileReader(new File(exercisePath.toFile(), fileName)))) {
String exerciseEntry = exerciseCompetencyManifest.readLine();
while (exerciseEntry != null) {
String[] exerciseEntyComponents = exerciseEntry.split(ExerciseCompetencyProfile.COMPETENCY_SEPARATOR);
ExerciseCompetencyProfile currentProfile = new ExerciseCompetencyProfile();
currentProfile.exerciseTopicName = exerciseEntyComponents[0];
currentProfile.exerciseName = exerciseEntyComponents[1];
currentProfile.exerciseURL = exerciseEntyComponents[2];
for (int competencyIndex = 0; competencyIndex < ExerciseCompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex] = Float.valueOf(exerciseEntyComponents[competencyIndex+3]);
}
currentProfile.difficulty = Float.parseFloat(exerciseEntyComponents[19]);
exerciseCompetencyProfiles.add(currentProfile);
exerciseEntry = exerciseCompetencyManifest.readLine();
}
exerciseCompetencyManifest.close();
LOG.info("Added " + exerciseCompetencyProfiles.size() + " test competency profiles from exercise competency manifest.");
} catch (FileNotFoundException e) {
LOG.info("Exercise competency manifest file not found.");
} catch (IOException e) {
LOG.info("Exercise competency manifest file unreadable.");
}
return exerciseCompetencyProfiles;
LOG.debug("Reading exercise competency profiles from path: {}, file: {}", exercisePath, fileName);
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = new ArrayList<>();
try (BufferedReader exerciseCompetencyManifest = new BufferedReader(new FileReader(new File(exercisePath.toFile(), fileName)))) {
String exerciseEntry = exerciseCompetencyManifest.readLine();
while (exerciseEntry != null) {
String[] exerciseEntyComponents = exerciseEntry.split(ExerciseCompetencyProfile.COMPETENCY_SEPARATOR);
ExerciseCompetencyProfile currentProfile = new ExerciseCompetencyProfile();
currentProfile.exerciseTopicName = exerciseEntyComponents[0];
currentProfile.exerciseName = exerciseEntyComponents[1];
currentProfile.exerciseURL = exerciseEntyComponents[2];
for (int competencyIndex = 0; competencyIndex < ExerciseCompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex] = Float.valueOf(exerciseEntyComponents[competencyIndex + 3]);
}
currentProfile.difficulty = Float.parseFloat(exerciseEntyComponents[19]);
exerciseCompetencyProfiles.add(currentProfile);
LOG.debug("Added exercise competency profile: {}", currentProfile.exerciseName);
exerciseEntry = exerciseCompetencyManifest.readLine();
}
LOG.info("Added {} exercise competency profiles from manifest.", exerciseCompetencyProfiles.size());
} catch (FileNotFoundException e) {
LOG.info("Exercise competency manifest file not found.");
} catch (IOException e) {
LOG.info("Exercise competency manifest file unreadable.");
}
return exerciseCompetencyProfiles;
}
public static String packFloats(float[] array) {
LOG.debug("Packing float array into string: {}", array);
return IntStream.range(0, array.length)
.mapToObj(i -> String.valueOf(array[i]))
.collect(Collectors.joining(";"));
.mapToObj(i -> String.valueOf(array[i]))
.collect(Collectors.joining(";"));
}
}
src/main/java/de/hftstuttgart/dtabackend/utils/ExecuteTestUtil.java
+ 143
- 89
@@ -71,7 +71,7 @@ public class ExecuteTestUtil {
assignmentBasePath + assignmentId + "_checkout",
testPath.toString() );
Files.copy(Paths.get(
assignmentBasePath, assignmentId + "_checkout",
assignmentBasePath, assignmentId + "_checkout",
CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME),
Paths.get(
testPath.toString(),
@@ -177,99 +177,153 @@ public class ExecuteTestUtil {
return resultSummary;
}
/*
* exercise recommendation part
*/
public List<Recommendation> recommendNextExercises(String assignmentId, Path testPathHost, List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary)
throws FileNotFoundException {
// fetch repo url from original test upload
Pattern pattern = Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX);
File file = Paths.get(assignmentBasePath, assignmentId + ".txt").toFile();
FileInputStream configFileStream = new FileInputStream(file);
Matcher config = RegexUtil.extractConfig(configFileStream, pattern);
String testRepoURL = config.group(1);
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = CompetencyAssessmentUtil.readExerciseCompetencyProfiles(testPathHost, CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME);
int currentTopicIndex=0;
float currentDifficulty=0.0f;
//build course topic order
Map<String, Integer> topicOrder = new HashMap<>();
int order = 1;
for (ExerciseCompetencyProfile e : exerciseCompetencyProfiles) {
if (!topicOrder.containsKey(e.exerciseTopicName)) {
topicOrder.put(e.exerciseTopicName, order++);
}
if (e.exerciseURL.equals(testRepoURL)) {
currentTopicIndex = order;
currentDifficulty = e.difficulty;
}
/*
* exercise recommendation part
*/
public List<Recommendation> recommendNextExercises(String assignmentId, Path testPathHost, List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary)
throws FileNotFoundException {
LOG.debug("Starting recommendNextExercises with assignmentId: {}", assignmentId);
// fetch repo url from original test upload
Pattern pattern = Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX);
LOG.debug("Compiled regex pattern for DTA_TESTCONFIGREGEX.");
File file = Paths.get(assignmentBasePath, assignmentId + ".txt").toFile();
LOG.debug("Resolved file path for assignmentId {}: {}", assignmentId, file.getAbsolutePath());
FileInputStream configFileStream = new FileInputStream(file);
LOG.debug("Opened FileInputStream for file: {}", file.getAbsolutePath());
Matcher config = RegexUtil.extractConfig(configFileStream, pattern);
LOG.debug("Extracted configuration using regex pattern.");
String testRepoURL = config.group(1) + config.group(4);
LOG.debug("Constructed testRepoURL: {}", testRepoURL);
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = CompetencyAssessmentUtil.readExerciseCompetencyProfiles(
testPathHost, CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME);
LOG.debug("Read exercise competency profiles from path: {}", testPathHost.resolve(CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME));
int currentTopicIndex = 0;
float currentDifficulty = 0.0f;
// build course topic order
LOG.debug("Building course topic order.");
Map<String, Integer> topicOrder = new HashMap<>();
int order = 1;
for (ExerciseCompetencyProfile e : exerciseCompetencyProfiles) {
if (!topicOrder.containsKey(e.exerciseTopicName)) {
topicOrder.put(e.exerciseTopicName, order++);
LOG.debug("Added topic {} to topicOrder with order {}", e.exerciseTopicName, order - 1);
}
//filter exercises according to success
float[] unsuccessful = CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, false);
List<ExerciseCompetencyProfile> filteredExercises = filterExercisesByTopicsAndDifficulty(
exerciseCompetencyProfiles, topicOrder, currentTopicIndex, testRepoURL,
currentDifficulty, unsuccessful, resultSummary);
//compute recommendations
List<Recommendation> recommendedExercises = new ArrayList<>();
for (ExerciseCompetencyProfile exerciseProfile : filteredExercises) {
Recommendation recommendation = new Recommendation(exerciseProfile.exerciseTopicName, exerciseProfile.exerciseURL, exerciseProfile.exerciseName,
exerciseProfile.difficulty, calculateScore(exerciseProfile, unsuccessful, topicOrder, currentDifficulty));
recommendedExercises.add(recommendation);
LOG.info("Recommending exercise "+recommendation.topic+"/"+recommendation.exerciseName+" with score "+recommendation.score);
if (e.exerciseURL.equals(testRepoURL)) {
currentTopicIndex = order;
currentDifficulty = e.difficulty;
LOG.debug("Matched current testRepoURL to topic: {}, index: {}, difficulty: {}", e.exerciseTopicName, currentTopicIndex, currentDifficulty);
}
//sort the recommendations for successful or resilient learners, otherwise reverse in display
recommendedExercises.stream().sorted(Recommendation.COMPARE_BY_SCORE).collect(Collectors.toList());
return recommendedExercises;
}
public static List<ExerciseCompetencyProfile> filterExercisesByTopicsAndDifficulty(List<ExerciseCompetencyProfile> exerciseCompetencyProfiles,
Map<String, Integer> topicOrder, int currentTopicIndex, String testRepoURL, float currentDifficulty, float[] unsuccessful,
ResultSummary resultSummary) {
//filter out all advanced topics in any case
//option for later: include next topic if fullsuccess and current difficulty == max difficulty
List<ExerciseCompetencyProfile> filteredExercises = exerciseCompetencyProfiles.stream()
.filter(testProfile -> topicOrder.get(testProfile.exerciseTopicName) <= currentTopicIndex)
.collect(Collectors.toList());
//filter by difficulty according to success
if (isFullSuccess(unsuccessful)) {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty >= currentDifficulty && !testRepoURL.equals(profile.exerciseURL)).collect(Collectors.toList());
} else {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty <= currentDifficulty).collect(Collectors.toList());
}
// filter exercises according to success
LOG.debug("Filtering exercises according to success.");
float[] unsuccessful = CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, false);
LOG.debug("Computed unsuccessful competency profile: {}", unsuccessful);
return filteredExercises;
}
public static boolean isFullSuccess(float[] unsuccessful) {
for (float value : unsuccessful) {
if (value != 0.0f) {
return false;
}
}
return true;
}
List<ExerciseCompetencyProfile> filteredExercises = filterExercisesByTopicsAndDifficulty(
exerciseCompetencyProfiles, topicOrder, currentTopicIndex, testRepoURL,
currentDifficulty, unsuccessful, resultSummary);
LOG.debug("Filtered exercises count: {}", filteredExercises.size());
public static float calculateScore(ExerciseCompetencyProfile exerciseProfile, float[] unsuccessful, Map<String, Integer> topicOrder, float currentDifficulty) {
//ensure factor 1 for full success not to blank out score, thus offset the base
float score = 1.0f;
//competency profile difference to not fully achieved competencies component
for (int i = 0; i < exerciseProfile.competencyAssessments.length-1; i++) {
score += exerciseProfile.competencyAssessments[i] * unsuccessful[i];
}
//difficulty component
score = score * (exerciseProfile.difficulty*(0.5f+Math.abs(currentDifficulty-exerciseProfile.difficulty)));
//topic component
score *= topicOrder.get(exerciseProfile.exerciseTopicName);
score = Math.round(score * 10.0f) / 10.0f;
return score;
// compute recommendations
LOG.debug("Computing recommendations from filtered exercises.");
List<Recommendation> recommendedExercises = new ArrayList<>();
for (ExerciseCompetencyProfile exerciseProfile : filteredExercises) {
Recommendation recommendation = new Recommendation(
exerciseProfile.exerciseTopicName, exerciseProfile.exerciseURL, exerciseProfile.exerciseName,
exerciseProfile.difficulty, calculateScore(exerciseProfile, unsuccessful, topicOrder, currentDifficulty));
recommendedExercises.add(recommendation);
LOG.info("Recommending exercise {}/{} with score {}", recommendation.topic, recommendation.exerciseName, recommendation.score);
}
// sort the recommendations for successful or resilient learners, otherwise reverse in display
LOG.debug("Sorting recommendations.");
recommendedExercises.stream().sorted(Recommendation.COMPARE_BY_SCORE).collect(Collectors.toList());
LOG.debug("Completed recommendNextExercises with {} recommendations.", recommendedExercises.size());
return recommendedExercises;
}
public static List<ExerciseCompetencyProfile> filterExercisesByTopicsAndDifficulty(
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles,
Map<String, Integer> topicOrder, int currentTopicIndex, String testRepoURL,
float currentDifficulty, float[] unsuccessful, ResultSummary resultSummary) {
LOG.debug("Starting filterExercisesByTopicsAndDifficulty with currentTopicIndex: {}, currentDifficulty: {}, testRepoURL: {}",
currentTopicIndex, currentDifficulty, testRepoURL);
// Filter out all advanced topics in any case
List<ExerciseCompetencyProfile> filteredExercises = exerciseCompetencyProfiles.stream()
.filter(testProfile -> topicOrder.get(testProfile.exerciseTopicName) <= currentTopicIndex)
.collect(Collectors.toList());
LOG.debug("Filtered exercises by topic index. Remaining exercises count: {}", filteredExercises.size());
// Filter by difficulty according to success
if (isFullSuccess(unsuccessful)) {
LOG.debug("Detected full success, filtering exercises with difficulty >= {} and excluding current testRepoURL.", currentDifficulty);
filteredExercises = filteredExercises.stream()
.filter(profile -> profile.difficulty >= currentDifficulty && !testRepoURL.equals(profile.exerciseURL))
.collect(Collectors.toList());
} else {
LOG.debug("Detected partial success, filtering exercises with difficulty <= {}.", currentDifficulty);
filteredExercises = filteredExercises.stream()
.filter(profile -> profile.difficulty <= currentDifficulty)
.collect(Collectors.toList());
}
LOG.debug("Filtered exercises count after difficulty filter: {}", filteredExercises.size());
return filteredExercises;
}
public static boolean isFullSuccess(float[] unsuccessful) {
LOG.debug("Checking for full success. Unsuccessful array: {}", unsuccessful);
for (float value : unsuccessful) {
if (value != 0.0f) {
LOG.debug("Found non-zero value in unsuccessful array: {}. Returning false for full success.", value);
return false;
}
}
LOG.debug("All values in unsuccessful array are zero. Returning true for full success.");
return true;
}
public static float calculateScore(ExerciseCompetencyProfile exerciseProfile, float[] unsuccessful,
Map<String, Integer> topicOrder, float currentDifficulty) {
LOG.debug("Starting calculateScore for exercise: {}, difficulty: {}, currentDifficulty: {}",
exerciseProfile.exerciseName, exerciseProfile.difficulty, currentDifficulty);
float score = 1.0f;
LOG.debug("Initial score set to 1.0");
// Competency profile difference to not fully achieved competencies component
for (int i = 0; i < exerciseProfile.competencyAssessments.length - 1; i++) {
float adjustment = exerciseProfile.competencyAssessments[i] * unsuccessful[i];
score += adjustment;
LOG.debug("Adjusted score by competency index {}: {}, new score: {}", i, adjustment, score);
}
// Difficulty component
float difficultyComponent = exerciseProfile.difficulty * (0.5f + Math.abs(currentDifficulty - exerciseProfile.difficulty));
score = score * difficultyComponent;
LOG.debug("Applied difficulty component: {}, updated score: {}", difficultyComponent, score);
// Topic component
float topicMultiplier = topicOrder.get(exerciseProfile.exerciseTopicName);
score *= topicMultiplier;
LOG.debug("Applied topic multiplier: {}, updated score: {}", topicMultiplier, score);
// Round score
score = Math.round(score * 10.0f) / 10.0f;
LOG.debug("Final rounded score: {}", score);
return score;
}
}
\ No newline at end of file

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal