Skip to content
GitLab
Commit 4fdae855 authored by mamunozgil's avatar mamunozgil
Browse files

Added debug logs

parent d2fbee0e
master amg-dev-volumes amg-svn+ssh
1 merge request!3Fix: dev recommendations
Pipeline #10595 passed with stage
in 18 seconds
This commit is part of merge request !3. Comments created here will be created in the context of that merge request.
Hide whitespace changes
Inline Side-by-side
Showing
with 241 additions and 170 deletions
+241 -170
src/main/java/de/hftstuttgart/dtabackend/utils/CompetencyAssessmentUtil.java
+ 98
- 81
View file @ 4fdae855
......@@ -27,110 +27,127 @@ import java.io.FileNotFoundException;
public class CompetencyAssessmentUtil {
private static final Logger LOG = LogManager.getLogger(CompetencyAssessmentUtil.class);
public static String TEST_COMPETENCY_MANIFEST_FILE_NAME="competency-tests.mft";
public static String EXERCISE_COMPETENCY_MANIFEST_FILE_NAME="exercise-tests.mft";
/*public static void main(String[] args) throws StreamReadException, DatabindException, MalformedURLException, IOException {
ResultSummary summary=ExecuteTestUtil.generateResult("1", Path.of(args[0]), Path.of(args[1]));
System.out.println(summary.successfulTestCompetencyProfile);
}
*/
public static String TEST_COMPETENCY_MANIFEST_FILE_NAME = "competency-tests.mft";
public static String EXERCISE_COMPETENCY_MANIFEST_FILE_NAME = "exercise-tests.mft";
public static float[] sumTestCompetencyProfiles(List<TestCompetencyProfile> testCompetencyProfiles) {
float[] tcpTotalProfile=new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for(TestCompetencyProfile currentProfile: testCompetencyProfiles) {
tcpTotalProfile=ICompetencyProfile.competencySum(tcpTotalProfile, currentProfile.competencyAssessments);
LOG.debug("Starting sumTestCompetencyProfiles with {} test profiles.", testCompetencyProfiles.size());
float[] tcpTotalProfile = new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for (TestCompetencyProfile currentProfile : testCompetencyProfiles) {
LOG.debug("Adding competency assessments from profile: {}", currentProfile.testName);
tcpTotalProfile = ICompetencyProfile.competencySum(tcpTotalProfile, currentProfile.competencyAssessments);
}
LOG.debug("Completed summing test competency profiles. Total profile: {}", packFloats(tcpTotalProfile));
return tcpTotalProfile;
}
public static float[] sumSuccessfulCompetencyProfiles(List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary, boolean success) {
float[] sumSuccessful=new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for(Result currentResult: resultSummary.results) {
LOG.debug("Starting sumSuccessfulCompetencyProfiles. Success flag: {}", success);
float[] sumSuccessful = new float[ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for (Result currentResult : resultSummary.results) {
boolean isSuccess = Integer.valueOf(currentResult.state).equals(Result.State.SUCCESS.ordinal());
if (isSuccess == success) {
TestCompetencyProfile currentProfile=new TestCompetencyProfile();
currentProfile.testPackageName=(currentResult.packageName!=null)?currentResult.packageName:"";
currentProfile.testClassName=(currentResult.className!=null)?currentResult.className:"";
currentProfile.testName=(currentResult.name!=null)?currentResult.name:"";
int testIndex=testCompetencyProfiles.indexOf(currentProfile);
if(testIndex!=-1) {
sumSuccessful=ICompetencyProfile.competencySum(sumSuccessful, testCompetencyProfiles.get(testIndex).competencyAssessments);
LOG.debug("Processing result: {} with success state: {}", currentResult.name, isSuccess);
if (isSuccess == success) {
TestCompetencyProfile currentProfile = new TestCompetencyProfile();
currentProfile.testPackageName = (currentResult.packageName != null) ? currentResult.packageName : "";
currentProfile.testClassName = (currentResult.className != null) ? currentResult.className : "";
currentProfile.testName = (currentResult.name != null) ? currentResult.name : "";
int testIndex = testCompetencyProfiles.indexOf(currentProfile);
if (testIndex != -1) {
LOG.debug("Found matching profile for result: {}. Adding competencies.", currentResult.name);
sumSuccessful = ICompetencyProfile.competencySum(sumSuccessful, testCompetencyProfiles.get(testIndex).competencyAssessments);
} else {
LOG.debug("No matching profile found for result: {}", currentResult.name);
}
}
}
LOG.debug("Completed summing successful competency profiles. Sum: {}", packFloats(sumSuccessful));
return sumSuccessful;
}
public static List<TestCompetencyProfile> readTestCompetencyProfiles(Path testPath, String fileName) {
List<TestCompetencyProfile> testCompetencyProfiles=new ArrayList<TestCompetencyProfile>();
try {
BufferedReader testCompetencyManifest=new BufferedReader(new FileReader(new File(testPath.toFile(), fileName)));
String testEntry=testCompetencyManifest.readLine();
while(testEntry!=null)
{
String[] testEntyComponents=testEntry.split(ICompetencyProfile.COMPETENCY_SEPARATOR);
TestCompetencyProfile currentProfile=new TestCompetencyProfile();
currentProfile.testPackageName=testEntyComponents[0];
currentProfile.testClassName=testEntyComponents[1];
currentProfile.testName=testEntyComponents[2];
for(int competencyIndex=0; competencyIndex<ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex]=Float.valueOf(testEntyComponents[competencyIndex+3]);
LOG.debug("Reading test competency profiles from path: {}, file: {}", testPath, fileName);
List<TestCompetencyProfile> testCompetencyProfiles = new ArrayList<>();
try (BufferedReader testCompetencyManifest = new BufferedReader(new FileReader(new File(testPath.toFile(), fileName)))) {
String testEntry = testCompetencyManifest.readLine();
while (testEntry != null) {
String[] testEntyComponents = testEntry.split(ICompetencyProfile.COMPETENCY_SEPARATOR);
TestCompetencyProfile currentProfile = new TestCompetencyProfile();
currentProfile.testPackageName = testEntyComponents[0];
currentProfile.testClassName = testEntyComponents[1];
currentProfile.testName = testEntyComponents[2];
for (int competencyIndex = 0; competencyIndex < ICompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex] = Float.valueOf(testEntyComponents[competencyIndex + 3]);
}
testCompetencyProfiles.add(currentProfile);
testEntry=testCompetencyManifest.readLine();
LOG.debug("Added test competency profile: {}", currentProfile.testName);
testEntry = testCompetencyManifest.readLine();
}
testCompetencyManifest.close();
LOG.info("Added "+testCompetencyProfiles.size()+" test competency profiles from test competency manifest. Optional agent functionality enabled.");
LOG.info("Added {} test competency profiles from manifest.", testCompetencyProfiles.size());
} catch (FileNotFoundException e) {
LOG.info("Test competency manifest file for agent feedback not found. Skipping optional functionality.");
testCompetencyProfiles=null;
LOG.info("Test competency manifest file not found. Skipping functionality.");
} catch (IOException e) {
LOG.info("Test competency manifest file for agent feedback unreadable. Skipping optional functionality.");
testCompetencyProfiles=null;
}
LOG.info("Test competency manifest file unreadable. Skipping functionality.");
}
return testCompetencyProfiles;
}
public static List<ExerciseCompetencyProfile> readExerciseCompetencyProfiles(Path exercisePath, String fileName) {
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = new ArrayList<>();
try (BufferedReader exerciseCompetencyManifest = new BufferedReader(new FileReader(new File(exercisePath.toFile(), fileName)))) {
String exerciseEntry = exerciseCompetencyManifest.readLine();
while (exerciseEntry != null) {
String[] exerciseEntyComponents = exerciseEntry.split(ExerciseCompetencyProfile.COMPETENCY_SEPARATOR);
ExerciseCompetencyProfile currentProfile = new ExerciseCompetencyProfile();
currentProfile.exerciseTopicName = exerciseEntyComponents[0];
currentProfile.exerciseName = exerciseEntyComponents[1];
currentProfile.exerciseURL = exerciseEntyComponents[2];
for (int competencyIndex = 0; competencyIndex < ExerciseCompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex] = Float.valueOf(exerciseEntyComponents[competencyIndex+3]);
}
currentProfile.difficulty = Float.parseFloat(exerciseEntyComponents[19]);
exerciseCompetencyProfiles.add(currentProfile);
exerciseEntry = exerciseCompetencyManifest.readLine();
}
exerciseCompetencyManifest.close();
LOG.info("Added " + exerciseCompetencyProfiles.size() + " test competency profiles from exercise competency manifest.");
} catch (FileNotFoundException e) {
LOG.info("Exercise competency manifest file not found.");
} catch (IOException e) {
LOG.info("Exercise competency manifest file unreadable.");
}
return exerciseCompetencyProfiles;
LOG.debug("Reading exercise competency profiles from path: {}, file: {}", exercisePath, fileName);
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = new ArrayList<>();
try (BufferedReader exerciseCompetencyManifest = new BufferedReader(new FileReader(new File(exercisePath.toFile(), fileName)))) {
String exerciseEntry = exerciseCompetencyManifest.readLine();
while (exerciseEntry != null) {
String[] exerciseEntyComponents = exerciseEntry.split(ExerciseCompetencyProfile.COMPETENCY_SEPARATOR);
ExerciseCompetencyProfile currentProfile = new ExerciseCompetencyProfile();
currentProfile.exerciseTopicName = exerciseEntyComponents[0];
currentProfile.exerciseName = exerciseEntyComponents[1];
currentProfile.exerciseURL = exerciseEntyComponents[2];
for (int competencyIndex = 0; competencyIndex < ExerciseCompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex] = Float.valueOf(exerciseEntyComponents[competencyIndex + 3]);
}
currentProfile.difficulty = Float.parseFloat(exerciseEntyComponents[19]);
exerciseCompetencyProfiles.add(currentProfile);
LOG.debug("Added exercise competency profile: {}", currentProfile.exerciseName);
exerciseEntry = exerciseCompetencyManifest.readLine();
}
LOG.info("Added {} exercise competency profiles from manifest.", exerciseCompetencyProfiles.size());
} catch (FileNotFoundException e) {
LOG.info("Exercise competency manifest file not found.");
} catch (IOException e) {
LOG.info("Exercise competency manifest file unreadable.");
}
return exerciseCompetencyProfiles;
}
public static String packFloats(float[] array) {
LOG.debug("Packing float array into string: {}", array);
return IntStream.range(0, array.length)
.mapToObj(i -> String.valueOf(array[i]))
.collect(Collectors.joining(";"));
.mapToObj(i -> String.valueOf(array[i]))
.collect(Collectors.joining(";"));
}
}
src/main/java/de/hftstuttgart/dtabackend/utils/ExecuteTestUtil.java
+ 143
- 89
View file @ 4fdae855
......@@ -71,7 +71,7 @@ public class ExecuteTestUtil {
assignmentBasePath + assignmentId + "_checkout",
testPath.toString() );
Files.copy(Paths.get(
assignmentBasePath, assignmentId + "_checkout",
assignmentBasePath, assignmentId + "_checkout",
CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME),
Paths.get(
testPath.toString(),
......@@ -177,99 +177,153 @@ public class ExecuteTestUtil {
return resultSummary;
}
/*
* exercise recommendation part
*/
public List<Recommendation> recommendNextExercises(String assignmentId, Path testPathHost, List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary)
throws FileNotFoundException {
// fetch repo url from original test upload
Pattern pattern = Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX);
File file = Paths.get(assignmentBasePath, assignmentId + ".txt").toFile();
FileInputStream configFileStream = new FileInputStream(file);
Matcher config = RegexUtil.extractConfig(configFileStream, pattern);
String testRepoURL = config.group(1)+config.group(4);
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = CompetencyAssessmentUtil.readExerciseCompetencyProfiles(testPathHost, CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME);
int currentTopicIndex=0;
float currentDifficulty=0.0f;
//build course topic order
Map<String, Integer> topicOrder = new HashMap<>();
int order = 1;
for (ExerciseCompetencyProfile e : exerciseCompetencyProfiles) {
if (!topicOrder.containsKey(e.exerciseTopicName)) {
topicOrder.put(e.exerciseTopicName, order++);
}
if (e.exerciseURL.equals(testRepoURL)) {
currentTopicIndex = order;
currentDifficulty = e.difficulty;
}
/*
* exercise recommendation part
*/
public List<Recommendation> recommendNextExercises(String assignmentId, Path testPathHost, List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary)
throws FileNotFoundException {
LOG.debug("Starting recommendNextExercises with assignmentId: {}", assignmentId);
// fetch repo url from original test upload
Pattern pattern = Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX);
LOG.debug("Compiled regex pattern for DTA_TESTCONFIGREGEX.");
File file = Paths.get(assignmentBasePath, assignmentId + ".txt").toFile();
LOG.debug("Resolved file path for assignmentId {}: {}", assignmentId, file.getAbsolutePath());
FileInputStream configFileStream = new FileInputStream(file);
LOG.debug("Opened FileInputStream for file: {}", file.getAbsolutePath());
Matcher config = RegexUtil.extractConfig(configFileStream, pattern);
LOG.debug("Extracted configuration using regex pattern.");
String testRepoURL = config.group(1) + config.group(4);
LOG.debug("Constructed testRepoURL: {}", testRepoURL);
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = CompetencyAssessmentUtil.readExerciseCompetencyProfiles(
testPathHost, CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME);
LOG.debug("Read exercise competency profiles from path: {}", testPathHost.resolve(CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME));
int currentTopicIndex = 0;
float currentDifficulty = 0.0f;
// build course topic order
LOG.debug("Building course topic order.");
Map<String, Integer> topicOrder = new HashMap<>();
int order = 1;
for (ExerciseCompetencyProfile e : exerciseCompetencyProfiles) {
if (!topicOrder.containsKey(e.exerciseTopicName)) {
topicOrder.put(e.exerciseTopicName, order++);
LOG.debug("Added topic {} to topicOrder with order {}", e.exerciseTopicName, order - 1);
}
//filter exercises according to success
float[] unsuccessful = CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, false);
List<ExerciseCompetencyProfile> filteredExercises = filterExercisesByTopicsAndDifficulty(
exerciseCompetencyProfiles, topicOrder, currentTopicIndex, testRepoURL,
currentDifficulty, unsuccessful, resultSummary);
//compute recommendations
List<Recommendation> recommendedExercises = new ArrayList<>();
for (ExerciseCompetencyProfile exerciseProfile : filteredExercises) {
Recommendation recommendation = new Recommendation(exerciseProfile.exerciseTopicName, exerciseProfile.exerciseURL, exerciseProfile.exerciseName,
exerciseProfile.difficulty, calculateScore(exerciseProfile, unsuccessful, topicOrder, currentDifficulty));
recommendedExercises.add(recommendation);
LOG.info("Recommending exercise "+recommendation.topic+"/"+recommendation.exerciseName+" with score "+recommendation.score);
if (e.exerciseURL.equals(testRepoURL)) {
currentTopicIndex = order;
currentDifficulty = e.difficulty;
LOG.debug("Matched current testRepoURL to topic: {}, index: {}, difficulty: {}", e.exerciseTopicName, currentTopicIndex, currentDifficulty);
}
//sort the recommendations for successful or resilient learners, otherwise reverse in display
recommendedExercises.stream().sorted(Recommendation.COMPARE_BY_SCORE).collect(Collectors.toList());
return recommendedExercises;
}
public static List<ExerciseCompetencyProfile> filterExercisesByTopicsAndDifficulty(List<ExerciseCompetencyProfile> exerciseCompetencyProfiles,
Map<String, Integer> topicOrder, int currentTopicIndex, String testRepoURL, float currentDifficulty, float[] unsuccessful,
ResultSummary resultSummary) {
//filter out all advanced topics in any case
//option for later: include next topic if fullsuccess and current difficulty == max difficulty
List<ExerciseCompetencyProfile> filteredExercises = exerciseCompetencyProfiles.stream()
.filter(testProfile -> topicOrder.get(testProfile.exerciseTopicName) <= currentTopicIndex)
.collect(Collectors.toList());
//filter by difficulty according to success
if (isFullSuccess(unsuccessful)) {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty >= currentDifficulty && !testRepoURL.equals(profile.exerciseURL)).collect(Collectors.toList());
} else {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty <= currentDifficulty).collect(Collectors.toList());
}
// filter exercises according to success
LOG.debug("Filtering exercises according to success.");
float[] unsuccessful = CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, false);
LOG.debug("Computed unsuccessful competency profile: {}", unsuccessful);
return filteredExercises;
}
public static boolean isFullSuccess(float[] unsuccessful) {
for (float value : unsuccessful) {
if (value != 0.0f) {
return false;
}
}
return true;
}
List<ExerciseCompetencyProfile> filteredExercises = filterExercisesByTopicsAndDifficulty(
exerciseCompetencyProfiles, topicOrder, currentTopicIndex, testRepoURL,
currentDifficulty, unsuccessful, resultSummary);
LOG.debug("Filtered exercises count: {}", filteredExercises.size());
public static float calculateScore(ExerciseCompetencyProfile exerciseProfile, float[] unsuccessful, Map<String, Integer> topicOrder, float currentDifficulty) {
//ensure factor 1 for full success not to blank out score, thus offset the base
float score = 1.0f;
//competency profile difference to not fully achieved competencies component
for (int i = 0; i < exerciseProfile.competencyAssessments.length-1; i++) {
score += exerciseProfile.competencyAssessments[i] * unsuccessful[i];
}
//difficulty component
score = score * (exerciseProfile.difficulty*(0.5f+Math.abs(currentDifficulty-exerciseProfile.difficulty)));
//topic component
score *= topicOrder.get(exerciseProfile.exerciseTopicName);
score = Math.round(score * 10.0f) / 10.0f;
return score;
// compute recommendations
LOG.debug("Computing recommendations from filtered exercises.");
List<Recommendation> recommendedExercises = new ArrayList<>();
for (ExerciseCompetencyProfile exerciseProfile : filteredExercises) {
Recommendation recommendation = new Recommendation(
exerciseProfile.exerciseTopicName, exerciseProfile.exerciseURL, exerciseProfile.exerciseName,
exerciseProfile.difficulty, calculateScore(exerciseProfile, unsuccessful, topicOrder, currentDifficulty));
recommendedExercises.add(recommendation);
LOG.info("Recommending exercise {}/{} with score {}", recommendation.topic, recommendation.exerciseName, recommendation.score);
}
// sort the recommendations for successful or resilient learners, otherwise reverse in display
LOG.debug("Sorting recommendations.");
recommendedExercises.stream().sorted(Recommendation.COMPARE_BY_SCORE).collect(Collectors.toList());
LOG.debug("Completed recommendNextExercises with {} recommendations.", recommendedExercises.size());
return recommendedExercises;
}
public static List<ExerciseCompetencyProfile> filterExercisesByTopicsAndDifficulty(
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles,
Map<String, Integer> topicOrder, int currentTopicIndex, String testRepoURL,
float currentDifficulty, float[] unsuccessful, ResultSummary resultSummary) {
LOG.debug("Starting filterExercisesByTopicsAndDifficulty with currentTopicIndex: {}, currentDifficulty: {}, testRepoURL: {}",
currentTopicIndex, currentDifficulty, testRepoURL);
// Filter out all advanced topics in any case
List<ExerciseCompetencyProfile> filteredExercises = exerciseCompetencyProfiles.stream()
.filter(testProfile -> topicOrder.get(testProfile.exerciseTopicName) <= currentTopicIndex)
.collect(Collectors.toList());
LOG.debug("Filtered exercises by topic index. Remaining exercises count: {}", filteredExercises.size());
// Filter by difficulty according to success
if (isFullSuccess(unsuccessful)) {
LOG.debug("Detected full success, filtering exercises with difficulty >= {} and excluding current testRepoURL.", currentDifficulty);
filteredExercises = filteredExercises.stream()
.filter(profile -> profile.difficulty >= currentDifficulty && !testRepoURL.equals(profile.exerciseURL))
.collect(Collectors.toList());
} else {
LOG.debug("Detected partial success, filtering exercises with difficulty <= {}.", currentDifficulty);
filteredExercises = filteredExercises.stream()
.filter(profile -> profile.difficulty <= currentDifficulty)
.collect(Collectors.toList());
}
LOG.debug("Filtered exercises count after difficulty filter: {}", filteredExercises.size());
return filteredExercises;
}
public static boolean isFullSuccess(float[] unsuccessful) {
LOG.debug("Checking for full success. Unsuccessful array: {}", unsuccessful);
for (float value : unsuccessful) {
if (value != 0.0f) {
LOG.debug("Found non-zero value in unsuccessful array: {}. Returning false for full success.", value);
return false;
}
}
LOG.debug("All values in unsuccessful array are zero. Returning true for full success.");
return true;
}
public static float calculateScore(ExerciseCompetencyProfile exerciseProfile, float[] unsuccessful,
Map<String, Integer> topicOrder, float currentDifficulty) {
LOG.debug("Starting calculateScore for exercise: {}, difficulty: {}, currentDifficulty: {}",
exerciseProfile.exerciseName, exerciseProfile.difficulty, currentDifficulty);
float score = 1.0f;
LOG.debug("Initial score set to 1.0");
// Competency profile difference to not fully achieved competencies component
for (int i = 0; i < exerciseProfile.competencyAssessments.length - 1; i++) {
float adjustment = exerciseProfile.competencyAssessments[i] * unsuccessful[i];
score += adjustment;
LOG.debug("Adjusted score by competency index {}: {}, new score: {}", i, adjustment, score);
}
// Difficulty component
float difficultyComponent = exerciseProfile.difficulty * (0.5f + Math.abs(currentDifficulty - exerciseProfile.difficulty));
score = score * difficultyComponent;
LOG.debug("Applied difficulty component: {}, updated score: {}", difficultyComponent, score);
// Topic component
float topicMultiplier = topicOrder.get(exerciseProfile.exerciseTopicName);
score *= topicMultiplier;
LOG.debug("Applied topic multiplier: {}, updated score: {}", topicMultiplier, score);
// Round score
score = Math.round(score * 10.0f) / 10.0f;
LOG.debug("Final rounded score: {}", score);
return score;
}
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal