-
mamunozgil authored257c0503
package de.hftstuttgart.dtabackend.utils;
import com.fasterxml.jackson.core.exc.StreamReadException;
import com.fasterxml.jackson.databind.DatabindException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Volume;
import de.hftstuttgart.dtabackend.models.ExerciseCompetencyProfile;
import de.hftstuttgart.dtabackend.models.Recommendation;
import de.hftstuttgart.dtabackend.models.ResultSummary;
import de.hftstuttgart.dtabackend.models.TestCompetencyProfile;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
import java.io.*;
import java.net.MalformedURLException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Component
public class ExecuteTestUtil {
private static final Logger LOG = LogManager.getLogger(ExecuteTestUtil.class);
private final DockerUtil dockerUtil;
private final String basePathProfessorUnitTests;
private final String basePathStudentsSubmissionCode;
public ExecuteTestUtil(Environment env, DockerUtil dockerUtil) {
this.dockerUtil = dockerUtil;
// set base path for assignments to be stored
Path p = Paths.get(
env.getProperty("data.dir"), ///data
env.getProperty("data.dir.test.folder.name")); //UnitTests
this.basePathProfessorUnitTests = p.toAbsolutePath().toString();
this.basePathStudentsSubmissionCode = env.getProperty( "tests.tmp.dir");// /dta-test-assignments
}
public ResultSummary runTests(String assignmentId, Path workDirectory) throws IOException, InterruptedException {
// Define paths for the submission-specific directories
String submissionDirectory = this.basePathStudentsSubmissionCode + workDirectory.getFileName(); // /dta-test-assignments/dta-submissionID
Path testPath = Paths.get(submissionDirectory, "test");
Path srcPath = Paths.get(submissionDirectory, "src");
Path resultPath = Paths.get(submissionDirectory, "result");
// Ensure directories exist
Files.createDirectories(testPath);
Files.createDirectories(srcPath);
Files.createDirectories(resultPath);
// Clone stored test to testPath
LOG.debug("Copying pre-downloaded unit test repo");
FileUtil.copyFolder(Paths.get(basePathProfessorUnitTests, assignmentId), testPath);
LOG.debug("Copying exercise manifest");
Files.copy(
Paths.get(basePathProfessorUnitTests, assignmentId + "_checkout", CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME),
testPath.resolve(CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME)
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
);
LOG.debug("Copying test config");
Files.copy(
Paths.get(basePathProfessorUnitTests, assignmentId + ".txt"),
Paths.get(submissionDirectory, "config.txt")
);
LOG.info("Reading test config");
Matcher config = RegexUtil.extractConfig(
new FileInputStream(Paths.get(submissionDirectory, "config.txt").toFile()),
Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX)
);
String image;
if (config == null) {
config = RegexUtil.extractConfig(
new FileInputStream(Paths.get(submissionDirectory, "config.txt").toFile()),
Pattern.compile(RegexUtil.TESTCONFIGREGEX)
);
if (config == null) {
throw new RuntimeException("Couldn't find repo config for unit test image extraction");
}
image = config.group(4);
} else {
image = config.group(5);
}
// Start the test-container with professor-given image and submission-specific volume mounts
dockerUtil.runContainerWithBinds(
image,
new Bind(testPath.toString(), new Volume("/data/test")),
new Bind(srcPath.toString(), new Volume("/data/src")),
new Bind(resultPath.toString(), new Volume("/data/result"))
);
return generateResult(assignmentId, resultPath, testPath);
}
private ResultSummary generateResult(String assignmentId, Path resultPath, Path testPath)
throws IOException, StreamReadException, DatabindException, MalformedURLException {
// Define expected result file
File resultFile = resultPath.resolve("result.json").toFile();
// Check if result file is there
if (!resultFile.exists() || !resultFile.isFile()) {
LOG.error("Could not find result file in {}", resultFile.getAbsolutePath());
throw new RuntimeException("No result file found");
}
LOG.debug("Parsing results JSON");
ObjectMapper objectMapper = new ObjectMapper();
ResultSummary resultSummary = objectMapper.readValue(resultFile.toURI().toURL(), ResultSummary.class);
LOG.debug("Result JSON returned time {} with {} test results.", resultSummary.timestamp, resultSummary.results.size());
LOG.info("Checking for optional test competency profile information for pedagogical agent functionality...");
List<TestCompetencyProfile> testCompetencyProfiles = CompetencyAssessmentUtil.readTestCompetencyProfiles(testPath, CompetencyAssessmentUtil.TEST_COMPETENCY_MANIFEST_FILE_NAME);
LOG.debug(String.format(
"Reading Test Competency Profiles: basePath=%s, fileName=%s",
testPath,
CompetencyAssessmentUtil.TEST_COMPETENCY_MANIFEST_FILE_NAME
));
if (testCompetencyProfiles != null) {
LOG.info("Found optional test competency profiles, generating agent profile data...");
resultSummary.overallTestCompetencyProfile = CompetencyAssessmentUtil.packFloats(CompetencyAssessmentUtil.sumTestCompetencyProfiles(testCompetencyProfiles));
resultSummary.successfulTestCompetencyProfile = CompetencyAssessmentUtil.packFloats(CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, true));
LOG.info("Checking for optional exercise competency profile information for pedagogical agent exercise recommendation functionality...");
Path exerciseManifestFile = Paths.get(testPath.toString(), CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME);
LOG.debug(String.format(
"Constructing Path for exercise manifest: testPath=%s, fileName=%s -> Resulting Path=%s",
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
testPath.toString(),
CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME,
exerciseManifestFile.toString()
));
if (Files.exists(exerciseManifestFile)) {
LOG.info("Found optional exercise competency profiles, generating recommendations...");
resultSummary.recommendations = recommendNextExercises(assignmentId, testPath, testCompetencyProfiles, resultSummary);
}
}
return resultSummary;
}
/*
* exercise recommendation part
*/
public List<Recommendation> recommendNextExercises(String assignmentId, Path testPath, List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary)
throws FileNotFoundException {
// fetch repo url from original test upload
Pattern pattern = Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX);
File configFile = Paths.get(basePathProfessorUnitTests, assignmentId + ".txt").toFile();
Matcher config = RegexUtil.extractConfig(new FileInputStream(configFile), pattern);
String testRepoURL = config.group(1);
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = CompetencyAssessmentUtil.readExerciseCompetencyProfiles(
testPath, CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME
);
int currentTopicIndex = 0;
float currentDifficulty = 0.0f;
//build course topic order
Map<String, Integer> topicOrder = new HashMap<>();
int order = 1;
for (ExerciseCompetencyProfile e : exerciseCompetencyProfiles) {
if (!topicOrder.containsKey(e.exerciseTopicName)) {
topicOrder.put(e.exerciseTopicName, order++);
}
if (e.exerciseURL.equals(testRepoURL)) {
currentTopicIndex = order;
currentDifficulty = e.difficulty;
}
}
//filter exercises according to success
float[] unsuccessful = CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, false);
List<ExerciseCompetencyProfile> filteredExercises = filterExercisesByTopicsAndDifficulty(
exerciseCompetencyProfiles, topicOrder, currentTopicIndex, testRepoURL,
currentDifficulty, unsuccessful, resultSummary);
//compute recommendations
List<Recommendation> recommendedExercises = new ArrayList<>();
for (ExerciseCompetencyProfile exerciseProfile : filteredExercises) {
Recommendation recommendation = new Recommendation(exerciseProfile.exerciseTopicName, exerciseProfile.exerciseURL, exerciseProfile.exerciseName,
exerciseProfile.difficulty, calculateScore(exerciseProfile, unsuccessful, topicOrder, currentDifficulty));
recommendedExercises.add(recommendation);
LOG.info("Recommending exercise "+recommendation.topic+"/"+recommendation.exerciseName+" with score "+recommendation.score);
}
//sort the recommendations for successful or resilient learners, otherwise reverse in display
recommendedExercises.stream().sorted(Recommendation.COMPARE_BY_SCORE).collect(Collectors.toList());
return recommendedExercises;
}
public static List<ExerciseCompetencyProfile> filterExercisesByTopicsAndDifficulty(List<ExerciseCompetencyProfile> exerciseCompetencyProfiles,
Map<String, Integer> topicOrder, int currentTopicIndex, String testRepoURL, float currentDifficulty, float[] unsuccessful,
ResultSummary resultSummary) {
//filter out all advanced topics in any case
//option for later: include next topic if fullsuccess and current difficulty == max difficulty
List<ExerciseCompetencyProfile> filteredExercises = exerciseCompetencyProfiles.stream()
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
.filter(testProfile -> topicOrder.get(testProfile.exerciseTopicName) <= currentTopicIndex)
.collect(Collectors.toList());
//filter by difficulty according to success
if (isFullSuccess(unsuccessful)) {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty >= currentDifficulty && !testRepoURL.equals(profile.exerciseURL)).collect(Collectors.toList());
} else {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty <= currentDifficulty).collect(Collectors.toList());
}
return filteredExercises;
}
public static boolean isFullSuccess(float[] unsuccessful) {
for (float value : unsuccessful) {
if (value != 0.0f) {
return false;
}
}
return true;
}
public static float calculateScore(ExerciseCompetencyProfile exerciseProfile, float[] unsuccessful, Map<String, Integer> topicOrder, float currentDifficulty) {
//ensure factor 1 for full success not to blank out score, thus offset the base
float score = 1.0f;
//competency profile difference to not fully achieved competencies component
for (int i = 0; i < exerciseProfile.competencyAssessments.length-1; i++) {
score += exerciseProfile.competencyAssessments[i] * unsuccessful[i];
}
//difficulty component
score = score * (exerciseProfile.difficulty*(0.5f+Math.abs(currentDifficulty-exerciseProfile.difficulty)));
//topic component
score *= topicOrder.get(exerciseProfile.exerciseTopicName);
score = Math.round(score * 10.0f) / 10.0f;
return score;
}
}