An error occurred while loading the file. Please try again.
-
Lückemeyer authoredee0c7e20
package de.hftstuttgart.dtabackend.utils;
import com.fasterxml.jackson.core.exc.StreamReadException;
import com.fasterxml.jackson.databind.DatabindException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Volume;
import de.hftstuttgart.dtabackend.models.ExerciseCompetencyProfile;
import de.hftstuttgart.dtabackend.models.ICompetencyProfile;
import de.hftstuttgart.dtabackend.models.Recommendation;
import de.hftstuttgart.dtabackend.models.ResultSummary;
import de.hftstuttgart.dtabackend.models.TestCompetencyProfile;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
import java.io.*;
import java.net.MalformedURLException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Component
public class ExecuteTestUtil {
private static final Logger LOG = LogManager.getLogger(ExecuteTestUtil.class);
private final DockerUtil dockerUtil;
private final String assignmentBasePath;
private final Path testTmpPathHost;
public ExecuteTestUtil(
Environment env,
DockerUtil dockerUtil
) {
this.dockerUtil = dockerUtil;
// set base path for assignments to be stored
Path p = Paths.get(
env.getProperty("data.dir"),
env.getProperty("data.dir.test.folder.name"));
this.assignmentBasePath = p.toAbsolutePath().toString();
// set path of temporary directory on host _and_ inside our container, _must_ be identical
this.testTmpPathHost = Paths.get(env.getProperty("host.tests.tmp.dir"));
}
public ResultSummary runTests(String assignmentId, Path workDirectory) throws IOException, InterruptedException {
// define paths for the test, the submission and where the result is to be expected afterwards
Path testPath = Paths.get(workDirectory.toAbsolutePath().toString(), "/test");
Path srcPath = Paths.get(workDirectory.toAbsolutePath().toString(), "/src");
Path resultPath = Paths.get(workDirectory.toAbsolutePath().toString(), "/result");
// clone stored test to tmpdir
LOG.debug("copying pre-downloaded unitttest repo");
FileUtil.copyFolder(
Paths.get(assignmentBasePath, assignmentId),
testPath);
LOG.debug("copying exercise manifest from: %s in testPath: %s",
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
assignmentBasePath + assignmentId + "_checkout",
testPath.toString() );
Files.copy(Paths.get(
assignmentBasePath, assignmentId + "_checkout",
CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME),
Paths.get(
testPath.toString(),
CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME));
LOG.debug("copy test config");
Files.copy(
Paths.get(assignmentBasePath, assignmentId + ".txt"),
Paths.get(workDirectory.toAbsolutePath().toString(), "config.txt"));
Files.createDirectory(resultPath);
LOG.info("reading test config");
Matcher config = RegexUtil.extractConfig(
new FileInputStream(Paths.get(workDirectory.toAbsolutePath().toString(), "config.txt").toFile()), Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX));
String image="";
if(config==null)
{
config = RegexUtil.extractConfig(
new FileInputStream(Paths.get(workDirectory.toAbsolutePath().toString(), "config.txt").toFile()), Pattern.compile(RegexUtil.TESTCONFIGREGEX));
if(config==null)
{
throw new RuntimeException("couldn't find repo config for unittest image extraction");
}
image=config.group(4);
}
else
{
image=config.group(5);
}
// define the paths to mount as Binds from Host to the test-container
Path testPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
testPath.getName(testPath.getNameCount()-1).toString()
);
Path srcPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
srcPath.getName(srcPath.getNameCount()-1).toString()
);
Path resultPathHost = Paths.get(
testTmpPathHost.toAbsolutePath().toString(),
workDirectory.getName(workDirectory.getNameCount()-1).toString(),
resultPath.getName(resultPath.getNameCount()-1).toString()
);
// start test-container with professor given image and bind mounts for test, submission and result
dockerUtil.runContainer(
image,
new Bind(testPathHost.toAbsolutePath().toString(), new Volume("/data/test")),
new Bind(srcPathHost.toAbsolutePath().toString(), new Volume("/data/src")),
new Bind(resultPathHost.toAbsolutePath().toString(), new Volume("/data/result"))
);
ResultSummary resultSummary = generateResult(assignmentId, resultPath, testPathHost);
return resultSummary;
}
private ResultSummary generateResult(String assignmentId, Path resultPath, Path testPathHost)
throws IOException, StreamReadException, DatabindException, MalformedURLException {
// define expected result file
File resultFile = Paths.get(resultPath.toAbsolutePath().toString(), "result.json").toFile();
// check if result file is there
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
if (!resultFile.exists() || !resultFile.isFile()) {
LOG.error(String.format("Could not find result file in %s", resultFile.getAbsolutePath()));
throw new RuntimeException("no resultfile found");
}
LOG.debug("parse results json");
ObjectMapper objectMapper = new ObjectMapper();
ResultSummary resultSummary = objectMapper.readValue(resultFile.toURI().toURL(), ResultSummary.class);
LOG.debug("result json returned time "+ resultSummary.timestamp + " with "+resultSummary.results.size()+ " test results.");
LOG.info("Checking for optional test competency profile information for paedagogical agent functionality...");
List<TestCompetencyProfile> testCompetencyProfiles=CompetencyAssessmentUtil.readTestCompetencyProfiles(testPathHost, CompetencyAssessmentUtil.TEST_COMPETENCY_MANIFEST_FILE_NAME);
LOG.debug(String.format(
"Reading Test Competency Profiles: basePath=%s, fileName=%s",
testPathHost,
CompetencyAssessmentUtil.TEST_COMPETENCY_MANIFEST_FILE_NAME
));
if(testCompetencyProfiles!=null) {
LOG.info("Found optional test competency profiles, generating agent profile data...");
resultSummary.overallTestCompetencyProfile=CompetencyAssessmentUtil.packFloats(CompetencyAssessmentUtil.sumTestCompetencyProfiles(testCompetencyProfiles));
resultSummary.successfulTestCompetencyProfile=CompetencyAssessmentUtil.packFloats(CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, true));
LOG.info("Checking for optional exercise competency profile information for paedagogical agent exercise recommendation functionality...");
//testPathHost or assignmentBasePath
Path exerciseManifestFile = Paths.get(testPathHost.toString(), CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME);
LOG.debug(String.format(
"Constructing Path for exercise manifest: testPath=%s, fileName=%s -> Resulting Path=%s",
testPathHost.toString(),
CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME,
exerciseManifestFile.toString()
));
if (Files.exists(exerciseManifestFile)) {
LOG.info("Found optional exercise competency profiles, generating recommendations...");
resultSummary.recommendations = recommendNextExercises(assignmentId, testPathHost, testCompetencyProfiles, resultSummary);
}
}
return resultSummary;
}
/*
* exercise recommendation part
*/
public List<Recommendation> recommendNextExercises(String assignmentId, Path testPathHost, List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary)
throws FileNotFoundException {
// fetch repo url from original test upload
Pattern pattern = Pattern.compile(RegexUtil.DTA_TESTCONFIGREGEX);
File file = Paths.get(assignmentBasePath, assignmentId + ".txt").toFile();
FileInputStream configFileStream = new FileInputStream(file);
Matcher config = RegexUtil.extractConfig(configFileStream, pattern);
String testRepoURL = config.group(1);
List<ExerciseCompetencyProfile> exerciseCompetencyProfiles = CompetencyAssessmentUtil.readExerciseCompetencyProfiles(testPathHost, CompetencyAssessmentUtil.EXERCISE_COMPETENCY_MANIFEST_FILE_NAME);
int currentTopicIndex=0;
float currentDifficulty=0.0f;
//build course topic order
Map<String, Integer> topicOrder = new HashMap<>();
int order = 1;
for (ExerciseCompetencyProfile e : exerciseCompetencyProfiles) {
if (!topicOrder.containsKey(e.exerciseTopicName)) {
topicOrder.put(e.exerciseTopicName, order++);
}
if (e.exerciseURL.equals(testRepoURL)) {
currentTopicIndex = order;
currentDifficulty = e.difficulty;
}
}
//filter exercises according to success
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
float[] unsuccessful = CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary, false);
List<ExerciseCompetencyProfile> filteredExercises = filterExercisesByTopicsAndDifficulty(
exerciseCompetencyProfiles, topicOrder, currentTopicIndex, testRepoURL,
currentDifficulty, unsuccessful, resultSummary);
//compute recommendations
List<Recommendation> recommendedExercises = new ArrayList<>();
for (ExerciseCompetencyProfile exerciseProfile : filteredExercises) {
Recommendation recommendation = new Recommendation(exerciseProfile.exerciseTopicName, exerciseProfile.exerciseURL, exerciseProfile.exerciseName,
exerciseProfile.difficulty, calculateScore(exerciseProfile, unsuccessful, topicOrder, currentDifficulty));
recommendedExercises.add(recommendation);
LOG.info("Recommending exercise "+recommendation.topic+"/"+recommendation.exerciseName+" with score "+recommendation.score);
}
//sort the recommendations for successful or resilient learners, otherwise reverse in display
recommendedExercises.stream().sorted(Recommendation.COMPARE_BY_SCORE).collect(Collectors.toList());
return recommendedExercises;
}
public static List<ExerciseCompetencyProfile> filterExercisesByTopicsAndDifficulty(List<ExerciseCompetencyProfile> exerciseCompetencyProfiles,
Map<String, Integer> topicOrder, int currentTopicIndex, String testRepoURL, float currentDifficulty, float[] unsuccessful,
ResultSummary resultSummary) {
//filter out all advanced topics in any case
//option for later: include next topic if fullsuccess and current difficulty == max difficulty
List<ExerciseCompetencyProfile> filteredExercises = exerciseCompetencyProfiles.stream()
.filter(testProfile -> topicOrder.get(testProfile.exerciseTopicName) <= currentTopicIndex)
.collect(Collectors.toList());
//filter by difficulty according to success
if (isFullSuccess(unsuccessful)) {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty >= currentDifficulty && !testRepoURL.equals(profile.exerciseURL)).collect(Collectors.toList());
} else {
filteredExercises = filteredExercises.stream().filter(profile -> profile.difficulty <= currentDifficulty).collect(Collectors.toList());
}
return filteredExercises;
}
public static boolean isFullSuccess(float[] unsuccessful) {
for (float value : unsuccessful) {
if (value != 0.0f) {
return false;
}
}
return true;
}
public static float calculateScore(ExerciseCompetencyProfile exerciseProfile, float[] unsuccessful, Map<String, Integer> topicOrder, float currentDifficulty) {
//ensure factor 1 for full success not to blank out score, thus offset the base
float score = 1.0f;
//competency profile difference to not fully achieved competencies component
for (int i = 0; i < exerciseProfile.competencyAssessments.length-1; i++) {
score += exerciseProfile.competencyAssessments[i] * unsuccessful[i];
}
//difficulty component
score = score * (exerciseProfile.difficulty*(0.5f+Math.abs(currentDifficulty-exerciseProfile.difficulty)));
//topic component
score *= topicOrder.get(exerciseProfile.exerciseTopicName);
score = Math.round(score * 10.0f) / 10.0f;
return score;
}
}