Commit 2330d090 authored by Lückemeyer's avatar Lückemeyer
Browse files

BREAKING: added competency assessment including result fields

parent d85d8dc3
......@@ -2,7 +2,9 @@ package de.hftstuttgart.dtabackend.models;
public class Result
{
public String name;
public String packageName;
public String className;
public String name;
public int state;
public String failureType;
......
......@@ -7,5 +7,7 @@ public class ResultSummary
{
public long timestamp = System.currentTimeMillis() / 1000;
public String globalStacktrace = null;
public String successfulTestCompetencyProfile;
public String overallTestCompetencyProfile;
public Set<Result> results = new HashSet<>();
}
package de.hftstuttgart.dtabackend.models;
public class TestCompetencyProfile {
public static final int MAX_COMPETENCY_DIMENSIONS = 16;
public static final String COMPETENCY_SEPARATOR=";";
public String testPackageName;
public String testClassName;
public String testName;
public float[] competencyAssessments=new float[MAX_COMPETENCY_DIMENSIONS];
public static float[] competencyProjection(float[] cp, float[] cp2) {
float z[] = new float[MAX_COMPETENCY_DIMENSIONS];
for (int i = 0; i < MAX_COMPETENCY_DIMENSIONS; i++) {
z[i] = cp[i] * cp2[i];
}
return z;
}
public static float[] competencyShare(float[] cp, float[] cpTotal) {
float z[] = new float[MAX_COMPETENCY_DIMENSIONS];
for (int i = 0; i < MAX_COMPETENCY_DIMENSIONS; i++) {
z[i] = cp[i] / cpTotal[i];
}
return z;
}
public static float[] competencySum(float[] cp, float[] cp2) {
float z[] = new float[MAX_COMPETENCY_DIMENSIONS];
for (int i = 0; i < MAX_COMPETENCY_DIMENSIONS; i++) {
z[i] = cp[i] + cp2[i];
}
return z;
}
@Override
public boolean equals(Object other) {
return other instanceof TestCompetencyProfile &&
testPackageName.equals(((TestCompetencyProfile)other).testPackageName) &&
testClassName.equals(((TestCompetencyProfile)other).testClassName) &&
testName.equals(((TestCompetencyProfile)other).testName);
}
}
......@@ -88,6 +88,8 @@ public class TaskUpload {
LOG.info("check for provided Ticketsystem information");
UnifiedTicketingUtil.reportResults(taskFileRef.getInputStream(), resultSummary);
}
taskFileRef.getInputStream().close();
LOG.info("submission tested successfully");
return resultSummary;
......
......@@ -65,7 +65,7 @@ public class UnitTestUpload {
unitTestFileRef.transferTo(file);
LOG.debug(String.format("saved config file to: %s", file.getAbsolutePath()));
Pattern pattern = Pattern.compile(this.TESTCONFIGREGEX);
Pattern pattern = Pattern.compile(TESTCONFIGREGEX);
Matcher config = null;
LOG.debug("reading test configuration file");
......
package de.hftstuttgart.dtabackend.utils;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import de.hftstuttgart.dtabackend.models.Result;
import de.hftstuttgart.dtabackend.models.ResultSummary;
import de.hftstuttgart.dtabackend.models.TestCompetencyProfile;
import java.io.FileNotFoundException;
public class CompetencyAssessmentUtil {
private static final Logger LOG = LogManager.getLogger(CompetencyAssessmentUtil.class);
public static String TEST_COMPETENCY_MANIFEST_FILE_NAME="competency-tests.mft";
/* public static void main(String[] args) throws StreamReadException, DatabindException, MalformedURLException, IOException {
List<TestCompetencyProfile> testCompetencyProfiles=readTestCompetencyProfiles(Path.of(args[0]), args[1]);
sumTestCompetencyProfiles(testCompetencyProfiles);
ObjectMapper objectMapper = new ObjectMapper();
ResultSummary resultSummary = objectMapper.readValue(
new File(Path.of(args[0]).toFile(), args[2]).toURI().toURL(),
ResultSummary.class);
sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary);
}
*/
public static float[] sumTestCompetencyProfiles(List<TestCompetencyProfile> testCompetencyProfiles) {
float[] tcpTotalProfile=new float[TestCompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for(TestCompetencyProfile currentProfile: testCompetencyProfiles) {
tcpTotalProfile=TestCompetencyProfile.competencySum(tcpTotalProfile, currentProfile.competencyAssessments);
}
return tcpTotalProfile;
}
public static float[] sumSuccessfulCompetencyProfiles(List<TestCompetencyProfile> testCompetencyProfiles, ResultSummary resultSummary) {
float[] sumSuccessful=new float[TestCompetencyProfile.MAX_COMPETENCY_DIMENSIONS];
for(Result currentResult: resultSummary.results) {
if(currentResult.state==Result.State.SUCCESS.ordinal()) {
TestCompetencyProfile currentProfile=new TestCompetencyProfile();
currentProfile.testPackageName=currentResult.packageName;
currentProfile.testClassName=currentResult.className;
currentProfile.testName=currentResult.name;
int testIndex=testCompetencyProfiles.indexOf(currentProfile);
sumSuccessful=TestCompetencyProfile.competencySum(sumSuccessful, testCompetencyProfiles.get(testIndex).competencyAssessments);
}
}
return sumSuccessful;
}
public static List<TestCompetencyProfile> readTestCompetencyProfiles(Path testPath, String fileName) {
List<TestCompetencyProfile> testCompetencyProfiles=new ArrayList<TestCompetencyProfile>();
try {
BufferedReader testCompetencyManifest=new BufferedReader(new FileReader(new File(testPath.toFile(), fileName)));
String testEntry=testCompetencyManifest.readLine();
while(testEntry!=null)
{
String[] testEntyComponents=testEntry.split(TestCompetencyProfile.COMPETENCY_SEPARATOR);
TestCompetencyProfile currentProfile=new TestCompetencyProfile();
currentProfile.testPackageName=testEntyComponents[0];
currentProfile.testClassName=testEntyComponents[1];
currentProfile.testName=testEntyComponents[2];
for(int competencyIndex=0; competencyIndex<TestCompetencyProfile.MAX_COMPETENCY_DIMENSIONS; competencyIndex++) {
currentProfile.competencyAssessments[competencyIndex]=Float.valueOf(testEntyComponents[competencyIndex+3]);
}
testCompetencyProfiles.add(currentProfile);
testEntry=testCompetencyManifest.readLine();
}
testCompetencyManifest.close();
LOG.info("Added "+testCompetencyProfiles.size()+" test competency profiles from test competency manifest. Optional agent functionality enabled.");
} catch (FileNotFoundException e) {
LOG.info("Test competency manifest file for agent feedback not found. Skipping optional functionality.");
testCompetencyProfiles=null;
} catch (IOException e) {
LOG.info("Test competency manifest file for agent feedback unreadable. Skipping optional functionality.");
testCompetencyProfiles=null;
}
return testCompetencyProfiles;
}
}
......@@ -8,7 +8,6 @@ import com.github.dockerjava.api.model.HostConfig;
import com.github.dockerjava.core.DefaultDockerClientConfig;
import com.github.dockerjava.core.DockerClientConfig;
import com.github.dockerjava.core.DockerClientImpl;
import com.github.dockerjava.transport.DockerHttpClient;
import com.github.dockerjava.zerodep.ZerodepDockerHttpClient;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......
......@@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Volume;
import de.hftstuttgart.dtabackend.models.ResultSummary;
import de.hftstuttgart.dtabackend.models.TestCompetencyProfile;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.core.env.Environment;
......@@ -13,7 +15,10 @@ import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.regex.Matcher;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@Component
public class ExecuteTestUtil {
......@@ -109,7 +114,7 @@ public class ExecuteTestUtil {
// check if result file is there
if (!resultFile.exists() || !resultFile.isFile()) {
LOG.error(String.format("couln't find result file in %s", resultFile.getAbsolutePath()));
LOG.error(String.format("Could not find result file in %s", resultFile.getAbsolutePath()));
throw new RuntimeException("no resultfile found");
}
......@@ -118,7 +123,21 @@ public class ExecuteTestUtil {
ResultSummary resultSummary = objectMapper.readValue(
resultFile.toURI().toURL(),
ResultSummary.class);
LOG.info("Checking for optional test competency profile information for paedagogical agent functionality...");
List<TestCompetencyProfile> testCompetencyProfiles=CompetencyAssessmentUtil.readTestCompetencyProfiles(testPathHost, CompetencyAssessmentUtil.TEST_COMPETENCY_MANIFEST_FILE_NAME);
if(testCompetencyProfiles!=null) {
LOG.info("Found optional test competency profiles, generating agent profile data...");
resultSummary.overallTestCompetencyProfile=packFloats(CompetencyAssessmentUtil.sumTestCompetencyProfiles(testCompetencyProfiles));
resultSummary.successfulTestCompetencyProfile=packFloats(CompetencyAssessmentUtil.sumSuccessfulCompetencyProfiles(testCompetencyProfiles, resultSummary));
}
return resultSummary;
}
private static String packFloats(float[] array) {
return IntStream.range(0, array.length)
.mapToObj(i -> String.valueOf(array[i]))
.collect(Collectors.joining(";"));
}
}
......@@ -15,7 +15,7 @@ import java.util.regex.Pattern;
public class RegexUtil {
public enum ConfigType {
PROFESSOR,
TEACHER,
STUDENT,
}
......@@ -26,13 +26,13 @@ public class RegexUtil {
}
public static Matcher findProfessorConfig(InputStream is) {
return findConfig(is, ConfigType.PROFESSOR);
return findConfig(is, ConfigType.TEACHER);
}
public static Matcher findConfig(InputStream is, ConfigType configType) {
Pattern pattern;
switch (configType) {
case PROFESSOR:
case TEACHER:
pattern = Pattern.compile(UnitTestUpload.TESTCONFIGREGEX);
break;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment