Skip to content
GitLab
Commit 81a9e9f3 authored by Artem Baranovskyi's avatar Artem Baranovskyi
Browse files

Final commit.

parents
asyst-moodle-plugin Siddique-Asyst-Moodle-Plugin issue-4-payload-sanitizing main master
1 merge request!1Final commit.
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1500 additions and 0 deletions
+1500 -0
.idea/workspace.xml 0 → 100644
+ 238
- 0
View file @ 81a9e9f3
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="AnalysisUIOptions">
<option name="ANALYZE_INJECTED_CODE" value="false" />
<option name="SCOPE_TYPE" value="3" />
</component>
<component name="AutoImportSettings">
<option name="autoReloadType" value="SELECTIVE" />
</component>
<component name="ChangeListManager">
<list default="true" id="4e8d1bce-352e-404c-8888-ed0a15a22933" name="Changes" comment="Initial commit.">
<change beforePath="$PROJECT_DIR$/../.env" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../.gitignore" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/lib.php" beforeDir="false" afterPath="$PROJECT_DIR$/lib.php" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../docker-compose.yml" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/Dockerfile" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/api.py" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/asyst/DE_Demo_Daten.xlsx" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/asyst/README.md" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/asyst/README_DE.md" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/asyst/README_EN.md" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/asyst/Source/LICENSE" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/asyst/Source/Skript/german/models/clf_BERT.pickle" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/asyst/Source/Skript/german/run_LR_SBERT.py" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../flask/requirements.txt" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../install_moodle.sh" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../moodle/Dockerfile" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../moodle/moodle_backup.sql" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../readme.md" beforeDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="ChangesViewManager">
<option name="groupingKeys">
<option value="directory" />
</option>
</component>
<component name="ComposerSettings" doNotAsk="true" synchronizationState="DONT_SYNCHRONIZE">
<pharConfigPath>/opt/moodle/composer.json</pharConfigPath>
<execution>
<interpreter composer="composer" />
</execution>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="GitHubPullRequestSearchHistory">{
&quot;lastFilter&quot;: {
&quot;state&quot;: &quot;OPEN&quot;,
&quot;assignee&quot;: &quot;ArtemBaranovsky&quot;
}
}</component>
<component name="GithubPullRequestsUISettings">{
&quot;selectedUrlAndAccountId&quot;: {
&quot;url&quot;: &quot;https://github.com/ArtemBaranovsky/moodle-asyst-sync&quot;,
&quot;accountId&quot;: &quot;97d19928-826d-4328-80a9-8ade832b802b&quot;
}
}</component>
<component name="MarkdownSettingsMigration">
<option name="stateVersion" value="1" />
</component>
<component name="PHPCodeSnifferProjectConfiguration">
<option name="selectedConfigurationId" value="20b27b07-37c3-46a2-9f8e-bc616ef2bf15" />
</component>
<component name="PhpWorkspaceProjectConfiguration" interpreter_name="PHP">
<include_path>
<path value="/opt/moodle" />
<path value="/opt/moodle/vendor/sebastian/environment" />
<path value="/opt/moodle/vendor/sebastian/exporter" />
<path value="/opt/moodle/vendor/sebastian/resource-operations" />
<path value="/opt/moodle/vendor/sebastian/code-unit" />
<path value="/opt/moodle/vendor/sebastian/code-unit-reverse-lookup" />
<path value="/opt/moodle/vendor/oleg-andreyev/mink-phpwebdriver" />
<path value="/opt/moodle/vendor/masterminds/html5" />
<path value="/opt/moodle/vendor/friends-of-behat/mink-extension" />
<path value="/opt/moodle/vendor/php-webdriver/webdriver" />
<path value="/opt/moodle/vendor/composer" />
<path value="/opt/moodle/vendor/psr/container" />
<path value="/opt/moodle/vendor/psr/log" />
<path value="/opt/moodle/vendor/psr/event-dispatcher" />
<path value="/opt/moodle/vendor/behat/mink-browserkit-driver" />
<path value="/opt/moodle/vendor/filp/whoops" />
<path value="/opt/moodle/vendor/behat/mink" />
<path value="/opt/moodle/vendor/behat/transliterator" />
<path value="/opt/moodle/vendor/behat/behat" />
<path value="/opt/moodle/vendor/behat/gherkin" />
<path value="/opt/moodle/vendor/myclabs/deep-copy" />
<path value="/opt/moodle/vendor/nikic/php-parser" />
<path value="/opt/moodle/vendor/phar-io/version" />
<path value="/opt/moodle/vendor/phar-io/manifest" />
<path value="/opt/moodle/vendor/phpunit/php-timer" />
<path value="/opt/moodle/vendor/phpunit/php-file-iterator" />
<path value="/opt/moodle/vendor/phpunit/php-invoker" />
<path value="/opt/moodle/vendor/phpunit/php-text-template" />
<path value="/opt/moodle/vendor/phpunit/phpunit" />
<path value="/opt/moodle/vendor/phpunit/php-code-coverage" />
<path value="/opt/moodle/vendor/symfony/dependency-injection" />
<path value="/opt/moodle/vendor/symfony/deprecation-contracts" />
<path value="/opt/moodle/vendor/symfony/service-contracts" />
<path value="/opt/moodle/vendor/symfony/string" />
<path value="/opt/moodle/vendor/symfony/polyfill-intl-idn" />
<path value="/opt/moodle/vendor/symfony/mime" />
<path value="/opt/moodle/vendor/symfony/http-client-contracts" />
<path value="/opt/moodle/vendor/symfony/browser-kit" />
<path value="/opt/moodle/vendor/symfony/event-dispatcher" />
<path value="/opt/moodle/vendor/symfony/dom-crawler" />
<path value="/opt/moodle/vendor/symfony/polyfill-ctype" />
<path value="/opt/moodle/vendor/symfony/css-selector" />
<path value="/opt/moodle/vendor/symfony/polyfill-intl-grapheme" />
<path value="/opt/moodle/vendor/symfony/event-dispatcher-contracts" />
<path value="/opt/moodle/vendor/symfony/console" />
<path value="/opt/moodle/vendor/symfony/yaml" />
<path value="/opt/moodle/vendor/symfony/polyfill-intl-normalizer" />
<path value="/opt/moodle/vendor/symfony/process" />
<path value="/opt/moodle/vendor/symfony/filesystem" />
<path value="/opt/moodle/vendor/symfony/polyfill-php72" />
<path value="/opt/moodle/vendor/symfony/translation" />
<path value="/opt/moodle/vendor/symfony/http-client" />
<path value="/opt/moodle/vendor/symfony/translation-contracts" />
<path value="/opt/moodle/vendor/symfony/polyfill-mbstring" />
<path value="/opt/moodle/vendor/symfony/var-exporter" />
<path value="/opt/moodle/vendor/symfony/config" />
<path value="/opt/moodle/vendor/doctrine/instantiator" />
<path value="/opt/moodle/vendor/theseer/tokenizer" />
<path value="/opt/moodle/vendor/sebastian/global-state" />
<path value="/opt/moodle/vendor/mikey179/vfsstream" />
<path value="/opt/moodle/vendor/sebastian/recursion-context" />
<path value="/opt/moodle/vendor/sebastian/complexity" />
<path value="/opt/moodle/vendor/sebastian/object-reflector" />
<path value="/opt/moodle/vendor/sebastian/object-enumerator" />
<path value="/opt/moodle/vendor/sebastian/diff" />
<path value="/opt/moodle/vendor/sebastian/cli-parser" />
<path value="/opt/moodle/vendor/sebastian/type" />
<path value="/opt/moodle/vendor/sebastian/lines-of-code" />
<path value="/opt/moodle/vendor/sebastian/comparator" />
<path value="/opt/moodle/vendor/sebastian/version" />
<path value="/opt/moodle/vendor/squizlabs/php_codesniffer" />
</include_path>
</component>
<component name="ProblemsViewState">
<option name="selectedTabId" value="CurrentFile" />
</component>
<component name="ProjectColorInfo">{
&quot;associatedIndex&quot;: 7
}</component>
<component name="ProjectId" id="2od1gEwZwY4gvpDftv2wPs3b5El" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">{
&quot;keyToString&quot;: {
&quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
&quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
&quot;WebServerToolWindowFactoryState&quot;: &quot;false&quot;,
&quot;git-widget-placeholder&quot;: &quot;master&quot;,
&quot;last_opened_file_path&quot;: &quot;/opt/moodle&quot;,
&quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
&quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
&quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
&quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
&quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
&quot;run.code.analysis.last.selected.profile&quot;: &quot;pProject Default&quot;,
&quot;settings.editor.selected.configurable&quot;: &quot;preferences.pluginManager&quot;,
&quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
}
}</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/classes/db" />
</key>
</component>
<component name="RunManager">
<configuration name="quiz_api_test" type="PHPUnitRunConfigurationType" factoryName="PHPUnit" temporary="true">
<TestRunner class="quiz_api_test" file="$PROJECT_DIR$/tests/quiz_api_test.php" scope="Class" />
<method v="2" />
</configuration>
<recent_temporary>
<list>
<item itemvalue="PHPUnit.quiz_api_test" />
</list>
</recent_temporary>
</component>
<component name="SharedIndexes">
<attachedChunks>
<set>
<option value="bundled-js-predefined-d6986cc7102b-5c90d61e3bab-JavaScript-PS-242.23726.107" />
<option value="bundled-php-predefined-a98d8de5180a-90914f2295cb-com.jetbrains.php.sharedIndexes-PS-242.23726.107" />
</set>
</attachedChunks>
</component>
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
<component name="SvnConfiguration">
<configuration>$USER_HOME$/.subversion</configuration>
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="4e8d1bce-352e-404c-8888-ed0a15a22933" name="Changes" comment="" />
<created>1731184747722</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1731184747722</updated>
<workItem from="1731184749032" duration="502000" />
<workItem from="1731230234321" duration="14106000" />
<workItem from="1731253295268" duration="8396000" />
<workItem from="1731779462740" duration="478000" />
<workItem from="1731779954953" duration="32450000" />
<workItem from="1731874192578" duration="7218000" />
</task>
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="3" />
</component>
<component name="Vcs.Log.Tabs.Properties">
<option name="TAB_STATES">
<map>
<entry key="MAIN">
<value>
<State />
</value>
</entry>
</map>
</option>
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value="Initial commit." />
<option name="LAST_COMMIT_MESSAGE" value="Initial commit." />
</component>
<component name="XSLT-Support.FileAssociations.UIState">
<expand />
<select />
</component>
</project>
\ No newline at end of file
LICENSE 0 → 100755
+ 0
- 0
View file @ 81a9e9f3
This diff is collapsed.
api.php 0 → 100755
+ 98
- 0
View file @ 81a9e9f3
<?php
// This file is part of Moodle - https://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <https://www.gnu.org/licenses/>.
/**
* This is an api endpoint that sends a request to a local container with a flask-based server.
* The plugin uses the ASYST grading tool <https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST>
* modified to work as a web endpoint.
*
* @package local_asystgrade
* @copyright 2024 Artem Baranovskyi <artem.baranovsky1980@gmail.com>
* @copyright based on work by 2023 Ulrike Pado <ulrike.pado@hft-stuttgart.de>,
* @copyright Yunus Eryilmaz & Larissa Kirschner <https://link.springer.com/article/10.1007/s40593-023-00383-w>
* @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
require_once($_SERVER['DOCUMENT_ROOT'] . '/config.php');
require_once('lib.php');
use local_asystgrade\api\client;
use local_asystgrade\api\http_client;
try {
require_login();
} catch (coding_exception | moodle_exception $e) {
debugging($e->getMessage());
redirect(
new moodle_url('/local/asystgrade/error.php'),
get_string('loginerror', 'local_asystgrade')
);
}
if ($_SERVER['REQUEST_METHOD'] === 'POST') {
$data = json_decode(file_get_contents('php://input'), true);
if ($data) {
// Preparing Flask API.
try {
$apiendpoint = get_config('local_asystgrade', 'apiendpoint');
if (!$apiendpoint) {
throw new dml_exception('missingconfig', 'local_asystgrade');
}
} catch (dml_exception $e) {
debugging('Config error: ' . $e->getMessage());
$apiendpoint = 'http://127.0.0.1:5001/api/autograde'; // Default value.
}
$httpclient = new http_client();
$apiclient = client::getInstance($apiendpoint, $httpclient);
$maxretries = 3;
$attempts = 0;
$success = false;
while ($attempts < $maxretries && !$success) {
try {
// Sending data to Flask and obtaining an answer.
$response = $apiclient->send_data($data);
$success = true;
} catch (Exception $e) {
$attempts++;
debugging('API request error: ' . $e->getMessage());
if ($attempts >= $maxretries) {
echo json_encode(['error' => 'A server error occurred. Please try again later.']);
exit; // Ensure to stop further processing.
}
}
}
if ($success) {
$grades = json_decode($response, true);
// Check JSON validity.
if (json_last_error() === JSON_ERROR_NONE) {
echo json_encode(['success' => true, 'grades' => $grades]);
} else {
debugging('JSON decode error: ' . json_last_error_msg());
echo json_encode(['error' => 'Invalid JSON from Flask API']);
}
}
} else {
echo json_encode(['error' => 'No data received']);
}
} else {
echo json_encode(['error' => 'Invalid request method']);
}
classes/api/client.php 0 → 100755
+ 85
- 0
View file @ 81a9e9f3
<?php
// This file is part of Moodle - https://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <https://www.gnu.org/licenses/>.
/**
* The plugin uses the ASYST grading tool <https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST>
* modified to work as a web endpoint.
*
* @package local_asystgrade
* @copyright 2024 Artem Baranovskyi <artem.baranovsky1980@gmail.com>
* @copyright based on work by 2023 Ulrike Padó <ulrike.pado@hft-stuttgart.de>,
* @copyright Yunus Eryilmaz & Larissa Kirschner <https://link.springer.com/article/10.1007/s40593-023-00383-w>
* @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace local_asystgrade\api;
use Exception;
use local_asystgrade\api\http_client_interface;
/**
* Client class for handling HTTP requests to Flask ML backend.
*
* This class provides methods for sending data to a specified endpoint using
* an HTTP client interface.
* @package local_asystgrade
*/
class client {
/** @var string This variable holds a domain or IP to attached flask ML backend */
private string $endpoint;
/** @var http_client_interface This variable holds an interface for http_client */
private http_client_interface $httpclient;
/** @var ?client This variable holds an object type for http_client */
private static ?client $instance = null;
/**
* Client class for handling HTTP requests to Flask ML backend.
*/
private function __construct(string $endpoint, http_client_interface $httpclient) {
$this->endpoint = $endpoint;
$this->httpclient = $httpclient;
}
/**
* Returns the singleton instance of the client.
*
* @param string $endpoint
* @param http_client_interface $httpClient
* @return client
*/
public static function getinstance(string $endpoint, http_client_interface $httpclient): client {
if (self::$instance === null) {
self::$instance = new client($endpoint, $httpclient);
}
return self::$instance;
}
/**
* Sends data to the endpoint.
*
* @param array $data
* @return bool|string
* @throws Exception
*/
public function send_data(array $data): bool|string {
try {
return $this->httpclient->post($this->endpoint, $data);
} catch (Exception $e) {
throw new Exception('HTTP request error: ' . $e->getMessage());
}
}
}
classes/api/http_client.php 0 → 100755
+ 72
- 0
View file @ 81a9e9f3
<?php
// This file is part of Moodle - https://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <https://www.gnu.org/licenses/>.
/**
* The plugin uses the ASYST grading tool <https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST>
* modified to work as a web endpoint.
*
* @package local_asystgrade
* @copyright 2024 Artem Baranovskyi <artem.baranovsky1980@gmail.com>
* @copyright based on work by 2023 Ulrike Padó <ulrike.pado@hft-stuttgart.de>,
* @copyright Yunus Eryilmaz & Larissa Kirschner <https://link.springer.com/article/10.1007/s40593-023-00383-w>
* @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace local_asystgrade\api;
use Exception;
/**
* HTTP client class for handling HTTP POST requests.
*/
class http_client implements http_client_interface {
/**
* Sends a POST request to the specified URL with the provided data.
*
* @param string $url An endpoint URL. Could be an IP, a domain, or a container name, like flask.
* @param array $data The request payload.
*
* @return bool|string
* @throws Exception
*/
public function post(string $url, array $data): bool|string {
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));
$response = curl_exec($ch);
if (curl_errno($ch)) {
throw new Exception('Curl error: ' . curl_error($ch));
}
$statuscode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if ($statuscode != 200) {
debugging("API Error: HTTP $statuscode - $response");
die('Error from API. Response code ' . $statuscode);
}
if ($response === false) {
throw new Exception('Error sending data to API');
}
return $response;
}
}
classes/api/http_client_interface.php 0 → 100755
+ 44
- 0
View file @ 81a9e9f3
<?php
// This file is part of Moodle - https://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <https://www.gnu.org/licenses/>.
/**
* The plugin uses the ASYST grading tool <https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST>
* modified to work as a web endpoint.
*
* @package local_asystgrade
* @copyright 2024 Artem Baranovskyi <artem.baranovsky1980@gmail.com>
* @copyright based on work by 2023 Ulrike Padó <ulrike.pado@hft-stuttgart.de>,
* @copyright Yunus Eryilmaz & Larissa Kirschner <https://link.springer.com/article/10.1007/s40593-023-00383-w>
* @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace local_asystgrade\api;
/**
* Interface for HTTP client implementations.
*/
interface http_client_interface {
/**
* Sends a POST request to the specified URL with the provided data.
*
* @param string $url An endpoint URL. Could be an IP, a domain, or a container name, like flask.
* @param array $data The request payload.
*
* @return bool|string
*/
public function post(string $url, array $data): bool|string;
}
classes/db/quizquery.php 0 → 100755
+ 127
- 0
View file @ 81a9e9f3
<?php
// This file is part of Moodle - https://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <https://www.gnu.org/licenses/>.
/**
* The plugin uses the ASYST grading tool <https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST>
* modified to work as a web endpoint.
*
* @package local_asystgrade
* @copyright 2024 Artem Baranovskyi <artem.baranovsky1980@gmail.com>
* @copyright based on work by 2023 Ulrike Padó <ulrike.pado@hft-stuttgart.de>,
* @copyright Yunus Eryilmaz & Larissa Kirschner <https://link.springer.com/article/10.1007/s40593-023-00383-w>
* @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace local_asystgrade\db;
use dml_exception;
use moodle_database;
use moodle_recordset;
use stdClass;
/**
* Class quizquery handles some SQL queries to fetch data about Quiz
*/
class quizquery implements quizquery_interface {
/**
* @var moodle_database $db $DB Holds Moodle Database connection
*/
private moodle_database $db;
/**
* Setting DB connection as a class property
*/
public function __construct() {
global $DB;
$this->db = $DB;
}
/**
* Function fetches set of question_attempt records
*
* @param int $qid
* @param int $slot
* @return moodle_recordset
* @throws dml_exception
*/
public function get_question_attempts(int $qid, int $slot): moodle_recordset {
return $this->db->get_recordset(
'question_attempts',
[
'questionid' => $qid,
'slot' => $slot
],
'',
'*'
);
}
/**
* Function fetches reference_answer
*
* @param int $qid
* @return stdClass
* @throws dml_exception
*/
public function get_reference_answer(int $qid): stdClass {
return $this->db->get_record(
'qtype_essay_options',
[
'questionid' => $qid
],
'*',
MUST_EXIST
)->graderinfo;
}
/**
* Function fetches set of question_attempt_steps records
*
* @param int $questionAttemptId
* @return moodle_recordset
* @throws dml_exception
*/
public function get_attempt_steps(int $questionattemptid): moodle_recordset {
return $this->db->get_recordset(
'question_attempt_steps',
[
'questionattemptid' => $questionattemptid
],
'',
'*'
);
}
/**
* Function fetches reference_answer
*
* @param int $attemptStepId
* @return stdClass
* @throws dml_exception
*/
public function get_student_answer(int $attemptstepid): stdClass {
return $this->db->get_record(
'question_attempt_step_data',
[
'attemptstepid' => $attemptstepid,
'name' => 'answer'
],
'*',
MUST_EXIST
)->value;
}
}
classes/db/quizquery_interface.php 0 → 100755
+ 67
- 0
View file @ 81a9e9f3
<?php
// This file is part of Moodle - https://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <https://www.gnu.org/licenses/>.
/**
* The plugin uses the ASYST grading tool <https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST>
* modified to work as a web endpoint.
*
* @package local_asystgrade
* @copyright 2024 Artem Baranovskyi <artem.baranovsky1980@gmail.com>
* @copyright based on work by 2023 Ulrike Padó <ulrike.pado@hft-stuttgart.de>,
* @copyright Yunus Eryilmaz & Larissa Kirschner <https://link.springer.com/article/10.1007/s40593-023-00383-w>
* @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace local_asystgrade\db;
/**
* Interface for executing queries related to quiz attempts.
*/
interface quizquery_interface {
/**
* Get question attempts for a given question ID and slot.
*
* @param int $qid The question ID.
* @param int $slot The question slot.
* @return moodle_recordset A moodle_recordset of question attempts.
*/
public function get_question_attempts(int $qid, int $slot): moodle_recordset;
/**
* Get the reference answer for a given question ID.
*
* @param int $qid The question ID.
* @return stdClass The reference answer.
*/
public function get_reference_answer(int $qid): stdClass;
/**
* Get the attempt steps for a given question attempt ID.
*
* @param int $questionAttemptId The question attempt ID.
* @return moodle_recordset A moodle_recordset of attempt steps.
*/
public function get_attempt_steps(int $questionattemptid): moodle_recordset;
/**
* Get the student answer for a given attempt step ID.
*
* @param int $attemptStepId The attempt step ID.
* @return stdClass The student answer.
*/
public function get_student_answer(int $attemptstepid): stdClass;
}
classes/privacy/provider.php 0 → 100755
+ 57
- 0
View file @ 81a9e9f3
<?php
// This file is part of Moodle - https://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <https://www.gnu.org/licenses/>.
/**
* The plugin uses the ASYST grading tool <https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST>
* modified to work as a web endpoint.
*
* @package local_asystgrade
* @copyright 2024 Artem Baranovskyi <artem.baranovsky1980@gmail.com>
* @copyright based on work by 2023 Ulrike Padó <ulrike.pado@hft-stuttgart.de>,
* @copyright Yunus Eryilmaz & Larissa Kirschner <https://link.springer.com/article/10.1007/s40593-023-00383-w>
* @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace local_asystgrade\privacy;
use coding_exception;
use core_privacy\local\metadata\null_provider;
/**
* https://moodledev.io/docs/4.5/apis/subsystems/privacy#implementation-requirements
* GDPR Implementation requirements
* In order to let Moodle know that you have audited your plugin, and that you do not store
* any personal user data, you must implement the \core_privacy\local\metadata\null_provider
* interface in your plugin's provider.
*/
/**
* Privacy Subsystem implementation for local_asystgrade.
*
* @package local_asystgrade
*/
class provider implements null_provider {
/**
* Get the reason why this plugin stores no data.
*
* @return string
* @throws coding_exception
*/
public static function get_reason(): string {
return get_string('privacy:metadata', 'local_asystgrade');
}
}
deploy_backend.sh 0 → 100755
+ 18
- 0
View file @ 81a9e9f3
#!/bin/bash
# Path to the folder with Flask backend relative to the Moodle root
BACKEND_PATH=${PWD}"/local/asystgrade/flask_ml_api"
echo $BACKEND_PATH
# Go to the desired folder
cd "$BACKEND_PATH" || {
echo "Folder with Flask API not found.";
exit 1;
}
# Run Docker Compose to deploy and run the container
docker-compose up -d --build
# Message about successful deployment
echo "Flask-бекенд развернут и запущен."
\ No newline at end of file
flask_ml_api/Dockerfile 0 → 100755
+ 32
- 0
View file @ 81a9e9f3
FROM python:3.11-slim
# Installing system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
python3-venv \
curl \
&& rm -rf /var/lib/apt/lists/*
# Creating and activating the virtual environment \
RUN python3 -m venv /opt/myenv
ENV PATH="/opt/myenv/bin:$PATH"
WORKDIR /app
COPY . /app
COPY ./asyst /app/asyst
# Installing dependencies
RUN /opt/myenv/bin/pip install -r /app/requirements.txt
RUN /opt/myenv/bin/python3 -m pip install --upgrade setuptools wheel
# Set permissions
RUN chown -R www-data:www-data /app/asyst
RUN chmod -R 755 /app/asyst
# Open port for Flask API
EXPOSE 5000
# Launch application
CMD ["python", "/app/api.py"]
\ No newline at end of file
flask_ml_api/api.py 0 → 100755
+ 60
- 0
View file @ 81a9e9f3
# api.py
# This file is part of Moodle - https://moodle.org/
#
# Moodle is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Moodle is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Moodle. If not, see <https://www.gnu.org/licenses/>.
# The plugin uses the ASYST grading tool <https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST> modified to work
# as a web endpoint.
#
# @package local_asystgrade
# @copyright 2024 Artem Baranovskyi <artem.baranovsky1980@gmail.com>
# @copyright based on work by 2023 Ulrike Padó <ulrike.pado@hft-stuttgart.de>, Yunus Eryilmaz & Larissa Kirschner
# @copyright <https://link.springer.com/article/10.1007/s40593-023-00383-w>
# @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
from flask import Flask, jsonify, request
import os
import sys
import logging
logging.basicConfig(level=logging.DEBUG)
# Adding a path of module to system path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'asyst/Source/Skript/german')))
from run_LR_SBERT import process_data
app = Flask(__name__)
@app.route('/api/autograde', methods=['POST'])
def get_data():
try:
data = request.get_json()
app.logger.debug(f"Received data: {data}")
if not data:
return jsonify({"error": "No data provided"}), 400
results = process_data(data)
app.logger.debug(f"Processed results: {results}")
return jsonify(results)
except Exception as e:
app.logger.error(f"Error during processing: {e}")
return jsonify({"error": str(e)}), 500
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True)
\ No newline at end of file
flask_ml_api/asyst/DE_Demo_Daten.xlsx 0 → 100755
+ 0
- 0
View file @ 81a9e9f3
File added
flask_ml_api/asyst/README.md 0 → 100755
+ 7
- 0
View file @ 81a9e9f3
ASYST ist ein Werkzeug zur Unterstützung beim Bewerten von Freitextantworten. <a href="README_DE.md">Mehr
Information</a>
ASYST is a tool to support grading of free text answers. <a href="README_EN.md">More information</a>
<a href="https://www.hft-stuttgart.com/research/projects/current/knight-project"><img src="images/csm_Knight_Logo_eac7003904.webp" alt="KNIGHT-Logo" width="200"/></a> <img src="images/csm_BMBF_gefoerdert_vom_deutsch_48a18b1887.webp" alt="BMBF-Logo" width="200"/>
flask_ml_api/asyst/README_DE.md 0 → 100755
+ 249
- 0
View file @ 81a9e9f3
<h2>Inhalt:</h2>
<ul>
<li><a href="#1"> Was ist ASYST? </a> </li>
<li><a href="#2"> Welche Sprachen unterstützt ASYST? </a></li>
<li><a href="#3"> Wie verwendet man ASYST? </a></li>
<ol>
<li><a href="#4"> Wie müssen auszuwertende Daten formatiert sein?</a> </li>
<li><a href="#5"> Wie führe ich das Programm unter Windows 11 aus? </a> </li>
</ol>
<li><a href="#6"> Wie arbeit man mit der Ausgabe von ASYST weiter? </a></li>
<li><a href="#7"> Wie kann ich ASYST ausführen, wenn ich kein Windows 11 nutze? </a>
<ul><li><a href="#8"> Ausführen von ASYST in der Entwicklungsumgebung Pycharm</a> </li></ul></li>
</ul>
<h2 id=1>Was ist ASYST?</h2>
ASYST ist ein Programm, das Lehrenden die Auswertung von Freitextantworten in Tests erleichtern soll: Mit Hilfe
künstlicher Intelligenz
macht ASYST Bewertungsvorschläge, die von den Lehrenden gezielt überprüft und ggf. verändert werden können.
ASYST ist für die Bewertung von Freitext-Kurzantworten gedacht - diese Kurzantworten sollten etwa ein bis drei Sätze
umfassen. Für längere Antworten ist die Anwendung nicht vorgesehen.
ASYST hilft der Lehrperson, indem es eine Bewertung vorschlägt. Diese Bewertung kann im Einzelfall durchaus auch falsch
sein; die Lehrperson kann sie prüfen und korrigieren.
Dabei spart man gegenüber der völlig manuellen Bewertung an zwei Stellen Zeit: Zum Einen ist das Überprüfen von
Bewertungen im Allgemeinen schneller als das Bewerten von Grund auf;
und zum anderen empfehlen wir, bei der Überprüfung gezielt die ASYST-Bewertungen auszuwählen, die eher fehleranfällig
sind (s. Abschnitt <a href="#6"> Wie arbeit man mit der Ausgabe von ASYST weiter? </a>).
Das Programm ist in Python geschrieben; der Quellcode ist öffentlich zugänglich. Um ASYST einfacher nutzbar zu machen,
wurden die Python-Skripte
in eine ausführbare Programmdatei umgewandelt, die in Windows 11 nutzbar ist.
Die technischen Hintergründe zu ASYST und eine Beispielrechnung zum Einsatz für das Deutsche finden sich
in <a href="https://rdcu.be/dxPLg">Pado, Eryilmaz und Kirschner, IJAIED 2023</a>.
<h2 id=2>Welche Sprachen unterstützt ASYST?</h2>
ASYST wurde für <a href="https://rdcu.be/dxPLg">Deutsch</a> und <a href="https://nlpado.de/~ulrike/papers/Pado22.pdf">
Englisch</a> getestet.
Das Sprachmodell, das Deutsch abdeckt, kann im Prinzip noch weitere Sprachen verarbeiten. Sie können also
grundsätzlich "Deutsch" als Spracheinstellung auswählen und Daten in einer der anderen unterstützten Sprachen hochladen.
Bitte prüfen Sie die Ergebnisse aber sorgfältig, es liegen keine Erfahrungen vor! (Die Sprachen
sind <a href="https://www.sbert.net/docs/pretrained_models.html#multi-lingual-models">lt. den Modellerstellern</a>: ar,
bg, ca, cs, da, de, el, en, es, et, fa, fi, fr, fr-ca, gl, gu, he, hi, hr, hu, hy, id, it, ja, ka, ko, ku, lt, lv, mk,
mn, mr, ms, my, nb, nl, pl, pt, pt-br, ro, ru, sk, sl, sq, sr, sv, th, tr, uk, ur, vi, zh-cn, zh-tw.)
<h2 id=3>Wie verwendet man ASYST?</h2>
Wir haben bei der Entwicklung von ASYST versucht, die Verwendung möglichst einfach zu machen sein.
<h3 id=4>Wie müssen auszuwertende Daten formatiert sein?</h3>
Das Programm arbeitet auf Basis Ihrer Daten im Excel-Format .xlsx (das auch von Libre Office Calc und anderen Programmen
erzeugt werden kann). Eine Beispieltabelle:
![table_input.png](images%2Ftable_input.png)
Dabei müssen die folgende Informationen in der **richtigen Reihenfolge** und mitem **richtigen Titel** der Spalten
enthalten sein:
1) **Question**: Die gestellte Frage
2) **referenceAnswer**: Eine korrekte Antwort / Musterlösung / Referenzantwort
3) **studentAnswer**: Die vom Prüfling gegebene Antwort, die bewertet werden soll.
5) (optional) **observed grade**: Hier kann die tatsächliche Bewertung durch die Lehrkraft eingetragen werden, um
Kennzahlen über die Richtigkeit der Vorhersagen zu bekommen.
Die Beispieltabelle finden Sie
unter <a href="https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST/-/blob/main/DE_Demo_Daten.xlsx">
DE_Demo_Daten.xlsx</a>. Sie enthält einige Fragen und Antworten aus dem CSSAG-Korpus (Computer Science Short Answers in
German) der HFT Stuttgart. Das Korpus is CC-BY-NC lizenziert.
<h3 id=5>Wie führe ich das Programm unter Windows 11 aus? </h3>
Zunächst muss die Datei
_ASYST.exe_ <a href="https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST/-/blob/main/ASYST.exe"> heruntergeladen
werden</a>.
Sobald dies geschehen ist, kann das Programm mittels Doppelklick gestartet werden.
Der Start des Programmes wird eine Weile dauern (ca 1 Minute). In dieser Zeit wird das System initialisiert.
**Hinweis**: Es kann passieren, dass Windows Defender davor warnt, die Anwendung auszuführen, da das Programm kein
Sicherheitszertifikat besitzt.
Durch Auswählen von _weitere Informationen_ und anschließend _Trotzdem ausführen_ verschwindet die Fehlermeldung und
ASYST kann ausgeführt werden. Der Quelltext von ASYST ist offen zugänglich, so dass Sie sich vergewissern können, dass
ASYST keine Schadsoftware ist.
<img src="images/win_def_de_1.JPG" width="450">
<img src="images/win_def_de_2.JPG" width="450">
Nachdem das Programm gestartet wurde, erscheint eine Oberfläche, auf der die Sprache der auszuwertenden Antworten
ausgewählt werden kann.
Anschließend kann über einen Klick auf das Feld "Input File" die zu verarbeitende Tabelle ausgewählt werden.
Hierbei sollten die Daten wie oben beschrieben angeordnet sein.
Nach einem Klick auf das "Start"-Feld beginnt ASYST mit der Verarbeitung der Daten. Dies kann wiederum eine Weile
dauern (1-2 Minuten, relativ unabhängig von der Menge der zu verarbeitenden Daten).
Sobald das Programm alle Einträge verarbeitet und Vorhersagen getroffen hat, öffnet sich eine Tabellenansicht mit der
Überschrift "Results" (Ergebnisse).
Die Ergebnistabelle enthält alle Spalten der eingelesenen Tabelle, sowie zusätzlich in der Spalte "predicted grade" die
von ASYST vorgeschlagene Bewertung der Antworten. Die "incorrect"-Einträge der als falsch eingestuften Antworten sind
rot hinterlegt. Sie können in dieser Tabelle allerdings noch keine Bewertungen verändern. Speichern Sie hierzu über
einen Klick auf "Save as" die erzeugte Tabelle und öffnen Sie sie dann mit einem Tabellenkalkulationsprogramm.
![table_results.png](images%2Ftable_results.png)
Sobald die Ergebnistabelle angezeigt wird, kann ASYST die nächste Tabelle einlesen und verarbeiten.
**ACHTUNG: Die Ergebnistabelle wird nicht automatisch gespeichert.** Werden die Ergebnisse nicht gespeichert,
wird die Erbgebnistabelle im nächsten Durchlauf überschrieben.
Daher sollte, um die Ergebnisse zu sichern, auf den **"Save as"**- Button geklickt und die Ausgabetabelle am gewünschten
Ort gespeichert werden.
<h2 id=6>Wie arbeitet man mit der Ausgabe von ASYST weiter?</h2>
Wir empfehlen die folgende **Vorgehensweise** beim Einsatz von ASYST:
(Weitere Informationen und eine konkretes Beispiel für das Vorgehen liefert der
Artikel <a href="https://nlpado.de/~ulrike/papers/Pado22.pdf">_Assessing the Practical Benefit of Automated Short-Answer
Graders_</a>.)
1) **Definition der Anforderungen**: Wie genau muss die Bewertung in meinem aktuellen Anwendungsfall sein?
<ul>
<li>Bei der Bewertung von Freitextfragen in eher informellen Testsituationen (keine Abschlussklausur o.ä.) unterscheiden sich auch <b>menschliche Bewertungen</b> in ca. 15% der Fälle - 0% Abweichung sind also auch für Menschen kaum erreichbar! </li>
<li>Wir empfehlen daher in solchen Situationen, eine Bewertungsgenauigkeit von mindestens 85% auch nach dem Einsatz von ASYST plus der menschlichen Korrektur anzustreben. </li>
<li>Zu Beachten ist zudem die Verteilung der Bewertungsfehler (Übermäßige Strenge/Milde)</li>
<li>Letztlich sollte die Verwendung des Tools den Anwender:innen eine Zeitersparnis bringen: Setzen Sie das verfügbare Budget oder eine angestrebte Mindestersparnis fest. </li>
</ul>
2) **Sammeln von** manuell bewerteten **Testdaten:**
Um einen Eindruck von der Genauigkeit und Zuverlässigkeit des automatischen Bewerters zu bekommen, werden annotierte
Testdaten benötigt,
d.h. Eingabe-Daten, für die eine korrekte Klassifizierung bereits festgelegt ist. Es werden also Daten im einlesbaren
Format benötigt, die bereits manuell bewertet wurden. Dies können z.B. Antworten aus früheren Tests sein.
Um den Datensatz möglichst robust gegenüber zufälligen Schwankungen zu machen, sollte er idealerweise einige hundert
Antworten umfassen -- aber kleinere Datensätze können natürlich ebenfalls verwendet werden.
4) **Analyse** der Leistung der automatischen Bewertung
Anhand der manuell bewerteten Testdaten kann nun gemessen werden, wie zuverlässig und treffsicher der Klassifizierer
für die spezifischen Fragen arbeitet. Damit bekommen Sie einen Eindruck davon, wie gut die Vorhersage für Ihren
eigenen Datensatz funktioniert.
Hierzu werden die Fragen und Antworten aus dem Testdatensatz von ASYST verarbeitet und anschließend die erhaltene
Klassifikation mit der manuellen Bewertung abgeglichen (z.B. in einer Tabellenkalkulation wie Excel oder Libre Office
Calc).
Dabei kann der Anteil der korrekt klassifizierten Antworten im gesamten Datensatz ermittelt werden - dieser sollte
85% oder höher betragen (das entspricht einer Fehlerquote von 15% oder weniger).
Sie können auch für die einzelnen Bewertungen (richtig/falsch) berechnen, wie groß die Präzision für die
verschiedenen Bewertungen jeweils ist. Die Präzision misst, wie viele Vorhersagen einer bestimmten Bewertung
tatsächlich richtig waren, d.h. wie vertrauenswürdig die Vorhersagen des Bewerters für ein bestimmtes Label sind. So
bedeutet eine Präzision von 75% für die Bewertung "korrekt", dass drei Viertel aller Vorhersagen von "korrekt"
gestimmt haben, aber in einem Viertel der Fälle die Antwort laut der manuellen Bewertung falsch war.
_(Die Funktion, diese Kenngrößen der Zuverlässigkeit automatisch in einem Testmodus zu generieren soll in Zukunft dem
Programm noch hinzugefügt werden.)_
5) **Entscheidung** wie der Ansatz genutzt werden soll.
Anhand der erhobenen Kenngrößen zur Zuverlässigkeit für die oben genannten Kriterien kann nun eine Entscheidung
getroffen werden.
<ul>
<li> Wie groß ist der Anteil der korrekt vorhergesagten Bewertungen? Beträgt er >85%, können Sie die ASYST-Vorhersagen sogar unverändert übernehmen, falls Sie dies wünschen. </li>
<li> Wie ist die Präzision der einzelnen Bewertungsklassen (richtig/falsch)? Wenn eine der Klassen deutlich zuverlässiger vorhergesagt wird, können Sie entscheiden, diese Vorhersagen ungeprüft zu übernehmen und <b>nur</b> die Vorhersagen für die weniger verlässlich erkannte Klasse zu überprüfen. Dies führt in der Praxis zu einer deutlichen Zeitersparnis. </li>
<li>Wie ist der Bewertungsfehler verteilt? Werden übermäßig viele korrekte Antworten als falsch bewertet, oder umgekehrt? Ist dies für Ihre Situation akzeptabel? </li>
<li> Wie viel Bewertungsaufwand hätten Sie für den Beispieldatensatz eingespart, z.B. indem Sie die verlässlichere Bewertungsklasse ungeprüft akzeptieren?
</ul>
<h3 id=7>Wie kann ich ASYST ausführen, wenn ich kein Windows 11 nutze?</h3>
Die klickbare Anwendung "ASYST.exe" eignet sich nur für die Ausführung unter Windows 11.
In anderen Betriebssystemen kann ASYST aus einer Entwicklungsumgebung heraus ausgeführt werden.
Der ASYST-Quellcode ist ursprünglich in Python geschrieben und kann daher robust in verschiedenen Umgebungen ausgeführt
werden.
Für Anwender, die mit dem Ausführen von Python-Programmen nicht vertraut sind, wird im folgenden eine Möglichkeit näher
beschrieben.
<h4 id=8>Ausführen von ASYST in der Entwicklungsumgebung Pycharm </h4>
<ol>
<li>Falls noch nicht geschehen, die Entwicklungsumgebung Pycharm aus dem Internet
<a href="https://www.jetbrains.com/pycharm/download/?section=mac"> herunterladen </a> und installieren.
Für mehr Informationen und Problemlösung siehe
<a href="https://www.jetbrains.com/help/pycharm/installation-guide.html"> Pycharm-Installationsguide</a>.</li>
<li>Python installieren
Die Entwicklung von ASYST erfolgte in Python 3.10 - daher wird diese Version für die Ausführung empfohlen.
Die zum Betriebssystem passende Version kann unter https://www.python.org/downloads ausgewählt und installiert werden.
</li>
<li> Den Quellcode aus Gitlab in die Entwicklungsumgebung herunterladen:
Get from VCS
<img src="images/get_from_vcs.png" width="450">
im Feld _url_ folgenden Pfad eintragen: git@transfer.hft-stuttgart.de:ulrike.pado/ASYST.git
<img src="images/svn_url.png" width="450">
Anschließend auf _clone_ klicken und warten
</li>
<li>Entwicklungsumgebung konfigurieren
**Python-Interpreter konfigurieren:**
Navigiere zu _Settings >> Project ASYST >> Python Interpreter >> Add Interpreter >> Add local Interpreter_
![add_interpreter.png](images%2Fadd_interpreter.png)
![create_venv.png](images%2Fcreate_venv.png)
_Location_: [Projektpfad]/[Projektname]/Source,
_Base interpreter_: Pfad zur installierten Pythonversion
*Benötigte Pakte installieren:*
Falls Pycharm nicht von sich aus vorschlägt, die in der requirements.txt aufgeführten Pakete zu installieren,
führe manuell über das Terminal von PyCharm folgende Befehle aus:
'''
> cd Source
>
>
> pip install -r requirements.txt
'''
</li>
<li>ASYST ausführen
![run_button.png](images%2Frun_button.png)
Nachdem über das Projektverzeichnis links die Datei _main.py_ ausgewählt wurde, wird der ausgegraute _Startknopf_ oben
rechts
im Fenster grün. Ein einfacher Klick genügt, und ASYST wird ausgeführt.
</li>
</ol>
flask_ml_api/asyst/README_EN.md 0 → 100755
+ 236
- 0
View file @ 81a9e9f3
<h2>Content:</h2>
<ul>
<li><a href="#1"> What is ASYST? </a> </li>
<li><a href="#2"> Which languages are supported by ASYST? </a></li>
<li><a href="#3"> How do I use ASYST? </a></li>
<ol>
<li><a href="#4"> What does the input look like?</a> </li>
<li><a href="#5"> How do I run ASYST on Windows 11? </a></li>
</ol>
<li><a href="#6"> How do I continue with the output from ASYST? </a></li>
<li><a href="#7"> How do I run ASYST if I don't use Windows 11? </a>
<ul><li><a href="#8"> Running ASYST in the Pycharm development environment</a> </li></ul></li>
</ol></ul>
<h2 id=1>What is ASYST?</h2>
ASYST is a program designed to support teachers as they grade free-text answers in tests: With the help of Artificial
Intelligence,
ASYST makes grade suggestions that can be reviewed and, if necessary, modified by the teachers.
ASYST is intended for the evaluation of short answers that are one to three sentences in long. It is not intended to be
used for longer responses.
ASYST helps the teacher by suggesting a grade. This assessment may well be incorrect in individual cases; the teacher
can check and correct it.
This saves time in two ways compared to completely manual grading: First, reviewing grades is generally faster than
grading from scratch;
and second, we recommend reviewing mostly those ASYST grades that are most prone to errors (see Section <a href="#6">
How do I continue with the output from ASYST? </a>).
The program is written in Python; the source code is publicly available. To make ASYST easier to use, the Python scripts
have been
converted into an executable that is usable in Windows 11.
The technical background and development history of ASYST are described in <a href="https://rdcu.be/dxPLg"> Pado,
Eryilmaz and Kirschner, IJAIED 2023</a> along with a worked example for German data. For English data, a similar example
is available in <a href="https://nlpado.de/~ulrike/papers/Pado22.pdf">Pado, AIED 2022</a>
<h2 id=2>Which languages are supported by ASYST?</h2>
ASYST has been tested for <a href="https://rdcu.be/dxPLg">German</a>
and <a href="https://nlpado.de/~ulrike/papers/Pado22.pdf">English</a>.
The language model that covers German can in principle handle other languages, as well. So, in principle, you could
select "German" as language setting and upload data in one of the other languages covered by the model. If you try this,
please check the results carefully, as this is
untested! (<a href="https://www.sbert.net/docs/pretrained_models.html#multi-lingual-models">According to the model
developers,</a> the covered languages are: ar, bg, ca, cs, da, de, el, en, es, et, fa, fi, fr, fr-ca, gl, gu, he, hi,
hr, hu, hy, id, it, ja, ka, ko, ku, lt, lv, mk, mn, mr, ms, my, nb, nl, pl, pt, pt-br, ro, ru, sk, sl, sq, sr, sv, th,
tr, uk, ur, vi, zh-cn, zh-tw.)
<h2 id=3>How do I use ASYST?</h2>
We developed ASYST to be as user-friendly as possible.
<h3 id=4>What does the input look like?</h3>
The program works based on your data in Excel's .xlsx format (which can also be generated by Libre Office Calc and other
programs). This is an example table:
![table_input.png](images%2Ftable_input.png)
The following information needs to be included in the **correct order** and with the **correct column headings**:
1) **question**: The question that was asked
2) **referenceAnswer**: A correct answer / reference answer
3) **studentAnswer**: The student answer that is to be evaluated
5) (optional) **observed grade**: The grade given by the teacher can be entered here in order to evaluate the accuracy
of the ASYST predictions.
The example table can be found
at <a href="https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST/-/blob/main/DE_Demo_Daten.xlsx">
DE_Demo_Daten.xlsx</a>. It contains some questions and answers from the CSSAG corpus (Computer Science Short Answers in
German) of HFT Stuttgart. The corpus is licensed as CC-BY-NC.
<h3 id=5>How do I run ASYST on Windows 11? </h3>
First, download <a href="https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST/-/blob/main/ASYST.exe">
_ASYST.exe_ </a>.
The program can be started by double-clicking its icon.
The program will take a while to start (approx. 1 minute). During this time the system is initialized.
**Note**: Windows Defender may warn against running the application because the program does not have a security
certificate.
By selecting _more information_ and then _Run anyway_ the error message disappears and ASYST can be executed. ASYST's
source code is open source so you can verify the code is not malicious.
<img src="images/win_def_de_1.JPG" width="450">
<img src="images/win_def_de_2.JPG" width="450">
After the program has been started, a window appears. First, select the language of the answers to be evaluated.
The table to be processed can then be selected by clicking on the “Input File” field. The data should be arranged as
described above.
After clicking on the “Start” field, ASYST begins processing the data. Again, this can take a while (1-2 minutes,
relatively independent of the amount of data being processed).
Once the program has processed all answers and made predictions, a table view headed "Results" opens.
The results table contains all columns of the input table, as well as the grades suggested by ASYST -- see the the "
predicted grade" column. The grades for answers classified as incorrect are highlighted in red. You cannot make change
in this table. Instead, save the data by clicking on “Save as” and then open the resulting .xlsx file with a spreadsheet
program.
![table_results.png](images%2Ftable_results.png)
As soon as the result table is displayed, ASYST can read and process the next input table.
**ATTENTION: The results table is not saved automatically.**
Therefore, to save the results, the **"Save as"** button should be clicked and the output table should be saved at the
desired location.
<h2 id=6>How do I continue with the output from ASYST?</h2>
We recommend the following **process** when using ASYST:
(Further information and a concrete example of the procedure can be found
in <a href="https://nlpado.de/~ulrike/papers/Pado22.pdf">_Assessing the Practical Benefit of Automated Short-Answer
Graders_</a>.)
1) **Define requirements**: How accurate does the grading need to be in my current use case?
<ul>
<li>When evaluating free text questions in low-stakes test situations (not in a final exam or similar), <b>human grades</b> differ in around 15% of cases! </li>
<li>In such situations, we therefore recommend aiming for a grading accuracy of at least 85% after using ASYST plus human review. </li>
<li>The distribution of grading errors (excessive strictness/leniency) should also be taken into account</li>
<li>Ultimately, using the tool should save users time: set the available time budget or a minimum requirement for time saved. </li>
</ul>
2) **Collect** manually evaluated **test data:**
To get an idea of the accuracy and reliability of the automated grader, annotated test data is needed.
This is input data for which a grade has already been determined. This can be answers from previous tests, for
example.
To make the data set as robust as possible against random fluctuations, it should ideally contain a few hundred
responses -- but smaller data sets can of course also be used.
4) **Analyze** the performance of the automated grading
The manually graded test data can be used to measure how reliable and accurate the automated grader is for the test
data. This will give you an idea of how well the grade prediction works for your own data set.
For this purpose, process the questions and answers from the test data set using ASYST and compare the grade
predictions with the manual assessment (e.g. in a spreadsheet such as Excel or Libre Office Calc).
The proportion of correctly classified answers in the entire data set gives you the system accuracy (which should be
at around 85% or higher, which means disagreement between the manual and machine grades of 15% or less).
You can also calculate the precision for each grade ("correct"/"incorrect"). Precision measures how many predictions
of a given grade were actually correct, i.e. how trustworthy the ASYST's predictions are for a given label. A
precision of 75% for the rating "correct" means that three quarters of all predictions of "correct" were in fact
right, but for a quarter of the cases the answer was actually wrong according to the manual grades.
_(We plan to add functionality to automatically generate these reliability parameters in the future.)_
5) **Decide** how to use ASYST's predictions.
A usage decision can now be made based on the reliability parameters collected for the criteria mentioned above.
<ul>
<li> How large is the proportion of correctly predicted reviews? If it is >85%, you can even adopt the ASYST predictions unchanged if you wish. </li>
<li> What is the precision of the grade labels ("correct"/"incorrect")? If one of the grade labels is likely to be significantly more reliable, you can decide to accept these predictions without review and <b>only</b> check the predictions for the less reliable grade predictions. In practice, this leads to significant time savings. </li>
<li>How is the grading error distributed? Are correct answers frequently predicted to be incorrect, or vice versa? Is this acceptable for your situation? </li>
<li> How much evaluation effort would you have saved for the example data set, e.g. by accepting the more reliable grade label without review?
</ul>
<h3 id=7>How can I run ASYST if I don't use Windows 11?</h3>
The clickable application “ASYST.exe” is only suitable for running on Windows 11.
On other operating systems, ASYST can be run from a development environment.
The ASYST source code is written in Python and can therefore be robustly executed in various development environments.
For users who are not familiar with running Python programs, one option is described in more detail below.
<h4 id=8>Running ASYST in the Pycharm development environment </h4>
<ol>
<li>If you haven't already done so, the Pycharm development environment from the Internet
Download <a href="https://www.jetbrains.com/pycharm/download/?section=mac"> </a> and install.
For more information and troubleshooting see
<a href="https://www.jetbrains.com/help/pycharm/installation-guide.html">Pycharm installation guide</a>.</li>
<li>Install Python
ASYST was developed in Python 3.10 - therefore this version is recommended for execution.
The version that matches the operating system can be selected and installed at https://www.python.org/downloads.
</li>
<li> Download the source code from Gitlab to the development environment:
Get from VCS
<img src="images/get_from_vcs.png" width="450">
Enter the following path in the _url_ field: git@transfer.hft-stuttgart.de:ulrike.pado/ASYST.git
<img src="images/svn_url.png" width="450">
Then click on _clone_ and wait
</li>
<li>Configure development environment
**Configure Python interpreter:**
Navigate to _Settings >> Project ASYST >> Python Interpreter >> Add Interpreter >> Add local Interpreter_
![add_interpreter.png](images%2Fadd_interpreter.png)
![create_venv.png](images%2Fcreate_venv.png)
_Location_: [Project Path]/[Project Name]/Source,
_Base interpreter_: Path to the installed Python version
*Install required packages:*
If Pycharm does not itself suggest installing the packages listed in the requirements.txt,
manually run the following commands in the PyCharm terminal:
'''
> cd Source
>
>
> pip install -r requirements.txt
'''
</li>
<li>Run ASYST
![run_button.png](images%2Frun_button.png)
After the file _main.py_ has been selected via the project directory on the left, the greyed out _Start button_ at the
top right of the window will appear green. A single click is enough to execute the ASYST code.
</li>
</ol>
flask_ml_api/asyst/Source/LICENSE 0 → 100755
+ 22
- 0
View file @ 81a9e9f3
MIT License
Copyright (c) 2022 Larissa Kirschner
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
\ No newline at end of file
flask_ml_api/asyst/Source/Skript/german/__pycache__/run_LR_SBERT.cpython-311.pyc 0 → 100755
+ 0
- 0
View file @ 81a9e9f3
File added
flask_ml_api/asyst/Source/Skript/german/models/clf_BERT.pickle 0 → 100755
+ 0
- 0
View file @ 81a9e9f3
File added
flask_ml_api/asyst/Source/Skript/german/run_LR_SBERT.py 0 → 100755
+ 88
- 0
View file @ 81a9e9f3
import os
import sys
import time
import numpy as np
import pandas as pd
# UP
import pickle
import argparse
from sklearn import metrics
from sentence_transformers import models, SentenceTransformer
from sklearn.linear_model import LogisticRegression, Perceptron
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import cross_validate, cross_val_predict
"""
This script has been adapted from the original script authored by Yunus Eryilmaz.
This script has been modified to adapt the source and structure of input-output data
for specific use case (data is given as params, result returns as an array instead of files).
"""
__author__ = "Yunus Eryilmaz"
__version__ = "1.0"
__date__ = "21.07.2021"
__source__ = "https://pypi.org/project/sentence-transformers/0.3.0/"
__source__ = "https://transfer.hft-stuttgart.de/gitlab/ulrike.pado/ASYST/-/blob/main/Source/Skript/german/run_LR_SBERT.py"
__adapted_by__ = "Artem Baranovskyi"
__adaptation_date__ = "14.09.2024"
__adaptation_version__ = "1.0"
def process_data(data):
parser = argparse.ArgumentParser()
parser.add_argument(
"--model_dir",
default="/app/asyst/Source/Skript/german/models",
type=str,
required=False,
help="The directory where the ML models are stored.",
)
args = parser.parse_args()
referenceAnswer = data['referenceAnswer']
studentAnswers = data['studentAnswers']
# Use BERT for mapping tokens to embeddings
word_embedding_model = models.Transformer('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2')
# pooling operation can choose by setting true (Apply mean pooling to get one fixed sized sentence vector)
pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),
pooling_mode_mean_tokens=True,
pooling_mode_cls_token=False,
pooling_mode_max_tokens=False)
# compute the sentence embeddings for both sentences
model = SentenceTransformer(modules=[word_embedding_model, pooling_model])
sentence_embeddings1 = model.encode([referenceAnswer] * len(studentAnswers), convert_to_tensor=True, show_progress_bar=False)
sentence_embeddings2 = model.encode(studentAnswers, convert_to_tensor=True, show_progress_bar=False)
computed_simis_test = similarity(sentence_embeddings1, sentence_embeddings2)
X_test = computed_simis_test
# UP: read pre-trained LR model
clf_log = pickle.load(open("/app/asyst/Source/Skript/german/models/clf_BERT.pickle", "rb"))
predictions = clf_log.predict(X_test)
results = []
for i in range(len(studentAnswers)):
result = {
"predicted_grade": "correct" if predictions[i] == 1 else "incorrect"
}
results.append(result)
return results
# Possible concatenations from the embedded sentences can be selected
def similarity(sentence_embeddings1, sentence_embeddings2):
# I2=(|u − v| + u ∗ v)
simi = abs(np.subtract(sentence_embeddings1, sentence_embeddings2)) + np.multiply(sentence_embeddings1,
sentence_embeddings2)
return simi
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal