Commit ec9210c4 authored by Artem Baranovskyi's avatar Artem Baranovskyi
Browse files

Fixed Supervisor & Flask infrastructure issues.

Added Plugin DB Migration.
-----------------------------------------------------------
TODO: Create an API to use script run_LR_SBERT.py on http://127.0.0.1:5000/api/data or like that.
parent 33081e26
...@@ -13,4 +13,6 @@ MOODLE_BASE_DIR_DATA=/var/www/html/moodledata ...@@ -13,4 +13,6 @@ MOODLE_BASE_DIR_DATA=/var/www/html/moodledata
MOODLE_WWWROOT=https://www.moodle.loc MOODLE_WWWROOT=https://www.moodle.loc
MOODLE_FULLNAME="Moodle LMS Site" MOODLE_FULLNAME="Moodle LMS Site"
MOODLE_SHORTNAME="Moodle" MOODLE_SHORTNAME="Moodle"
\ No newline at end of file
FLASK_APP=/var/www/html/moodle/api.py
\ No newline at end of file
...@@ -44,6 +44,8 @@ RUN python3 -m venv /opt/myenv ...@@ -44,6 +44,8 @@ RUN python3 -m venv /opt/myenv
RUN /opt/myenv/bin/python3 -m pip install --upgrade pip RUN /opt/myenv/bin/python3 -m pip install --upgrade pip
#RUN /opt/myenv/bin/python3 -m pip install matplotlib Flask torch sklearn-learn #RUN /opt/myenv/bin/python3 -m pip install matplotlib Flask torch sklearn-learn
COPY requirements.txt /opt/myenv/ COPY requirements.txt /opt/myenv/
COPY sentence-transformers-paraphrase-multilingual-MiniLM-L12-v2 /var/www/html/moodle/sentence-transformers-paraphrase-multilingual-MiniLM-L12-v2
WORKDIR /var/www/html/moodle
RUN /opt/myenv/bin/python3 -m pip install -r /opt/myenv/requirements.txt RUN /opt/myenv/bin/python3 -m pip install -r /opt/myenv/requirements.txt
RUN /opt/myenv/bin/python3 -m pip install --upgrade setuptools wheel RUN /opt/myenv/bin/python3 -m pip install --upgrade setuptools wheel
...@@ -149,11 +151,6 @@ RUN a2enmod ssl ...@@ -149,11 +151,6 @@ RUN a2enmod ssl
#Opening ports #Opening ports
EXPOSE 80 443 5000 EXPOSE 80 443 5000
#RUN /var/www/html/moodle/install_moodle.sh
# Run the Flask app (modify entrypoint or use a custom script if needed)
#CMD ["bash", "-c", "service apache2 start && source /opt/myenv/bin/activate && python3 /var/www/html/moodle/install_moodle.sh /var/www/html/moodle/api.py "]
#CMD ["bash", "-c", "service apache2 start && source /opt/myenv/bin/activate && python3 /var/www/html/moodle/api.py && /var/www/html/moodle/install_moodle.sh"]
# Setting up supervisord # Setting up supervisord
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
......
...@@ -2,20 +2,33 @@ ...@@ -2,20 +2,33 @@
from flask import Flask, jsonify from flask import Flask, jsonify
# TODO: Adding correct path to ASYST script run_LR_SBERT.py
# from run_LR_SBERT import main
# sys.path.append(os.path.join(os.path.dirname(__file__), 'asyst/Source/Skript/german'))
app = Flask(__name__) app = Flask(__name__)
# Example of an endpoint returning JSON data
@app.route('/api/data', methods=['GET']) @app.route('/api/data', methods=['GET'])
def get_data(): def get_data():
# TODO: use run_LR_SBERT.py ASYST script instead # Using path to data and model
data = { data_path = '/var/www/html/moodle/asyst/Source/Skript/outputs/test.tsv'
model_dir = '/var/www/html/moodle/asyst/Source/Skript/german/models'
# Obtaining results from run_asyst function
# results => run_asyst(data_path, model_dir)
# Demo dummy API output
results = {
'message': 'Hello from Python API!', 'message': 'Hello from Python API!',
'data': { 'data': {
'key1': 'value1', 'key1': 'value1',
'key2': 'value2' 'key2': 'value2'
} }
} }
return jsonify(data)
# Returning result in JSON format
return jsonify(results)
if __name__ == '__main__': if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True) app.run(host='0.0.0.0', port=5000, debug=True)
\ No newline at end of file
...@@ -36,7 +36,6 @@ def main(): ...@@ -36,7 +36,6 @@ def main():
#default=None, #default=None,
default="/var/www/html/moodle/asyst/Source/Skript/outputs/test.tsv", default="/var/www/html/moodle/asyst/Source/Skript/outputs/test.tsv",
type=str, type=str,
# required=True,
required=False, required=False,
help="The input data file for the task.", help="The input data file for the task.",
) )
...@@ -45,7 +44,6 @@ def main(): ...@@ -45,7 +44,6 @@ def main():
# default=None, # default=None,
default="/var/www/html/moodle/asyst/Source/Skript/outputs", default="/var/www/html/moodle/asyst/Source/Skript/outputs",
type=str, type=str,
# required=True,
required=False, required=False,
help="The output directory where predictions will be written.", help="The output directory where predictions will be written.",
) )
...@@ -54,32 +52,38 @@ def main(): ...@@ -54,32 +52,38 @@ def main():
# default=None, # default=None,
default=location+"/Skript/german/models", default=location+"/Skript/german/models",
type=str, type=str,
# required=True,
required=False, required=False,
help="The directory where the ML models are stored.", help="The directory where the ML models are stored.",
) )
parser.add_argument(
"--transformer_model_dir",
default="/var/www/html/moodle/sentence-transformers-paraphrase-multilingual-MiniLM-L12-v2",
type=str,
required=False,
help="The directory where the SentenceTransformer model is stored.",
)
args = parser.parse_args() args = parser.parse_args()
# open a log file next to the executable with line buffering # open a log file next to the executable with line buffering
# out = open("log.txt", "a",buffering=1); # out = open("log.txt", "a",buffering=1);
# print("Started German processing in",location,file=out); # print("Started German processing in", location, file=out);
# import SentenceTransformer-model # import SentenceTransformer-model
start_time = time.time() start_time = time.time()
# print("Reading from",args.data, file=out); # print("Reading from", args.data, file=out);
with open(os.path.join(location,args.data)) as ft: with open(args.data) as ft:
dft = pd.read_csv(ft, delimiter='\t') dft = pd.read_csv(ft, delimiter='\t')
# Sentences we want sentence embeddings for # Sentences we want sentence embeddings for
sentences1_test = dft['referenceAnswer'].values.tolist() sentences1_test = dft['referenceAnswer'].values.tolist()
sentences2_test = dft['studentAnswer'].values.tolist() sentences2_test = dft['studentAnswer'].values.tolist()
# print("Input read:",sentences2_test, file=out); # print("Input read:", sentences2_test, file=out);
# Use BERT for mapping tokens to embeddings # Use BERT for mapping tokens to embeddings
word_embedding_model = models.Transformer('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2') word_embedding_model = models.Transformer(args.transformer_model_dir)
# pooling operation can choose by setting true (Apply mean pooling to get one fixed sized sentence vector) # pooling operation can choose by setting true (Apply mean pooling to get one fixed sized sentence vector)
pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(), pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),
pooling_mode_mean_tokens=True, pooling_mode_mean_tokens=True,
...@@ -91,7 +95,7 @@ def main(): ...@@ -91,7 +95,7 @@ def main():
# print("Model loaded", file=out); # print("Model loaded", file=out);
sentence_embeddings1_test = model.encode(sentences1_test, convert_to_tensor=True, show_progress_bar=False) sentence_embeddings1_test = model.encode(sentences1_test, convert_to_tensor=True, show_progress_bar=False)
# print("Embeddings RefA:",sentence_embeddings1_test,file=out); # print("Embeddings RefA:", sentence_embeddings1_test, file=out);
sentence_embeddings2_test = model.encode(sentences2_test, convert_to_tensor=True, show_progress_bar=False) sentence_embeddings2_test = model.encode(sentences2_test, convert_to_tensor=True, show_progress_bar=False)
# print("Embeddings found", file=out); # print("Embeddings found", file=out);
...@@ -107,7 +111,7 @@ def main(): ...@@ -107,7 +111,7 @@ def main():
# calls the similarity function and get the concatenated values between the sentence embeddings # calls the similarity function and get the concatenated values between the sentence embeddings
computed_simis_test = similarity(sentence_embeddings1_test, sentence_embeddings2_test) computed_simis_test = similarity(sentence_embeddings1_test, sentence_embeddings2_test)
# get the sentence embeddings and the labels fpr train and test # get the sentence embeddings and the labels for train and test
X_test = computed_simis_test X_test = computed_simis_test
# Y_test = np.array(dft['label']) # Y_test = np.array(dft['label'])
...@@ -115,12 +119,12 @@ def main(): ...@@ -115,12 +119,12 @@ def main():
# UP: read pre-trained LR model # UP: read pre-trained LR model
clf_log = pickle.load(open("/var/www/html/moodle/asyst/Source/Skript/german/models/clf_BERT.pickle", "rb")) clf_log = pickle.load(open("/var/www/html/moodle/asyst/Source/Skript/german/models/clf_BERT.pickle", "rb"))
# print('--------Evaluate on Testset------- ', file=out) # print('--------Evaluate on Testset------- ', file=out)
predictions = clf_log.predict(X_test) predictions = clf_log.predict(X_test)
# UP print results # UP print results
with open(args.output_dir + "/predictions.txt", "w") as writer: with open(args.output_dir + "/predictions.txt", "w") as writer:
# TODO: write results to plugins DB Table
writer.write("question\treferenceAnswer\tstudentAnswer\tsuggested grade\tobserved grade\n") writer.write("question\treferenceAnswer\tstudentAnswer\tsuggested grade\tobserved grade\n")
for i in range(len(dft)): for i in range(len(dft)):
hrpred = "incorrect" hrpred = "incorrect"
...@@ -139,7 +143,7 @@ def main(): ...@@ -139,7 +143,7 @@ def main():
+ "\n" + "\n"
) )
# print('\nExecution time:', time.strftime("%H:%M:%S", time.gmtime(time.time() - start_time)),file=out) # print('\nExecution time:', time.strftime("%H:%M:%S", time.gmtime(time.time() - start_time)), file=out)
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -42,6 +42,9 @@ services: ...@@ -42,6 +42,9 @@ services:
- 80:80 - 80:80
- 443:443 - 443:443
- 5000:5000 - 5000:5000
dns:
- 8.8.8.8
- 8.8.4.4
depends_on: # Dependency from MariaDB service depends_on: # Dependency from MariaDB service
- mariadb - mariadb
volumes: volumes:
...@@ -51,12 +54,6 @@ services: ...@@ -51,12 +54,6 @@ services:
- moodledata:${MOODLE_BASE_DIR_DATA} # Том для данных Moodle - moodledata:${MOODLE_BASE_DIR_DATA} # Том для данных Moodle
networks: networks:
- network # Добавление к сети - network # Добавление к сети
healthcheck:
test: /bin/bash -c 'while ! nc -z mariadb 3306; do echo "Ждем подключения к базе данных..."; sleep 5; done'
interval: 10s
timeout: 5s
retries: 5
command: [ "bash", "-c", "apache2ctl -D FOREGROUND" ]
volumes: volumes:
moodledata: moodledata:
driver: local # Локальный драйвер томов driver: local # Локальный драйвер томов
......
...@@ -24,8 +24,8 @@ docker-compose exec moodle php ${MOODLE_BASE_DIR}/admin/cli/install.php \ ...@@ -24,8 +24,8 @@ docker-compose exec moodle php ${MOODLE_BASE_DIR}/admin/cli/install.php \
docker-compose exec moodle chmod -R 755 /var/www/html/moodle docker-compose exec moodle chmod -R 755 /var/www/html/moodle
# Set correct access rules for the plugin # Set correct access rules for the plugin
#docker-compose exec moodle chown -R www-data:www-data /var/www/html/moodle/mod/yourplugin docker-compose exec moodle chown -R www-data:www-data /var/www/html/moodle/mod/yourplugin
#docker-compose exec moodle chmod -R 775 /var/www/html/moodle/mod/yourplugin docker-compose exec moodle chmod -R 775 /var/www/html/moodle/mod/yourplugin
# Create the run_sag script file # Create the run_sag script file
docker-compose exec moodle bash -c 'echo "#!/bin/bash" > /usr/local/bin/run_sag' docker-compose exec moodle bash -c 'echo "#!/bin/bash" > /usr/local/bin/run_sag'
......
...@@ -4,11 +4,16 @@ Run these commands at CLI to use ASYST with universal BERT model based on German ...@@ -4,11 +4,16 @@ Run these commands at CLI to use ASYST with universal BERT model based on German
~~~bash ~~~bash
docker-compose up -d --build docker-compose up -d --build
install_moodle /bin/bash install_moodle.sh
~~~ ~~~
Use these creds to access Moodle admin page
admin:rootpassword
Database connection URL: jdbc:mariadb://localhost:3306/moodle
It is suggested to use our moodle plugin to communicate with ASYST script using such a It is suggested to use our moodle plugin to communicate with ASYST script using such a
route http://127.0.0.1:5000/api/data route http://127.0.0.1:5000
Now the preinstalled MOODLE LMS is available at https://www.moodle.loc Now the preinstalled MOODLE LMS is available at https://www.moodle.loc
......
...@@ -5,9 +5,13 @@ nodaemon=true ...@@ -5,9 +5,13 @@ nodaemon=true
command=/usr/sbin/apache2ctl -D FOREGROUND command=/usr/sbin/apache2ctl -D FOREGROUND
autostart=true autostart=true
autorestart=true autorestart=true
stdout_logfile=/var/log/supervisor/flask.log
stderr_logfile=/var/log/supervisor/flask_err.log
[program:flask] [program:flask]
command=/opt/myenv/bin/python /var/www/html/moodle/api.py command=/bin/bash -c 'source /opt/myenv/bin/activate && FLASK_APP=/var/www/html/moodle/api.py /opt/myenv/bin/flask run --host=0.0.0.0'
directory=/var/www/html/moodle directory=/var/www/html/moodle
autostart=true autostart=true
autorestart=true autorestart=true
\ No newline at end of file stdout_logfile=/var/log/supervisor/moodle.log
stderr_logfile=/var/log/supervisor/moodle_err.log
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8" ?>
<XMLDB PATH="mod/yourplugin/db" VERSION="2024032201" COMMENT="XMLDB file for mod yourplugin plugin"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="../../../lib/xmldb/xmldb.xsd"
>
<TABLES>
<TABLE NAME="mod_yourplugin" COMMENT="Table for storing yourplugin data">
<FIELDS>
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" UNSIGNED="true" SEQUENCE="true"/>
<FIELD NAME="question" TYPE="text" NOTNULL="true"/>
<FIELD NAME="referenceanswer" TYPE="text" NOTNULL="true"/>
<FIELD NAME="studentanswer" TYPE="text" NOTNULL="true"/>
<FIELD NAME="observed_grade" TYPE="int" LENGTH="3" NOTNULL="true"/>
<FIELD NAME="studentid" TYPE="int" LENGTH="10" NOTNULL="false"/>
<FIELD NAME="recordid" TYPE="int" LENGTH="10" NOTNULL="false"/>
</FIELDS>
<KEYS>
<KEY NAME="primary" TYPE="primary" FIELDS="id"/>
</KEYS>
</TABLE>
</TABLES>
</XMLDB>
...@@ -10,32 +10,36 @@ ...@@ -10,32 +10,36 @@
$string['pluginname'] = 'New local plugin'; $string['pluginname'] = 'New local plugin';
// Path to the Python 3 executable // Path to the Python 3 executable
$python_executable = '/usr/bin/python3'; $python_executable = '/opt/myenv/bin/python3';
// Path to the moodlemlbackend script to be executed. First we use test API api.py, then run_LR_SBERT.py // Path to the moodlemlbackend script to be executed. First we use test API api.py, then run_LR_SBERT.py
$python_script = '/var/www/html/moodle/api.py'; $python_script = '/var/www/html/moodle/api.py';
//$python_script = '/var/www/html/moodle/asyst/Source/Skript/german/run_LR_SBERT.py'; //$python_script = '/var/www/html/moodle/asyst/Source/Skript/german/run_LR_SBERT.py';
// Python command you want to execute // Python command you want to execute
$python_command = 'print("Hello, world!!!")'; //$python_command = 'print("Hello, world!!!")';
// Execution the command and getting the result
//shell_exec($python_command);
// Formation of a command to execute // Formation of a command to execute
//$full_command = $python_executable . ' -c \'' . $python_command . '\''; //$full_command = $python_executable . ' -c \'' . $python_command . '\'';
$full_command = $python_executable . ' ' . $python_script; //$full_command = $python_executable . ' ' . $python_script;
// Execution the command and getting the result // Now call the API
$result = shell_exec($full_command); $api_url = 'http://127.0.0.1:5000/api/data';
$response = file_get_contents($api_url);
// Output the result // Output the result (assuming backend returns JSON)
echo $result; if ($response !== false) {
// Output the result (assuming moodlemlbackend returns JSON) $data = json_decode($response, true);
$data = json_decode($result, true); if ($data !== null) {
if ($data !== null) {
// Data processing // Data processing
// Example: output results // Example: output results echo "<pre>";
echo "<pre>"; print_r($data);
print_r($data); echo "</pre>";
echo "</pre>"; } else {
echo "Error on data processing from moodlemlbackend!";
}
} else { } else {
echo "Error on data processing from moodlemlbackend!"; echo "Execution error: result is not a string.";
} }
<?php
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* @package mod_yourplugin
* @author Artem Baranovskyi
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
function mod_yourplugin_before_footer()
{
if (!get_config('mod_yourplugin', 'enabled')) {
return;
}
}
\ No newline at end of file
...@@ -2,14 +2,14 @@ ...@@ -2,14 +2,14 @@
/** /**
* Version details. * Version details.
* *
* @package local_yourplugin * @package mod_yourplugin
* @copyright 2024 Artem Baranovskyi * @copyright 2024 Artem Baranovskyi
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/ */
defined('MOODLE_INTERNAL') || die(); defined('MOODLE_INTERNAL') || die();
$plugin->version = 2022051101; // The current module version (Date: YYYYMMDDXX). $plugin->version = 2024032201; // The current module version (Date: YYYYMMDDXX).
$plugin->requires = 2014050800; // Requires this Moodle version. $plugin->requires = 2014050800; // Requires this Moodle version.
$plugin->component = 'yourplugin';// Full name of the plugin (used for diagnostics). $plugin->component = 'mod_yourplugin'; // Full name of the plugin (used for diagnostics).
$plugin->cron = 0; $plugin->cron = 0;
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment