initial commit of ASYST project to public repository

Source/LICENSE

+MIT License
+
+Copyright (c) 2022 Larissa Kirschner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file

Source/Skript/english/run_SAG_mnli.py

+# coding=utf-8
+
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Code is originally from https://github.com/nlpyang/pytorch-transformers/blob/master/examples/run_glue.py
+# Adapted to the SAG task by Ulrike Pado, HFT Stuttgart: Run a fine-tuned model on given input data to predict short-answer grades.
+
+
+
+
+from __future__ import absolute_import, division, print_function
+
+import argparse
+import os
+import random
+import sys
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader, SequentialSampler, TensorDataset
+
+from sklearn.metrics import f1_score, accuracy_score
+
+from transformers import (
+    BertConfig,
+    BertForSequenceClassification,
+    BertTokenizer,
+)
+
+from transformers import glue_compute_metrics as compute_metrics
+from transformers import (
+    glue_convert_examples_to_features as convert_examples_to_features,
+)
+from transformers.data.processors.utils import (
+    DataProcessor,
+    InputExample,
+)
+
+#logger = logging.getLogger(__name__)
+
+MODEL_CLASSES = {
+    "bert": (BertConfig, BertForSequenceClassification, BertTokenizer),
+}
+
+
+def set_seed():
+    random.seed(42)
+    np.random.seed(42)
+    torch.manual_seed(42)
+
+
+def evaluate(args, model, tokenizer, prefix=""):
+    # Loop to handle MNLI double evaluation (matched, mis-matched)
+    # and SemEval evaluation (unseen questions, unseen answers, unseen domains)
+    eval_task_names = ("sag",
+                       )
+
+    eval_outputs_dirs = (
+        (args.output_dir, )
+    )
+
+    results = {}
+    for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs):
+
+        eval_dataset = load_and_cache_examples(
+            args, eval_task, tokenizer
+        )
+
+        if not os.path.exists(eval_output_dir):
+            os.makedirs(eval_output_dir)
+
+        args.eval_batch_size = 8
+
+        # Note that DistributedSampler samples randomly
+        eval_sampler = SequentialSampler(eval_dataset)
+        eval_dataloader = DataLoader(
+            eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size
+        )
+
+        # Eval!
+        #logger.info("***** Running evaluation {} *****".format(prefix))
+        #logger.info("  Task name = {}".format(eval_task))
+        #logger.info("  Num examples = %d", len(eval_dataset))
+        #logger.info("  Batch size = %d", args.eval_batch_size)
+
+        eval_loss = 0.0
+        nb_eval_steps = 0
+        preds = None
+        out_label_ids = None
+
+        for batch in eval_dataloader:
+            #logger.info(" Starting eval for batch")
+            model.eval()
+            batch = tuple(t.to(args.device) for t in batch)
+            #logger.info("  Batch converted to tuple")
+
+            with torch.no_grad():
+                inputs = {
+                    "input_ids": batch[0],
+                    "attention_mask": batch[1],
+                    "token_type_ids": batch[2],
+                    "labels": batch[3],
+                }
+
+                outputs = model(**inputs)
+                tmp_eval_loss, logits = outputs[:2]
+
+                eval_loss += tmp_eval_loss.mean().item()
+                #logger.info("Eval loss: %d", eval_loss)
+            nb_eval_steps += 1
+            if preds is None:
+                preds = logits.detach().cpu().numpy()
+                out_label_ids = inputs["labels"].detach().cpu().numpy()
+            else:
+                preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
+                out_label_ids = np.append(
+                    out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0
+                )
+
+        #logger.info("Prediction generation done")
+
+       # classification task; choose maximum label
+        preds = np.argmax(preds, axis=1)
+
+        # if evaluating SAG, return both accuracy and F1
+        task = "sag"
+        # logger.info("starting to compute metrics")
+        result = my_compute_metrics(task, preds, out_label_ids)
+        results.update(result)
+
+        # print predictions made by the current model
+        if args.do_print_predictions:
+            print_predictions(args, preds)
+
+        output_eval_file = os.path.join(
+            eval_output_dir, prefix + "-eval_results.txt")
+
+        #logger.info("sending output to "+str(output_eval_file));
+
+        with open(output_eval_file, "w") as writer:
+            #logger.info("***** Eval results {} *****".format(prefix))
+            for key in sorted(result.keys()):
+                #logger.info("  %s = %s", key, str(result[key]))
+                writer.write("%s = %s\n" % (key, str(result[key])))
+
+    return results
+
+
+def load_and_cache_examples(args, task, tokenizer):
+    examples = []
+
+    # choose the correct processor to read the data
+
+    processor = (
+        SemEvalProcessor()
+    )
+    output_mode = "classification"
+
+    #logger.info("Creating features from dataset file at %s", args.data_dir)
+    label_list = processor.get_labels()
+
+    examples = (
+        processor.get_test_examples(args.data_dir)
+    )
+
+    # We are continuing to train mnli models, so task = mnli to create
+    # the correct type of features
+    feature_task = "mnli" if task.startswith("sag") else task
+
+    features = convert_examples_to_features(
+        examples,
+        tokenizer,
+        label_list=label_list,
+        max_length=args.max_seq_length,
+        output_mode=output_mode,
+        task=feature_task
+    )
+
+    # Convert to Tensors and build dataset
+    all_input_ids = torch.tensor(
+        [f.input_ids for f in features], dtype=torch.long)
+    all_attention_mask = torch.tensor(
+        [f.attention_mask for f in features], dtype=torch.long
+    )
+    
+    all_token_type_ids = torch.tensor(
+        [f.token_type_ids for f in features], dtype=torch.long
+    )
+    
+    # do classification setup
+    all_labels = torch.tensor(
+        [f.label for f in features], dtype=torch.long)
+
+    dataset = TensorDataset(
+        all_input_ids, all_attention_mask, all_token_type_ids, all_labels
+    )
+    return dataset
+
+
+def main():
+
+    # Where are we?
+    location=".";
+    if getattr(sys, 'frozen', False):
+        # running in a bundle
+        location = sys._MEIPASS
+
+    # open a log file next to the executable with line buffering
+    #out = open("log.txt", "a", buffering=1);
+
+    #print("Started English processing in", location, file=out);
+
+    parser = argparse.ArgumentParser()
+
+    # Required parameters - adapt to current directory
+    parser.add_argument(
+        "--data_dir",
+        # default=None,
+        default=location+"\\Skript\\outputs\\",
+        type=str,
+        # required=True,
+        required=False,
+        help="The input data dir. Should contain the .tsv files (or other data files) for the task.",
+    )
+    parser.add_argument(
+        "--model_type",
+        # default=None,
+        default="bert",
+        type=str,
+        # required=True,
+        required=False,
+        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES),
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        # default=None,
+        #default= "textattack/bert-base-uncased-MNLI",
+        default=location+"\\Skript\\english\\seb-bert-mnli",
+        type=str,
+        # required=True,
+        required=False,
+        help="Path to pre-trained model",
+    )
+
+    parser.add_argument(
+        "--tokenizer_name",
+        default="textattack/bert-base-uncased-MNLI",
+        type=str,
+        help="Pretrained tokenizer name or path if not the same as model_name",
+    )
+
+    parser.add_argument(
+        "--output_dir",
+        # default=None,
+        default=location+"\\Skript\\english\\seb-bert-mnli",
+        type=str,
+        # required=True,
+        required=False,
+        help="The output directory where checkpoints will be written.",
+    )
+
+    parser.add_argument(
+        "--config_name",
+        default=location+"\\Skript\\english\\seb-bert-mnli\\config.json",
+        type=str,
+        help="Pretrained config name or path if not the same as model_name",
+    )
+    parser.add_argument(
+        "--cache_dir",
+        default="",
+        type=str,
+        help="Where do you want to store the pre-trained models downloaded from s3",
+    )
+    parser.add_argument(
+        "--max_seq_length",
+        # default=128,
+        default=256,
+        type=int,
+        help="The maximum total input sequence length after tokenization. Sequences longer "
+        "than this will be truncated, sequences shorter will be padded.",
+    )
+    parser.add_argument(
+        # "--do_test", action="store_true", help="Whether to run eval on the test set."
+        "--do_test", action="store_false", help="Whether to run eval on the test set."
+    ),
+    parser.add_argument(
+        #"--do_print_predictions",action="store_true",help="Whether to print the model predictions for manual inspection.",
+        "--do_print_predictions",
+        action="store_false",
+        help="Whether to print the model predictions for manual inspection.",
+    ),
+    parser.add_argument(
+        "--do_lower_case",
+        # action="store_true",
+        action="store_false",
+        help="Set this flag if you are using an uncased model.",
+    )
+
+    parser.add_argument(
+        "--overwrite_output_dir",
+        # action="store_true",
+        action="store_false",
+        help="Overwrite the content of the output directory",
+    )
+
+    args = parser.parse_args()
+
+    if (
+        os.path.exists(args.output_dir)
+        and os.listdir(args.output_dir)
+        and not args.overwrite_output_dir
+    ):
+        raise ValueError(
+            "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
+                args.output_dir
+            )
+        )
+
+
+
+    # Setup CPU processing
+    
+    device = torch.device("cpu")
+    args.device = device
+
+    # Setup logging
+    #logging.basicConfig(
+    #    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+    #    datefmt="%m/%d/%Y %H:%M:%S",
+    #    filename='log.txt',
+    #    filemode='a',
+    #    level=logging.INFO,
+    #)
+
+    #logger.warning(
+    #    "Device: %s",
+    #    device
+    #)
+
+    # Set seed to 42
+    set_seed()
+
+    processor = (
+        SemEvalProcessor()
+    )
+    args.output_mode = (
+        "classification"
+    )
+    label_list = processor.get_labels()
+    num_labels = len(label_list)
+
+    args.model_type = args.model_type.lower()
+    #logger.info("Model %s", args.model_type)
+
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(
+        args.config_name if args.config_name else args.model_name_or_path,
+        num_labels=num_labels,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+    tokenizer = tokenizer_class.from_pretrained(
+        args.tokenizer_name if args.tokenizer_name else
+            args.model_name_or_path,
+        do_lower_case=args.do_lower_case,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+    model = model_class.from_pretrained(
+        args.model_name_or_path,
+        from_tf=bool(".ckpt" in args.model_name_or_path),
+        config=config,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+
+    model.to(args.device)
+
+    #logger.info("Training/evaluation parameters %s", args)
+
+    # Evaluation
+    results = {}
+    if args.do_test:
+        tokenizer = tokenizer_class.from_pretrained(
+            args.tokenizer_name if args.tokenizer_name else
+                args.model_name_or_path,
+            do_lower_case=args.do_lower_case,
+        )
+        checkpoints = [args.output_dir]
+        #logger.info("Evaluate the following checkpoints: %s", checkpoints)
+        for checkpoint in checkpoints:
+            global_step = checkpoint.split(
+                "-")[-1] if len(checkpoints) > 1 else ""
+            
+            prefix = str(global_step)
+
+            model = model_class.from_pretrained(checkpoint)
+
+            model.to(args.device)
+            result = evaluate(args, model, tokenizer, prefix=prefix)
+            result = dict((k + "_{}".format(global_step), v)
+                          for k, v in result.items())
+            results.update(result)
+    else:  # use currently active model
+        result = evaluate(args, model, tokenizer, prefix="test")
+        #results.update(result)
+    return results
+
+
+# define a new data processor for the SemEval data/SAG task
+
+
+class SemEvalProcessor(DataProcessor):
+    def get_train_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train"
+        )
+
+    def get_dev_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev"
+        )
+
+    def get_test_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "test.tsv")), "test"
+        )
+
+    def get_labels(self):
+        """See base class."""
+        return ["correct", "incorrect", "NONE"]
+
+    def _create_examples(self, lines, set_type):
+        """Creates examples for the test set."""
+        examples = []
+        for (i, line) in enumerate(lines):
+            if i == 0:
+                continue
+            guid = line[0]
+            text_a = line[1]
+            text_b = line[2]
+            label = line[-1]
+            examples.append(
+                InputExample(guid=guid, text_a=text_a,
+                             text_b=text_b, label=label)
+            )
+            
+        return examples
+
+
+# custom metrics for SAG: F1 and Accuracy
+
+
+def my_compute_metrics(eval_task, preds, labels):
+    result = {}
+    if eval_task == "sag":
+        acc = accuracy_score(y_pred=preds, y_true=labels)
+        f1_weighted = f1_score(y_pred=preds, y_true=labels, average="weighted")
+        f1_macro = f1_score(y_pred=preds, y_true=labels, average="macro")
+        result = {"f1-weighted": f1_weighted,
+                  "f1-macro": f1_macro, "accuracy": acc}
+    else:
+        result = compute_metrics(eval_task, preds, labels)
+    return result
+
+
+def print_predictions(args, preds):
+    # generate data set part of output path
+    dir_name = (""
+                )
+
+    # get examples
+    processor = (
+        SemEvalProcessor()
+    )
+
+    examples = (
+        processor.get_test_examples(args.data_dir)
+    )
+
+    with open(args.data_dir + "/" + dir_name + "/predictions.txt", "w", encoding="utf8") as writer:
+        # print("# examples: " + str(len(examples)))
+        # print("# labels: " + str(len(labels)))
+        # print("# preds: " + str(len(preds)))
+
+        writer.write(
+            "question\treferenceAnswer\tstudentAnswer\tsuggested grade\tobserved grade\n")
+        for i in range(len(examples)):
+
+            # iterate over data
+            # print prediction as a text-based label
+            hrpred = "incorrect"
+            if preds[i] == 0:
+                hrpred = "correct"
+
+            # get guid, text, from inputExample
+            writer.write(
+                str(examples[i].guid)
+                + "\t"
+                + examples[i].text_a
+                + "\t"
+                + examples[i].text_b
+                + "\t"
+                + hrpred
+                + "\t"
+                + examples[i].label
+                + "\n"
+            )
+            # else: print("Labels don't match! "+str(i)+": "+str(examples[i].label)+" "+str(labels[i]))
+
+
+if __name__ == "__main__":
+    main()

Source/Skript/english/seb-bert-mnli/config.json

+{
+  "_name_or_path": "textattack/bert-base-uncased-MNLI",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "finetuning_task": "sag-seb",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.2.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

Source/Skript/german/LICENSE

+MIT License
+
+Copyright (c) 2022 Yunus Eryilmaz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.