LLM-ASYST

asyst-backend/app/schemas/__init__.py

+"""Pydantic schemas."""

asyst-backend/app/schemas/grading.py

+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field, SecretStr
+
+
+LLMProfile = Literal["local_gemma", "chat_ai", "openai_custom"]
+
+
+class GradingExampleRequest(BaseModel):
+    question: str
+    answer: str
+    score: str
+
+
+class GradingStartRequest(BaseModel):
+    strategy: str | None = None
+    custom_grading_rules: str | None = None
+    examples: list[GradingExampleRequest] = Field(default_factory=list)
+    include_feedback: bool = False
+    llm_profile: LLMProfile | None = None
+    provider: str | None = None
+    model: str | None = None
+    base_url: str | None = None
+    api_key: SecretStr | None = Field(default=None, repr=False)
+    timeout_seconds: float | None = Field(default=None, gt=0)
+    retry_max_attempts: int | None = Field(default=None, ge=1)
+    retry_initial_seconds: float | None = Field(default=None, ge=0)
+    retry_max_seconds: float | None = Field(default=None, ge=0)
+    max_tokens: int | None = Field(default=None, ge=1)
+    temperature: float | None = Field(default=None, ge=0)

asyst-backend/app/schemas/label_evaluation.py

+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel
+
+
+LabelScaleKey = Literal["full", "whole"]
+
+
+class LabelMetricRow(BaseModel):
+    label: float
+    true_positive: int
+    predicted_count: int
+    support: int
+    precision: float | None
+    recall: float | None
+    low_support: bool
+
+
+class LabelScaleReport(BaseModel):
+    scale: LabelScaleKey
+    labels: list[LabelMetricRow]
+
+
+class LabelQuestionReport(BaseModel):
+    question_id: int
+    question_number: int
+    question_text: str
+    max_points: float
+    total_answer_count: int
+    evaluated_answer_count: int
+    failed_llm_count: int
+    excluded_invalid_count: int
+    full_scale: LabelScaleReport
+    whole_scale: LabelScaleReport | None
+    default_scale: LabelScaleKey
+
+
+class LabelEvaluationWarning(BaseModel):
+    code: str
+    message: str
+    count: int
+    question_id: int | None = None
+
+
+class LabelEvaluationTotals(BaseModel):
+    total_questions: int
+    total_answer_count: int
+    evaluated_answer_count: int
+    failed_llm_count: int
+    excluded_invalid_count: int
+    skipped_question_count: int
+    coverage_percent: float | None
+
+
+class LabelEvaluationResponse(BaseModel):
+    run_id: str
+    filename: str
+    run_type: str
+    low_support_threshold: int
+    questions: list[LabelQuestionReport]
+    totals: LabelEvaluationTotals
+    warnings: list[LabelEvaluationWarning]

asyst-backend/app/schemas/review.py

+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel
+
+
+class AnswerReviewRequest(BaseModel):
+    action: Literal["accept_llm", "override_score"]
+    human_score: float | None = None
+    review_note: str | None = None
+    reviewed_by: str | None = None
+
+
+class AnswerReviewResponse(BaseModel):
+    answer_id: int
+    run_id: str
+    student_id: int
+    question_id: int
+    processing_status: str
+    review_status: str
+    review_required: bool
+    llm_score: float | None
+    human_score: float | None
+    final_score: float | None
+    accepted_llm: bool
+    review_note: str | None
+    reviewed_by: str | None
+    reviewed_at: datetime | None
+    graded_at: datetime | None
+
+
+class RunReviewSummaryResponse(BaseModel):
+    run_id: str
+    filename: str
+    grading_status: str
+    total_students: int
+    total_questions: int
+    total_answers: int
+    total_graded: int
+    total_failed: int
+    total_review_pending: int
+    total_accepted: int
+    total_overridden: int
+    review_completion_percent: float
+    export_ready: bool
+
+
+class RunHistoryItem(BaseModel):
+    run_id: str
+    filename: str
+    run_type: str
+    status: str
+    created_at: datetime
+    total_students: int
+    total_questions: int
+    total_answers: int
+
+
+class RunHistoryResponse(BaseModel):
+    items: list[RunHistoryItem]
+
+
+class PaginationMeta(BaseModel):
+    page: int
+    page_size: int
+    total: int
+
+
+class StudentReviewListItem(BaseModel):
+    student_id: int
+    student_code: str
+    first_name: str
+    last_name: str
+    email: str
+    answered_count: int
+    reviewed_count: int
+    pending_review_count: int
+    total_final_score: float
+    total_max_points: float
+    review_complete: bool
+
+
+class StudentReviewListResponse(BaseModel):
+    run_id: str
+    items: list[StudentReviewListItem]
+    pagination: PaginationMeta
+
+
+class StudentAnswerReviewItem(BaseModel):
+    answer_id: int
+    question_id: int
+    question_number: int
+    question_text: str
+    answer_text: str
+    reference_answer: str | None
+    max_points: float | None
+    llm_score: float | None
+    human_score: float | None
+    final_score: float | None
+    review_status: str
+    accepted_llm: bool
+    reviewed_at: datetime | None
+    review_required: bool
+    processing_status: str
+    llm_feedback: str | None
+
+
+class StudentAnswerReviewResponse(BaseModel):
+    run_id: str
+    student_id: int
+    student_code: str
+    first_name: str
+    last_name: str
+    email: str
+    answers: list[StudentAnswerReviewItem]
+
+
+class PredictedScoreCount(BaseModel):
+    score: float
+    count: int
+
+
+class QuestionReviewListItem(BaseModel):
+    question_id: int
+    question_number: int
+    question_text: str
+    max_points: float | None
+    attempted_count: int
+    reviewed_count: int
+    pending_review_count: int
+    avg_llm_score: float | None
+    avg_final_score: float | None
+    predicted_score_counts: list[PredictedScoreCount]
+
+
+class QuestionReviewListResponse(BaseModel):
+    run_id: str
+    items: list[QuestionReviewListItem]
+
+
+class QuestionAnswerReviewItem(BaseModel):
+    answer_id: int
+    student_id: int
+    student_name: str
+    student_code: str
+    email: str
+    answer_text: str
+    llm_score: float | None
+    observed_score: float | None
+    human_score: float | None
+    final_score: float | None
+    review_status: str
+    review_required: bool
+    accepted_llm: bool
+    reviewed_at: datetime | None
+    processing_status: str
+    llm_feedback: str | None
+
+
+class QuestionAnswerReviewResponse(BaseModel):
+    run_id: str
+    question_id: int
+    question_number: int
+    question_text: str
+    max_points: float | None
+    items: list[QuestionAnswerReviewItem]
+    pagination: PaginationMeta
+
+
+class StudentResultRow(BaseModel):
+    student_id: int
+    name: str
+    email: str
+    total_final_score: float
+    total_max_points: float
+    percentage: float
+    review_complete: bool
+
+
+class StudentResultListResponse(BaseModel):
+    run_id: str
+    items: list[StudentResultRow]
+
+
+class RunDeleteResponse(BaseModel):
+    run_id: str
+    deleted: bool
+    answers_deleted: int
+    students_deleted: int
+    questions_deleted: int
+    grading_run_deleted: int
+    file_deleted: bool
+    file_path: str | None

asyst-backend/app/schemas/upload.py

+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class UploadResponse(BaseModel):
+    message: str
+    original_filename: str
+    stored_filename: str
+    file_type: str
+    relative_path: str
+    storage_directory: str
+    upload_id: str

asyst-backend/app/services/__init__.py

+"""Service layer package."""

asyst-backend/app/services/export_service.py

+from __future__ import annotations
+
+import csv
+import io
+from pathlib import Path
+
+import pandas as pd
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.models.grading import Answer, GradingRun, Question, Student
+from app.services.result_service import ResultError, get_student_result_rows
+
+
+class ExportError(ValueError):
+    pass
+
+
+def build_moodle_export_csv(db: Session, run_id: str) -> str:
+    pending_count = (
+        db.query(Answer)
+        .filter(
+            Answer.run_id == run_id,
+            (Answer.review_status == "pending") | (Answer.processing_status != "graded"),
+        )
+        .count()
+    )
+    if pending_count > 0:
+        raise ExportError(
+            f"Export blocked: {pending_count} answers are still pending review or not graded"
+        )
+
+    try:
+        rows = get_student_result_rows(db, run_id)
+    except ResultError as exc:
+        raise ExportError(str(exc)) from exc
+
+    run = db.query(GradingRun).filter(GradingRun.run_id == run_id).one_or_none()
+    if run is None:
+        raise ExportError("Run Id not found")
+
+    source_df = _read_source_dataframe(run)
+    language = _detect_source_language(source_df)
+    questions = (
+        db.query(Question)
+        .filter(Question.run_id == run_id)
+        .order_by(Question.question_number.asc())
+        .all()
+    )
+    students = (
+        db.query(Student)
+        .filter(Student.run_id == run_id)
+        .order_by(Student.row_index.asc(), Student.id.asc())
+        .all()
+    )
+    answers = (
+        db.query(Answer)
+        .filter(Answer.run_id == run_id)
+        .all()
+    )
+    answer_by_student_question = {(answer.student_id, answer.question_id): answer for answer in answers}
+    totals_by_student_id = {int(row["student_id"]): row for row in rows}
+
+    buffer = io.StringIO()
+    writer = csv.writer(buffer)
+    writer.writerow(_export_headers(language, questions))
+    for student in students:
+        writer.writerow(
+            _export_student_row(
+                language=language,
+                source_df=source_df,
+                student=student,
+                questions=questions,
+                answer_by_student_question=answer_by_student_question,
+                total_final_score=totals_by_student_id.get(student.id, {}).get("total_final_score", 0.0),
+            )
+        )
+    return buffer.getvalue()
+
+
+def _read_source_dataframe(run: GradingRun) -> pd.DataFrame | None:
+    path = _resolve_file_path(run.file_path)
+    if path is None:
+        return None
+
+    try:
+        if run.file_type.lower() == "csv":
+            return pd.read_csv(path)
+        if run.file_type.lower() == "xlsx":
+            return pd.read_excel(path, engine="openpyxl")
+    except Exception:
+        return None
+    return None
+
+
+def _resolve_file_path(file_path: str) -> Path | None:
+    candidate = Path(file_path)
+    if candidate.exists():
+        return candidate
+
+    fallback = settings.upload_dir / candidate.name
+    if fallback.exists():
+        return fallback
+    return None
+
+
+def _detect_source_language(source_df: pd.DataFrame | None) -> str:
+    if source_df is None:
+        return "english"
+    normalized = {_normalize_column_name(column) for column in source_df.columns}
+    if {"nachname", "vorname", "emailadresse"} & normalized or "frage1" in normalized:
+        return "german"
+    return "english"
+
+
+def _export_headers(language: str, questions: list[Question]) -> list[str]:
+    if language == "german":
+        headers = ["Nachname", "Vorname", "E-Mail-Adresse", "Status", "Begonnen", "Beendet", "Dauer"]
+        for question in questions:
+            number = question.question_number
+            headers.extend([f"Frage {number}", f"Antwort {number}", f"Richtige Antwort {number}", f"Punktzahl {number}"])
+        headers.append("Gesamtbewertung")
+        return headers
+
+    headers = ["LastName", "FirstName", "Email-Address", "Status"]
+    for question in questions:
+        number = question.question_number
+        headers.extend([f"Question {number}", f"Answer {number}", f"Reference Answer {number}", f"Score {number}"])
+    headers.append("Overall Grade")
+    return headers
+
+
+def _export_student_row(
+    *,
+    language: str,
+    source_df: pd.DataFrame | None,
+    student: Student,
+    questions: list[Question],
+    answer_by_student_question: dict[tuple[int, int], Answer],
+    total_final_score: object,
+) -> list[object]:
+    if language == "german":
+        row: list[object] = [
+            _source_or_default(source_df, student.row_index, "Nachname", student.last_name),
+            _source_or_default(source_df, student.row_index, "Vorname", student.first_name),
+            _source_or_default(source_df, student.row_index, "E-Mail-Adresse", student.email),
+            _source_or_default(source_df, student.row_index, "Status", student.status or ""),
+            _source_or_default(source_df, student.row_index, "Begonnen", ""),
+            _source_or_default(source_df, student.row_index, "Beendet", ""),
+            _source_or_default(source_df, student.row_index, "Dauer", ""),
+        ]
+    else:
+        row = [
+            _source_or_default(source_df, student.row_index, "LastName", student.last_name),
+            _source_or_default(source_df, student.row_index, "FirstName", student.first_name),
+            _source_or_default(source_df, student.row_index, "Email-Address", student.email),
+            _source_or_default(source_df, student.row_index, "Status", student.status or ""),
+        ]
+
+    for question in questions:
+        answer = answer_by_student_question.get((student.id, question.id))
+        score = answer.final_score if answer and answer.final_score is not None else ""
+        row.extend(
+            [
+                question.question_text,
+                answer.answer_text if answer else "",
+                question.reference_answer or "",
+                score,
+            ]
+        )
+
+    row.append(total_final_score)
+    return row
+
+
+def _source_or_default(
+    source_df: pd.DataFrame | None,
+    row_index: int,
+    column_name: str,
+    default: object,
+) -> object:
+    if source_df is None or row_index >= len(source_df.index):
+        return default
+    source_column = _find_source_column(source_df, column_name)
+    if source_column is None:
+        return default
+    value = source_df.iloc[row_index].get(source_column)
+    if value is None or pd.isna(value):
+        return default
+    return value
+
+
+def _find_source_column(source_df: pd.DataFrame, column_name: str) -> str | None:
+    target = _normalize_column_name(column_name)
+    for candidate in source_df.columns:
+        if _normalize_column_name(candidate) == target:
+            return candidate
+    return None
+
+
+def _normalize_column_name(value: object) -> str:
+    return "".join(ch for ch in str(value).strip().casefold() if ch.isalnum())

asyst-backend/app/services/file_storage.py

+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+from app.core.config import settings
+from app.utils.file_utils import sanitize_filename
+
+
+class FileStorageService:
+    @staticmethod
+    def store_temp_file(original_filename: str, content: bytes, file_extension: str) -> dict[str, str]:
+        settings.upload_dir.mkdir(parents=True, exist_ok=True)
+
+        upload_id = uuid.uuid4().hex
+        sanitized = sanitize_filename(original_filename)
+        stem = Path(sanitized).stem[:80] or "upload"
+        stored_filename = f"{stem}_{upload_id}.{file_extension}"
+
+        destination = settings.upload_dir / stored_filename
+        destination.write_bytes(content)
+
+        return {
+            "message": "File uploaded successfully",
+            "original_filename": original_filename,
+            "stored_filename": stored_filename,
+            "file_type": file_extension,
+            "relative_path": str(Path(settings.upload_dir.name) / stored_filename),
+            "storage_directory": str(settings.upload_dir.resolve()),
+            "upload_id": upload_id,
+        }

asyst-backend/app/services/grading.py

+from __future__ import annotations
+
+from dataclasses import dataclass
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy import func
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.db import SessionLocal
+from app.models.grading import Answer, Question
+from app.services.llm.base import GradeRequest, GradingExample, LLMRuntimeConfig
+from app.services.llm.factory import build_llm_provider
+from app.services.llm.strategies import normalize_strategy_name
+
+STATUS_PENDING = "pending"
+STATUS_PROCESSING = "processing"
+STATUS_COMPLETED = "graded"
+STATUS_FAILED = "failed"
+STATUS_REVIEW_REQUIRED = "review_required"
+
+
+def utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+@dataclass
+class RunJobState:
+    run_id: str
+    strategy: str
+    status: str
+    started_at: str
+    finished_at: str | None = None
+    processed: int = 0
+    completed: int = 0
+    failed: int = 0
+    review_required: int = 0
+    total_to_process: int = 0
+    error: str | None = None
+
+
+@dataclass(frozen=True)
+class PromptRunOptions:
+    custom_grading_rules: str | None = None
+    examples: tuple[GradingExample, ...] = ()
+    include_feedback: bool = False
+
+
+_job_lock = threading.Lock()
+_jobs: dict[str, RunJobState] = {}
+
+
+class SafeRateLimiter:
+    def __init__(self, requests_per_minute: int) -> None:
+        self.min_interval = 60.0 / max(requests_per_minute, 1)
+        self.lock = threading.Lock()
+        self.next_allowed_at = 0.0
+
+    def wait(self) -> None:
+        with self.lock:
+            now = time.time()
+            wait = max(0.0, self.next_allowed_at - now)
+            if wait > 0:
+                time.sleep(wait)
+            self.next_allowed_at = max(self.next_allowed_at, time.time()) + self.min_interval
+
+    def apply_header_backoff(self, headers: dict[str, str]) -> None:
+        retry_after = headers.get("retry-after")
+        if not retry_after:
+            return
+        try:
+            delay = float(retry_after)
+        except ValueError:
+            return
+        with self.lock:
+            self.next_allowed_at = max(self.next_allowed_at, time.time() + max(delay, 0.0))
+
+
+def validate_prompt_run_options(
+    *,
+    strategy: str,
+    custom_grading_rules: str | None,
+    examples: list[Any] | tuple[Any, ...] | None,
+    include_feedback: bool = False,
+) -> PromptRunOptions:
+    normalized_strategy = normalize_strategy_name(strategy)
+    strategy_uses_rules = normalized_strategy in {"p2_reference_discrete", "p3_true_one_shot"}
+    rules = ((custom_grading_rules or "").strip() or None) if strategy_uses_rules else None
+    raw_examples = (examples or []) if normalized_strategy == "p3_true_one_shot" else []
+    parsed_examples = tuple(_coerce_grading_example(example) for example in raw_examples)
+
+    if len(parsed_examples) > 3:
+        raise ValueError("Example-guided grading accepts at most 3 examples")
+    if normalized_strategy == "p3_true_one_shot" and len(parsed_examples) == 0:
+        raise ValueError("Example-guided grading requires at least 1 scored example")
+
+    return PromptRunOptions(
+        custom_grading_rules=rules,
+        examples=parsed_examples,
+        include_feedback=bool(include_feedback),
+    )
+
+
+def _coerce_grading_example(example: Any) -> GradingExample:
+    question = _example_value(example, "question")
+    answer = _example_value(example, "answer")
+    score_text = _example_value(example, "score")
+
+    if not question:
+        raise ValueError("Each grading example needs a question")
+    if not answer:
+        raise ValueError("Each grading example needs a student answer")
+    if not score_text:
+        raise ValueError("Each grading example needs an awarded score")
+
+    try:
+        score = float(score_text.replace(",", "."))
+    except ValueError as exc:
+        raise ValueError("Example score must be a number") from exc
+
+    if score < 0:
+        raise ValueError("Example score must be greater than or equal to 0")
+
+    return GradingExample(question=question, answer=answer, score=score)
+
+
+def _example_value(example: Any, key: str) -> str:
+    if isinstance(example, dict):
+        value = example.get(key)
+    else:
+        value = getattr(example, key, None)
+    return str(value).strip() if value is not None else ""
+
+
+def grade_run(
+    run_id: str,
+    *,
+    limit: int | None = None,
+    strategy: str | None = None,
+    runtime_config: LLMRuntimeConfig | None = None,
+    prompt_options: PromptRunOptions | None = None,
+) -> dict[str, int | str]:
+    selected_strategy = normalize_strategy_name(strategy or settings.llm_prompt_strategy)
+    selected_prompt_options = prompt_options or validate_prompt_run_options(
+        strategy=selected_strategy,
+        custom_grading_rules=None,
+        examples=None,
+        include_feedback=False,
+    )
+    provider = build_llm_provider(selected_strategy, runtime_config=runtime_config)
+    limiter = SafeRateLimiter(settings.llm_requests_per_minute)
+
+    db = SessionLocal()
+    try:
+        query = db.query(Answer).filter(
+            Answer.run_id == run_id,
+            Answer.processing_status.in_([STATUS_PENDING, STATUS_FAILED, STATUS_PROCESSING]),
+            Answer.review_status == STATUS_PENDING,
+        ).order_by(Answer.id.asc())
+
+        if limit is not None:
+            query = query.limit(max(limit, 0))
+
+        answers = query.all()
+        _set_total_to_process(run_id, len(answers))
+
+        processed = 0
+        completed = 0
+        failed = 0
+        review_required = 0
+
+        for answer in answers:
+            processed += 1
+            _mark_processing(db, answer)
+
+            question = db.query(Question).filter(Question.id == answer.question_id).one()
+            request = GradeRequest(
+                question_text=question.question_text,
+                reference_answer=question.reference_answer,
+                student_answer=answer.answer_text,
+                max_points=question.max_points,
+                custom_grading_rules=selected_prompt_options.custom_grading_rules,
+                examples=selected_prompt_options.examples,
+                include_feedback=selected_prompt_options.include_feedback,
+            )
+
+            limiter.wait()
+            try:
+                result = provider.grade_answer(request)
+                limiter.apply_header_backoff(result.headers)
+                _apply_grade(
+                    db,
+                    answer,
+                    result.score,
+                    question.max_points,
+                    feedback=result.feedback,
+                    raw_response=result.raw_response,
+                    parser_warning=result.parser_warning,
+                )
+                if answer.processing_status == STATUS_REVIEW_REQUIRED:
+                    review_required += 1
+                else:
+                    completed += 1
+            except Exception as exc:
+                _mark_failed(db, answer, _safe_error_message(exc))
+                failed += 1
+
+            _update_progress(
+                run_id,
+                processed=processed,
+                completed=completed,
+                failed=failed,
+                review_required=review_required,
+            )
+
+        return {
+            "run_id": run_id,
+            "strategy": selected_strategy,
+            "processed": processed,
+            "completed": completed,
+            "failed": failed,
+            "review_required": review_required,
+        }
+    finally:
+        db.close()
+
+
+def _mark_processing(db: Session, answer: Answer) -> None:
+    answer.processing_status = STATUS_PROCESSING
+    answer.attempt_count += 1
+    answer.last_error = None
+    db.commit()
+
+
+def _apply_grade(
+    db: Session,
+    answer: Answer,
+    score: float | None,
+    max_points: float | None,
+    *,
+    feedback: str | None = None,
+    raw_response: str | None = None,
+    parser_warning: str | None = None,
+) -> None:
+    if score is None:
+        raise ValueError("Model did not return a numeric score")
+
+    if max_points is not None and (score < 0 or score > max_points):
+        raise ValueError(f"Model score {score} is out of range [0, {max_points}]")
+
+    answer.llm_score = score
+    answer.llm_feedback = feedback
+    answer.llm_raw_response = raw_response
+    answer.llm_parse_warning = parser_warning
+    answer.human_score = None
+    answer.final_score = None
+    answer.accepted_llm = False
+    answer.review_status = STATUS_PENDING
+    answer.reviewed_by = None
+    answer.review_note = None
+    answer.reviewed_at = None
+    answer.graded_at = utc_now()
+    answer.review_required = True
+    answer.processing_status = STATUS_COMPLETED
+    db.commit()
+
+
+def _mark_failed(db: Session, answer: Answer, message: str) -> None:
+    answer.processing_status = STATUS_FAILED
+    answer.last_error = message[:2000]
+    db.commit()
+
+
+def _safe_error_message(exc: Exception) -> str:
+    message = str(exc).strip() or exc.__class__.__name__
+    cause = exc.__cause__ or exc.__context__
+    if cause is not None:
+        cause_message = str(cause).strip() or cause.__class__.__name__
+        if cause_message and cause_message not in message:
+            message = f"{message} ({cause.__class__.__name__}: {cause_message})"
+    return message
+
+
+def start_grade_run(
+    run_id: str,
+    *,
+    limit: int | None = None,
+    strategy: str | None = None,
+    runtime_config: LLMRuntimeConfig | None = None,
+    custom_grading_rules: str | None = None,
+    examples: list[Any] | None = None,
+    include_feedback: bool = False,
+) -> dict[str, str]:
+    selected_strategy = normalize_strategy_name(strategy or settings.llm_prompt_strategy)
+    prompt_options = validate_prompt_run_options(
+        strategy=selected_strategy,
+        custom_grading_rules=custom_grading_rules,
+        examples=examples,
+        include_feedback=include_feedback,
+    )
+    with _job_lock:
+        existing = _jobs.get(run_id)
+        if existing and existing.status == "running":
+            return {
+                "run_id": run_id,
+                "status": "already_running",
+                "strategy": existing.strategy,
+            }
+
+        _jobs[run_id] = RunJobState(
+            run_id=run_id,
+            strategy=selected_strategy,
+            status="running",
+            started_at=utc_now().isoformat(),
+        )
+
+    worker = threading.Thread(
+        target=_run_grading_worker,
+        kwargs={
+            "run_id": run_id,
+            "limit": limit,
+            "strategy": selected_strategy,
+            "runtime_config": runtime_config,
+            "prompt_options": prompt_options,
+        },
+        daemon=True,
+        name=f"grading-{run_id}",
+    )
+    worker.start()
+    return {"run_id": run_id, "status": "grading_started", "strategy": selected_strategy}
+
+
+def get_grade_run_status(run_id: str) -> dict[str, int | str | None]:
+    counts = _fetch_db_status_counts(run_id)
+    with _job_lock:
+        state = _jobs.get(run_id)
+
+    if state:
+        return {
+            "run_id": run_id,
+            "strategy": state.strategy,
+            "job_status": state.status,
+            "started_at": state.started_at,
+            "finished_at": state.finished_at,
+            "processed": state.processed,
+            "completed": state.completed,
+            "failed": state.failed,
+            "review_required": state.review_required,
+            "total_to_process": state.total_to_process,
+            "error": state.error,
+            "db_pending": counts.get(STATUS_PENDING, 0),
+            "db_processing": counts.get(STATUS_PROCESSING, 0),
+            "db_completed": counts.get(STATUS_COMPLETED, 0),
+            "db_failed": counts.get(STATUS_FAILED, 0),
+            "db_review_required": counts.get(STATUS_REVIEW_REQUIRED, 0),
+            "db_total": sum(counts.values()),
+        }
+
+    return {
+        "run_id": run_id,
+        "strategy": None,
+        "job_status": "not_started",
+        "started_at": None,
+        "finished_at": None,
+        "processed": 0,
+        "completed": 0,
+        "failed": 0,
+        "review_required": 0,
+        "total_to_process": 0,
+        "error": None,
+        "db_pending": counts.get(STATUS_PENDING, 0),
+        "db_processing": counts.get(STATUS_PROCESSING, 0),
+        "db_completed": counts.get(STATUS_COMPLETED, 0),
+        "db_failed": counts.get(STATUS_FAILED, 0),
+        "db_review_required": counts.get(STATUS_REVIEW_REQUIRED, 0),
+        "db_total": sum(counts.values()),
+    }
+
+
+def _run_grading_worker(
+    run_id: str,
+    limit: int | None,
+    strategy: str,
+    runtime_config: LLMRuntimeConfig | None,
+    prompt_options: PromptRunOptions,
+) -> None:
+    try:
+        result = grade_run(
+            run_id,
+            limit=limit,
+            strategy=strategy,
+            runtime_config=runtime_config,
+            prompt_options=prompt_options,
+        )
+        with _job_lock:
+            state = _jobs.get(run_id)
+            if state:
+                state.status = "completed"
+                state.finished_at = utc_now().isoformat()
+                state.processed = int(result.get("processed", 0))
+                state.completed = int(result.get("completed", 0))
+                state.failed = int(result.get("failed", 0))
+                state.review_required = int(result.get("review_required", 0))
+    except Exception as exc:  # pragma: no cover
+        with _job_lock:
+            state = _jobs.get(run_id)
+            if state:
+                state.status = "failed"
+                state.error = str(exc)
+                state.finished_at = utc_now().isoformat()
+
+
+def _set_total_to_process(run_id: str, total: int) -> None:
+    with _job_lock:
+        state = _jobs.get(run_id)
+        if state:
+            state.total_to_process = total
+
+
+def _update_progress(run_id: str, *, processed: int, completed: int, failed: int, review_required: int) -> None:
+    with _job_lock:
+        state = _jobs.get(run_id)
+        if state:
+            state.processed = processed
+            state.completed = completed
+            state.failed = failed
+            state.review_required = review_required
+
+
+def _fetch_db_status_counts(run_id: str) -> dict[str, int]:
+    db = SessionLocal()
+    try:
+        rows = (
+            db.query(Answer.processing_status, func.count(Answer.id))
+            .filter(Answer.run_id == run_id)
+            .group_by(Answer.processing_status)
+            .all()
+        )
+        return {status: int(count) for status, count in rows}
+    finally:
+        db.close()
+
+
+def is_grade_run_running(run_id: str) -> bool:
+    with _job_lock:
+        state = _jobs.get(run_id)
+        return bool(state and state.status == "running")

asyst-backend/app/services/label_evaluation_service.py

+from __future__ import annotations
+
+from collections import defaultdict
+from math import floor, isfinite
+
+from sqlalchemy.orm import Session
+
+from app.models.grading import Answer, GradingRun, Question
+
+GRADE_TOLERANCE = 0.001
+LOW_SUPPORT_THRESHOLD = 5
+
+
+class LabelEvaluationError(ValueError):
+    pass
+
+
+def get_label_evaluation(db: Session, run_id: str) -> dict[str, object]:
+    run = db.query(GradingRun).filter(GradingRun.run_id == run_id).one_or_none()
+    if run is None:
+        raise LabelEvaluationError("Run not found")
+
+    query_rows = (
+        db.query(Answer, Question)
+        .join(Question, Question.id == Answer.question_id)
+        .filter(Answer.run_id == run_id)
+        .order_by(Question.question_number, Answer.id)
+        .all()
+    )
+    grouped: dict[int, list[Answer]] = defaultdict(list)
+    questions: dict[int, Question] = {}
+    for answer, question in query_rows:
+        grouped[question.id].append(answer)
+        questions[question.id] = question
+
+    reports: list[dict[str, object]] = []
+    warnings: list[dict[str, object]] = []
+    failed_total = 0
+    invalid_total = 0
+    skipped_questions = 0
+
+    for question in sorted(questions.values(), key=lambda item: item.question_number):
+        answers = grouped[question.id]
+        max_points = _finite_float(question.max_points)
+        if max_points is None or max_points <= 0 or not grade_on_half_step(max_points):
+            skipped_questions += 1
+            invalid_total += len(answers)
+            warnings.append(
+                {
+                    "code": "INVALID_MAX_POINTS",
+                    "message": f"Question {question.question_number} was skipped because its maximum points must be a positive 0.5-step value.",
+                    "count": len(answers),
+                    "question_id": question.id,
+                }
+            )
+            continue
+
+        valid_rows: list[tuple[float, float]] = []
+        failed_count = 0
+        invalid_count = 0
+        for answer in answers:
+            professor_grade = _finite_float(answer.observed_score)
+            if (
+                professor_grade is None
+                or professor_grade < 0
+                or professor_grade > max_points + GRADE_TOLERANCE
+                or not grade_on_half_step(professor_grade)
+            ):
+                invalid_count += 1
+                continue
+
+            llm_grade = _finite_float(answer.llm_score)
+            if (
+                answer.processing_status != "graded"
+                or llm_grade is None
+                or llm_grade < 0
+                or llm_grade > max_points + GRADE_TOLERANCE
+                or not grade_on_half_step(llm_grade)
+            ):
+                failed_count += 1
+                continue
+            valid_rows.append((professor_grade, llm_grade))
+
+        failed_total += failed_count
+        invalid_total += invalid_count
+        if failed_count:
+            warnings.append(
+                {
+                    "code": "FAILED_LLM_ROWS",
+                    "message": f"{failed_count} answer(s) for Question {question.question_number} had no valid AI grade and were excluded.",
+                    "count": failed_count,
+                    "question_id": question.id,
+                }
+            )
+        if invalid_count:
+            warnings.append(
+                {
+                    "code": "INVALID_GRADE_ROWS",
+                    "message": f"{invalid_count} answer(s) for Question {question.question_number} had invalid professor grades and were excluded.",
+                    "count": invalid_count,
+                    "question_id": question.id,
+                }
+            )
+        if not valid_rows:
+            warnings.append(
+                {
+                    "code": "EMPTY_QUESTION",
+                    "message": f"Question {question.question_number} has no valid answer pairs to evaluate.",
+                    "count": len(answers),
+                    "question_id": question.id,
+                }
+            )
+            continue
+
+        full_scale = build_label_scale(valid_rows, "full")
+        whole_scale = None
+        if grades_equal(max_points, round(max_points)) and max_points >= 2:
+            whole_rows = [
+                (round_half_up_grade(professor), round_half_up_grade(llm))
+                for professor, llm in valid_rows
+            ]
+            whole_scale = build_label_scale(whole_rows, "whole")
+
+        possible_full_label_count = int(round(max_points * 2)) + 1
+        default_scale = "whole" if whole_scale is not None and possible_full_label_count > 6 else "full"
+        reports.append(
+            {
+                "question_id": question.id,
+                "question_number": question.question_number,
+                "question_text": question.question_text,
+                "max_points": max_points,
+                "total_answer_count": len(answers),
+                "evaluated_answer_count": len(valid_rows),
+                "failed_llm_count": failed_count,
+                "excluded_invalid_count": invalid_count,
+                "full_scale": full_scale,
+                "whole_scale": whole_scale,
+                "default_scale": default_scale,
+            }
+        )
+
+    evaluated_total = sum(int(report["evaluated_answer_count"]) for report in reports)
+    total_answers = len(query_rows)
+    return {
+        "run_id": run.run_id,
+        "filename": run.filename,
+        "run_type": run.run_type,
+        "low_support_threshold": LOW_SUPPORT_THRESHOLD,
+        "questions": reports,
+        "totals": {
+            "total_questions": len(reports),
+            "total_answer_count": total_answers,
+            "evaluated_answer_count": evaluated_total,
+            "failed_llm_count": failed_total,
+            "excluded_invalid_count": invalid_total,
+            "skipped_question_count": skipped_questions,
+            "coverage_percent": _percent(evaluated_total, total_answers),
+        },
+        "warnings": warnings,
+    }
+
+
+def build_label_scale(rows: list[tuple[float, float]], scale: str) -> dict[str, object]:
+    labels = sorted({value for professor, llm in rows for value in (professor, llm)})
+    metrics: list[dict[str, object]] = []
+    for label in labels:
+        true_positive = sum(
+            1 for professor, llm in rows if grades_equal(professor, label) and grades_equal(llm, label)
+        )
+        predicted_count = sum(1 for _, llm in rows if grades_equal(llm, label))
+        support = sum(1 for professor, _ in rows if grades_equal(professor, label))
+        metrics.append(
+            {
+                "label": label,
+                "true_positive": true_positive,
+                "predicted_count": predicted_count,
+                "support": support,
+                "precision": _ratio(true_positive, predicted_count),
+                "recall": _ratio(true_positive, support),
+                "low_support": support < LOW_SUPPORT_THRESHOLD,
+            }
+        )
+    return {"scale": scale, "labels": metrics}
+
+
+def round_half_up_grade(value: float) -> float:
+    return float(floor(value + 0.5))
+
+
+def grades_equal(left: float, right: float) -> bool:
+    return abs(left - right) < GRADE_TOLERANCE
+
+
+def grade_on_half_step(value: float) -> bool:
+    return abs(value * 2 - round(value * 2)) < GRADE_TOLERANCE
+
+
+def _finite_float(value: object) -> float | None:
+    if value is None:
+        return None
+    try:
+        result = float(value)
+    except (TypeError, ValueError):
+        return None
+    return result if isfinite(result) else None
+
+
+def _ratio(numerator: int, denominator: int) -> float | None:
+    if denominator == 0:
+        return None
+    return round(numerator / denominator, 4)
+
+
+def _percent(numerator: int, denominator: int) -> float | None:
+    if denominator == 0:
+        return None
+    return round((numerator / denominator) * 100.0, 2)

asyst-backend/app/services/llm/__init__.py

+"""LLM provider abstractions and implementations."""

asyst-backend/app/services/llm/base.py

+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+ChatMessage = dict[str, str]
+
+
+@dataclass(frozen=True)
+class GradingExample:
+    question: str
+    answer: str
+    score: float
+
+
+@dataclass
+class GradeRequest:
+    question_text: str
+    reference_answer: str | None
+    student_answer: str
+    max_points: float | None
+    custom_grading_rules: str | None = None
+    examples: tuple[GradingExample, ...] = ()
+    include_feedback: bool = False
+
+
+@dataclass
+class GradeResult:
+    score: float | None
+    raw_response: str
+    headers: dict[str, str]
+    feedback: str | None = None
+    parser_warning: str | None = None
+
+
+@dataclass(frozen=True)
+class LLMRuntimeConfig:
+    provider: str | None = None
+    model: str | None = None
+    base_url: str | None = None
+    api_key: str | None = None
+    timeout_seconds: float | None = None
+    retry_max_attempts: int | None = None
+    retry_initial_seconds: float | None = None
+    retry_max_seconds: float | None = None
+    max_tokens: int | None = None
+    temperature: float | None = None
+
+
+class LLMProvider(Protocol):
+    def grade_answer(self, request: GradeRequest) -> GradeResult:
+        ...

asyst-backend/app/services/llm/factory.py

+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from app.core.config import settings
+from app.services.llm.base import LLMProvider, LLMRuntimeConfig
+from app.services.llm.providers.openai_compatible import OpenAICompatibleProvider
+from app.services.llm.strategies import get_prompt_strategy
+
+_OPENAI_COMPATIBLE_PROVIDERS = {
+    "openai_compatible",
+    "chat_ai",
+    "openai",
+    "local",
+    "local_llm",
+    "llama_cpp",
+}
+
+
+def build_llm_provider(
+    strategy_name: str | None = None,
+    runtime_config: LLMRuntimeConfig | None = None,
+) -> LLMProvider:
+    backend_root = Path(__file__).resolve().parents[3]
+    load_dotenv(backend_root / ".env", override=True)
+
+    selected_strategy = strategy_name or os.getenv("LLM_PROMPT_STRATEGY") or settings.llm_prompt_strategy
+    prompt_strategy = get_prompt_strategy(selected_strategy)
+
+    base_url = _pick("LLM_BASE_URL", settings.llm_base_url, runtime_config.base_url if runtime_config else None)
+    provider_name = _pick("LLM_PROVIDER", settings.llm_provider, runtime_config.provider if runtime_config else None)
+    normalized_provider = provider_name.strip().lower()
+    api_key = _pick_api_key(
+        normalized_provider,
+        base_url,
+        runtime_config.api_key if runtime_config else None,
+    )
+    model = _pick("LLM_MODEL", settings.llm_model, runtime_config.model if runtime_config else None)
+    timeout_seconds = _pick_float(
+        "LLM_TIMEOUT_SECONDS",
+        settings.llm_timeout_seconds,
+        runtime_config.timeout_seconds if runtime_config else None,
+    )
+    retry_max_attempts = _pick_int(
+        "LLM_RETRY_MAX_ATTEMPTS",
+        settings.llm_retry_max_attempts,
+        runtime_config.retry_max_attempts if runtime_config else None,
+    )
+    retry_initial_seconds = _pick_float(
+        "LLM_RETRY_INITIAL_SECONDS",
+        settings.llm_retry_initial_seconds,
+        runtime_config.retry_initial_seconds if runtime_config else None,
+    )
+    retry_max_seconds = _pick_float(
+        "LLM_RETRY_MAX_SECONDS",
+        settings.llm_retry_max_seconds,
+        runtime_config.retry_max_seconds if runtime_config else None,
+    )
+    max_tokens = _pick_int("LLM_MAX_TOKENS", settings.llm_max_tokens, runtime_config.max_tokens if runtime_config else None)
+    temperature = _pick_float(
+        "LLM_TEMPERATURE",
+        settings.llm_temperature,
+        runtime_config.temperature if runtime_config else None,
+    )
+
+    if normalized_provider not in _OPENAI_COMPATIBLE_PROVIDERS:
+        raise ValueError(f"Unsupported LLM provider: {provider_name}")
+
+    if not api_key and _requires_api_key(normalized_provider, base_url):
+        raise ValueError("Missing LLM_API_KEY/API_KEY for selected LLM provider")
+
+    return OpenAICompatibleProvider(
+        api_key=api_key or "local-not-required",
+        base_url=base_url,
+        model=model,
+        timeout_seconds=timeout_seconds,
+        retry_max_attempts=retry_max_attempts,
+        retry_initial_seconds=retry_initial_seconds,
+        retry_max_seconds=retry_max_seconds,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        prompt_strategy=prompt_strategy,
+    )
+
+
+def _pick(env_name: str, default: str | None, runtime_value: str | None) -> str:
+    if runtime_value is not None and runtime_value.strip():
+        return runtime_value.strip()
+    env_value = os.getenv(env_name)
+    if env_value is not None and env_value.strip():
+        return env_value.strip()
+    return (default or "").strip()
+
+
+def _pick_api_key(provider_name: str, base_url: str, runtime_value: str | None) -> str:
+    if runtime_value is not None and runtime_value.strip():
+        return runtime_value.strip()
+
+    if provider_name == "chat_ai":
+        return (os.getenv("CHAT_AI_API_KEY") or os.getenv("API_KEY") or "").strip()
+
+    if provider_name in {"local", "local_llm", "llama_cpp"} or not _requires_api_key(provider_name, base_url):
+        return (os.getenv("LOCAL_LLM_API_KEY") or "local-not-required").strip()
+
+    return (
+        os.getenv("OPENAI_API_KEY")
+        or os.getenv("LLM_API_KEY")
+        or os.getenv("API_KEY")
+        or settings.llm_api_key
+        or ""
+    ).strip()
+
+
+def _pick_int(env_name: str, default: int, runtime_value: int | None) -> int:
+    if runtime_value is not None:
+        return int(runtime_value)
+    env_value = os.getenv(env_name)
+    if env_value is not None and env_value.strip():
+        return int(env_value)
+    return int(default)
+
+
+def _pick_float(env_name: str, default: float, runtime_value: float | None) -> float:
+    if runtime_value is not None:
+        return float(runtime_value)
+    env_value = os.getenv(env_name)
+    if env_value is not None and env_value.strip():
+        return float(env_value)
+    return float(default)
+
+
+def _requires_api_key(provider_name: str, base_url: str) -> bool:
+    if provider_name in {"local", "local_llm", "llama_cpp"}:
+        return False
+    return "localhost" not in base_url and "127.0.0.1" not in base_url and "local-llm" not in base_url

asyst-backend/app/services/llm/profiles.py

+from __future__ import annotations
+
+import os
+
+from app.services.llm.base import LLMRuntimeConfig
+
+
+def build_runtime_config_for_profile(
+    *,
+    llm_profile: str | None,
+    provider: str | None = None,
+    model: str | None = None,
+    base_url: str | None = None,
+    api_key: str | None = None,
+    timeout_seconds: float | None = None,
+    retry_max_attempts: int | None = None,
+    retry_initial_seconds: float | None = None,
+    retry_max_seconds: float | None = None,
+    max_tokens: int | None = None,
+    temperature: float | None = None,
+) -> LLMRuntimeConfig | None:
+    if llm_profile is None:
+        if not any(
+            [
+                provider,
+                model,
+                base_url,
+                api_key,
+                timeout_seconds,
+                retry_max_attempts,
+                retry_initial_seconds,
+                retry_max_seconds,
+                max_tokens,
+                temperature,
+            ]
+        ):
+            return None
+        return LLMRuntimeConfig(
+            provider=provider,
+            model=model,
+            base_url=base_url,
+            api_key=api_key,
+            timeout_seconds=timeout_seconds,
+            retry_max_attempts=retry_max_attempts,
+            retry_initial_seconds=retry_initial_seconds,
+            retry_max_seconds=retry_max_seconds,
+            max_tokens=max_tokens,
+            temperature=temperature,
+        )
+
+    normalized = llm_profile.strip().lower()
+    if normalized == "local_gemma":
+        return LLMRuntimeConfig(
+            provider="local_llm",
+            model=os.getenv("LOCAL_LLM_MODEL", "gemma-4-E4B-it-GGUF"),
+            base_url=os.getenv("LOCAL_LLM_BASE_URL", "http://127.0.0.1:8080/v1"),
+            api_key=os.getenv("LOCAL_LLM_API_KEY", "local-not-required"),
+            timeout_seconds=timeout_seconds,
+            retry_max_attempts=retry_max_attempts,
+            retry_initial_seconds=retry_initial_seconds,
+            retry_max_seconds=retry_max_seconds,
+            max_tokens=max_tokens,
+            temperature=temperature,
+        )
+
+    if normalized == "chat_ai":
+        return LLMRuntimeConfig(
+            provider="chat_ai",
+            model=model or os.getenv("CHAT_AI_MODEL", "gemma-4-31b-it"),
+            base_url=os.getenv("CHAT_AI_BASE_URL", "https://chat-ai.academiccloud.de/v1"),
+            api_key=os.getenv("CHAT_AI_API_KEY") or os.getenv("API_KEY"),
+            timeout_seconds=timeout_seconds,
+            retry_max_attempts=retry_max_attempts,
+            retry_initial_seconds=retry_initial_seconds,
+            retry_max_seconds=retry_max_seconds,
+            max_tokens=max_tokens,
+            temperature=temperature,
+        )
+
+    if normalized == "openai_custom":
+        return LLMRuntimeConfig(
+            provider="openai",
+            model=model,
+            base_url=base_url or "https://api.openai.com/v1",
+            api_key=api_key,
+            timeout_seconds=timeout_seconds,
+            retry_max_attempts=retry_max_attempts,
+            retry_initial_seconds=retry_initial_seconds,
+            retry_max_seconds=retry_max_seconds,
+            max_tokens=max_tokens,
+            temperature=temperature,
+        )
+
+    raise ValueError(f"Unsupported LLM profile: {llm_profile}")

asyst-backend/app/services/llm/providers/__init__.py

+"""Concrete LLM provider adapters."""

asyst-backend/app/services/llm/providers/openai_compatible.py

+from __future__ import annotations
+
+import time
+
+from openai import OpenAI
+
+from app.services.llm.base import GradeRequest, GradeResult
+from app.services.llm.score_parser import parse_grade_response
+from app.services.llm.strategies.base import PromptStrategy
+
+
+class OpenAICompatibleProvider:
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str,
+        model: str,
+        timeout_seconds: float,
+        retry_max_attempts: int,
+        retry_initial_seconds: float,
+        retry_max_seconds: float,
+        max_tokens: int,
+        temperature: float,
+        prompt_strategy: PromptStrategy,
+    ) -> None:
+        self.model = model
+        self.retry_max_attempts = max(1, retry_max_attempts)
+        self.retry_initial_seconds = max(0.0, retry_initial_seconds)
+        self.retry_max_seconds = max(0.0, retry_max_seconds)
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+        self.prompt_strategy = prompt_strategy
+        self.client = OpenAI(
+            api_key=api_key,
+            base_url=base_url,
+            timeout=timeout_seconds,
+            max_retries=0,
+        )
+
+    def grade_answer(self, request: GradeRequest) -> GradeResult:
+        raw = self._create_chat_completion_with_retry(
+            {
+                "model": self.model,
+                "temperature": self.temperature,
+                "max_tokens": self.max_tokens,
+                "messages": self.prompt_strategy.build_messages(request),
+            }
+        )
+
+        completion = raw.parse()
+        content = self._extract_content(completion)
+        max_points = request.max_points if request.max_points is not None else 5.0
+        parsed = parse_grade_response(content, max_points=max_points, snap=True)
+
+        return GradeResult(
+            score=parsed.score,
+            raw_response=content,
+            headers={k.lower(): v for k, v in raw.headers.items()},
+            feedback=parsed.feedback,
+            parser_warning=parsed.warning,
+        )
+
+    def _create_chat_completion_with_retry(self, payload: dict[str, object]) -> object:
+        last_exc: Exception | None = None
+        for attempt in range(1, self.retry_max_attempts + 1):
+            try:
+                return self.client.chat.completions.with_raw_response.create(**payload)
+            except Exception as exc:
+                last_exc = exc
+                if attempt >= self.retry_max_attempts or not self._is_retryable_exception(exc):
+                    raise
+                time.sleep(self._retry_delay_seconds(attempt))
+
+        if last_exc is not None:
+            raise last_exc
+        raise RuntimeError("LLM request failed before it was sent")
+
+    def _retry_delay_seconds(self, attempt: int) -> float:
+        if self.retry_initial_seconds <= 0:
+            return 0.0
+        delay = self.retry_initial_seconds * (2 ** max(attempt - 1, 0))
+        if self.retry_max_seconds > 0:
+            delay = min(delay, self.retry_max_seconds)
+        return delay
+
+    @staticmethod
+    def _is_retryable_exception(exc: Exception) -> bool:
+        status_code = getattr(exc, "status_code", None)
+        if status_code in {408, 409, 425, 429, 500, 502, 503, 504}:
+            return True
+
+        exc_name = exc.__class__.__name__.lower()
+        return any(part in exc_name for part in ("connection", "timeout"))
+
+    @staticmethod
+    def _extract_content(completion: object) -> str:
+        choices = getattr(completion, "choices", None)
+        if not choices:
+            return ""
+
+        message = getattr(choices[0], "message", None)
+        if message is None:
+            return ""
+
+        content = getattr(message, "content", "")
+        if content is None:
+            return ""
+        if isinstance(content, str):
+            return content
+
+        if isinstance(content, list):
+            parts: list[str] = []
+            for item in content:
+                if isinstance(item, dict):
+                    text_value = item.get("text")
+                else:
+                    text_value = getattr(item, "text", None)
+                if text_value:
+                    parts.append(str(text_value))
+            return "\n".join(parts)
+
+        return str(content)

asyst-backend/app/services/llm/score_parser.py

+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+import re
+
+from app.services.llm.strategies.defaults import valid_scores
+
+MAX_FEEDBACK_CHARS = 280
+
+
+@dataclass(frozen=True)
+class ParsedGradeResponse:
+    score: float | None
+    feedback: str | None
+    warning: str | None
+
+
+def parse_score(response: str | None, max_points: float, snap: bool = True) -> tuple[float | None, str | None]:
+    parsed = parse_grade_response(response, max_points=max_points, snap=snap)
+    return parsed.score, parsed.warning
+
+
+def parse_grade_response(response: str | None, max_points: float, snap: bool = True) -> ParsedGradeResponse:
+    if not response:
+        return ParsedGradeResponse(score=None, feedback=None, warning="empty_response")
+
+    raw_text = _strip_markdown_json_fence(response.strip())
+
+    if raw_text.startswith("{"):
+        try:
+            payload = json.loads(raw_text)
+            score_value = payload.get("score")
+            if isinstance(score_value, str):
+                score = float(score_value.strip().replace(",", "."))
+            else:
+                score = float(score_value)
+            validated_score, warning = _validate_score(score, max_points, snap=snap)
+            return ParsedGradeResponse(
+                score=validated_score,
+                feedback=_clean_feedback(payload.get("feedback")),
+                warning=warning,
+            )
+        except Exception as exc:
+            return ParsedGradeResponse(score=None, feedback=None, warning=f"json_parse_error: {exc}")
+
+    text = raw_text.replace(",", ".")
+    match = re.fullmatch(r"\s*(-?\d+(?:\.\d+)?)\s*", text)
+    if not match:
+        match = re.search(r"-?\d+(?:\.\d+)?", text)
+        if not match:
+            return ParsedGradeResponse(score=None, feedback=None, warning="no_numeric_score_found")
+
+    try:
+        score = float(match.group(0))
+    except ValueError:
+        return ParsedGradeResponse(score=None, feedback=None, warning="invalid_numeric_score")
+
+    validated_score, warning = _validate_score(score, max_points, snap=snap)
+    return ParsedGradeResponse(score=validated_score, feedback=None, warning=warning)
+
+
+def _validate_score(score: float, max_points: float, snap: bool = True) -> tuple[float | None, str | None]:
+    if score < 0 or score > max_points:
+        score = max(0.0, min(score, max_points))
+
+    allowed = valid_scores(max_points)
+    if score in allowed:
+        return round(score, 2), None
+
+    nearest = min(allowed, key=lambda allowed_score: abs(allowed_score - score))
+    if snap:
+        return round(nearest, 2), f"snapped_from_{score}"
+    return None, f"score_not_on_valid_grid: {score}"
+
+
+def _clean_feedback(value: object) -> str | None:
+    if value is None:
+        return None
+    feedback = str(value).strip()
+    if not feedback:
+        return None
+    feedback = re.sub(r"\s+", " ", feedback)
+    return feedback[:MAX_FEEDBACK_CHARS]
+
+
+def _strip_markdown_json_fence(text: str) -> str:
+    fenced = re.fullmatch(r"```(?:json)?\s*(.*?)\s*```", text, flags=re.IGNORECASE | re.DOTALL)
+    if fenced:
+        return fenced.group(1).strip()
+    return text

asyst-backend/app/services/llm/strategies/__init__.py

+"""Prompt strategy implementations for grading prompts."""
+
+from app.services.llm.strategies.base import PromptStrategy
+from app.services.llm.strategies.defaults import get_prompt_strategy, normalize_strategy_name
+
+__all__ = ["PromptStrategy", "get_prompt_strategy", "normalize_strategy_name"]

asyst-backend/app/services/llm/strategies/base.py

+from __future__ import annotations
+
+from typing import Protocol
+
+from app.services.llm.base import ChatMessage, GradeRequest
+
+
+class PromptStrategy(Protocol):
+    def build_messages(self, request: GradeRequest) -> list[ChatMessage]:
+        ...

asyst-backend/app/services/llm/strategies/defaults.py

+from __future__ import annotations
+
+from app.services.llm.base import ChatMessage, GradeRequest
+from app.services.llm.strategies.base import PromptStrategy
+
+P0_INSTRUCTION_ONLY = "p0_instruction_only"
+P1_REFERENCE_ZERO_SHOT = "p1_reference_zero_shot"
+P2_REFERENCE_DISCRETE = "p2_reference_discrete"
+P3_TRUE_ONE_SHOT = "p3_true_one_shot"
+P0_INSTRUCTION_ONLY_WITH_FEEDBACK = "p0_instruction_only_with_feedback"
+P1_REFERENCE_ZERO_SHOT_WITH_FEEDBACK = "p1_reference_zero_shot_with_feedback"
+P2_REFERENCE_DISCRETE_WITH_FEEDBACK = "p2_reference_discrete_with_feedback"
+P3_TRUE_ONE_SHOT_WITH_FEEDBACK = "p3_true_one_shot_with_feedback"
+
+STRATEGY_ALIASES = {
+    "zero_shot": P0_INSTRUCTION_ONLY,
+    "one_shot": P1_REFERENCE_ZERO_SHOT,
+    "reference_rules": P2_REFERENCE_DISCRETE,
+    "example_guided": P3_TRUE_ONE_SHOT,
+}
+
+
+def normalize_strategy_name(name: str) -> str:
+    normalized = name.strip().lower()
+    return STRATEGY_ALIASES.get(normalized, normalized)
+
+
+def valid_scores(max_points: float, step: float = 0.5) -> list[float]:
+    count = int(round(float(max_points) / step))
+    return [round(i * step, 2) for i in range(count + 1)]
+
+
+def valid_scores_text(max_points: float) -> str:
+    return ", ".join(f"{score:.1f}" for score in valid_scores(max_points))
+
+
+def _base_system() -> str:
+    return (
+        "Du bist ein erfahrener Korrektor fuer kurze studentische Antworten in der Informatik.\n"
+        "Bewerte ausschliesslich die fachliche Korrektheit.\n"
+        "Ignoriere Rechtschreibung, Grammatik, Stil und Zeichensetzung.\n"
+        "Bewerte nur belegte fachliche Inhalte.\n"
+        "Vergib keine Punkte fuer Annahmen, die nicht in der Antwort stehen."
+    )
+
+
+def _max_points_or_default(request: GradeRequest) -> float:
+    return request.max_points if request.max_points is not None else 5.0
+
+
+def _score_only_suffix() -> str:
+    return """Waehle genau einen gueltigen Punktwert aus der Liste.
+Gib nur die Zahl aus. Keine Begruendung.
+Punktzahl:"""
+
+
+def _feedback_suffix() -> str:
+    return """Waehle genau einen gueltigen Punktwert aus der Liste.
+Antworte ausschliesslich als JSON in diesem Format:
+{"score": 1.5, "feedback": "- Kurzer fachlicher Grund.\\n- Wichtigster fehlender oder richtiger Punkt."}
+Die Begruendung muss knapp sein: maximal 2 kurze Stichpunkte, keine langen Saetze."""
+
+
+def _suffix_for_request(request: GradeRequest) -> str:
+    return _feedback_suffix() if request.include_feedback else _score_only_suffix()
+
+
+def _default_grading_rules() -> str:
+    return """- Volle Punktzahl nur, wenn die wesentlichen fachlichen Kernaussagen korrekt enthalten sind.
+- Teilpunkte vergeben, wenn nur ein Teil der erwarteten Inhalte korrekt ist.
+- Synonyme und andere Formulierungen sind erlaubt, wenn die fachliche Bedeutung stimmt.
+- Rechtschreibung, Grammatik, Stil und Zeichensetzung nicht bewerten.
+- Waehle die Punktzahl, die am besten durch die explizit vorhandenen Inhalte der Antwort gestuetzt wird."""
+
+
+def _grading_rules_text(request: GradeRequest) -> str:
+    custom_rules = (request.custom_grading_rules or "").strip()
+    return custom_rules or _default_grading_rules()
+
+
+def _format_score(score: float) -> str:
+    return f"{score:.1f}" if score == round(score, 1) else f"{score:.2f}"
+
+
+def _format_examples(request: GradeRequest) -> str:
+    chunks: list[str] = []
+    for index, example in enumerate(request.examples, start=1):
+        chunks.append(
+            f"""Bewertungsbeispiel {index}:
+Frage:
+<<<BEISPIEL_FRAGE>>>
+{example.question}
+<<<END_BEISPIEL_FRAGE>>>
+
+Studentische Antwort:
+<<<BEISPIEL_ANTWORT>>>
+{example.answer}
+<<<END_BEISPIEL_ANTWORT>>>
+
+Punktzahl: {_format_score(example.score)}"""
+        )
+    return "\n\n".join(chunks)
+
+
+def _instruction_only_prompt(request: GradeRequest, suffix: str) -> str:
+    max_points = _max_points_or_default(request)
+    return f"""Frage:
+<<<FRAGE>>>
+{request.question_text}
+<<<END_FRAGE>>>
+
+Gueltige Punktwerte:
+[{valid_scores_text(max_points)}]
+
+Studentische Antwort:
+<<<ANTWORT>>>
+{request.student_answer or ''}
+<<<END_ANTWORT>>>
+
+{suffix}"""
+
+
+def _reference_prompt(request: GradeRequest, suffix: str) -> str:
+    max_points = _max_points_or_default(request)
+    return f"""Frage:
+<<<FRAGE>>>
+{request.question_text}
+<<<END_FRAGE>>>
+
+Musterloesung / Korrekturhinweise:
+<<<MUSTERLOESUNG>>>
+{request.reference_answer or ''}
+<<<END_MUSTERLOESUNG>>>
+
+Gueltige Punktwerte:
+[{valid_scores_text(max_points)}]
+
+Studentische Antwort:
+<<<ANTWORT>>>
+{request.student_answer or ''}
+<<<END_ANTWORT>>>
+
+{suffix}"""
+
+
+def _reference_rules_prompt(request: GradeRequest, suffix: str) -> str:
+    max_points = _max_points_or_default(request)
+    return f"""Frage:
+<<<FRAGE>>>
+{request.question_text}
+<<<END_FRAGE>>>
+
+Musterloesung / Korrekturhinweise:
+<<<MUSTERLOESUNG>>>
+{request.reference_answer or ''}
+<<<END_MUSTERLOESUNG>>>
+
+Gueltige Punktwerte:
+[{valid_scores_text(max_points)}]
+
+Bewertungsregeln:
+{_grading_rules_text(request)}
+
+Studentische Antwort:
+<<<ANTWORT>>>
+{request.student_answer or ''}
+<<<END_ANTWORT>>>
+
+{suffix}"""
+
+
+def _example_guided_prompt(request: GradeRequest, suffix: str) -> str:
+    max_points = _max_points_or_default(request)
+    return f"""Frage:
+<<<FRAGE>>>
+{request.question_text}
+<<<END_FRAGE>>>
+
+Musterloesung / Korrekturhinweise:
+<<<MUSTERLOESUNG>>>
+{request.reference_answer or ''}
+<<<END_MUSTERLOESUNG>>>
+
+Gueltige Punktwerte:
+[{valid_scores_text(max_points)}]
+
+Bewertungsregeln:
+{_grading_rules_text(request)}
+
+Die Beispiele zeigen ein Bewertungsmuster, sind aber nicht die einzigen gueltigen Formulierungen.
+
+Bewertungsbeispiele:
+{_format_examples(request)}
+
+Zu bewertende Antwort:
+<<<ANTWORT>>>
+{request.student_answer or ''}
+<<<END_ANTWORT>>>
+
+{suffix}"""
+
+
+class InstructionOnlyPromptStrategy:
+    def build_messages(self, request: GradeRequest) -> list[ChatMessage]:
+        return [
+            {"role": "system", "content": _base_system()},
+            {"role": "user", "content": _instruction_only_prompt(request, _suffix_for_request(request))},
+        ]
+
+
+class ReferenceZeroShotPromptStrategy:
+    def build_messages(self, request: GradeRequest) -> list[ChatMessage]:
+        return [
+            {"role": "system", "content": _base_system()},
+            {"role": "user", "content": _reference_prompt(request, _suffix_for_request(request))},
+        ]
+
+
+class ReferenceRulesPromptStrategy:
+    def build_messages(self, request: GradeRequest) -> list[ChatMessage]:
+        return [
+            {"role": "system", "content": _base_system()},
+            {"role": "user", "content": _reference_rules_prompt(request, _suffix_for_request(request))},
+        ]
+
+
+class ExampleGuidedPromptStrategy:
+    def build_messages(self, request: GradeRequest) -> list[ChatMessage]:
+        if not request.examples:
+            raise ValueError("Example-guided grading requires at least one scored example")
+        return [
+            {"role": "system", "content": _base_system()},
+            {"role": "user", "content": _example_guided_prompt(request, _suffix_for_request(request))},
+        ]
+
+
+class InstructionOnlyFeedbackPromptStrategy:
+    def build_messages(self, request: GradeRequest) -> list[ChatMessage]:
+        request.include_feedback = True
+        return [
+            {"role": "system", "content": _base_system()},
+            {"role": "user", "content": _instruction_only_prompt(request, _feedback_suffix())},
+        ]
+
+
+class ReferenceZeroShotFeedbackPromptStrategy:
+    def build_messages(self, request: GradeRequest) -> list[ChatMessage]:
+        request.include_feedback = True
+        return [
+            {"role": "system", "content": _base_system()},
+            {"role": "user", "content": _reference_prompt(request, _feedback_suffix())},
+        ]
+
+
+class ReferenceRulesFeedbackPromptStrategy:
+    def build_messages(self, request: GradeRequest) -> list[ChatMessage]:
+        request.include_feedback = True
+        return ReferenceRulesPromptStrategy().build_messages(request)
+
+
+class ExampleGuidedFeedbackPromptStrategy:
+    def build_messages(self, request: GradeRequest) -> list[ChatMessage]:
+        request.include_feedback = True
+        return ExampleGuidedPromptStrategy().build_messages(request)
+
+
+def get_prompt_strategy(name: str) -> PromptStrategy:
+    normalized = normalize_strategy_name(name)
+    registry: dict[str, PromptStrategy] = {
+        P0_INSTRUCTION_ONLY: InstructionOnlyPromptStrategy(),
+        P1_REFERENCE_ZERO_SHOT: ReferenceZeroShotPromptStrategy(),
+        P2_REFERENCE_DISCRETE: ReferenceRulesPromptStrategy(),
+        P3_TRUE_ONE_SHOT: ExampleGuidedPromptStrategy(),
+        P0_INSTRUCTION_ONLY_WITH_FEEDBACK: InstructionOnlyFeedbackPromptStrategy(),
+        P1_REFERENCE_ZERO_SHOT_WITH_FEEDBACK: ReferenceZeroShotFeedbackPromptStrategy(),
+        P2_REFERENCE_DISCRETE_WITH_FEEDBACK: ReferenceRulesFeedbackPromptStrategy(),
+        P3_TRUE_ONE_SHOT_WITH_FEEDBACK: ExampleGuidedFeedbackPromptStrategy(),
+    }
+    strategy = registry.get(normalized)
+    if strategy is None:
+        raise ValueError(f"Unsupported prompt strategy: {name}")
+    return strategy