Skip to content
GitLab
Commit e8105cb5 authored by Kantz's avatar Kantz
Browse files

formating mit autopep8

parent 5b144250
Hide whitespace changes
Inline Side-by-side
math-tutor/backend/app/api/canvas.py
View file @ e8105cb5
......@@ -17,7 +17,8 @@ settings = config.get_mathpix_settings()
mathpix_client = None
if settings.app_id and settings.app_key:
mathpix_client = MathpixClient(app_id=settings.app_id, app_key=settings.app_key)
mathpix_client = MathpixClient(
app_id=settings.app_id, app_key=settings.app_key)
class CanvasSaveRequest(BaseModel):
......@@ -34,7 +35,8 @@ class CanvasSaveResponse(BaseModel):
@router.post("/api/canvas/save", response_model=CanvasSaveResponse)
def save_canvas(request: CanvasSaveRequest) -> CanvasSaveResponse:
if not request.data_url.startswith("data:image"):
raise HTTPException(status_code=400, detail="data_url must be an image")
raise HTTPException(
status_code=400, detail="data_url must be an image")
header, encoded = request.data_url.split(",", 1)
if "image/png" not in header:
......@@ -43,14 +45,16 @@ def save_canvas(request: CanvasSaveRequest) -> CanvasSaveResponse:
try:
raw = base64.b64decode(encoded)
except base64.binascii.Error as exc:
raise HTTPException(status_code=400, detail="invalid base64 payload") from exc
raise HTTPException(
status_code=400, detail="invalid base64 payload") from exc
drawings_dir = Path(__file__).resolve().parents[1] / "storage" / "drawings"
drawings_dir.mkdir(parents=True, exist_ok=True)
timestamp = int(time.time())
safe_hint = request.filename_hint or "drawing"
safe_hint = "".join(ch for ch in safe_hint if ch.isalnum() or ch in ("-", "_"))
safe_hint = "".join(
ch for ch in safe_hint if ch.isalnum() or ch in ("-", "_"))
if not safe_hint:
safe_hint = "drawing"
......@@ -62,7 +66,8 @@ def save_canvas(request: CanvasSaveRequest) -> CanvasSaveResponse:
handle.write(raw)
try:
image = mathpix_client.image_new(file_path=str(img_path), include_line_data=True)
image = mathpix_client.image_new(
file_path=str(img_path), include_line_data=True)
lines = image.lines_json()
line_type = lines[0]["type"] if lines else None
if line_type != "math":
......
math-tutor/backend/app/api/chat.py
View file @ e8105cb5
......@@ -14,7 +14,7 @@ import app.config as config
if config.get_orchestrator() == "tutor":
from app.deterministic_services.orchestrators import orchestrator_tutor as orchestrator
else:
from app.deterministic_services.orchestrators import orchestrator_qa as orchestrator
from app.deterministic_services.orchestrators import orchestrator_qa as orchestrator
router = APIRouter()
......@@ -70,7 +70,8 @@ def chat(request: ChatRequest) -> ChatResponse:
raise HTTPException(status_code=500, detail=str(exc)) from exc
except Exception as exc:
logger.exception("Chat request failed")
raise HTTPException(status_code=502, detail="chat provider failed") from exc
raise HTTPException(
status_code=502, detail="chat provider failed") from exc
return ChatResponse(reply=reply, sources=sources)
......@@ -81,7 +82,8 @@ def list_archives(limit: int = Query(20, ge=1, le=200)) -> List[ChatArchiveSumma
return session_store.list_archives(limit=limit)
except Exception as exc:
logger.exception("Chat archive list failed")
raise HTTPException(status_code=502, detail="chat archive list failed") from exc
raise HTTPException(
status_code=502, detail="chat archive list failed") from exc
@router.get("/api/chat/archive/{chat_id}", response_model=ChatArchiveDetail)
......@@ -90,7 +92,8 @@ def get_archive(chat_id: str = Path(..., min_length=1)) -> ChatArchiveDetail:
record = session_store.load_archive(chat_id)
except Exception as exc:
logger.exception("Chat archive load failed")
raise HTTPException(status_code=502, detail="chat archive load failed") from exc
raise HTTPException(
status_code=502, detail="chat archive load failed") from exc
if not record:
raise HTTPException(status_code=404, detail="chat archive not found")
......@@ -98,7 +101,8 @@ def get_archive(chat_id: str = Path(..., min_length=1)) -> ChatArchiveDetail:
return ChatArchiveDetail(
chat_id=record["chat_id"],
saved_at=record.get("saved_at", ""),
history=[ChatMessage(role=item["role"], text=item["text"]) for item in record["history"]],
history=[ChatMessage(role=item["role"], text=item["text"])
for item in record["history"]],
)
......@@ -114,6 +118,7 @@ def archive_chat(request: ChatRequest) -> ChatArchiveResponse:
)
except Exception as exc:
logger.exception("Chat archive failed")
raise HTTPException(status_code=502, detail="chat archive failed") from exc
raise HTTPException(
status_code=502, detail="chat archive failed") from exc
return ChatArchiveResponse(status="ok", chat_id=chat_id)
math-tutor/backend/app/api/context.py
View file @ e8105cb5
......@@ -7,7 +7,6 @@ from pydantic import BaseModel, Field
from app.deterministic_services import context_store
router = APIRouter()
......@@ -20,6 +19,7 @@ class ContextRequest(BaseModel):
messages: List[ChatMessage]
draft: Optional[str] = None
@router.post("/api/context/retrieval")
def get_retrieval_context(request: ContextRequest) -> List[dict]:
if not request or not request.messages:
......
math-tutor/backend/app/config.py
View file @ e8105cb5
......@@ -7,6 +7,7 @@ from typing import Optional
load_dotenv()
class EmbeddingSettings(BaseModel):
embedding_type: str # "openai-like" oder "sentence-transformer"
base_url: Optional[str] = None
......@@ -25,12 +26,14 @@ def get_retrieval_impl() -> str:
return value
return "child"
def get_embedding_settings() -> EmbeddingSettings:
embedding_type = os.getenv("EMBEDDING_TYPE", "openai-like")
if embedding_type == "sentence-transformer":
return EmbeddingSettings(
embedding_type=embedding_type,
model=os.getenv("SENTENCE_TRANSFORMER_MODEL", "jinaai/jina-embeddings-v4"),
model=os.getenv("SENTENCE_TRANSFORMER_MODEL",
"jinaai/jina-embeddings-v4"),
target_dim=int(os.getenv("EMBEDDING_DIM", "512")),
)
if embedding_type == "openai-like":
......@@ -59,6 +62,7 @@ class OllamaSettings:
keepalive: str | None
temperature: float | None
@dataclass(frozen=True)
class OpenAIChatSettings:
base_url: str
......@@ -67,11 +71,13 @@ class OpenAIChatSettings:
timeout: float | None
temperature: float | None
@dataclass(frozen=True)
class OpenAIBaseSettings:
base_url: str
api_key: str
@dataclass(frozen=True)
class MathpixSettings:
app_id: str
......@@ -86,18 +92,21 @@ def _read_float(value: str | None) -> float | None:
except ValueError:
return None
def get_mathpix_settings() -> MathpixSettings:
return MathpixSettings(
app_id=os.getenv("MATHPIX_APP_ID"),
app_key=os.getenv("MATHPIX_APP_KEY")
app_key=os.getenv("MATHPIX_APP_KEY")
)
def get_frontend_url() -> str:
url = os.getenv("FRONTEND_URL")
if not url:
raise ValueError("Missing FRONTEND_URL")
return url
def get_ollama_settings() -> OllamaSettings:
return OllamaSettings(
base_url=os.getenv("OLLAMA_URL", "http://localhost:11434"),
......@@ -107,6 +116,7 @@ def get_ollama_settings() -> OllamaSettings:
temperature=_read_float(os.getenv("OLLAMA_TEMPERATURE")),
)
def get_openai_base_settings() -> OpenAIBaseSettings | None:
base_url = os.getenv("OPENAI_BASE_URL")
api_key = os.getenv("OPENAI_API_KEY")
......@@ -117,6 +127,7 @@ def get_openai_base_settings() -> OpenAIBaseSettings | None:
api_key=api_key,
)
def get_openai_chat_settings() -> OpenAIChatSettings | None:
model = os.getenv("OPENAI_CHAT_MODEL")
if not model:
......
math-tutor/backend/app/deterministic_services/context_store.py
View file @ e8105cb5
......@@ -26,7 +26,8 @@ def _utc_now() -> str:
def get_chat_id(messages: list[dict], draft: str | None = None) -> str:
if draft:
return f"draft_{draft}"
first_user = next((m for m in messages if m.get("role") == "user" and m.get("content")), None)
first_user = next((m for m in messages if m.get("role")
== "user" and m.get("content")), None)
if not first_user:
return "unknown"
digest = hashlib.sha1(first_user["content"].encode("utf-8")).hexdigest()
......@@ -42,7 +43,7 @@ def init_sheet(chat_id: str, messages: list[dict]) -> dict[str, Any]:
"history": messages[:],
"retrieval_contexts": [],
"math_solutions": [],
"LLM_solutions":[],
"LLM_solutions": [],
"decisions": [],
"initialized": False,
}
......@@ -62,6 +63,7 @@ def load_sheet(chat_id: str) -> dict[str, Any] | None:
return sheet
return None
def save_sheet(sheet: dict[str, Any]) -> None:
os.makedirs(_LOG_DIR, exist_ok=True)
sheet["updated_at"] = _utc_now()
......@@ -76,6 +78,7 @@ def save_sheet(sheet: dict[str, Any]) -> None:
with _LOCK:
_CACHE[chat_id] = sheet
def format_sheet(sheet: dict[str, Any]) -> str:
parts = []
history = format_history(sheet.get("history", []))
......@@ -87,12 +90,12 @@ def format_sheet(sheet: dict[str, Any]) -> str:
for item in retrievals:
query = item.get('query', '')
sources = item.get('sources', [])
# Erstelle eine formatierte Liste der Quellen mit ihren Scores
source_blocks = []
for source in sources:
source_blocks.append(Source.model_validate(source).to_string())
source_info = "\n".join(source_blocks) if source_blocks else ""
blocks.append(f"QUERY: {query}\nSOURCES:\n{source_info}")
parts.append("RETRIEVAL_CONTEXT:\n" + "\n\n".join(blocks))
......@@ -115,17 +118,18 @@ def format_sheet(sheet: dict[str, Any]) -> str:
else:
parts.append("MATH_SOLUTIONS:\n(leer)")
return "\n\n".join(parts)
# ---------------------------------------------------------------------------------------------------
# History related
# ---------------------------------------------------------------------------------------------------
def update_history(sheet: dict[str, Any], messages: list[dict]) -> None:
sheet["history"] = messages[:]
sheet["updated_at"] = _utc_now()
def format_history(messages: list[dict]) -> str:
lines = []
for msg in messages:
......@@ -134,12 +138,15 @@ def format_history(messages: list[dict]) -> str:
lines.append(f"{role}: {content}")
return "\n".join(lines)
def get_history(sheet: dict[str, Any]) -> str:
return format_history(sheet.get("history", []))
def get_history_turns(sheet: dict[str, Any]) -> list[dict]:
return sheet.get("history", [])
def get_task(sheet: dict[str, Any]) -> str:
history = sheet.get("history", [])
if history:
......@@ -159,22 +166,25 @@ def update_retrieval_context(
retrievals = sheet.get("retrieval_contexts", [])
if retrievals:
latest_retrieval = retrievals[-1]
latest_retrieval["sources"] = [source.model_dump() for source in sources]
latest_retrieval["sources"] = [source.model_dump()
for source in sources]
latest_retrieval["query"] = query
sheet["updated_at"] = _utc_now()
def get_retrieval(sheet: dict[str, Any]) -> list[Source]:
retrievals = sheet.get("retrieval_contexts", [])
if not retrievals:
return []
latest_retrieval = retrievals[-1]
sources= latest_retrieval.get("sources", [])
sources = latest_retrieval.get("sources", [])
return [Source.model_validate(source) for source in sources]
# ---------------------------------------------------------------------------------------------------
# Math-solution related
# ---------------------------------------------------------------------------------------------------
def add_math_solution(
sheet: dict[str, Any],
solution: str,
......@@ -186,6 +196,7 @@ def add_math_solution(
)
sheet["updated_at"] = _utc_now()
def first_math_solution(sheet: dict[str, Any]) -> str:
if not sheet["math_solutions"]:
return ""
......@@ -207,6 +218,7 @@ def add_LLM_solution(
)
sheet["updated_at"] = _utc_now()
def last_LLM_solution(sheet: dict[str, Any]) -> str:
if not sheet["LLM_solutions"]:
return ""
......@@ -216,7 +228,8 @@ def last_LLM_solution(sheet: dict[str, Any]) -> str:
# decision related
# ---------------------------------------------------------------------------------------------------
def add_decision(sheet: dict[str, Any], decision: dict[str, Any]) -> None:
entry = {"timestamp": _utc_now(), **decision}
sheet["decisions"].append(entry)
sheet["updated_at"] = _utc_now()
\ No newline at end of file
sheet["updated_at"] = _utc_now()
math-tutor/backend/app/deterministic_services/embeddings.py
View file @ e8105cb5
......@@ -20,27 +20,34 @@ class EmbeddingType(str, Enum):
class OpenAILikeConfig(BaseModel):
"""Konfiguration für OpenAI-ähnliche APIs."""
base_url: str = Field(..., description="Base URL der API (z. B. http://localhost:11434/v1)")
api_key: str = Field(..., description="API-Key (z. B. 'ollama' für Ollama)")
model: str = Field(..., description="Modellname (z. B. 'nomic-embed-text')")
base_url: str = Field(...,
description="Base URL der API (z. B. http://localhost:11434/v1)")
api_key: str = Field(...,
description="API-Key (z. B. 'ollama' für Ollama)")
model: str = Field(...,
description="Modellname (z. B. 'nomic-embed-text')")
target_dim: int = Field(1024, description="Ziel-Dimension der Embeddings")
class SentenceTransformerConfig(BaseModel):
"""Konfiguration für lokale SentenceTransformer-Modelle."""
model: str = Field(..., description="Name des SentenceTransformer-Modells (z. B. 'all-MiniLM-L6-v2')")
model: str = Field(...,
description="Name des SentenceTransformer-Modells (z. B. 'all-MiniLM-L6-v2')")
target_dim: int = Field(384, description="Ziel-Dimension der Embeddings")
class EmbeddingConfig(BaseModel):
"""Gemeinsame Konfiguration für die Factory."""
embedding_type: EmbeddingType = Field(..., description="Typ der Embeddings")
config: Union[OpenAILikeConfig, SentenceTransformerConfig] = Field(..., description="Spezifische Konfiguration")
embedding_type: EmbeddingType = Field(...,
description="Typ der Embeddings")
config: Union[OpenAILikeConfig, SentenceTransformerConfig] = Field(
..., description="Spezifische Konfiguration")
# -----------------------------
# Basisklasse für Embeddings
# -----------------------------
class BaseEmbeddings:
"""
Basisklasse für Embedding-Generierung mit gemeinsamen Methoden.
......@@ -59,7 +66,8 @@ class BaseEmbeddings:
def _truncate(self, vec: List[float]) -> List[float]:
"""Trunziert oder füllt den Vektor auf die Ziel-Dimension."""
if len(vec) < self.target_dim:
raise ValueError(f"Embedding dimension {len(vec)} < target {self.target_dim}")
raise ValueError(
f"Embedding dimension {len(vec)} < target {self.target_dim}")
if len(vec) > self.target_dim:
vec = vec[: self.target_dim]
return self._normalize(vec)
......@@ -114,7 +122,8 @@ class OpenAILikeEmbeddings(BaseEmbeddings):
}
with httpx.Client(timeout=60.0) as client:
response = client.post(self.endpoint, headers=headers, json=payload)
response = client.post(
self.endpoint, headers=headers, json=payload)
response.raise_for_status()
data = response.json().get("data")
......@@ -148,7 +157,8 @@ class SentenceTransformerEmbeddings(BaseEmbeddings):
def model(self) -> SentenceTransformer:
"""Liefert das SentenceTransformer-Modell (lazy load)."""
if self._model is None:
self._model = SentenceTransformer(self.model_name, trust_remote_code=True)
self._model = SentenceTransformer(
self.model_name, trust_remote_code=True)
self._model.max_seq_length = 512
return self._model
......@@ -171,7 +181,6 @@ class SentenceTransformerEmbeddings(BaseEmbeddings):
return self._truncate([float(x) for x in query_embeddings[0]])
# -----------------------------
# Factory: Erzeugt die richtige Embeddings-Instanz
# -----------------------------
......@@ -200,4 +209,5 @@ class EmbeddingFactory:
elif config.embedding_type == EmbeddingType.SENTENCE_TRANSFORMER:
return SentenceTransformerEmbeddings(config=config)
else:
raise ValueError(f"Unsupported embedding type: {config.embedding_type}")
raise ValueError(
f"Unsupported embedding type: {config.embedding_type}")
math-tutor/backend/app/deterministic_services/llm_client.py
View file @ e8105cb5
......@@ -22,7 +22,8 @@ def _chat_openai(messages: list[dict]) -> dict:
return {}
timeout = settings.timeout or 60.0
client = OpenAI(api_key=settings.api_key, base_url=settings.base_url, timeout=timeout)
client = OpenAI(api_key=settings.api_key,
base_url=settings.base_url, timeout=timeout)
kwargs = {"messages": messages, "model": settings.model}
if settings.temperature is not None:
kwargs["temperature"] = settings.temperature
......@@ -70,6 +71,7 @@ def _extract_message(response) -> object:
return response.message
return {}
def get_message_content(result: dict | object) -> str:
message = result.get("message") if isinstance(result, dict) else result
if isinstance(message, dict):
......@@ -119,8 +121,10 @@ def _apply_tool_calls(
except Exception as exc: # pragma: no cover - defensive
output = f"Tool error: {exc}"
outputs.append({"name": name, "arguments": arguments, "result": output})
messages.append({"role": "tool", "tool_name": name, "content": str(output)})
outputs.append(
{"name": name, "arguments": arguments, "result": output})
messages.append(
{"role": "tool", "tool_name": name, "content": str(output)})
return outputs
......
math-tutor/backend/app/deterministic_services/orchestrators/orchestrator_base.py
View file @ e8105cb5
......@@ -61,7 +61,9 @@ def bootstrap_retrieval(sheet: dict, query_text: str, tool_log: list[dict]) -> N
source_dump = {"sources": [source.to_string() for source in sources]}
context_store.update_retrieval_context(sheet, query_text, sources)
append_tool_log(tool_log, "update_retrieve_context", {"query": query_text}, source_dump)
append_tool_log(tool_log, "update_retrieve_context",
{"query": query_text}, source_dump)
def init_chat_state(messages: list[dict], draft: str | None = None) -> ChatState:
if not messages:
......
math-tutor/backend/app/deterministic_services/orchestrators/orchestrator_tutor.py
View file @ e8105cb5
......@@ -10,7 +10,8 @@ def _on_bootstrap(state: base.ChatState, query_text: str) -> None:
math_solution = math_intent_LLM.solve_with_tools(query_text)
if math_solution:
context_store.add_math_solution(state.sheet, math_solution)
base.append_tool_log(state.tool_log, "math_intent_LLM", {"query": query_text}, math_solution)
base.append_tool_log(state.tool_log, "math_intent_LLM", {
"query": query_text}, math_solution)
def _on_turn_logic(state: base.ChatState) -> None:
......@@ -29,7 +30,8 @@ def _on_turn_logic(state: base.ChatState) -> None:
return
decision = decision_LLM.needs_more_context(sheet_text)
base.append_tool_log(state.tool_log, "decision", {"sheet": sheet_text}, decision)
base.append_tool_log(state.tool_log, "decision", {
"sheet": sheet_text}, decision)
context_store.add_decision(state.sheet, decision)
if decision.get("needs_more_context"):
......
math-tutor/backend/app/deterministic_services/referenz_decoder.py
View file @ e8105cb5
......@@ -85,4 +85,3 @@ def decode_references(text: str, sources: Iterable[Source]) -> Tuple[str, int]:
return f"[{ref}]({href})"
return _BRACKET_RE.sub(replace, text), replacements
math-tutor/backend/app/deterministic_services/session_store.py
View file @ e8105cb5
......@@ -100,7 +100,8 @@ def load_archive(chat_id: str) -> dict[str, Any] | None:
continue
if record.get("chat_id") == chat_id:
history = [
{"role": entry.get("role", "user"), "text": entry.get("content", "")}
{"role": entry.get("role", "user"),
"text": entry.get("content", "")}
for entry in record.get("history", [])
]
return {
......
math-tutor/backend/app/deterministic_services/tool_logging.py
View file @ e8105cb5
......@@ -37,7 +37,8 @@ def write_tool_log(entries: list[dict], created_at: str | None = None, chat_id:
existing = json.load(f)
if isinstance(existing, dict) and isinstance(existing.get("entries"), list):
payload["entries"] = existing["entries"] + payload["entries"]
payload["timestamp"] = existing.get("timestamp", payload["timestamp"])
payload["timestamp"] = existing.get(
"timestamp", payload["timestamp"])
except (OSError, json.JSONDecodeError):
pass
with open(path, "w", encoding="utf-8") as f:
......
math-tutor/backend/app/deterministic_services/vector_store.py
View file @ e8105cb5
......@@ -20,6 +20,7 @@ embedding_dim = app.config.get_embedding_settings().target_dim
# Einlesen der Dokumente
# --------------------------------------------------------------------------------------------------------------------
@dataclass
class DocRecord:
doc_type: str
......@@ -118,6 +119,7 @@ def load_docs(base_dir: Path) -> List[DocRecord]:
# Init der Databse
# --------------------------------------------------------------------------------------------------------------------
DATABASE_CREATION_SQL = f"""
CREATE EXTENSION IF NOT EXISTS vector;
......@@ -148,6 +150,7 @@ CREATE INDEX IF NOT EXISTS docs_section_idx ON docs(section_index);
CREATE INDEX IF NOT EXISTS docs_subsection_idx ON docs(section_index, subsection_index);
"""
def init_db(pg_url: str) -> None:
with psycopg.connect(pg_url) as conn:
with conn.cursor() as cur:
......@@ -159,6 +162,7 @@ def init_db(pg_url: str) -> None:
# Einfügen der Dokumente und Embeddings
# --------------------------------------------------------------------------------------------------------------------
UPSERT_SQL = """
INSERT INTO docs (
uid, doc_type,
......@@ -186,6 +190,7 @@ ON CONFLICT (uid) DO UPDATE SET
;
"""
def _meta_int(meta: Dict[str, Any], key: str) -> Optional[int]:
value = meta.get(key)
if value is None or value == "":
......@@ -234,6 +239,7 @@ def clear_docs(pg_url: str) -> None:
cur.execute("TRUNCATE TABLE docs;")
conn.commit()
def embed_documents(embedder: EmbeddingLike, texts: List[str]) -> List[List[float]]:
return embedder.embed_documents(texts)
......@@ -245,6 +251,7 @@ def embed_query(embedder: EmbeddingLike, text: str) -> List[float]:
# Retrival der Dokumente
# --------------------------------------------------------------------------------------------------------------------
@dataclass
class Retrieved:
uid: str
......@@ -262,6 +269,7 @@ class Retrieved:
"markdown": self.markdown,
}
def _row_to_retrieved(row: Dict[str, Any], source_type: Optional[str] = None) -> Retrieved:
meta = {
"uid": row["uid"],
......@@ -352,7 +360,8 @@ def retrieve(
sec_sub_counts[key] = sec_sub_counts.get(key, 0) + 1
if sec_sub_counts:
most_common_sec_sub = max(sec_sub_counts.items(), key=lambda x: x[1])[0]
most_common_sec_sub = max(
sec_sub_counts.items(), key=lambda x: x[1])[0]
most_common_sec, most_common_sub = most_common_sec_sub
cur.execute(
......@@ -368,9 +377,11 @@ def retrieve(
AND d.section_index = %(sec)s
AND d.subsection_index = %(sub)s
""",
{"sec": most_common_sec, "sub": most_common_sub, "sub_doc_types": ["subsection", "chapter"]},
{"sec": most_common_sec, "sub": most_common_sub,
"sub_doc_types": ["subsection", "chapter"]},
)
subsections = [_row_to_retrieved(row, source_type="subsection") for row in cur.fetchall()]
subsections = [_row_to_retrieved(
row, source_type="subsection") for row in cur.fetchall()]
cur.execute(
"""
......@@ -384,9 +395,11 @@ def retrieve(
WHERE doc_type = ANY(%(sec_doc_types)s)
AND section_index = %(sec)s
""",
{"sec": most_common_sec, "sec_doc_types": ["section", "oberchapter"]},
{"sec": most_common_sec, "sec_doc_types": [
"section", "oberchapter"]},
)
sections_docs = [_row_to_retrieved(row, source_type="section") for row in cur.fetchall()]
sections_docs = [_row_to_retrieved(
row, source_type="section") for row in cur.fetchall()]
cur.execute(
"""
......@@ -403,12 +416,15 @@ def retrieve(
ORDER BY d.embedding <=> %(qvec)s
LIMIT 5;
""",
{"qvec": qvec, "sec": most_common_sec, "sub": most_common_sub},
{"qvec": qvec, "sec": most_common_sec,
"sub": most_common_sub},
)
additional_children = [_row_to_retrieved(row) for row in cur.fetchall()]
additional_children = [_row_to_retrieved(
row) for row in cur.fetchall()]
existing_uids = {child.uid for child in children}
new_children = [child for child in additional_children if child.uid not in existing_uids]
new_children = [
child for child in additional_children if child.uid not in existing_uids]
children.extend(new_children)
children_expanded.extend(new_children)
......@@ -444,12 +460,15 @@ def retrieve(
ON d.section_index = u.sec AND d.subsection_index = u.sub AND d.child_index = u.child
WHERE d.doc_type = 'child'
""",
{"sec_arr": sec_arr, "sub_arr": sub_arr, "child_arr": child_arr},
{"sec_arr": sec_arr, "sub_arr": sub_arr,
"child_arr": child_arr},
)
neighbors = [_row_to_retrieved(row) for row in cur.fetchall()]
neighbors = [_row_to_retrieved(
row) for row in cur.fetchall()]
child_uids = {child.uid for child in children}
neighbors = [neighbor for neighbor in neighbors if neighbor.uid not in child_uids]
neighbors = [
neighbor for neighbor in neighbors if neighbor.uid not in child_uids]
retrivla_dict = {
"children_direct": children_direct,
"children_expanded": children_expanded,
......@@ -464,6 +483,7 @@ def retrieve(
# Retrival in Sources umwandeln
# --------------------------------------------------------------------------------------------------------------------
class Source(BaseModel):
source_id: SourceID
retrieved_as: str
......@@ -489,6 +509,7 @@ class Source(BaseModel):
f" markdown={self.markdown}"
)
class SourceID(BaseModel):
chapter_title: Optional[str] = None
section_title: Optional[str] = None
......@@ -515,13 +536,14 @@ class SourceID(BaseModel):
string_rep = f"{self.chapter_title}|{string_rep}"
return f"[{string_rep}|{self.doc_type}]"
def _retrivla_to_sources(retrievd : Dict[str, List[Retrieved]]) -> List[Source]:
def _retrivla_to_sources(retrievd: Dict[str, List[Retrieved]]) -> List[Source]:
sources = []
for name, retrieved_grouep in retrievd.items():
for retrival in retrieved_grouep:
sources.append(Source(
source_id=SourceID(
chapter_title = "none",
chapter_title="none",
section_title=retrival.metadata.get("section_title"),
subsection_title=retrival.metadata.get("subsection_title"),
title=retrival.metadata.get("title"),
......@@ -531,13 +553,14 @@ def _retrivla_to_sources(retrievd : Dict[str, List[Retrieved]]) -> List[Source]:
source_type=retrival.metadata.get("source_type") or "unknown",
score=retrival.score,
markdown=retrival.markdown
))
))
return sources
# --------------------------------------------------------------------------------------------------------------------
# Listen für Filterung
# --------------------------------------------------------------------------------------------------------------------
def list_sections(pg_url: str) -> List[Dict[str, Any]]:
sql = """
SELECT DISTINCT ON (section_index)
......
math-tutor/backend/app/deterministic_services/vector_store_subsection.py
View file @ e8105cb5
......@@ -72,5 +72,6 @@ def retrieve(
cur.execute(sql, params)
rows = cur.fetchall()
subsections = [_row_to_retrieved(row, source_type="subsection") for row in rows]
subsections = [_row_to_retrieved(
row, source_type="subsection") for row in rows]
return _retrivla_to_sources({"subsections_direct": subsections})
math-tutor/backend/scripts/retrieval_cli.py
View file @ e8105cb5
......@@ -2,6 +2,8 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from app.deterministic_services import retrieval_store, vector_store
from app.deterministic_services.embeddings import BaseEmbeddings, EmbeddingFactory
import argparse
import os
......@@ -17,13 +19,11 @@ ROOT_DIR = Path(__file__).resolve().parents[1]
if str(ROOT_DIR) not in sys.path:
sys.path.insert(0, str(ROOT_DIR))
from app.deterministic_services.embeddings import BaseEmbeddings, EmbeddingFactory
from app.deterministic_services import retrieval_store, vector_store
def build_embedder() -> BaseEmbeddings:
return EmbeddingFactory.create(config.get_embedding_settings())
def cli_init_db(args: argparse.Namespace) -> None:
pg_url = args.pg or os.getenv("POSTGRES_URL")
if not pg_url:
......@@ -36,7 +36,8 @@ def cli_ingest(args: argparse.Namespace) -> None:
embedder = build_embedder()
base_dir = Path(args.base)
docs = vector_store.load_docs(base_dir)
embeddings = vector_store.embed_documents(embedder, [doc.markdown for doc in docs])
embeddings = vector_store.embed_documents(
embedder, [doc.markdown for doc in docs])
pg_url = args.pg or os.getenv("POSTGRES_URL")
if not pg_url:
raise ValueError("Missing POSTGRES_URL")
......@@ -70,18 +71,24 @@ def main() -> int:
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(required=True)
ap_init = sub.add_parser("init-db", help="Create tables and indexes in Postgres")
ap_init.add_argument("--pg", default=None, help="Postgres URL (or set POSTGRES_URL)")
ap_init = sub.add_parser(
"init-db", help="Create tables and indexes in Postgres")
ap_init.add_argument("--pg", default=None,
help="Postgres URL (or set POSTGRES_URL)")
ap_init.set_defaults(func=cli_init_db)
ap_ing = sub.add_parser("ingest", help="Embed and upsert markdown docs")
ap_ing.add_argument("--pg", default=None, help="Postgres URL (or set POSTGRES_URL)")
ap_ing.add_argument("--base", default="markdown", help="Base folder with sections/subsections/childs")
ap_ing.add_argument("--clear", action="store_true", help="Clear docs table before ingest")
ap_ing.add_argument("--pg", default=None,
help="Postgres URL (or set POSTGRES_URL)")
ap_ing.add_argument("--base", default="markdown",
help="Base folder with sections/subsections/childs")
ap_ing.add_argument("--clear", action="store_true",
help="Clear docs table before ingest")
ap_ing.set_defaults(func=cli_ingest)
ap_q = sub.add_parser("query", help="Run retrieval")
ap_q.add_argument("--pg", default=None, help="Postgres URL (or set POSTGRES_URL)")
ap_q.add_argument("--pg", default=None,
help="Postgres URL (or set POSTGRES_URL)")
ap_q.add_argument("--q", required=True, help="Query text")
ap_q.add_argument("--k", type=int, default=8)
ap_q.add_argument("--expand", action="store_true")
......
math-tutor/backend/test/decision_test.py
View file @ e8105cb5
......@@ -17,14 +17,18 @@ def _resolve_sheet(args: argparse.Namespace) -> dict[str, Any]:
if args.chat_id:
sheet = context_store.load_sheet(args.chat_id)
if sheet is None:
raise FileNotFoundError(f"Kein Context-Sheet gefunden fuer chat_id={args.chat_id}")
raise FileNotFoundError(
f"Kein Context-Sheet gefunden fuer chat_id={args.chat_id}")
return sheet
raise ValueError("Bitte --sheet oder --chat-id angeben.")
def main() -> None:
parser = argparse.ArgumentParser(description="Isolierter Decision-LLM Test mit Context Sheet.")
parser = argparse.ArgumentParser(
description="Isolierter Decision-LLM Test mit Context Sheet.")
parser.add_argument("--sheet", help="Pfad zu einem Context Sheet JSON.")
parser.add_argument("--chat-id", help="Chat-ID zum Laden aus logs/context_sheets.")
parser.add_argument(
"--chat-id", help="Chat-ID zum Laden aus logs/context_sheets.")
args = parser.parse_args()
......
math-tutor/backend/test/hint_test.py
View file @ e8105cb5
......@@ -17,19 +17,23 @@ def _resolve_sheet(args: argparse.Namespace) -> dict[str, Any]:
if args.chat_id:
sheet = context_store.load_sheet(args.chat_id)
if sheet is None:
raise FileNotFoundError(f"Kein Context-Sheet gefunden fuer chat_id={args.chat_id}")
raise FileNotFoundError(
f"Kein Context-Sheet gefunden fuer chat_id={args.chat_id}")
return sheet
raise ValueError("Bitte --sheet oder --chat-id angeben.")
def main() -> None:
parser = argparse.ArgumentParser(description="Isolierter Hint-LLM Test mit Context Sheet.")
parser = argparse.ArgumentParser(
description="Isolierter Hint-LLM Test mit Context Sheet.")
parser.add_argument("--sheet", help="Pfad zu einem Context Sheet JSON.")
parser.add_argument("--chat-id", help="Chat-ID zum Laden aus logs/context_sheets.")
parser.add_argument(
"--chat-id", help="Chat-ID zum Laden aus logs/context_sheets.")
args = parser.parse_args()
sheet = _resolve_sheet(args)
history_turns = context_store.get_history_turns(sheet)
args = {
"query": history_turns[-1]["content"] if history_turns else "",
......
math-tutor/backend/test/math_intent_test.py
View file @ e8105cb5
......@@ -21,7 +21,8 @@ def _iter_inputs(args: argparse.Namespace) -> Iterable[str]:
def main() -> None:
parser = argparse.ArgumentParser(description="Isolierter Math-Intent Test.")
parser = argparse.ArgumentParser(
description="Isolierter Math-Intent Test.")
parser.add_argument(
"--input",
action="append",
......
math-tutor/backend/test/retrieval_store_test.py
View file @ e8105cb5
......@@ -6,9 +6,11 @@ from app.deterministic_services import retrieval_store
def main() -> None:
parser = argparse.ArgumentParser(description="Isolierter Vector-Store Retrieval Test.")
parser = argparse.ArgumentParser(
description="Isolierter Vector-Store Retrieval Test.")
parser.add_argument("--query", required=True, help="Query text")
parser.add_argument("--pg", default=None, help="Postgres URL (oder set POSTGRES_URL)")
parser.add_argument("--pg", default=None,
help="Postgres URL (oder set POSTGRES_URL)")
parser.add_argument("--k", type=int, default=8)
parser.add_argument("--expand", action="store_true")
parser.add_argument("--section-index", type=int, default=None)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal