pg-databank verbunden

math-tutor/.gitignore

@@ -2,3 +2,4 @@
 .env
 __pycache__/
 drawings/
+markdown/
\ No newline at end of file

math-tutor/README.md

@@ -15,3 +15,36 @@ cd math-tutor/frontend
 npm install
 npm run dev
 ```
+
+## Database (pgvector) setup
+
+Ensure `POSTGRES_URL` and embedding env vars are in `backend/.env`:
+```
+POSTGRES_URL=postgresql://user:pass@host:5432/db
+OPENAI_BASE_URL=...
+OPENAI_API_KEY=...
+OPENAI_EMBED_MODEL=...
+```
+
+Create the Postgres database before running init.
+
+Init DB schema:
+```powershell
+cd math-tutor/backend
+.\.venv\Scripts\Activate.ps1
+python .\scripts\retrieval_cli.py init-db
+```
+
+Ingest markdown docs (expects `markdown/sections`, `markdown/subsections`, `markdown/childs`):
+```powershell
+cd math-tutor/backend
+.\.venv\Scripts\Activate.ps1
+python .\scripts\retrieval_cli.py ingest --base markdown
+```
+
+Query via CLI:
+```powershell
+cd math-tutor/backend
+.\.venv\Scripts\Activate.ps1
+python .\scripts\retrieval_cli.py query --q "Was ist eine Teilmenge?" --k 8 --expand
+```

math-tutor/backend/app/__init__.py

+# Package marker for backend app

math-tutor/backend/app/api/retrieval.py

 from __future__ import annotations
 
-from fastapi import APIRouter
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import List, Optional
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel, Field
+from dotenv import load_dotenv
+
+from app.services.embeddings import OpenAILikeEmbeddings
+from app.services import vector_store
 
 router = APIRouter()
 
+load_dotenv()
+
+
+def _get_embedder() -> OpenAILikeEmbeddings:
+    base_url = os.getenv("OPENAI_BASE_URL")
+    api_key = os.getenv("OPENAI_API_KEY")
+    model = os.getenv("OPENAI_EMBED_MODEL", "text-embedding-3-large")
+    if not base_url or not api_key:
+        raise HTTPException(status_code=500, detail="Missing OPENAI_BASE_URL or OPENAI_API_KEY")
+    return OpenAILikeEmbeddings(base_url=base_url, api_key=api_key, model=model, target_dim=1024)
+
+
+class InitDbRequest(BaseModel):
+    pg_url: Optional[str] = None
+
+
+class IngestRequest(BaseModel):
+    pg_url: Optional[str] = None
+    base_dir: str = Field(default="markdown")
+
+
+class QueryRequest(BaseModel):
+    pg_url: Optional[str] = None
+    query: str = Field(..., min_length=1)
+    k: int = 8
+    expand_links: bool = True
+    section_index: Optional[int] = None
+    subsection_index: Optional[int] = None
+    type_filter: Optional[List[str]] = None
+    neighbor_expand: int = 0
+
+
+@router.post("/api/retrieval/init-db")
+def init_db(request: InitDbRequest) -> dict:
+    pg_url = request.pg_url or os.getenv("POSTGRES_URL")
+    if not pg_url:
+        raise HTTPException(status_code=500, detail="Missing POSTGRES_URL")
+    vector_store.init_db(pg_url)
+    return {"status": "ok"}
+
+
+@router.post("/api/retrieval/ingest")
+def ingest(request: IngestRequest) -> dict:
+    pg_url = request.pg_url or os.getenv("POSTGRES_URL")
+    if not pg_url:
+        raise HTTPException(status_code=500, detail="Missing POSTGRES_URL")
+    base_dir = Path(request.base_dir)
+    if not base_dir.exists():
+        raise HTTPException(status_code=400, detail="base_dir does not exist")
+
+    embedder = _get_embedder()
+    docs = vector_store.load_docs(base_dir)
+    embeddings = vector_store.embed_passages(embedder, [doc.markdown for doc in docs])
+    upserted = vector_store.upsert_docs(pg_url, docs, embeddings)
+    return {"status": "ok", "upserted": upserted}
+
+
+@router.post("/api/retrieval/query")
+def query(request: QueryRequest) -> dict:
+    pg_url = request.pg_url or os.getenv("POSTGRES_URL")
+    if not pg_url:
+        raise HTTPException(status_code=500, detail="Missing POSTGRES_URL")
+    embedder = _get_embedder()
+    result = vector_store.retrieve(
+        pg_url=pg_url,
+        embedder=embedder,
+        query=request.query,
+        k=request.k,
+        section_index=request.section_index,
+        subsection_index=request.subsection_index,
+        type_filter=request.type_filter,
+        expand_links=request.expand_links,
+        neighbor_expand=request.neighbor_expand,
+    )
+
+    def pack(items: list[vector_store.Retrieved]) -> list[dict]:
+        return [item.to_dict() for item in items]
+
+    return {
+        "query": result["query"],
+        "children": pack(result["children"]),
+        "children_direct": pack(result["children_direct"]),
+        "children_expanded": pack(result["children_expanded"]),
+        "subsections": pack(result["subsections"]),
+        "sections": pack(result["sections"]),
+        "neighbors": pack(result["neighbors"]),
+    }
+
+
+@router.get("/api/retrieval/sections")
+def sections(pg_url: Optional[str] = None) -> list[dict]:
+    url = pg_url or os.getenv("POSTGRES_URL")
+    if not url:
+        raise HTTPException(status_code=500, detail="Missing POSTGRES_URL")
+    return vector_store.list_sections(url)
+
+
+@router.get("/api/retrieval/subsections")
+def subsections(pg_url: Optional[str] = None, section_index: Optional[int] = None) -> list[dict]:
+    url = pg_url or os.getenv("POSTGRES_URL")
+    if not url:
+        raise HTTPException(status_code=500, detail="Missing POSTGRES_URL")
+    return vector_store.list_subsections(url, section_index)
+
 
 @router.get("/api/retrieval/health")
 def retrieval_health() -> dict:

math-tutor/backend/app/services/__init__.py

+# Package marker for services

math-tutor/backend/app/services/embeddings.py

+from __future__ import annotations
+
+import math
+from typing import List
+
+import httpx
+
+
+class OpenAILikeEmbeddings:
+    def __init__(self, base_url: str, api_key: str, model: str, target_dim: int = 1024) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.model = model
+        self.target_dim = target_dim
+        self.endpoint = self._embedding_endpoint()
+
+    def _embedding_endpoint(self) -> str:
+        if self.base_url.endswith("/embeddings"):
+            return self.base_url
+        if self.base_url.endswith("/v1"):
+            return f"{self.base_url}/embeddings"
+        return f"{self.base_url}/v1/embeddings"
+
+    def _normalize(self, vec: List[float]) -> List[float]:
+        norm = math.sqrt(sum(x * x for x in vec))
+        if norm == 0.0:
+            return vec
+        return [x / norm for x in vec]
+
+    def _truncate(self, vec: List[float]) -> List[float]:
+        if len(vec) < self.target_dim:
+            raise ValueError(f"Embedding dimension {len(vec)} < target {self.target_dim}")
+        if len(vec) > self.target_dim:
+            vec = vec[: self.target_dim]
+        return self._normalize(vec)
+
+    def _embed(self, inputs: List[str] | str) -> List[List[float]]:
+        payload = {
+            "input": inputs,
+            "model": self.model,
+            "encoding_format": "float",
+        }
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+        }
+        with httpx.Client(timeout=60.0) as client:
+            response = client.post(self.endpoint, headers=headers, json=payload)
+            response.raise_for_status()
+
+        data = response.json().get("data")
+        if not isinstance(data, list):
+            raise ValueError("Embedding response missing 'data' list.")
+
+        data_sorted = sorted(data, key=lambda item: item.get("index", 0))
+        embeddings: List[List[float]] = []
+        for item in data_sorted:
+            emb = item.get("embedding")
+            if not isinstance(emb, list):
+                raise ValueError("Embedding item missing 'embedding' list.")
+            embeddings.append(self._truncate([float(x) for x in emb]))
+        return embeddings
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        return self._embed(texts)
+
+    def embed_query(self, text: str) -> List[float]:
+        return self._embed(text)[0]

math-tutor/backend/app/services/vector_store.py

+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Protocol, Tuple
+
+import psycopg
+from psycopg.rows import dict_row
+from pgvector import Vector
+from pgvector.psycopg import register_vector
+import yaml
+
+
+@dataclass
+class DocRecord:
+    doc_type: str
+    path: str
+    uid: str
+    markdown: str
+    metadata: Dict[str, Any]
+
+
+class EmbeddingLike(Protocol):
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        ...
+
+    def embed_query(self, text: str) -> List[float]:
+        ...
+
+
+def parse_markdown_with_frontmatter(text: str) -> Tuple[Dict[str, Any], str]:
+    text = text.lstrip("\ufeff")
+    if not text.startswith("---"):
+        return {}, text
+
+    parts = text.split("\n---\n", 1)
+    if len(parts) != 2:
+        m = text.split("---", 2)
+        if len(m) >= 3:
+            fm = m[1]
+            body = "---".join(m[2:])
+            return (yaml.safe_load(fm) or {}), body.lstrip()
+        return {}, text
+
+    fm_block = parts[0].strip("-\n ")
+    body = parts[1]
+    meta = yaml.safe_load(fm_block) or {}
+    return meta, body.lstrip()
+
+
+def stable_uid(doc_type: str, path: str, meta: Dict[str, Any]) -> str:
+    sec = meta.get("section_index")
+    sub = meta.get("subsection_index")
+    child = meta.get("child_index")
+
+    if doc_type == "section" and sec is not None:
+        raw = f"section|s{int(sec):03d}"
+    elif doc_type == "subsection" and sec is not None and sub is not None:
+        raw = f"subsection|s{int(sec):03d}|ss{int(sub):03d}"
+    elif doc_type == "child" and sec is not None and sub is not None and child is not None:
+        raw = f"child|s{int(sec):03d}|ss{int(sub):03d}|c{int(child):03d}"
+    else:
+        raw = f"{doc_type}|{path}"
+
+    h = hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16]
+    return f"{raw}|{h}"
+
+
+def load_docs(base_dir: Path) -> List[DocRecord]:
+    docs: List[DocRecord] = []
+    mapping = [
+        ("section", [base_dir / "sections", base_dir / "section"]),
+        ("subsection", [base_dir / "subsections", base_dir / "subsection"]),
+        ("child", [base_dir / "childs"]),
+    ]
+
+    for doc_type, folders in mapping:
+        for folder in folders:
+            if not folder.exists():
+                continue
+            for path in sorted(folder.glob("*.md")):
+                text = path.read_text(encoding="utf-8")
+                meta, body = parse_markdown_with_frontmatter(text)
+
+                meta_norm: Dict[str, Any] = dict(meta)
+                meta_norm["markdown"] = body
+                meta_norm["doc_type"] = doc_type
+                meta_norm["path"] = str(path)
+
+                uid = stable_uid(doc_type, str(path), meta_norm)
+                docs.append(
+                    DocRecord(
+                        doc_type=doc_type,
+                        path=str(path),
+                        uid=uid,
+                        markdown=body,
+                        metadata=meta_norm,
+                    )
+                )
+
+    if not docs:
+        raise FileNotFoundError(
+            f"No markdown docs found under {base_dir}. Expected sections/subsections/childs."
+        )
+
+    return docs
+
+
+def embed_passages(embedder: EmbeddingLike, texts: List[str]) -> List[List[float]]:
+    prefixed = [f"passage: {t}" for t in texts]
+    return embedder.embed_documents(prefixed)
+
+
+def embed_query(embedder: EmbeddingLike, text: str) -> List[float]:
+    return embedder.embed_query(f"query: {text}")
+
+
+DDL = """
+CREATE EXTENSION IF NOT EXISTS vector;
+
+CREATE TABLE IF NOT EXISTS docs (
+  uid TEXT PRIMARY KEY,
+  doc_type TEXT NOT NULL,
+  section_index INT NULL,
+  subsection_index INT NULL,
+  child_index INT NULL,
+
+  section_title TEXT NULL,
+  subsection_title TEXT NULL,
+  title TEXT NULL,
+  type TEXT NULL,
+  box_hint TEXT NULL,
+
+  path TEXT NOT NULL,
+  markdown TEXT NOT NULL,
+
+  embedding VECTOR(1024) NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS docs_embedding_cos_idx
+  ON docs USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
+
+CREATE INDEX IF NOT EXISTS docs_type_idx ON docs(type);
+CREATE INDEX IF NOT EXISTS docs_doc_type_idx ON docs(doc_type);
+CREATE INDEX IF NOT EXISTS docs_section_idx ON docs(section_index);
+CREATE INDEX IF NOT EXISTS docs_subsection_idx ON docs(section_index, subsection_index);
+"""
+
+UPSERT_SQL = """
+INSERT INTO docs (
+  uid, doc_type,
+  section_index, subsection_index, child_index,
+  section_title, subsection_title, title, type, box_hint,
+  path, markdown, embedding
+) VALUES (
+  %(uid)s, %(doc_type)s,
+  %(section_index)s, %(subsection_index)s, %(child_index)s,
+  %(section_title)s, %(subsection_title)s, %(title)s, %(type)s, %(box_hint)s,
+  %(path)s, %(markdown)s, %(embedding)s
+)
+ON CONFLICT (uid) DO UPDATE SET
+  doc_type = EXCLUDED.doc_type,
+  section_index = EXCLUDED.section_index,
+  subsection_index = EXCLUDED.subsection_index,
+  child_index = EXCLUDED.child_index,
+  section_title = EXCLUDED.section_title,
+  subsection_title = EXCLUDED.subsection_title,
+  title = EXCLUDED.title,
+  type = EXCLUDED.type,
+  box_hint = EXCLUDED.box_hint,
+  path = EXCLUDED.path,
+  markdown = EXCLUDED.markdown,
+  embedding = EXCLUDED.embedding
+;
+"""
+
+
+def init_db(pg_url: str) -> None:
+    with psycopg.connect(pg_url) as conn:
+        with conn.cursor() as cur:
+            cur.execute(DDL)
+        register_vector(conn)
+        conn.commit()
+
+
+def _meta_int(meta: Dict[str, Any], key: str) -> Optional[int]:
+    value = meta.get(key)
+    if value is None or value == "":
+        return None
+    try:
+        return int(value)
+    except Exception:
+        return None
+
+
+def upsert_docs(pg_url: str, docs: List[DocRecord], embeddings: List[List[float]]) -> int:
+    if len(docs) != len(embeddings):
+        raise ValueError("Docs and embeddings length mismatch")
+
+    rows = []
+    for doc, emb in zip(docs, embeddings):
+        m = doc.metadata
+        rows.append(
+            {
+                "uid": doc.uid,
+                "doc_type": doc.doc_type,
+                "section_index": _meta_int(m, "section_index"),
+                "subsection_index": _meta_int(m, "subsection_index"),
+                "child_index": _meta_int(m, "child_index"),
+                "section_title": m.get("section_title"),
+                "subsection_title": m.get("subsection_title"),
+                "title": m.get("title"),
+                "type": m.get("type"),
+                "box_hint": m.get("box_hint"),
+                "path": doc.path,
+                "markdown": doc.markdown,
+                "embedding": Vector(emb),
+            }
+        )
+
+    with psycopg.connect(pg_url) as conn:
+        register_vector(conn)
+        with conn.cursor() as cur:
+            cur.executemany(UPSERT_SQL, rows)
+        conn.commit()
+    return len(rows)
+
+
+@dataclass
+class Retrieved:
+    uid: str
+    doc_type: str
+    score: float
+    metadata: Dict[str, Any]
+    markdown: str
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "uid": self.uid,
+            "doc_type": self.doc_type,
+            "score": self.score,
+            "metadata": self.metadata,
+            "markdown": self.markdown,
+        }
+
+
+def _row_to_retrieved(row: Dict[str, Any]) -> Retrieved:
+    meta = {
+        "uid": row["uid"],
+        "doc_type": row["doc_type"],
+        "section_index": row["section_index"],
+        "subsection_index": row["subsection_index"],
+        "child_index": row["child_index"],
+        "section_title": row["section_title"],
+        "subsection_title": row["subsection_title"],
+        "title": row["title"],
+        "type": row["type"],
+        "box_hint": row["box_hint"],
+        "path": row["path"],
+    }
+    return Retrieved(
+        uid=row["uid"],
+        doc_type=row["doc_type"],
+        score=float(row["score"]),
+        metadata=meta,
+        markdown=row["markdown"],
+    )
+
+
+def retrieve(
+    pg_url: str,
+    embedder: EmbeddingLike,
+    query: str,
+    k: int = 8,
+    section_index: Optional[int] = None,
+    subsection_index: Optional[int] = None,
+    type_filter: Optional[List[str]] = None,
+    expand_links: bool = True,
+    neighbor_expand: int = 0,
+) -> Dict[str, Any]:
+    qvec = Vector(embed_query(embedder, query))
+
+    where = ["doc_type = 'child'"]
+    params: Dict[str, Any] = {"qvec": qvec, "k": k}
+
+    if section_index is not None:
+        where.append("section_index = %(section_index)s")
+        params["section_index"] = section_index
+
+    if subsection_index is not None:
+        where.append("subsection_index = %(subsection_index)s")
+        params["subsection_index"] = subsection_index
+
+    if type_filter:
+        where.append("type = ANY(%(type_filter)s)")
+        params["type_filter"] = type_filter
+
+    where_sql = " AND ".join(where)
+
+    sql = f"""
+    SELECT
+      uid, doc_type,
+      section_index, subsection_index, child_index,
+      section_title, subsection_title, title, type, box_hint,
+      path, markdown,
+      1 - (embedding <=> %(qvec)s) AS score
+    FROM docs
+    WHERE {where_sql}
+    ORDER BY embedding <=> %(qvec)s
+    LIMIT %(k)s;
+    """
+
+    children: List[Retrieved] = []
+    neighbors: List[Retrieved] = []
+    subsections: List[Retrieved] = []
+    sections_docs: List[Retrieved] = []
+    children_direct: List[Retrieved] = []
+    children_expanded: List[Retrieved] = []
+
+    with psycopg.connect(pg_url, row_factory=dict_row) as conn:
+        register_vector(conn)
+        with conn.cursor() as cur:
+            cur.execute(sql, params)
+            rows = cur.fetchall()
+            children = [_row_to_retrieved(row) for row in rows]
+            children_direct = list(children)
+
+            if expand_links and children:
+                sec_sub_counts: Dict[tuple[int, int], int] = {}
+                for child in children:
+                    sec_idx = child.metadata.get("section_index")
+                    sub_idx = child.metadata.get("subsection_index")
+                    if sec_idx is not None and sub_idx is not None:
+                        key = (int(sec_idx), int(sub_idx))
+                        sec_sub_counts[key] = sec_sub_counts.get(key, 0) + 1
+
+                if sec_sub_counts:
+                    most_common_sec_sub = max(sec_sub_counts.items(), key=lambda x: x[1])[0]
+                    most_common_sec, most_common_sub = most_common_sec_sub
+
+                    cur.execute(
+                        """
+                        SELECT
+                          d.uid, d.doc_type,
+                          d.section_index, d.subsection_index, d.child_index,
+                          d.section_title, d.subsection_title, d.title, d.type, d.box_hint,
+                          d.path, d.markdown,
+                          1.0 AS score
+                        FROM docs d
+                        WHERE d.doc_type = ANY(%(sub_doc_types)s)
+                          AND d.section_index = %(sec)s
+                          AND d.subsection_index = %(sub)s
+                        """,
+                        {"sec": most_common_sec, "sub": most_common_sub, "sub_doc_types": ["subsection", "chapter"]},
+                    )
+                    subsections = [_row_to_retrieved(row) for row in cur.fetchall()]
+
+                    cur.execute(
+                        """
+                        SELECT
+                          uid, doc_type,
+                          section_index, subsection_index, child_index,
+                          section_title, subsection_title, title, type, box_hint,
+                          path, markdown,
+                          1.0 AS score
+                        FROM docs
+                        WHERE doc_type = ANY(%(sec_doc_types)s)
+                          AND section_index = %(sec)s
+                        """,
+                        {"sec": most_common_sec, "sec_doc_types": ["section", "oberchapter"]},
+                    )
+                    sections_docs = [_row_to_retrieved(row) for row in cur.fetchall()]
+
+                    cur.execute(
+                        """
+                        SELECT
+                          d.uid, d.doc_type,
+                          d.section_index, d.subsection_index, d.child_index,
+                          d.section_title, d.subsection_title, d.title, d.type, d.box_hint,
+                          d.path, d.markdown,
+                          1 - (d.embedding <=> %(qvec)s) AS score
+                        FROM docs d
+                        WHERE d.doc_type = 'child'
+                          AND d.section_index = %(sec)s
+                          AND d.subsection_index = %(sub)s
+                        ORDER BY d.embedding <=> %(qvec)s
+                        LIMIT 5;
+                        """,
+                        {"qvec": qvec, "sec": most_common_sec, "sub": most_common_sub},
+                    )
+
+                    additional_children = [_row_to_retrieved(row) for row in cur.fetchall()]
+                    existing_uids = {child.uid for child in children}
+                    new_children = [child for child in additional_children if child.uid not in existing_uids]
+                    children.extend(new_children)
+                    children_expanded.extend(new_children)
+
+                    if neighbor_expand and neighbor_expand > 0:
+                        wanted: set[tuple[int, int, int]] = set()
+                        for child in children:
+                            si = child.metadata.get("section_index")
+                            ssi = child.metadata.get("subsection_index")
+                            ci = child.metadata.get("child_index")
+                            if si is None or ssi is None or ci is None:
+                                continue
+                            for dx in range(-neighbor_expand, neighbor_expand + 1):
+                                if dx == 0:
+                                    continue
+                                wanted.add((int(si), int(ssi), int(ci) + dx))
+
+                        if wanted:
+                            triples = sorted(wanted)
+                            sec_arr = [a for (a, b, cidx) in triples]
+                            sub_arr = [b for (a, b, cidx) in triples]
+                            child_arr = [cidx for (a, b, cidx) in triples]
+
+                            cur.execute(
+                                """
+                                SELECT
+                                  d.uid, d.doc_type,
+                                  d.section_index, d.subsection_index, d.child_index,
+                                  d.section_title, d.subsection_title, d.title, d.type, d.box_hint,
+                                  d.path, d.markdown,
+                                  0.9 AS score
+                                FROM docs d
+                                JOIN unnest(%(sec_arr)s::int[], %(sub_arr)s::int[], %(child_arr)s::int[]) AS u(sec, sub, child)
+                                ON d.section_index = u.sec AND d.subsection_index = u.sub AND d.child_index = u.child
+                                WHERE d.doc_type = 'child'
+                                """,
+                                {"sec_arr": sec_arr, "sub_arr": sub_arr, "child_arr": child_arr},
+                            )
+                            neighbors = [_row_to_retrieved(row) for row in cur.fetchall()]
+
+    child_uids = {child.uid for child in children}
+    neighbors = [neighbor for neighbor in neighbors if neighbor.uid not in child_uids]
+
+    return {
+        "children": children,
+        "children_direct": children_direct,
+        "children_expanded": children_expanded,
+        "subsections": subsections,
+        "sections": sections_docs,
+        "neighbors": neighbors,
+        "query": query,
+    }
+
+
+def list_sections(pg_url: str) -> List[Dict[str, Any]]:
+    sql = """
+    SELECT DISTINCT ON (section_index)
+      section_index,
+      COALESCE(NULLIF(section_title, ''), '') AS section_title
+    FROM docs
+    WHERE section_index IS NOT NULL
+    ORDER BY section_index,
+      CASE WHEN section_title IS NULL OR section_title = '' THEN 1 ELSE 0 END,
+      section_title
+    """
+    with psycopg.connect(pg_url, row_factory=dict_row) as conn:
+        register_vector(conn)
+        with conn.cursor() as cur:
+            cur.execute(sql)
+            return cur.fetchall()
+
+
+def list_subsections(pg_url: str, section_idx: Optional[int] = None) -> List[Dict[str, Any]]:
+    if section_idx is None:
+        sql = """
+        SELECT DISTINCT ON (section_index, subsection_index)
+          section_index, subsection_index,
+          COALESCE(NULLIF(subsection_title, ''), '') AS subsection_title
+        FROM docs
+        WHERE subsection_index IS NOT NULL
+        ORDER BY section_index, subsection_index,
+          CASE WHEN subsection_title IS NULL OR subsection_title = '' THEN 1 ELSE 0 END,
+          subsection_title
+        """
+        params: Dict[str, Any] = {}
+    else:
+        sql = """
+        SELECT DISTINCT ON (section_index, subsection_index)
+          section_index, subsection_index,
+          COALESCE(NULLIF(subsection_title, ''), '') AS subsection_title
+        FROM docs
+        WHERE subsection_index IS NOT NULL AND section_index = %(sec)s
+        ORDER BY section_index, subsection_index,
+          CASE WHEN subsection_title IS NULL OR subsection_title = '' THEN 1 ELSE 0 END,
+          subsection_title
+        """
+        params = {"sec": section_idx}
+
+    with psycopg.connect(pg_url, row_factory=dict_row) as conn:
+        register_vector(conn)
+        with conn.cursor() as cur:
+            cur.execute(sql, params)
+            return cur.fetchall()

math-tutor/backend/requirements.txt

@@ -4,3 +4,6 @@ python-dotenv
 mpxpy
 pillow
 httpx
+psycopg[binary]
+pgvector
+pyyaml

math-tutor/backend/scripts/retrieval_cli.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+from app.services.embeddings import OpenAILikeEmbeddings
+from app.services import vector_store
+
+
+def build_embedder() -> OpenAILikeEmbeddings:
+    base_url = os.getenv("OPENAI_BASE_URL")
+    api_key = os.getenv("OPENAI_API_KEY")
+    model = os.getenv("OPENAI_EMBED_MODEL", "text-embedding-3-large")
+    if not base_url or not api_key:
+        raise ValueError("Missing OPENAI_BASE_URL or OPENAI_API_KEY")
+    return OpenAILikeEmbeddings(base_url=base_url, api_key=api_key, model=model, target_dim=1024)
+
+
+def cli_init_db(args: argparse.Namespace) -> None:
+    pg_url = args.pg or os.getenv("POSTGRES_URL")
+    if not pg_url:
+        raise ValueError("Missing POSTGRES_URL")
+    vector_store.init_db(pg_url)
+    print("DB initialized.")
+
+
+def cli_ingest(args: argparse.Namespace) -> None:
+    embedder = build_embedder()
+    base_dir = Path(args.base)
+    docs = vector_store.load_docs(base_dir)
+    embeddings = vector_store.embed_passages(embedder, [doc.markdown for doc in docs])
+    pg_url = args.pg or os.getenv("POSTGRES_URL")
+    if not pg_url:
+        raise ValueError("Missing POSTGRES_URL")
+    upserted = vector_store.upsert_docs(pg_url, docs, embeddings)
+    print(f"Ingested/upserted docs: {upserted}")
+
+
+def cli_query(args: argparse.Namespace) -> None:
+    embedder = build_embedder()
+    pg_url = args.pg or os.getenv("POSTGRES_URL")
+    if not pg_url:
+        raise ValueError("Missing POSTGRES_URL")
+    result = vector_store.retrieve(
+        pg_url=pg_url,
+        embedder=embedder,
+        query=args.q,
+        k=args.k,
+        expand_links=args.expand,
+        section_index=args.section_index,
+        subsection_index=args.subsection_index,
+        type_filter=args.type_filter,
+    )
+    print("=== SECTIONS ===")
+    for item in result["sections"]:
+        meta = item.metadata
+        print(f"- [{item.score:.3f}] s{meta.get('section_index')} {meta.get('title') or meta.get('section_title')}")
+    print("=== SUBSECTIONS ===")
+    for item in result["subsections"]:
+        meta = item.metadata
+        print(
+            f"- [{item.score:.3f}] s{meta.get('section_index')}, ss{meta.get('subsection_index')} {meta.get('title')}"
+        )
+    print("=== CHILDREN ===")
+    for item in result["children"]:
+        meta = item.metadata
+        print(
+            f"- [{item.score:.3f}] s{meta.get('section_index')}, ss{meta.get('subsection_index')}, c{meta.get('child_index')}"
+        )
+
+
+def main() -> int:
+    load_dotenv()
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(required=True)
+
+    ap_init = sub.add_parser("init-db", help="Create tables and indexes in Postgres")
+    ap_init.add_argument("--pg", default=None, help="Postgres URL (or set POSTGRES_URL)")
+    ap_init.set_defaults(func=cli_init_db)
+
+    ap_ing = sub.add_parser("ingest", help="Embed and upsert markdown docs")
+    ap_ing.add_argument("--pg", default=None, help="Postgres URL (or set POSTGRES_URL)")
+    ap_ing.add_argument("--base", default="markdown", help="Base folder with sections/subsections/childs")
+    ap_ing.set_defaults(func=cli_ingest)
+
+    ap_q = sub.add_parser("query", help="Run retrieval")
+    ap_q.add_argument("--pg", default=None, help="Postgres URL (or set POSTGRES_URL)")
+    ap_q.add_argument("--q", required=True, help="Query text")
+    ap_q.add_argument("--k", type=int, default=8)
+    ap_q.add_argument("--expand", action="store_true")
+    ap_q.add_argument("--section-index", type=int, default=None)
+    ap_q.add_argument("--subsection-index", type=int, default=None)
+    ap_q.add_argument("--type-filter", nargs="*", default=None)
+    ap_q.set_defaults(func=cli_query)
+
+    args = parser.parse_args()
+    args.func(args)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

math-tutor/frontend/README.md

-# React + TypeScript + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
-- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) (or [oxc](https://oxc.rs) when used in [rolldown-vite](https://vite.dev/guide/rolldown)) for Fast Refresh
-- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
-
-## React Compiler
-
-The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation).
-
-## Expanding the ESLint configuration
-
-If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules:
-
-```js
-export default defineConfig([
-  globalIgnores(['dist']),
-  {
-    files: ['**/*.{ts,tsx}'],
-    extends: [
-      // Other configs...
-
-      // Remove tseslint.configs.recommended and replace with this
-      tseslint.configs.recommendedTypeChecked,
-      // Alternatively, use this for stricter rules
-      tseslint.configs.strictTypeChecked,
-      // Optionally, add this for stylistic rules
-      tseslint.configs.stylisticTypeChecked,
-
-      // Other configs...
-    ],
-    languageOptions: {
-      parserOptions: {
-        project: ['./tsconfig.node.json', './tsconfig.app.json'],
-        tsconfigRootDir: import.meta.dirname,
-      },
-      // other options...
-    },
-  },
-])
-```
-
-You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules:
-
-```js
-// eslint.config.js
-import reactX from 'eslint-plugin-react-x'
-import reactDom from 'eslint-plugin-react-dom'
-
-export default defineConfig([
-  globalIgnores(['dist']),
-  {
-    files: ['**/*.{ts,tsx}'],
-    extends: [
-      // Other configs...
-      // Enable lint rules for React
-      reactX.configs['recommended-typescript'],
-      // Enable lint rules for React DOM
-      reactDom.configs.recommended,
-    ],
-    languageOptions: {
-      parserOptions: {
-        project: ['./tsconfig.node.json', './tsconfig.app.json'],
-        tsconfigRootDir: import.meta.dirname,
-      },
-      // other options...
-    },
-  },
-])
-```