chapter gefixed

math-tutor/backend/app/deterministic_services/vector_store.py

@@ -64,13 +64,13 @@ def stable_uid(doc_type: str, path: str, meta: Dict[str, Any]) -> str:
     sub = meta.get("subsection_index")
     child = meta.get("child_index")
 
-    if doc_type == "chapter" and sub is not None:
+    if doc_type == "chapter" and chapter is not None:
         raw = f"chapter|c{int(chapter):03d}"
-    if doc_type == "section" and sec is not None:
+    if doc_type == "section" and chapter is not None and sec is not None:
         raw = f"section|c{int(chapter):03d}|s{int(sec):03d}"
-    elif doc_type == "subsection" and sec is not None and sub is not None:
+    elif doc_type == "subsection" and chapter is not None and sec is not None and sub is not None:
         raw = f"subsection|c{int(chapter):03d}|s{int(sec):03d}|ss{int(sub):03d}"
-    elif doc_type == "child" and sec is not None and sub is not None and child is not None:
+    elif doc_type == "child" and chapter is not None and sec is not None and sub is not None and child is not None:
         raw = f"child|c{int(chapter):03d}|s{int(sec):03d}|ss{int(sub):03d}|c{int(child):03d}"
     else:
         raw = f"{doc_type}|{path}"
@@ -82,7 +82,7 @@ def stable_uid(doc_type: str, path: str, meta: Dict[str, Any]) -> str:
 def load_docs(base_dir: Path) -> List[DocRecord]:
     docs: List[DocRecord] = []
     mapping = [
-        ("chapter", [base_dir / "chapter", base_dir / "chapter"]),
+        ("chapter", [base_dir / "chapters", base_dir / "chapter"]),
         ("section", [base_dir / "sections", base_dir / "section"]),
         ("subsection", [base_dir / "subsections", base_dir / "subsection"]),
         ("child", [base_dir / "childs"]),
@@ -289,6 +289,7 @@ def _row_to_retrieved(row: Dict[str, Any], source_type: Optional[str] = None) ->
         "section_index": row["section_index"],
         "subsection_index": row["subsection_index"],
         "child_index": row["child_index"],
+        "chapter_title": row["chapter_title"],
         "section_title": row["section_title"],
         "subsection_title": row["subsection_title"],
         "title": row["title"],
@@ -317,9 +318,10 @@ def retrieve(
     neighbor_expand: int = 0,
 ) -> List[Source]:
     qvec = Vector(embed_query(embedder, query))
+    vector_k = max(k * 4, k + 16)
 
     where = ["doc_type = 'child'"]
-    params: Dict[str, Any] = {"qvec": qvec, "k": k}
+    params: Dict[str, Any] = {"qvec": qvec, "vector_k": vector_k}
 
     if chapter_index is not None:
         where.append("chapter_index = %(chapter_index)s")
@@ -349,28 +351,40 @@ def retrieve(
     FROM docs
     WHERE {where_sql}
     ORDER BY embedding <=> %(qvec)s
-    LIMIT %(k)s;
+    LIMIT %(vector_k)s;
     """
 
     children: List[Retrieved] = []
     neighbors: List[Retrieved] = []
-    chapter: List[Retrieved] = []
+    chapter_docs: List[Retrieved] = []
     subsections: List[Retrieved] = []
     sections_docs: List[Retrieved] = []
     children_direct: List[Retrieved] = []
     children_expanded: List[Retrieved] = []
 
+    def _top_k_unique(values: List[Retrieved], wanted_k: int) -> List[Retrieved]:
+        selected: List[Retrieved] = []
+        seen: set[str] = set()
+        for item in values:
+            if item.uid in seen:
+                continue
+            selected.append(item)
+            seen.add(item.uid)
+            if len(selected) >= wanted_k:
+                break
+        return selected
+
     with psycopg.connect(pg_url, row_factory=dict_row) as conn:
         register_vector(conn)
         with conn.cursor() as cur:
             cur.execute(sql, params)
             rows = cur.fetchall()
             children = [_row_to_retrieved(row) for row in rows]
-            children_direct = list(children)
+            children_direct = _top_k_unique(children, k)
 
-            if expand_links and children:
+            if expand_links and children_direct:
                 cpt_sec_sub_counts: Dict[tuple[int, int, int], int] = {}
-                for child in children:
+                for child in children_direct:
                     cpt_idx = child.metadata.get("chapter_index")
                     sec_idx = child.metadata.get("section_index")
                     sub_idx = child.metadata.get("subsection_index")
@@ -392,13 +406,12 @@ def retrieve(
                           d.path, d.markdown,
                           1.0 AS score
                         FROM docs d
-                        WHERE d.doc_type = ANY(%(sub_doc_types)s)
+                        WHERE d.doc_type = 'subsection'
                           AND d.chapter_index = %(cpt)s
                           AND d.section_index = %(sec)s
                           AND d.subsection_index = %(sub)s
                         """,
-                        {"cpt": most_common_cpt, "sec": most_common_sec, "sub": most_common_sub,
-                            "sub_doc_types": ["chapter", "section", "subsection"]},
+                        {"cpt": most_common_cpt, "sec": most_common_sec, "sub": most_common_sub},
                     )
                     subsections = [_row_to_retrieved(
                         row, source_type="subsection") for row in cur.fetchall()]
@@ -418,11 +431,11 @@ def retrieve(
                           AND d.section_index = %(sec)s
                         """,
                         {"cpt": most_common_cpt, "sec": most_common_sec, "sec_doc_types": [
-                            "section", "chapter"]},
+                            "section"]},
                     )
                     sections_docs = [_row_to_retrieved(
                         row, source_type="section") for row in cur.fetchall()]
-                    
+
                     cur.execute(
                         """
                         SELECT
@@ -432,11 +445,10 @@ def retrieve(
                           d.path, d.markdown,
                           1.0 AS score
                         FROM docs d
-                        WHERE d.doc_type = ANY(%(cpt_doc_types)s)
+                        WHERE d.doc_type = 'chapter'
                           AND d.chapter_index = %(cpt)s
                         """,
-                        {"cpt": most_common_cpt, "cpt_doc_types": [
-                            "chapter"]},
+                        {"cpt": most_common_cpt},
                     )
                     chapter_docs = [_row_to_retrieved(
                         row, source_type="chapter") for row in cur.fetchall()]
@@ -455,22 +467,27 @@ def retrieve(
                           AND d.section_index = %(sec)s
                           AND d.subsection_index = %(sub)s
                         ORDER BY d.embedding <=> %(qvec)s
-                        LIMIT 5;
+                        LIMIT %(fill_k)s;
                         """,
                         {"qvec": qvec, "cpt": most_common_cpt, "sec": most_common_sec,
-                            "sub": most_common_sub},
+                            "sub": most_common_sub, "fill_k": max(k, 5)},
                     )
 
                     additional_children = [_row_to_retrieved(
                         row) for row in cur.fetchall()]
-                    existing_uids = {child.uid for child in children}
+                    existing_uids = {child.uid for child in children_direct}
                     new_children = [
                         child for child in additional_children if child.uid not in existing_uids]
-                    children.extend(new_children)
-                    children_expanded.extend(new_children)
+                    needed = max(0, k - len(children_direct))
+                    if needed:
+                        fill_children = new_children[:needed]
+                        children_expanded.extend(fill_children)
+                        children = [*children_direct, *fill_children]
+                    else:
+                        children = list(children_direct)
 
                     if neighbor_expand and neighbor_expand > 0:
-                        wanted: set[tuple[int, int, int]] = set()
+                        wanted: set[tuple[int, int, int, int]] = set()
                         for child in children:
                             cpti = child.metadata.get("chapter_index")
                             si = child.metadata.get("section_index")
@@ -509,6 +526,7 @@ def retrieve(
                             neighbors = [_row_to_retrieved(
                                 row) for row in cur.fetchall()]
 
+    children = [*children_direct, *children_expanded]
     child_uids = {child.uid for child in children}
     neighbors = [
         neighbor for neighbor in neighbors if neighbor.uid not in child_uids]
@@ -521,7 +539,7 @@ def retrieve(
         "neighbors": neighbors,
     }
     sources = _retrivla_to_sources(retrivla_dict)
-    return sources
+    return sorted(sources, key=lambda source: source.score, reverse=True)
 
 # --------------------------------------------------------------------------------------------------------------------
 # Retrival mit Subsection Referenzen
@@ -553,8 +571,8 @@ def load_children_for_subsections(
     sql = """
     SELECT
       d.uid, d.doc_type,
-      d.section_index, d.subsection_index, d.child_index,
-      d.section_title, d.subsection_title, d.title, d.source_type,
+      d.chapter_index, d.section_index, d.subsection_index, d.child_index,
+      d.chapter_title, d.section_title, d.subsection_title, d.title, d.source_type,
       d.path, d.markdown,
       1.0 AS score
     FROM docs d

math-tutor/backend/app/deterministic_services/vector_store_subsection.py

@@ -59,8 +59,8 @@ def retrieve(
     sql = f"""
     SELECT
       uid, doc_type,
-      section_index, subsection_index, child_index,
-      section_title, subsection_title, title, source_type,
+      chapter_index, section_index, subsection_index, child_index,
+      chapter_title, section_title, subsection_title, title, source_type,
       path, markdown,
       1 - (embedding <=> %(qvec)s) AS score
     FROM docs