hinzufügen eines neune Retrievals das basiert auf übergebenen Subsections

math-tutor/README.md

@@ -69,5 +69,6 @@ Retrieval settings:
 ## Testing
 python -m test.hint_test --chat-id draft_session_mlgmxxzc_avmjfb
 python -m test.retrieval_store_test --query "Was ist eine Teilmenge?" --k 8 --expand
+python -m test.retrieval_store_test --query "Was ist eine Teilmenge?" --k 3 --subsections 1:1
 python -m test.math_intent_test --input "Integrate x^2" --input "Was ist 2+2?"
 python -m test.decision_test --chat-id draft_session_mlgmxxzc_avmjfb

math-tutor/backend/app/deterministic_services/retrieval_store.py

@@ -46,3 +46,43 @@ def retrieve(
         expand_links=expand_links,
         neighbor_expand=neighbor_expand,
     )
+
+
+def retrieve_with_subsections(
+    pg_url: str,
+    embedder: EmbeddingLike,
+    query: str,
+    subsection_refs: list[vector_store.SubsectionRef] | None = None,
+    k: int = 4,
+    section_index: int | None = None,
+    subsection_index: int | None = None,
+    source_type_filter: list[str] | None = None,
+    expand_links: bool = True,
+    neighbor_expand: int = 0,
+) -> List[Source]:
+    if _use_subsection_retrieval():
+        return vector_store_subsection.retrieve_with_subsections(
+            pg_url=pg_url,
+            embedder=embedder,
+            query=query,
+            subsection_refs=subsection_refs,
+            k=k,
+            section_index=section_index,
+            subsection_index=subsection_index,
+            source_type_filter=source_type_filter,
+            expand_links=expand_links,
+            neighbor_expand=neighbor_expand,
+        )
+
+    return vector_store.retrieve_with_subsections(
+        pg_url=pg_url,
+        embedder=embedder,
+        query=query,
+        subsection_refs=subsection_refs,
+        k=k,
+        section_index=section_index,
+        subsection_index=subsection_index,
+        source_type_filter=source_type_filter,
+        expand_links=expand_links,
+        neighbor_expand=neighbor_expand,
+    )

math-tutor/backend/app/deterministic_services/vector_store.py

@@ -302,6 +302,8 @@ def retrieve(
     source_type_filter: Optional[List[str]] = None,
     expand_links: bool = True,
     neighbor_expand: int = 0,
+    exclude_subsection_refs: Optional[List["SubsectionRef"]] = None,
+    exclude_child_subsection_refs: Optional[List["SubsectionRef"]] = None,
 ) -> List[Source]:
     qvec = Vector(embed_query(embedder, query))
 
@@ -320,6 +322,18 @@ def retrieve(
         where.append("source_type = ANY(%(source_type_filter)s)")
         params["source_type_filter"] = source_type_filter
 
+    excluded_child_subsections = _normalize_subsection_refs(exclude_child_subsection_refs)
+    if excluded_child_subsections:
+        sec_arr = [sec for sec, _ in excluded_child_subsections]
+        sub_arr = [sub for _, sub in excluded_child_subsections]
+        where.append(
+            "NOT EXISTS ("
+            "SELECT 1 FROM unnest(%(exclude_sec_arr)s::int[], %(exclude_sub_arr)s::int[]) AS ex(sec, sub) "
+            "WHERE ex.sec = section_index AND ex.sub = subsection_index)"
+        )
+        params["exclude_sec_arr"] = sec_arr
+        params["exclude_sub_arr"] = sub_arr
+
     where_sql = " AND ".join(where)
 
     sql = f"""
@@ -341,6 +355,7 @@ def retrieve(
     sections_docs: List[Retrieved] = []
     children_direct: List[Retrieved] = []
     children_expanded: List[Retrieved] = []
+    excluded_subsections = set(_normalize_subsection_refs(exclude_subsection_refs))
 
     with psycopg.connect(pg_url, row_factory=dict_row) as conn:
         register_vector(conn)
@@ -364,24 +379,25 @@ def retrieve(
                         sec_sub_counts.items(), key=lambda x: x[1])[0]
                     most_common_sec, most_common_sub = most_common_sec_sub
 
-                    cur.execute(
-                        """
-                        SELECT
-                          d.uid, d.doc_type,
-                          d.section_index, d.subsection_index, d.child_index,
-                          d.section_title, d.subsection_title, d.title, d.source_type,
-                          d.path, d.markdown,
-                          1.0 AS score
-                        FROM docs d
-                        WHERE d.doc_type = ANY(%(sub_doc_types)s)
-                          AND d.section_index = %(sec)s
-                          AND d.subsection_index = %(sub)s
-                        """,
-                        {"sec": most_common_sec, "sub": most_common_sub,
-                            "sub_doc_types": ["subsection", "chapter"]},
-                    )
-                    subsections = [_row_to_retrieved(
-                        row, source_type="subsection") for row in cur.fetchall()]
+                    if (most_common_sec, most_common_sub) not in excluded_subsections:
+                        cur.execute(
+                            """
+                            SELECT
+                              d.uid, d.doc_type,
+                              d.section_index, d.subsection_index, d.child_index,
+                              d.section_title, d.subsection_title, d.title, d.source_type,
+                              d.path, d.markdown,
+                              1.0 AS score
+                            FROM docs d
+                            WHERE d.doc_type = ANY(%(sub_doc_types)s)
+                              AND d.section_index = %(sec)s
+                              AND d.subsection_index = %(sub)s
+                            """,
+                            {"sec": most_common_sec, "sub": most_common_sub,
+                                "sub_doc_types": ["subsection", "chapter"]},
+                        )
+                        subsections = [_row_to_retrieved(
+                            row, source_type="subsection") for row in cur.fetchall()]
 
                     cur.execute(
                         """
@@ -479,6 +495,114 @@ def retrieve(
     sources = _retrivla_to_sources(retrivla_dict)
     return sources
 
+# --------------------------------------------------------------------------------------------------------------------
+# Retrival mit Subsection Referenzen
+# --------------------------------------------------------------------------------------------------------------------
+
+
+SubsectionRef = Tuple[int, int]
+
+
+def _normalize_subsection_refs(
+    subsection_refs: Optional[List[SubsectionRef]],
+) -> List[SubsectionRef]:
+    if not subsection_refs:
+        return []
+    normalized = {(int(sec), int(sub)) for sec, sub in subsection_refs}
+    return sorted(normalized)
+
+
+def load_children_for_subsections(
+    pg_url: str,
+    subsection_refs: Optional[List[SubsectionRef]],
+) -> List[Source]:
+    refs = _normalize_subsection_refs(subsection_refs)
+    if not refs:
+        return []
+
+    sec_arr = [sec for sec, _ in refs]
+    sub_arr = [sub for _, sub in refs]
+    sql = """
+    SELECT
+      d.uid, d.doc_type,
+      d.section_index, d.subsection_index, d.child_index,
+      d.section_title, d.subsection_title, d.title, d.source_type,
+      d.path, d.markdown,
+      1.0 AS score
+    FROM docs d
+    JOIN unnest(%(sec_arr)s::int[], %(sub_arr)s::int[]) AS u(sec, sub)
+      ON d.section_index = u.sec AND d.subsection_index = u.sub
+    WHERE d.doc_type = 'child'
+    ORDER BY d.section_index, d.subsection_index, d.child_index
+    """
+    with psycopg.connect(pg_url, row_factory=dict_row) as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql, {"sec_arr": sec_arr, "sub_arr": sub_arr})
+            rows = cur.fetchall()
+
+    children = [_row_to_retrieved(row) for row in rows]
+    return _retrivla_to_sources({"children_from_subsections": children})
+
+
+def merge_sources(primary: List[Source], additional: List[Source]) -> List[Source]:
+    merged: List[Source] = []
+    seen: set[Tuple[str, str, str, str, str, str]] = set()
+
+    def source_key(source: Source) -> Tuple[str, str, str, str, str, str]:
+        return (
+            source.source_id.doc_type or "",
+            source.source_id.section_title or "",
+            source.source_id.subsection_title or "",
+            source.source_id.title or "",
+            source.source_type or "",
+            source.markdown or "",
+        )
+
+    for source in [*primary, *additional]:
+        key = source_key(source)
+        if key in seen:
+            continue
+        seen.add(key)
+        merged.append(source)
+    return merged
+
+
+def retrieve_with_subsections(
+    pg_url: str,
+    embedder: EmbeddingLike,
+    query: str,
+    subsection_refs: Optional[List[SubsectionRef]] = None,
+    k: int = 4,
+    section_index: Optional[int] = None,
+    subsection_index: Optional[int] = None,
+    source_type_filter: Optional[List[str]] = None,
+    expand_links: bool = True,
+    neighbor_expand: int = 0,
+) -> List[Source]:
+    # Oversampling improves recall with ivfflat when additional filters exclude
+    # close hits. We trim back to k after retrieval.
+    vector_k = max(k * 4, k + 16)
+    vector_sources = retrieve(
+        pg_url=pg_url,
+        embedder=embedder,
+        query=query,
+        k=vector_k,
+        section_index=section_index,
+        subsection_index=subsection_index,
+        source_type_filter=source_type_filter,
+        expand_links=expand_links,
+        neighbor_expand=neighbor_expand,
+        exclude_subsection_refs=subsection_refs,
+        exclude_child_subsection_refs=subsection_refs,
+    )
+    vector_sources = vector_sources[:k]
+    subsection_children = load_children_for_subsections(
+        pg_url=pg_url,
+        subsection_refs=subsection_refs,
+    )
+    return merge_sources(vector_sources, subsection_children)
+
+
 # --------------------------------------------------------------------------------------------------------------------
 # Retrival in Sources umwandeln
 # --------------------------------------------------------------------------------------------------------------------

math-tutor/backend/app/deterministic_services/vector_store_subsection.py

@@ -10,9 +10,12 @@ from psycopg.rows import dict_row
 from app.deterministic_services.vector_store import (
     EmbeddingLike,
     Source,
+    SubsectionRef,
     _retrivla_to_sources,
     _row_to_retrieved,
     embed_query,
+    load_children_for_subsections,
+    merge_sources,
 )
 
 
@@ -26,6 +29,7 @@ def retrieve(
     source_type_filter: Optional[List[str]] = None,
     expand_links: bool = False,
     neighbor_expand: int = 0,
+    exclude_subsection_refs: Optional[List[SubsectionRef]] = None,
 ) -> List[Source]:
     # Parameters kept for drop-in compatibility with child-level retrieve.
     _ = expand_links
@@ -52,6 +56,19 @@ def retrieve(
         where.append("source_type = ANY(%(source_type_filter)s)")
         params["source_type_filter"] = source_type_filter
 
+    excluded = set((int(sec), int(sub)) for sec, sub in (exclude_subsection_refs or []))
+    if excluded:
+        exclude_clauses: List[str] = []
+        for idx, (sec, sub) in enumerate(sorted(excluded)):
+            sec_key = f"exclude_sec_{idx}"
+            sub_key = f"exclude_sub_{idx}"
+            exclude_clauses.append(
+                f"(section_index = %({sec_key})s AND subsection_index = %({sub_key})s)"
+            )
+            params[sec_key] = sec
+            params[sub_key] = sub
+        where.append(f"NOT ({' OR '.join(exclude_clauses)})")
+
     where_sql = " AND ".join(where)
     sql = f"""
     SELECT
@@ -75,3 +92,36 @@ def retrieve(
     subsections = [_row_to_retrieved(
         row, source_type="subsection") for row in rows]
     return _retrivla_to_sources({"subsections_direct": subsections})
+
+
+def retrieve_with_subsections(
+    pg_url: str,
+    embedder: EmbeddingLike,
+    query: str,
+    subsection_refs: Optional[List[SubsectionRef]] = None,
+    k: int = 4,
+    section_index: Optional[int] = None,
+    subsection_index: Optional[int] = None,
+    source_type_filter: Optional[List[str]] = None,
+    expand_links: bool = False,
+    neighbor_expand: int = 0,
+) -> List[Source]:
+    vector_k = max(k * 4, k + 16)
+    vector_sources = retrieve(
+        pg_url=pg_url,
+        embedder=embedder,
+        query=query,
+        k=vector_k,
+        section_index=section_index,
+        subsection_index=subsection_index,
+        source_type_filter=source_type_filter,
+        expand_links=expand_links,
+        neighbor_expand=neighbor_expand,
+        exclude_subsection_refs=subsection_refs,
+    )
+    vector_sources = vector_sources[:k]
+    subsection_children = load_children_for_subsections(
+        pg_url=pg_url,
+        subsection_refs=subsection_refs,
+    )
+    return merge_sources(vector_sources, subsection_children)

math-tutor/backend/test/retrieval_store_test.py

@@ -5,6 +5,33 @@ from app.deterministic_services.embeddings import EmbeddingFactory
 from app.deterministic_services import retrieval_store
 
 
+def _parse_subsections(
+    raw_values: list[str] | None,
+    section_index: int | None,
+) -> list[tuple[int, int]]:
+    if not raw_values:
+        return []
+
+    refs: set[tuple[int, int]] = set()
+    for raw in raw_values:
+        token = str(raw).strip()
+        if not token:
+            continue
+
+        if ":" in token:
+            sec_raw, sub_raw = token.split(":", 1)
+            refs.add((int(sec_raw), int(sub_raw)))
+            continue
+
+        if section_index is None:
+            raise ValueError(
+                "Subsection ohne Section ist ungueltig. Nutze '<sec>:<sub>' oder --section-index."
+            )
+        refs.add((int(section_index), int(token)))
+
+    return sorted(refs)
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(
         description="Isolierter Vector-Store Retrieval Test.")
@@ -15,6 +42,12 @@ def main() -> None:
     parser.add_argument("--expand", action="store_true")
     parser.add_argument("--section-index", type=int, default=None)
     parser.add_argument("--subsection-index", type=int, default=None)
+    parser.add_argument(
+        "--subsections",
+        nargs="*",
+        default=None,
+        help="Liste von Subsections: '<sec>:<sub>' oder '<sub>' zusammen mit --section-index",
+    )
     parser.add_argument("--source-type-filter", nargs="*", default=None)
     parser.add_argument("--neighbor-expand", type=int, default=0)
 
@@ -23,17 +56,32 @@ def main() -> None:
     pg_url = args.pg or config.get_postgres_url()
     embedder = EmbeddingFactory.create(config.get_embedding_settings())
 
-    sources = retrieval_store.retrieve(
-        pg_url=pg_url,
-        embedder=embedder,
-        query=args.query,
-        k=args.k,
-        expand_links=args.expand,
-        section_index=args.section_index,
-        subsection_index=args.subsection_index,
-        source_type_filter=args.source_type_filter,
-        neighbor_expand=args.neighbor_expand,
-    )
+    subsection_refs = _parse_subsections(args.subsections, args.section_index)
+    if subsection_refs:
+        sources = retrieval_store.retrieve_with_subsections(
+            pg_url=pg_url,
+            embedder=embedder,
+            query=args.query,
+            subsection_refs=subsection_refs,
+            k=args.k,
+            expand_links=args.expand,
+            section_index=args.section_index,
+            subsection_index=args.subsection_index,
+            source_type_filter=args.source_type_filter,
+            neighbor_expand=args.neighbor_expand,
+        )
+    else:
+        sources = retrieval_store.retrieve(
+            pg_url=pg_url,
+            embedder=embedder,
+            query=args.query,
+            k=args.k,
+            expand_links=args.expand,
+            section_index=args.section_index,
+            subsection_index=args.subsection_index,
+            source_type_filter=args.source_type_filter,
+            neighbor_expand=args.neighbor_expand,
+        )
 
     if not sources:
         print("Keine Quellen gefunden.")
@@ -41,7 +89,9 @@ def main() -> None:
 
     print(f"Gefundene Quellen: {len(sources)}")
     for source in sources:
-        print(f"[{source.score:.4f}] {source.source_id.title} ({source.source_type})")
+        print(
+            f"[{source.score:.4f}] {source.source_id.title} ({source.source_type}) [{source.retrieved_as}]"
+        )
 
 
 if __name__ == "__main__":