Skip to content
GitLab
Commit 75a26eb5 authored by Kantz's avatar Kantz
Browse files

anpassung aug neue preprocessing struktur

parent a27a4ff5
Expand all Hide whitespace changes
Inline Side-by-side
math-tutor/backend/app/deterministic_services/orchestrators/orchestrator_socratic.py
View file @ 75a26eb5
......@@ -21,6 +21,7 @@ def _apply_selected_subsection(
def _retrieve_context_for_subsection(state: base.ChatState, query_text: str) -> int:
# Deprecated: socratic retrieval still uses subsection refs until it is migrated to the new parent model.
refs = task_catalog.get_selected_subsection_refs(state.sheet)
if not refs:
return 0
......
math-tutor/backend/app/deterministic_services/orchestrators/orchestrator_task.py
View file @ 75a26eb5
......@@ -55,6 +55,7 @@ def _ensure_context_task_fields(state: base.ChatState, query_text: str) -> tuple
def _retrieve_context_for_task(state: base.ChatState, query_text: str) -> int:
# Deprecated: this still uses subsection refs until task retrieval is migrated to the new parent model.
refs = task_catalog.get_selected_task_subsection_refs(state.sheet)
if not refs:
return 0
......
math-tutor/backend/app/deterministic_services/referenz_decoder.py
View file @ 75a26eb5
......@@ -12,6 +12,7 @@ class _SourceIDLike(Protocol):
chapter_title: str | None
section_title: str | None
subsection_title: str | None
subsubsection_title: str | None
title: str | None
doc_type: str | None
......@@ -27,6 +28,7 @@ def _source_id_key(source_id: _SourceIDLike) -> str:
source_id.chapter_title or "",
source_id.section_title or "",
source_id.subsection_title or "",
source_id.subsubsection_title or "",
source_id.title or "",
source_id.doc_type or "",
]
......@@ -58,6 +60,7 @@ def _build_source_index(sources: Iterable[_SourceLike]) -> Tuple[Dict[str, str],
source.source_id.chapter_title,
source.source_id.section_title,
source.source_id.subsection_title,
source.source_id.subsubsection_title,
source.source_id.title,
]
for start in range(len(parts)):
......
math-tutor/backend/app/deterministic_services/retrieval_store.py
View file @ 75a26eb5
......@@ -19,6 +19,7 @@ def retrieve(
chapter_index: int | None = None,
section_index: int | None = None,
subsection_index: int | None = None,
subsubsection_index: int | None = None,
source_type_filter: list[str] | None = None,
expand_links: bool = True,
neighbor_expand: int = 0,
......@@ -32,6 +33,7 @@ def retrieve(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -45,6 +47,7 @@ def retrieve(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -60,6 +63,7 @@ def retrieve_with_subsections(
chapter_index: int | None = None,
section_index: int | None = None,
subsection_index: int | None = None,
subsubsection_index: int | None = None,
source_type_filter: list[str] | None = None,
expand_links: bool = True,
neighbor_expand: int = 0,
......@@ -74,6 +78,7 @@ def retrieve_with_subsections(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -88,6 +93,7 @@ def retrieve_with_subsections(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -98,6 +104,7 @@ def retrieve_for_subsections(
pg_url: str,
subsection_refs: list[vector_store.SubsectionRef] | None = None,
) -> List[Source]:
# Deprecated: subsection-ref retrieval is kept only for legacy task/socratic flows.
return vector_store.load_children_for_subsections(
pg_url=pg_url,
subsection_refs=subsection_refs,
......
math-tutor/backend/app/deterministic_services/vector_store_subsection.py
View file @ 75a26eb5
......@@ -18,6 +18,8 @@ from app.deterministic_services.vector_store import (
merge_sources,
)
# Deprecated: this module keeps the legacy subsection-centric retrieval path for compatibility.
def retrieve(
pg_url: str,
......@@ -27,13 +29,15 @@ def retrieve(
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
subsubsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
expand_links: bool = False,
neighbor_expand: int = 0,
) -> List[Source]:
# Parameters kept for drop-in compatibility with child-level retrieve.
# Deprecated compatibility path. Parameters kept for drop-in compatibility with child-level retrieve.
_ = expand_links
_ = neighbor_expand
_ = subsubsection_index
qvec = Vector(embed_query(embedder, query))
......@@ -64,8 +68,8 @@ def retrieve(
sql = f"""
SELECT
uid, doc_type,
chapter_index, section_index, subsection_index, child_index,
chapter_title, section_title, subsection_title, title, source_type,
chapter_index, section_index, subsection_index, subsubsection_index, child_index,
chapter_title, section_title, subsection_title, subsubsection_title, title, source_type,
path, markdown,
1 - (embedding <=> %(qvec)s) AS score
FROM docs
......@@ -94,10 +98,12 @@ def retrieve_with_subsections(
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
subsubsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
expand_links: bool = False,
neighbor_expand: int = 0,
) -> List[Source]:
# Deprecated compatibility path for subsection-only retrieval composition.
vector_k = max(k * 4, k + 16)
vector_sources = retrieve(
pg_url=pg_url,
......@@ -107,6 +113,7 @@ def retrieve_with_subsections(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......
math-tutor/backend/test/vector_store_pipeline_unit_test.py
View file @ 75a26eb5
......@@ -24,6 +24,7 @@ def _mk_retrieved(
chapter_index: int,
section_index: int,
subsection_index: int | None,
subsubsection_index: int | None = 0,
child_index: int = 1,
) -> Retrieved:
return Retrieved(
......@@ -34,10 +35,12 @@ def _mk_retrieved(
"chapter_index": chapter_index,
"section_index": section_index,
"subsection_index": subsection_index,
"subsubsection_index": subsubsection_index,
"child_index": child_index,
"chapter_title": "C",
"section_title": "S",
"subsection_title": "SS",
"subsubsection_title": "SSS" if subsubsection_index else "",
"title": uid,
"source_type": "child",
"path": "",
......@@ -91,37 +94,77 @@ class VectorStorePipelineUnitTest(unittest.TestCase):
self.assertTrue(cfg.enable_scoped_child_search)
self.assertTrue(cfg.enable_context_docs)
def test_select_dominant_scope_prefers_count(self) -> None:
def test_select_dominant_scope_prefers_subsection_count(self) -> None:
children = [
_mk_retrieved("a", 0.9, 1, 1, 1),
_mk_retrieved("b", 0.8, 1, 1, 1),
_mk_retrieved("c", 0.95, 1, 1, 2),
]
scope = select_dominant_scope(children, level="subsection")
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "subsection")
self.assertEqual((scope.chapter_index, scope.section_index, scope.subsection_index), (1, 1, 1))
def test_select_dominant_scope_resolves_section_parent(self) -> None:
children = [
_mk_retrieved("a", 0.9, 1, 1, 0, 0),
_mk_retrieved("b", 0.8, 1, 1, 0, 0),
_mk_retrieved("c", 0.95, 1, 1, 2, 0),
]
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "section")
self.assertEqual((scope.chapter_index, scope.section_index), (1, 1))
self.assertIsNone(scope.subsection_index)
self.assertIsNone(scope.subsubsection_index)
def test_select_dominant_scope_resolves_subsubsection_parent(self) -> None:
children = [
_mk_retrieved("a", 0.9, 1, 1, 2, 1),
_mk_retrieved("b", 0.8, 1, 1, 2, 1),
_mk_retrieved("c", 0.95, 1, 1, 2, 0),
]
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "subsubsection")
self.assertEqual((scope.chapter_index, scope.section_index, scope.subsection_index, scope.subsubsection_index), (1, 1, 2, 1))
def test_select_dominant_scope_tiebreak_avg_score(self) -> None:
children = [
_mk_retrieved("a", 0.7, 1, 1, 1),
_mk_retrieved("b", 0.8, 1, 1, 2),
_mk_retrieved("a", 0.7, 1, 1, 1, 0),
_mk_retrieved("b", 0.8, 1, 1, 2, 0),
]
scope = select_dominant_scope(children, level="subsection")
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "subsection")
self.assertEqual((scope.chapter_index, scope.section_index, scope.subsection_index), (1, 1, 2))
def test_select_dominant_scope_tiebreak_lexicographic(self) -> None:
children = [
_mk_retrieved("a", 0.8, 2, 1, 1),
_mk_retrieved("b", 0.8, 1, 2, 3),
_mk_retrieved("a", 0.8, 2, 1, 1, 0),
_mk_retrieved("b", 0.8, 1, 2, 3, 0),
]
scope = select_dominant_scope(children, level="subsection")
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual((scope.chapter_index, scope.section_index, scope.subsection_index), (1, 2, 3))
def test_select_dominant_scope_keeps_exact_parent_levels_separate(self) -> None:
children = [
_mk_retrieved("section-hit", 0.91, 1, 1, 2, 0),
_mk_retrieved("subsub-hit", 0.92, 1, 1, 2, 1),
]
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "subsubsection")
self.assertEqual(scope.subsection_index, 2)
self.assertEqual(scope.subsubsection_index, 1)
def test_merge_retrieval_groups_dedup_max_score_and_trim_children(self) -> None:
groups = {
"children_direct": [
......@@ -146,10 +189,55 @@ class VectorStorePipelineUnitTest(unittest.TestCase):
self.assertEqual(merged["neighbors"], [])
def test_expand_neighbor_children_returns_empty_for_zero_expand(self) -> None:
children = [_mk_retrieved("u1", 0.8, 1, 1, 1, child_index=3)]
children = [_mk_retrieved("u1", 0.8, 1, 1, 1, 4, child_index=3)]
result = expand_neighbor_children("postgresql://unused", children, neighbor_expand=0)
self.assertEqual(result, [])
def test_source_id_to_string_includes_subsubsection_title(self) -> None:
source_id = SourceID(
chapter_title="Kapitel",
section_title="Section",
subsection_title="Subsection",
subsubsection_title="Subsubsection",
title="Child",
doc_type="child",
)
self.assertEqual(source_id.to_string(), "[Kapitel|Section|Subsection|Subsubsection|Child|child]")
def test_merge_sources_distinguishes_subsubsection_title(self) -> None:
first = Source(
source_id=SourceID(
chapter_title="Kapitel",
section_title="Section",
subsection_title="Subsection",
subsubsection_title="A",
title="Child A",
doc_type="child",
),
retrieved_as="children_direct",
source_type="child",
score=0.7,
markdown="same-md",
)
second = Source(
source_id=SourceID(
chapter_title="Kapitel",
section_title="Section",
subsection_title="Subsection",
subsubsection_title="B",
title="Child A",
doc_type="child",
),
retrieved_as="children_direct",
source_type="child",
score=0.6,
markdown="same-md",
)
merged = merge_sources([first], [second])
self.assertEqual(len(merged), 2)
if __name__ == "__main__":
unittest.main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal