Skip to content
GitLab
Commit b055a154 authored by Kantz's avatar Kantz
Browse files

chapter weitere geupdatet

parent bb0dd6ee
Hide whitespace changes
Inline Side-by-side
math-tutor/README.md
View file @ b055a154
......@@ -115,7 +115,7 @@ Retrieval settings:
## Testing
currently there a multiple test to test some components seperatly. Check the test files for specific calling. Here are some example calls.
currently there a multiple test to test some components seperatly. Check the test files for specific calling. Here are some example calls.
```powershell
python -m test.hint_test --chat-id draft_session_mlgmxxzc_avmjfb
......
math-tutor/backend/app/deterministic_services/retrieval_store.py
View file @ b055a154
......@@ -16,6 +16,7 @@ def retrieve(
embedder: EmbeddingLike,
query: str,
k: int = 4,
chapter_index: int | None = None,
section_index: int | None = None,
subsection_index: int | None = None,
source_type_filter: list[str] | None = None,
......@@ -28,6 +29,7 @@ def retrieve(
embedder=embedder,
query=query,
k=k,
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
source_type_filter=source_type_filter,
......@@ -40,6 +42,7 @@ def retrieve(
embedder=embedder,
query=query,
k=k,
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
source_type_filter=source_type_filter,
......@@ -54,6 +57,7 @@ def retrieve_with_subsections(
query: str,
subsection_refs: list[vector_store.SubsectionRef] | None = None,
k: int = 4,
chapter_index: int | None = None,
section_index: int | None = None,
subsection_index: int | None = None,
source_type_filter: list[str] | None = None,
......@@ -67,6 +71,7 @@ def retrieve_with_subsections(
query=query,
subsection_refs=subsection_refs,
k=k,
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
source_type_filter=source_type_filter,
......@@ -80,6 +85,7 @@ def retrieve_with_subsections(
query=query,
subsection_refs=subsection_refs,
k=k,
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
source_type_filter=source_type_filter,
......
math-tutor/backend/app/deterministic_services/task_catalog.py
View file @ b055a154
......@@ -24,14 +24,14 @@ def _normalize_subsection_key(value: str) -> str:
return re.sub(r"\s+", " ", collapsed)
def _parse_subsection_ref(value: str) -> tuple[int, int] | None:
match = re.match(r"^\s*(\d+)\s*[:.]\s*(\d+)\s*$", str(value))
def _parse_subsection_ref(value: str) -> tuple[int, int, int] | None:
match = re.match(r"^\s*(\d+)\s*[:.]\s*(\d+)\s*[:.]\s*(\d+)\s*$", str(value))
if not match:
return None
return int(match.group(1)), int(match.group(2))
return int(match.group(1)), int(match.group(2)), int(match.group(3))
def load_subsection_map(path: Path = SUBSECTION_MAP_PATH) -> dict[str, tuple[int, int]]:
def load_subsection_map(path: Path = SUBSECTION_MAP_PATH) -> dict[str, tuple[int, int, int]]:
if not path.exists():
return {}
try:
......@@ -41,7 +41,7 @@ def load_subsection_map(path: Path = SUBSECTION_MAP_PATH) -> dict[str, tuple[int
if not isinstance(content, dict):
return {}
mapped: dict[str, tuple[int, int]] = {}
mapped: dict[str, tuple[int, int, int]] = {}
for raw_key, raw_ref in content.items():
key = _normalize_subsection_key(str(raw_key))
parsed = _parse_subsection_ref(str(raw_ref))
......@@ -53,21 +53,21 @@ def load_subsection_map(path: Path = SUBSECTION_MAP_PATH) -> dict[str, tuple[int
def _resolve_task_subsection_refs(
task_file: dict[str, Any],
subsection_map: dict[str, tuple[int, int]] | None = None,
) -> list[tuple[int, int]]:
subsection_map: dict[str, tuple[int, int, int]] | None = None,
) -> list[tuple[int, int, int]]:
mapping = subsection_map if subsection_map is not None else load_subsection_map()
subsections = task_file.get("subsections", [])
if not isinstance(subsections, list):
return []
refs: set[tuple[int, int]] = set()
refs: set[tuple[int, int, int]] = set()
for subsection in subsections:
key = _normalize_subsection_key(str(subsection))
if not key:
continue
ref = mapping.get(key)
if ref is not None:
refs.add((int(ref[0]), int(ref[1])))
refs.add((int(ref[0]), int(ref[1]), int(ref[2])))
return sorted(refs)
......@@ -141,7 +141,7 @@ def set_selected_task(
sheet["task_file_id"] = str(task_file.get("_file_id", ""))
sheet["task_id"] = str(task_entry.get("id", "")).zfill(2)
refs = _resolve_task_subsection_refs(task_file)
sheet["task_subsection_refs"] = [[sec, sub] for sec, sub in refs]
sheet["task_subsection_refs"] = [[chap, sec, sub] for chap, sec, sub in refs]
def select_task_by_ids(
......@@ -167,16 +167,16 @@ def get_selected_task_ids(sheet: dict[str, Any]) -> tuple[str | None, str | None
return (file_id or None, task_id or None)
def get_selected_task_subsection_refs(sheet: dict[str, Any]) -> list[tuple[int, int]]:
def get_selected_task_subsection_refs(sheet: dict[str, Any]) -> list[tuple[int, int, int]]:
refs_raw = sheet.get("task_subsection_refs", [])
if not isinstance(refs_raw, list):
return []
refs: set[tuple[int, int]] = set()
refs: set[tuple[int, int, int]] = set()
for item in refs_raw:
if isinstance(item, (list, tuple)) and len(item) >= 2:
if isinstance(item, (list, tuple)) and len(item) >= 3:
try:
refs.add((int(item[0]), int(item[1])))
refs.add((int(item[0]), int(item[1]), int(item[2])))
except Exception:
continue
return sorted(refs)
......
math-tutor/backend/app/deterministic_services/vector_store.py
View file @ b055a154
......@@ -546,7 +546,7 @@ def retrieve(
# --------------------------------------------------------------------------------------------------------------------
SubsectionRef = Tuple[int, int]
SubsectionRef = Tuple[int, int, int]
def _normalize_subsection_refs(
......@@ -554,7 +554,9 @@ def _normalize_subsection_refs(
) -> List[SubsectionRef]:
if not subsection_refs:
return []
normalized = {(int(sec), int(sub)) for sec, sub in subsection_refs}
normalized = {
(int(chap), int(sec), int(sub)) for chap, sec, sub in subsection_refs
}
return sorted(normalized)
......@@ -566,8 +568,9 @@ def load_children_for_subsections(
if not refs:
return []
sec_arr = [sec for sec, _ in refs]
sub_arr = [sub for _, sub in refs]
chap_arr = [chap for chap, _, _ in refs]
sec_arr = [sec for _, sec, _ in refs]
sub_arr = [sub for _, _, sub in refs]
sql = """
SELECT
d.uid, d.doc_type,
......@@ -576,14 +579,21 @@ def load_children_for_subsections(
d.path, d.markdown,
1.0 AS score
FROM docs d
JOIN unnest(%(sec_arr)s::int[], %(sub_arr)s::int[]) AS u(sec, sub)
ON d.section_index = u.sec AND d.subsection_index = u.sub
JOIN unnest(%(chap_arr)s::int[], %(sec_arr)s::int[], %(sub_arr)s::int[]) AS u(chap, sec, sub)
ON d.chapter_index = u.chap AND d.section_index = u.sec AND d.subsection_index = u.sub
WHERE d.doc_type = 'child'
ORDER BY d.section_index, d.subsection_index, d.child_index
ORDER BY d.chapter_index, d.section_index, d.subsection_index, d.child_index
"""
with psycopg.connect(pg_url, row_factory=dict_row) as conn:
with conn.cursor() as cur:
cur.execute(sql, {"sec_arr": sec_arr, "sub_arr": sub_arr})
cur.execute(
sql,
{
"chap_arr": chap_arr,
"sec_arr": sec_arr,
"sub_arr": sub_arr,
},
)
rows = cur.fetchall()
children = [_row_to_retrieved(row) for row in rows]
......@@ -619,6 +629,7 @@ def retrieve_with_subsections(
query: str,
subsection_refs: Optional[List[SubsectionRef]] = None,
k: int = 4,
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
......@@ -633,6 +644,7 @@ def retrieve_with_subsections(
embedder=embedder,
query=query,
k=vector_k,
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
source_type_filter=source_type_filter,
......
math-tutor/backend/app/deterministic_services/vector_store_subsection.py
View file @ b055a154
......@@ -24,6 +24,7 @@ def retrieve(
embedder: EmbeddingLike,
query: str,
k: int = 4,
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
......@@ -43,6 +44,10 @@ def retrieve(
"sub_doc_types": ["subsection", "chapter"],
}
if chapter_index is not None:
where.append("chapter_index = %(chapter_index)s")
params["chapter_index"] = chapter_index
if section_index is not None:
where.append("section_index = %(section_index)s")
params["section_index"] = section_index
......@@ -86,6 +91,7 @@ def retrieve_with_subsections(
query: str,
subsection_refs: Optional[List[SubsectionRef]] = None,
k: int = 4,
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
......@@ -98,6 +104,7 @@ def retrieve_with_subsections(
embedder=embedder,
query=query,
k=vector_k,
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
source_type_filter=source_type_filter,
......
math-tutor/backend/test/retrieval_store_test.py
View file @ b055a154
......@@ -7,27 +7,38 @@ from app.deterministic_services import retrieval_store
def _parse_subsections(
raw_values: list[str] | None,
chapter_index: int | None,
section_index: int | None,
) -> list[tuple[int, int]]:
) -> list[tuple[int, int, int]]:
if not raw_values:
return []
refs: set[tuple[int, int]] = set()
refs: set[tuple[int, int, int]] = set()
for raw in raw_values:
token = str(raw).strip()
if not token:
continue
if ":" in token:
sec_raw, sub_raw = token.split(":", 1)
refs.add((int(sec_raw), int(sub_raw)))
parts = token.split(":")
if len(parts) == 3:
chap_raw, sec_raw, sub_raw = parts
refs.add((int(chap_raw), int(sec_raw), int(sub_raw)))
continue
if section_index is None:
if len(parts) == 2:
if chapter_index is None:
raise ValueError(
"Subsection ohne Chapter ist ungueltig. Nutze '<chap>:<sec>:<sub>' oder --chapter-index."
)
sec_raw, sub_raw = parts
refs.add((int(chapter_index), int(sec_raw), int(sub_raw)))
continue
if section_index is None or chapter_index is None:
raise ValueError(
"Subsection ohne Section ist ungueltig. Nutze '<sec>:<sub>' oder --section-index."
"Subsection ohne Chapter/Section ist ungueltig. Nutze '<chap>:<sec>:<sub>' oder --chapter-index und --section-index."
)
refs.add((int(section_index), int(token)))
refs.add((int(chapter_index), int(section_index), int(token)))
return sorted(refs)
......@@ -40,13 +51,14 @@ def main() -> None:
help="Postgres URL (oder set POSTGRES_URL)")
parser.add_argument("--k", type=int, default=8)
parser.add_argument("--expand", action="store_true")
parser.add_argument("--chapter-index", type=int, default=None)
parser.add_argument("--section-index", type=int, default=None)
parser.add_argument("--subsection-index", type=int, default=None)
parser.add_argument(
"--subsections",
nargs="*",
default=None,
help="Liste von Subsections: '<sec>:<sub>' oder '<sub>' zusammen mit --section-index",
help="Liste von Subsections: '<chap>:<sec>:<sub>'; alternativ '<sec>:<sub>' mit --chapter-index oder '<sub>' mit --chapter-index und --section-index",
)
parser.add_argument("--source-type-filter", nargs="*", default=None)
parser.add_argument("--neighbor-expand", type=int, default=0)
......@@ -56,7 +68,9 @@ def main() -> None:
pg_url = args.pg or config.get_postgres_url()
embedder = EmbeddingFactory.create(config.get_embedding_settings())
subsection_refs = _parse_subsections(args.subsections, args.section_index)
subsection_refs = _parse_subsections(
args.subsections, args.chapter_index, args.section_index
)
if subsection_refs:
sources = retrieval_store.retrieve_with_subsections(
pg_url=pg_url,
......@@ -65,6 +79,7 @@ def main() -> None:
subsection_refs=subsection_refs,
k=args.k,
expand_links=args.expand,
chapter_index=args.chapter_index,
section_index=args.section_index,
subsection_index=args.subsection_index,
source_type_filter=args.source_type_filter,
......@@ -77,6 +92,7 @@ def main() -> None:
query=args.query,
k=args.k,
expand_links=args.expand,
chapter_index=args.chapter_index,
section_index=args.section_index,
subsection_index=args.subsection_index,
source_type_filter=args.source_type_filter,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal