Commit e31b3ff9 authored by Kantz's avatar Kantz
Browse files

Merge branch 'dev' into 'main'

Dev

See merge request kantz/tutor_react!24
parents c2af99d6 b61d4201
......@@ -10,7 +10,7 @@
if you want to use the task mode you need :
- task folder in sources with your tasks
- `_subsection_map.yaml`.
- `_subsection_map.yaml` as the topic-to-index map for section, subsection, or subsubsection references.
## Setup
......
......@@ -26,6 +26,13 @@ class TaskItem(BaseModel):
images: List[TaskImage] = Field(default_factory=list)
class TopicEntry(BaseModel):
topic_key: str
label: str
refs: List[List[int]]
summary: str = ""
class SubsectionEntry(BaseModel):
subsection_key: str
label: str
......@@ -38,14 +45,15 @@ class TaskFile(BaseModel):
title: str
intro: str
tasks: List[TaskItem]
subsections: List[str] = Field(default_factory=list)
topics: List[str] = Field(default_factory=list)
topic_options: List[TopicEntry] = Field(default_factory=list)
class TasksResponse(BaseModel):
orchestrator: str
enabled: bool
task_files: List[TaskFile]
subsections: List[SubsectionEntry] = Field(default_factory=list)
topics: List[TopicEntry] = Field(default_factory=list)
class SocraticResponse(BaseModel):
......@@ -104,12 +112,12 @@ def get_task_config() -> dict[str, object]:
def list_tasks() -> TasksResponse:
orchestrator = config.get_orchestrator()
task_files = task_catalog.build_task_catalog()
subsections = task_catalog.build_subsection_catalog()
topics = task_catalog.build_topic_catalog()
return TasksResponse(
orchestrator=orchestrator,
enabled=orchestrator in TASK_ORCHESTRATORS,
task_files=task_files,
subsections=subsections,
topics=topics,
)
......
......@@ -27,7 +27,6 @@ class OpenAILikeConfig(BaseModel):
model: str = Field(...,
description="Modellname (z. B. 'nomic-embed-text')")
target_dim: int = Field(1024, description="Ziel-Dimension der Embeddings")
timeout: float | None = Field(None, description="Request timeout in seconds")
class SentenceTransformerConfig(BaseModel):
......@@ -100,14 +99,12 @@ class OpenAILikeEmbeddings(BaseEmbeddings):
self.base_url = config.base_url.rstrip("/")
self.api_key = config.api_key
self.model = config.model
self.timeout = config.timeout or 60.0
def _embed(self, inputs: List[str] | str) -> List[List[float]]:
"""Ruft die externe Embedding-API auf."""
client = OpenAI(
api_key=self.api_key,
base_url=self.base_url,
timeout=self.timeout,
)
response = client.embeddings.create(
input=inputs,
......
......@@ -21,6 +21,7 @@ def _apply_selected_subsection(
def _retrieve_context_for_subsection(state: base.ChatState, query_text: str) -> int:
# Deprecated: socratic retrieval still uses subsection refs until it is migrated to the new parent model.
refs = task_catalog.get_selected_subsection_refs(state.sheet)
if not refs:
return 0
......
......@@ -55,27 +55,27 @@ def _ensure_context_task_fields(state: base.ChatState, query_text: str) -> tuple
def _retrieve_context_for_task(state: base.ChatState, query_text: str) -> int:
refs = task_catalog.get_selected_task_subsection_refs(state.sheet)
refs = task_catalog.get_selected_task_parent_refs(state.sheet)
if not refs:
return 0
def _retrieve() -> dict:
sources = retrieval_store.retrieve_for_subsections(
sources = retrieval_store.retrieve_for_parent_refs(
pg_url=config.get_postgres_url(),
subsection_refs=refs,
parent_refs=refs,
)
context_store.update_retrieval_context(state.sheet, sources)
return {
"subsection_refs": refs,
"parent_refs": refs,
"source_count": len(sources),
}
result = base.log_timed_call(
state.tool_log,
"retrieve_context_with_task_subsections",
"retrieve_context_with_task_parents",
{
"query": query_text,
"subsection_refs": refs,
"parent_refs": refs,
},
_retrieve,
)
......
......@@ -12,6 +12,7 @@ class _SourceIDLike(Protocol):
chapter_title: str | None
section_title: str | None
subsection_title: str | None
subsubsection_title: str | None
title: str | None
doc_type: str | None
......@@ -27,6 +28,7 @@ def _source_id_key(source_id: _SourceIDLike) -> str:
source_id.chapter_title or "",
source_id.section_title or "",
source_id.subsection_title or "",
source_id.subsubsection_title or "",
source_id.title or "",
source_id.doc_type or "",
]
......@@ -58,6 +60,7 @@ def _build_source_index(sources: Iterable[_SourceLike]) -> Tuple[Dict[str, str],
source.source_id.chapter_title,
source.source_id.section_title,
source.source_id.subsection_title,
source.source_id.subsubsection_title,
source.source_id.title,
]
for start in range(len(parts)):
......
......@@ -19,6 +19,7 @@ def retrieve(
chapter_index: int | None = None,
section_index: int | None = None,
subsection_index: int | None = None,
subsubsection_index: int | None = None,
source_type_filter: list[str] | None = None,
expand_links: bool = True,
neighbor_expand: int = 0,
......@@ -32,6 +33,7 @@ def retrieve(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -45,6 +47,7 @@ def retrieve(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -60,6 +63,7 @@ def retrieve_with_subsections(
chapter_index: int | None = None,
section_index: int | None = None,
subsection_index: int | None = None,
subsubsection_index: int | None = None,
source_type_filter: list[str] | None = None,
expand_links: bool = True,
neighbor_expand: int = 0,
......@@ -74,6 +78,7 @@ def retrieve_with_subsections(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -88,6 +93,7 @@ def retrieve_with_subsections(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -98,7 +104,18 @@ def retrieve_for_subsections(
pg_url: str,
subsection_refs: list[vector_store.SubsectionRef] | None = None,
) -> List[Source]:
# Deprecated: subsection-ref retrieval is kept only for legacy task/socratic flows.
return vector_store.load_children_for_subsections(
pg_url=pg_url,
subsection_refs=subsection_refs,
)
def retrieve_for_parent_refs(
pg_url: str,
parent_refs: list[vector_store.ParentRef] | None = None,
) -> List[Source]:
return vector_store.load_sources_for_parent_refs(
pg_url=pg_url,
parent_refs=parent_refs,
)
......@@ -16,8 +16,14 @@ from app.deterministic_services.vector_store import parse_markdown_with_frontmat
TASKS_DIR = config.get_task_folder()
TASK_IMAGES_DIR = TASKS_DIR / "images"
SOURCES_DIR = Path(__file__).resolve().parents[2] / "sources"
SUBSECTION_MAP_PATH = TASKS_DIR / "_subsection_map.yaml"
TOPIC_MAP_PATH = TASKS_DIR / "_topic_to_index_map.yaml"
SUBSECTION_MAP_PATH = TOPIC_MAP_PATH
TASK_ASSET_URL_PREFIX = "/api/tasks/assets"
ParentRef = tuple[int, int, int, int]
def _normalize_parent_ref_key(value: str) -> str:
return _normalize_subsection_key(value)
def _normalize_text(value: str) -> str:
return re.sub(r"\s+", " ", value.strip().lower())
......@@ -42,7 +48,80 @@ def _parse_subsection_ref(value: str) -> tuple[int, int, int] | None:
return int(match.group(1)), int(match.group(2)), int(match.group(3))
def load_subsection_map(path: Path = SUBSECTION_MAP_PATH) -> dict[str, tuple[int, int, int]]:
def _parse_parent_ref(value: Any) -> ParentRef | None:
if isinstance(value, (list, tuple)):
parts = [str(item).strip() for item in value if str(item).strip()]
elif isinstance(value, dict):
try:
chapter = int(value.get("chapter_index"))
section = int(value.get("section_index"))
except Exception:
return None
subsection = int(value.get("subsection_index") or 0)
subsubsection = int(value.get("subsubsection_index") or 0)
return _normalize_parent_ref((chapter, section, subsection, subsubsection))
else:
token = str(value).strip()
if not token:
return None
parts = [part for part in re.split(r"\s*[-.:]\s*", token) if part]
try:
ints = [int(part) for part in parts]
except Exception:
return None
if len(ints) == 2:
return _normalize_parent_ref((ints[0], ints[1], 0, 0))
if len(ints) == 3:
return _normalize_parent_ref((ints[0], ints[1], ints[2], 0))
if len(ints) == 4:
return _normalize_parent_ref((ints[0], ints[1], ints[2], ints[3]))
return None
def _normalize_parent_ref(ref: ParentRef) -> ParentRef:
chap, sec, sub, subsub = [int(part) for part in ref]
if sub <= 0:
return (chap, sec, 0, 0)
if subsub <= 0:
return (chap, sec, sub, 0)
return (chap, sec, sub, subsub)
def _parent_ref_level(ref: ParentRef) -> str:
_, _, sub, subsub = _normalize_parent_ref(ref)
if sub <= 0:
return "section"
if subsub <= 0:
return "subsection"
return "subsubsection"
def _parent_ref_to_key(ref: ParentRef) -> str:
chap, sec, sub, subsub = _normalize_parent_ref(ref)
if sub <= 0:
return f"{chap}-{sec}"
if subsub <= 0:
return f"{chap}-{sec}-{sub}"
return f"{chap}-{sec}-{sub}-{subsub}"
def _parent_ref_to_list(ref: ParentRef) -> list[int]:
chap, sec, sub, subsub = _normalize_parent_ref(ref)
return [chap, sec, sub, subsub]
def _format_parent_ref_label(ref: ParentRef) -> str:
chap, sec, sub, subsub = _normalize_parent_ref(ref)
if sub <= 0:
return f"Section {chap}.{sec}"
if subsub <= 0:
return f"Subsection {chap}.{sec}.{sub}"
return f"Subsubsection {chap}.{sec}.{sub}.{subsub}"
def load_topic_map(path: Path = TOPIC_MAP_PATH) -> dict[str, ParentRef]:
if not path.exists():
return {}
try:
......@@ -52,13 +131,24 @@ def load_subsection_map(path: Path = SUBSECTION_MAP_PATH) -> dict[str, tuple[int
if not isinstance(content, dict):
return {}
mapped: dict[str, tuple[int, int, int]] = {}
mapped: dict[str, ParentRef] = {}
for raw_key, raw_ref in content.items():
key = _normalize_subsection_key(str(raw_key))
parsed = _parse_subsection_ref(str(raw_ref))
key = _normalize_parent_ref_key(str(raw_key))
parsed = _parse_parent_ref(raw_ref)
if not key or parsed is None:
continue
mapped[key] = parsed
mapped[key] = _normalize_parent_ref(parsed)
return mapped
def load_subsection_map(path: Path = SUBSECTION_MAP_PATH) -> dict[str, tuple[int, int, int]]:
topic_map = load_topic_map(path)
mapped: dict[str, tuple[int, int, int]] = {}
for key, ref in topic_map.items():
chap, sec, sub, subsub = ref
if sub <= 0 or subsub > 0:
continue
mapped[key] = (chap, sec, sub)
return mapped
......@@ -79,13 +169,18 @@ def _extract_subsection_summary(text: str) -> str:
@lru_cache(maxsize=1)
def load_subsection_summaries(base_dir: Path = SOURCES_DIR) -> dict[str, str]:
def load_topic_summaries(base_dir: Path = SOURCES_DIR) -> dict[str, str]:
summaries: dict[str, str] = {}
for folder in (
folders = (
base_dir / "lecture_script_new" / "sections",
base_dir / "lecture_script_new" / "subsections",
base_dir / "lecture_script_new" / "subsubsections",
base_dir / "with_chapters" / "sections",
base_dir / "with_chapters" / "subsections",
base_dir / "child_lvl" / "subsections",
base_dir / "subsection_lvl" / "subsections",
):
)
for folder in folders:
if not folder.exists():
continue
for path in sorted(folder.glob("*.md")):
......@@ -94,8 +189,14 @@ def load_subsection_summaries(base_dir: Path = SOURCES_DIR) -> dict[str, str]:
except Exception:
continue
meta, body = parse_markdown_with_frontmatter(text)
title = str(meta.get("title") or meta.get("subsection_title") or meta.get("section_title") or "").strip()
key = _normalize_subsection_key(title)
title = str(
meta.get("title")
or meta.get("subsubsection_title")
or meta.get("subsection_title")
or meta.get("section_title")
or ""
).strip()
key = _normalize_parent_ref_key(title)
summary = _extract_subsection_summary(body)
if not key or not summary:
continue
......@@ -103,24 +204,114 @@ def load_subsection_summaries(base_dir: Path = SOURCES_DIR) -> dict[str, str]:
return summaries
def load_subsection_summaries(base_dir: Path = SOURCES_DIR) -> dict[str, str]:
return load_topic_summaries(base_dir)
def _normalize_topic_ref_entries(
raw_entries: Any,
topic_map: dict[str, ParentRef],
) -> list[dict[str, Any]]:
if not isinstance(raw_entries, list):
return []
options: list[dict[str, Any]] = []
seen_refs: set[ParentRef] = set()
for raw_entry in raw_entries:
normalized_key = ""
label = ""
summary = ""
ref: ParentRef | None = None
if isinstance(raw_entry, str):
normalized_key = _normalize_parent_ref_key(raw_entry)
label = _format_subsection_label(raw_entry)
ref = topic_map.get(normalized_key)
elif isinstance(raw_entry, dict):
raw_key = str(raw_entry.get("key") or raw_entry.get("topic_key") or raw_entry.get("subsection_key") or "").strip()
normalized_key = _normalize_parent_ref_key(raw_key) if raw_key else ""
label = str(raw_entry.get("label") or "").strip()
summary = str(raw_entry.get("summary") or "").strip()
ref = _parse_parent_ref(raw_entry)
if ref is None and normalized_key:
ref = topic_map.get(normalized_key)
else:
continue
if ref is None:
continue
normalized_ref = _normalize_parent_ref(ref)
if normalized_ref in seen_refs:
continue
seen_refs.add(normalized_ref)
key = normalized_key or _parent_ref_to_key(normalized_ref)
options.append(
{
"topic_key": key,
"label": label or _format_parent_ref_label(normalized_ref),
"summary": summary,
"level": _parent_ref_level(normalized_ref),
"refs": [_parent_ref_to_list(normalized_ref)],
}
)
return options
def _resolve_task_topic_options(
task_file: dict[str, Any],
topic_map: dict[str, ParentRef] | None = None,
topic_summaries: dict[str, str] | None = None,
) -> list[dict[str, Any]]:
mapping = topic_map if topic_map is not None else load_topic_map()
summaries = topic_summaries if topic_summaries is not None else load_topic_summaries()
raw_entries: list[Any] = []
legacy_topics = task_file.get("subsections", [])
if isinstance(legacy_topics, list):
raw_entries.extend(legacy_topics)
explicit_refs = task_file.get("topic_refs", [])
if isinstance(explicit_refs, list):
raw_entries.extend(explicit_refs)
options = _normalize_topic_ref_entries(raw_entries, mapping)
for option in options:
if not option.get("summary"):
option["summary"] = summaries.get(str(option.get("topic_key", "")).strip(), "")
return options
def _resolve_task_topic_refs(
task_file: dict[str, Any],
topic_map: dict[str, ParentRef] | None = None,
) -> list[ParentRef]:
refs: set[ParentRef] = set()
for option in _resolve_task_topic_options(task_file, topic_map=topic_map):
refs_raw = option.get("refs", [])
if not isinstance(refs_raw, list):
continue
for item in refs_raw:
parsed = _parse_parent_ref(item)
if parsed is not None:
refs.add(_normalize_parent_ref(parsed))
return sorted(refs)
def _resolve_task_subsection_refs(
task_file: dict[str, Any],
subsection_map: dict[str, tuple[int, int, int]] | None = None,
) -> list[tuple[int, int, int]]:
mapping = subsection_map if subsection_map is not None else load_subsection_map()
subsections = task_file.get("subsections", [])
if not isinstance(subsections, list):
return []
refs: set[tuple[int, int, int]] = set()
for subsection in subsections:
key = _normalize_subsection_key(str(subsection))
if not key:
mapping: dict[str, ParentRef]
if subsection_map is None:
mapping = load_topic_map()
else:
mapping = {key: (int(ref[0]), int(ref[1]), int(ref[2]), 0) for key, ref in subsection_map.items()}
refs = _resolve_task_topic_refs(task_file, topic_map=mapping)
normalized: list[tuple[int, int, int]] = []
for chap, sec, sub, subsub in refs:
if sub <= 0 or subsub > 0:
continue
ref = mapping.get(key)
if ref is not None:
refs.add((int(ref[0]), int(ref[1]), int(ref[2])))
return sorted(refs)
normalized.append((chap, sec, sub))
return sorted(normalized)
def _format_subsection_label(value: str) -> str:
......@@ -131,17 +322,36 @@ def _format_subsection_label(value: str) -> str:
return cleaned.title()
def build_subsection_catalog(path: Path = SUBSECTION_MAP_PATH) -> list[dict[str, Any]]:
subsection_map = load_subsection_map(path)
subsection_summaries = load_subsection_summaries()
def build_topic_catalog(path: Path = TOPIC_MAP_PATH) -> list[dict[str, Any]]:
topic_map = load_topic_map(path)
topic_summaries = load_topic_summaries()
response: list[dict[str, Any]] = []
for key, ref in sorted(subsection_map.items(), key=lambda item: (item[0], item[1])):
for key, ref in sorted(topic_map.items(), key=lambda item: (item[0], item[1])):
response.append(
{
"subsection_key": key,
"topic_key": key,
"label": _format_subsection_label(key),
"level": _parent_ref_level(ref),
"refs": [_parent_ref_to_list(ref)],
"summary": topic_summaries.get(key, ""),
}
)
return response
def build_subsection_catalog(path: Path = SUBSECTION_MAP_PATH) -> list[dict[str, Any]]:
response: list[dict[str, Any]] = []
for item in build_topic_catalog(path):
refs = item.get("refs", [])
if item.get("level") != "subsection" or not refs:
continue
ref = refs[0]
response.append(
{
"subsection_key": item["topic_key"],
"label": item["label"],
"refs": [[int(ref[0]), int(ref[1]), int(ref[2])]],
"summary": subsection_summaries.get(key, ""),
"summary": item["summary"],
}
)
return response
......@@ -151,33 +361,28 @@ def _resolve_task_subsection_options(
task_file: dict[str, Any],
subsection_map: dict[str, tuple[int, int, int]] | None = None,
) -> list[dict[str, Any]]:
mapping = subsection_map if subsection_map is not None else load_subsection_map()
subsections = task_file.get("subsections", [])
if not isinstance(subsections, list):
return []
mapping: dict[str, ParentRef]
if subsection_map is None:
mapping = load_topic_map()
else:
mapping = {key: (int(ref[0]), int(ref[1]), int(ref[2]), 0) for key, ref in subsection_map.items()}
options: list[dict[str, Any]] = []
seen_refs: set[tuple[int, int, int]] = set()
for subsection in subsections:
raw_label = str(subsection).strip()
key = _normalize_subsection_key(raw_label)
if not key:
continue
ref = mapping.get(key)
if ref is None:
continue
normalized_ref = (int(ref[0]), int(ref[1]), int(ref[2]))
if normalized_ref in seen_refs:
options = _resolve_task_topic_options(task_file, topic_map=mapping)
response: list[dict[str, Any]] = []
for option in options:
refs = option.get("refs", [])
if option.get("level") != "subsection" or not refs:
continue
seen_refs.add(normalized_ref)
options.append(
ref = refs[0]
response.append(
{
"subsection_key": key,
"label": _format_subsection_label(raw_label) or raw_label,
"refs": [[normalized_ref[0], normalized_ref[1], normalized_ref[2]]],
"subsection_key": option["topic_key"],
"label": option["label"],
"refs": [[int(ref[0]), int(ref[1]), int(ref[2])]],
"summary": option.get("summary", ""),
}
)
return options
return response
def _match_score(query_text: str, candidate_text: str) -> int:
......@@ -249,7 +454,7 @@ def _normalize_yaml_task_entry(task_entry: Any, position: int) -> dict[str, Any]
return None
statement_parts, statement_images = _extract_text_and_images(task_entry.get("aufgabe", []))
hint_parts, hint_images = _extract_text_and_images(task_entry.get("hinweise", []))
hint_parts, _ = _extract_text_and_images(task_entry.get("hinweise", []))
solution_parts, _ = _extract_text_and_images(task_entry.get("loesung", []))
return {
......@@ -257,7 +462,7 @@ def _normalize_yaml_task_entry(task_entry: Any, position: int) -> dict[str, Any]
"statement": "\n".join(statement_parts).strip(),
"hints": hint_parts,
"solution": "\n".join(solution_parts).strip(),
"images": statement_images + hint_images,
"images": statement_images,
}
......@@ -287,10 +492,10 @@ def _normalize_yaml_task_file(content: Any, path: Path) -> dict[str, Any] | None
intro = str(content.get("description", "")).strip()
file_id = str(content.get("id", "")).strip() or path.stem
slug = str(content.get("slug", "")).strip() or _slugify(title or file_id)
level = str(content.get("level", "")).strip()
subsections = _normalize_exercise_topics(content.get("exercise_topic", []))
topic_refs_raw = content.get("topic_refs", [])
if not title or not file_id or not slug or not level:
if not title or not file_id or not slug:
return None
tasks: list[dict[str, Any]] = []
......@@ -306,8 +511,8 @@ def _normalize_yaml_task_file(content: Any, path: Path) -> dict[str, Any] | None
"title": title,
"intro": intro,
"slug": slug,
"level": level,
"subsections": subsections,
"topic_refs": topic_refs_raw if isinstance(topic_refs_raw, list) else [],
"tasks": tasks,
}
......@@ -381,8 +586,9 @@ def set_selected_task(
store_new.set_solution(sheet, solution)
sheet["task_file_id"] = str(task_file.get("_file_id", ""))
sheet["task_id"] = str(task_entry.get("id", "")).zfill(2)
refs = _resolve_task_subsection_refs(task_file)
sheet["task_subsection_refs"] = [[chap, sec, sub] for chap, sec, sub in refs]
refs = _resolve_task_topic_refs(task_file)
sheet["task_parent_refs"] = [_parent_ref_to_list(ref) for ref in refs]
sheet.pop("task_subsection_refs", None)
sheet.pop("selected_subsection_key", None)
sheet.pop("selected_subsection_label", None)
sheet.pop("selected_subsection_refs", None)
......@@ -410,6 +616,7 @@ def set_selected_subsection(
sheet["selected_subsection_refs"] = [[chap, sec, sub] for chap, sec, sub in sorted({*refs})]
sheet.pop("task_id", None)
sheet.pop("task_parent_refs", None)
sheet.pop("task_subsection_refs", None)
sheet.pop("task", None)
sheet.pop("hints", None)
......@@ -475,6 +682,7 @@ def select_subsection_by_key(
sheet["selected_subsection_refs"] = [[int(ref[0]), int(ref[1]), int(ref[2])]]
sheet.pop("task_file_id", None)
sheet.pop("task_id", None)
sheet.pop("task_parent_refs", None)
sheet.pop("task_subsection_refs", None)
sheet.pop("task", None)
sheet.pop("hints", None)
......@@ -508,18 +716,25 @@ def get_selected_subsection_refs(sheet: dict[str, Any]) -> list[tuple[int, int,
return sorted(refs)
def get_selected_task_subsection_refs(sheet: dict[str, Any]) -> list[tuple[int, int, int]]:
refs_raw = sheet.get("task_subsection_refs", [])
def get_selected_task_parent_refs(sheet: dict[str, Any]) -> list[ParentRef]:
refs_raw = sheet.get("task_parent_refs", [])
if not isinstance(refs_raw, list):
return []
refs: set[tuple[int, int, int]] = set()
refs: set[ParentRef] = set()
for item in refs_raw:
if isinstance(item, (list, tuple)) and len(item) >= 3:
try:
refs.add((int(item[0]), int(item[1]), int(item[2])))
except Exception:
continue
parsed = _parse_parent_ref(item)
if parsed is not None:
refs.add(_normalize_parent_ref(parsed))
return sorted(refs)
def get_selected_task_subsection_refs(sheet: dict[str, Any]) -> list[tuple[int, int, int]]:
refs: list[tuple[int, int, int]] = []
for chap, sec, sub, subsub in get_selected_task_parent_refs(sheet):
if sub <= 0 or subsub > 0:
continue
refs.append((chap, sec, sub))
return sorted(refs)
......@@ -600,8 +815,12 @@ def build_task_catalog(task_files: list[dict[str, Any]] | None = None) -> list[d
file_id = str(task_file.get("_file_id", ""))
title = str(task_file.get("title", "")).strip()
intro = str(task_file.get("intro", "")).strip()
subsections = task_file.get("subsections", [])
subsection_options = _resolve_task_subsection_options(task_file)
topic_options = _resolve_task_topic_options(task_file)
topics_raw = task_file.get("subsections", [])
if isinstance(topics_raw, list) and topics_raw:
topics = [str(item).strip() for item in topics_raw if str(item).strip()]
else:
topics = [str(item.get("topic_key", "")).strip() for item in topic_options if str(item.get("topic_key", "")).strip()]
tasks: list[dict[str, str]] = []
for item in task_file.get("tasks", []):
if not isinstance(item, dict):
......@@ -625,8 +844,8 @@ def build_task_catalog(task_files: list[dict[str, Any]] | None = None) -> list[d
"title": title,
"intro": intro,
"tasks": tasks,
"subsections": subsections,
"subsection_options": subsection_options,
"topics": topics,
"topic_options": topic_options,
}
)
return response
......@@ -62,6 +62,7 @@ def stable_uid(doc_type: str, path: str, meta: Dict[str, Any]) -> str:
chapter = meta.get("chapter_index")
sec = meta.get("section_index")
sub = meta.get("subsection_index")
subsub = meta.get("subsubsection_index")
child = meta.get("child_index")
if doc_type == "chapter" and chapter is not None:
......@@ -70,8 +71,10 @@ def stable_uid(doc_type: str, path: str, meta: Dict[str, Any]) -> str:
raw = f"section|c{int(chapter):03d}|s{int(sec):03d}"
elif doc_type == "subsection" and chapter is not None and sec is not None and sub is not None:
raw = f"subsection|c{int(chapter):03d}|s{int(sec):03d}|ss{int(sub):03d}"
elif doc_type == "child" and chapter is not None and sec is not None and sub is not None and child is not None:
raw = f"child|c{int(chapter):03d}|s{int(sec):03d}|ss{int(sub):03d}|c{int(child):03d}"
elif doc_type == "subsubsection" and chapter is not None and sec is not None and sub is not None and subsub is not None:
raw = f"subsubsection|c{int(chapter):03d}|s{int(sec):03d}|ss{int(sub):03d}|sss{int(subsub):03d}"
elif doc_type == "child" and chapter is not None and sec is not None and sub is not None and subsub is not None and child is not None:
raw = f"child|c{int(chapter):03d}|s{int(sec):03d}|ss{int(sub):03d}|sss{int(subsub):03d}|c{int(child):03d}"
else:
raw = f"{doc_type}|{path}"
......@@ -85,6 +88,7 @@ def load_docs(base_dir: Path) -> List[DocRecord]:
("chapter", [base_dir / "chapters", base_dir / "chapter"]),
("section", [base_dir / "sections", base_dir / "section"]),
("subsection", [base_dir / "subsections", base_dir / "subsection"]),
("subsubsection", [base_dir / "subsubsections", base_dir / "subsubsection"]),
("child", [base_dir / "childs"]),
]
......@@ -133,11 +137,13 @@ CREATE TABLE IF NOT EXISTS docs (
chapter_index INT NULL,
section_index INT NULL,
subsection_index INT NULL,
subsubsection_index INT NULL,
child_index INT NULL,
chapter_title TEXT NULL,
section_title TEXT NULL,
subsection_title TEXT NULL,
subsubsection_title TEXT NULL,
title TEXT NULL,
source_type TEXT NULL,
......@@ -155,6 +161,7 @@ CREATE INDEX IF NOT EXISTS docs_doc_type_idx ON docs(doc_type);
CREATE INDEX IF NOT EXISTS docs_chapter_idx ON docs(chapter_index);
CREATE INDEX IF NOT EXISTS docs_section_idx ON docs(chapter_index, section_index);
CREATE INDEX IF NOT EXISTS docs_subsection_idx ON docs(chapter_index, section_index, subsection_index);
CREATE INDEX IF NOT EXISTS docs_subsubsection_idx ON docs(chapter_index, section_index, subsection_index, subsubsection_index);
"""
......@@ -173,13 +180,13 @@ def init_db(pg_url: str) -> None:
UPSERT_SQL = """
INSERT INTO docs (
uid, doc_type,
chapter_index, section_index, subsection_index, child_index,
chapter_title, section_title, subsection_title, title, source_type,
chapter_index, section_index, subsection_index, subsubsection_index, child_index,
chapter_title, section_title, subsection_title, subsubsection_title, title, source_type,
path, markdown, embedding
) VALUES (
%(uid)s, %(doc_type)s,
%(chapter_index)s, %(section_index)s, %(subsection_index)s, %(child_index)s,
%(chapter_title)s, %(section_title)s, %(subsection_title)s, %(title)s, %(source_type)s,
%(chapter_index)s, %(section_index)s, %(subsection_index)s, %(subsubsection_index)s, %(child_index)s,
%(chapter_title)s, %(section_title)s, %(subsection_title)s, %(subsubsection_title)s, %(title)s, %(source_type)s,
%(path)s, %(markdown)s, %(embedding)s
)
ON CONFLICT (uid) DO UPDATE SET
......@@ -187,10 +194,12 @@ ON CONFLICT (uid) DO UPDATE SET
chapter_index = EXCLUDED.chapter_index,
section_index = EXCLUDED.section_index,
subsection_index = EXCLUDED.subsection_index,
subsubsection_index = EXCLUDED.subsubsection_index,
child_index = EXCLUDED.child_index,
chapter_title = EXCLUDED.chapter_title,
section_title = EXCLUDED.section_title,
subsection_title = EXCLUDED.subsection_title,
subsubsection_title = EXCLUDED.subsubsection_title,
title = EXCLUDED.title,
source_type = EXCLUDED.source_type,
path = EXCLUDED.path,
......@@ -228,10 +237,12 @@ def upsert_docs(pg_url: str, docs: List[DocRecord], embeddings: List[List[float]
"chapter_index": _meta_int(m, "chapter_index"),
"section_index": _meta_int(m, "section_index"),
"subsection_index": _meta_int(m, "subsection_index"),
"subsubsection_index": _meta_int(m, "subsubsection_index"),
"child_index": _meta_int(m, "child_index"),
"chapter_title": m.get("chapter_title"),
"section_title": m.get("section_title"),
"subsection_title": m.get("subsection_title"),
"subsubsection_title": m.get("subsubsection_title"),
"title": m.get("title"),
"source_type": m.get("source_type"),
"path": doc.path,
......@@ -292,10 +303,12 @@ def _row_to_retrieved(row: Dict[str, Any], source_type: Optional[str] = None) ->
"chapter_index": row["chapter_index"],
"section_index": row["section_index"],
"subsection_index": row["subsection_index"],
"subsubsection_index": row["subsubsection_index"],
"child_index": row["child_index"],
"chapter_title": row["chapter_title"],
"section_title": row["section_title"],
"subsection_title": row["subsection_title"],
"subsubsection_title": row["subsubsection_title"],
"title": row["title"],
"source_type": row["source_type"] or source_type,
"path": row["path"],
......@@ -319,11 +332,11 @@ class RetrievalPipelineConfig:
enable_dominant_scope: bool = True
enable_scoped_child_search: bool = True
enable_context_docs: bool = True
dominance_level: Literal["subsection", "section"] = "subsection"
scope_fill_k: int = 5
chapter_index: Optional[int] = None
section_index: Optional[int] = None
subsection_index: Optional[int] = None
subsubsection_index: Optional[int] = None
source_type_filter: Optional[List[str]] = None
......@@ -332,7 +345,8 @@ class DominantScope:
chapter_index: int
section_index: int
subsection_index: Optional[int]
level: Literal["subsection", "section"]
subsubsection_index: Optional[int]
level: Literal["section", "subsection", "subsubsection"]
def build_default_pipeline_config(
......@@ -340,6 +354,7 @@ def build_default_pipeline_config(
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
subsubsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
expand_links: bool = True,
neighbor_expand: int = 0,
......@@ -354,11 +369,11 @@ def build_default_pipeline_config(
enable_dominant_scope=True,
enable_scoped_child_search=True,
enable_context_docs=True,
dominance_level="subsection",
scope_fill_k=max(k, 5),
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
)
......@@ -386,6 +401,10 @@ def run_global_child_vector_search(
where.append("subsection_index = %(subsection_index)s")
params["subsection_index"] = config.subsection_index
if config.subsubsection_index is not None:
where.append("subsubsection_index = %(subsubsection_index)s")
params["subsubsection_index"] = config.subsubsection_index
if config.source_type_filter:
where.append("source_type = ANY(%(source_type_filter)s)")
params["source_type_filter"] = config.source_type_filter
......@@ -394,8 +413,8 @@ def run_global_child_vector_search(
sql = f"""
SELECT
uid, doc_type,
chapter_index, section_index, subsection_index, child_index,
chapter_title, section_title, subsection_title, title, source_type,
chapter_index, section_index, subsection_index, subsubsection_index, child_index,
chapter_title, section_title, subsection_title, subsubsection_title, title, source_type,
path, markdown,
1 - (embedding <=> %(qvec)s) AS score
FROM docs
......@@ -424,51 +443,94 @@ def select_top_k_unique(values: List[Retrieved], wanted_k: int) -> List[Retrieve
return selected
def _normalized_level_index(value: Any) -> int:
if value is None:
return 0
try:
return int(value)
except Exception:
return 0
def _build_scope_from_indices(
chapter_index: Any,
section_index: Any,
subsection_index: Any,
subsubsection_index: Any,
) -> Optional[DominantScope]:
if chapter_index is None or section_index is None:
return None
sub = _normalized_level_index(subsection_index)
subsub = _normalized_level_index(subsubsection_index)
if sub <= 0:
return DominantScope(
chapter_index=int(chapter_index),
section_index=int(section_index),
subsection_index=None,
subsubsection_index=None,
level="section",
)
if subsub <= 0:
return DominantScope(
chapter_index=int(chapter_index),
section_index=int(section_index),
subsection_index=sub,
subsubsection_index=None,
level="subsection",
)
return DominantScope(
chapter_index=int(chapter_index),
section_index=int(section_index),
subsection_index=sub,
subsubsection_index=subsub,
level="subsubsection",
)
def _scope_key(scope: DominantScope) -> Tuple[int, ...]:
if scope.level == "section":
return (scope.chapter_index, scope.section_index)
if scope.level == "subsection":
return (scope.chapter_index, scope.section_index, int(scope.subsection_index or 0))
return (
scope.chapter_index,
scope.section_index,
int(scope.subsection_index or 0),
int(scope.subsubsection_index or 0),
)
def select_dominant_scope(
children_direct: List[Retrieved],
level: Literal["subsection", "section"] = "subsection",
) -> Optional[DominantScope]:
if not children_direct:
return None
counts: Dict[Tuple[int, ...], Tuple[int, float]] = {}
counts: Dict[Tuple[str, Tuple[int, ...]], Tuple[DominantScope, int, float]] = {}
for child in children_direct:
chapter_index = child.metadata.get("chapter_index")
section_index = child.metadata.get("section_index")
subsection_index = child.metadata.get("subsection_index")
if chapter_index is None or section_index is None:
scope = _build_scope_from_indices(
child.metadata.get("chapter_index"),
child.metadata.get("section_index"),
child.metadata.get("subsection_index"),
child.metadata.get("subsubsection_index"),
)
if scope is None:
continue
if level == "subsection":
if subsection_index is None:
continue
key = (int(chapter_index), int(section_index), int(subsection_index))
else:
key = (int(chapter_index), int(section_index))
count, score_sum = counts.get(key, (0, 0.0))
counts[key] = (count + 1, score_sum + float(child.score))
key = (scope.level, _scope_key(scope))
_, count, score_sum = counts.get(key, (scope, 0, 0.0))
counts[key] = (scope, count + 1, score_sum + float(child.score))
if not counts:
return None
def _rank(item: Tuple[Tuple[int, ...], Tuple[int, float]]) -> Tuple[int, float, Tuple[int, ...]]:
key, (count, score_sum) = item
def _rank(item: Tuple[Tuple[str, Tuple[int, ...]], Tuple[DominantScope, int, float]]) -> Tuple[int, float, Tuple[int, ...]]:
_, (scope, count, score_sum) = item
avg_score = score_sum / count if count else 0.0
return (count, avg_score, tuple([-part for part in key]))
return (count, avg_score, tuple([-part for part in _scope_key(scope)]))
winner_key, _ = max(counts.items(), key=_rank)
if level == "subsection":
return DominantScope(
chapter_index=winner_key[0],
section_index=winner_key[1],
subsection_index=winner_key[2],
level="subsection",
)
return DominantScope(
chapter_index=winner_key[0],
section_index=winner_key[1],
subsection_index=None,
level="section",
)
_, (winner_scope, _, _) = max(counts.items(), key=_rank)
return winner_scope
def fetch_scope_context_docs(
......@@ -478,16 +540,17 @@ def fetch_scope_context_docs(
chapter_docs: List[Retrieved] = []
section_docs: List[Retrieved] = []
subsection_docs: List[Retrieved] = []
subsubsection_docs: List[Retrieved] = []
with psycopg.connect(pg_url, row_factory=dict_row) as conn:
with conn.cursor() as cur:
if scope.level == "subsection" and scope.subsection_index is not None:
if scope.level in ("subsection", "subsubsection") and scope.subsection_index is not None:
cur.execute(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.title, d.source_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
......@@ -502,12 +565,33 @@ def fetch_scope_context_docs(
subsection_docs = [_row_to_retrieved(
row, source_type="subsection") for row in cur.fetchall()]
if scope.level == "subsubsection" and scope.subsection_index is not None and scope.subsubsection_index is not None:
cur.execute(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
WHERE d.doc_type = 'subsubsection'
AND d.chapter_index = %(cpt)s
AND d.section_index = %(sec)s
AND d.subsection_index = %(sub)s
AND d.subsubsection_index = %(subsub)s
""",
{"cpt": scope.chapter_index, "sec": scope.section_index, "sub": scope.subsection_index, "subsub": scope.subsubsection_index},
)
subsubsection_docs = [_row_to_retrieved(
row, source_type="subsubsection") for row in cur.fetchall()]
cur.execute(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.title, d.source_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
......@@ -524,8 +608,8 @@ def fetch_scope_context_docs(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.title, d.source_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
......@@ -541,6 +625,7 @@ def fetch_scope_context_docs(
"chapters": chapter_docs,
"sections": section_docs,
"subsections": subsection_docs,
"subsubsections": subsubsection_docs,
}
......@@ -561,16 +646,19 @@ def run_scoped_child_vector_search(
"sec": scope.section_index,
"fill_k": max(1, int(fill_k)),
}
if scope.level == "subsection" and scope.subsection_index is not None:
if scope.level in ("subsection", "subsubsection") and scope.subsection_index is not None:
where.append("d.subsection_index = %(sub)s")
params["sub"] = scope.subsection_index
if scope.level == "subsubsection" and scope.subsubsection_index is not None:
where.append("d.subsubsection_index = %(subsub)s")
params["subsub"] = scope.subsubsection_index
where_sql = " AND ".join(where)
sql = f"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.title, d.source_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1 - (d.embedding <=> %(qvec)s) AS score
FROM docs d
......@@ -594,27 +682,29 @@ def expand_neighbor_children(
if neighbor_expand <= 0:
return []
wanted: set[Tuple[int, int, int, int]] = set()
wanted: set[Tuple[int, int, int, int, int]] = set()
for child in children:
cpti = child.metadata.get("chapter_index")
si = child.metadata.get("section_index")
ssi = child.metadata.get("subsection_index")
sssi = child.metadata.get("subsubsection_index")
ci = child.metadata.get("child_index")
if cpti is None or si is None or ssi is None or ci is None:
if cpti is None or si is None or ssi is None or sssi is None or ci is None:
continue
for dx in range(-neighbor_expand, neighbor_expand + 1):
if dx == 0:
continue
wanted.add((int(cpti), int(si), int(ssi), int(ci) + dx))
wanted.add((int(cpti), int(si), int(ssi), int(sssi), int(ci) + dx))
if not wanted:
return []
quadruple = sorted(wanted)
cpt_arr = [a for (a, b, c, cidx) in quadruple]
sec_arr = [b for (a, b, c, cidx) in quadruple]
sub_arr = [c for (a, b, c, cidx) in quadruple]
child_arr = [cidx for (a, b, c, cidx) in quadruple]
quintuple = sorted(wanted)
cpt_arr = [a for (a, b, c, d, cidx) in quintuple]
sec_arr = [b for (a, b, c, d, cidx) in quintuple]
sub_arr = [c for (a, b, c, d, cidx) in quintuple]
subsub_arr = [d for (a, b, c, d, cidx) in quintuple]
child_arr = [cidx for (a, b, c, d, cidx) in quintuple]
with psycopg.connect(pg_url, row_factory=dict_row) as conn:
with conn.cursor() as cur:
......@@ -622,16 +712,16 @@ def expand_neighbor_children(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.title, d.source_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
0 AS score
FROM docs d
JOIN unnest(%(cpt_arr)s::int[],%(sec_arr)s::int[], %(sub_arr)s::int[], %(child_arr)s::int[]) AS u(cpt, sec, sub, child)
ON d.chapter_index = u.cpt AND d.section_index = u.sec AND d.subsection_index = u.sub AND d.child_index = u.child
JOIN unnest(%(cpt_arr)s::int[],%(sec_arr)s::int[], %(sub_arr)s::int[], %(subsub_arr)s::int[], %(child_arr)s::int[]) AS u(cpt, sec, sub, subsub, child)
ON d.chapter_index = u.cpt AND d.section_index = u.sec AND d.subsection_index = u.sub AND d.subsubsection_index = u.subsub AND d.child_index = u.child
WHERE d.doc_type = 'child'
""",
{"cpt_arr": cpt_arr, "sec_arr": sec_arr, "sub_arr": sub_arr,
{"cpt_arr": cpt_arr, "sec_arr": sec_arr, "sub_arr": sub_arr, "subsub_arr": subsub_arr,
"child_arr": child_arr},
)
rows = cur.fetchall()
......@@ -687,6 +777,7 @@ def run_child_retrieval_pipeline(
"children_expanded": [],
"chapters": [],
"subsections": [],
"subsubsections": [],
"sections": [],
"neighbors": [],
}
......@@ -698,13 +789,14 @@ def run_child_retrieval_pipeline(
dominant_scope: Optional[DominantScope] = None
if config.expand_links and config.enable_dominant_scope:
dominant_scope = select_dominant_scope(children_direct, config.dominance_level)
dominant_scope = select_dominant_scope(children_direct)
if dominant_scope and config.expand_links and config.enable_context_docs:
context_docs = fetch_scope_context_docs(pg_url, dominant_scope)
groups["chapters"] = context_docs.get("chapters", [])
groups["sections"] = context_docs.get("sections", [])
groups["subsections"] = context_docs.get("subsections", [])
groups["subsubsections"] = context_docs.get("subsubsections", [])
if dominant_scope and config.expand_links and config.enable_scoped_child_search:
additional_children = run_scoped_child_vector_search(
......@@ -739,6 +831,7 @@ def retrieve(
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
subsubsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
expand_links: bool = True,
neighbor_expand: int = 0,
......@@ -748,6 +841,7 @@ def retrieve(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -766,12 +860,42 @@ def retrieve(
# --------------------------------------------------------------------------------------------------------------------
# Deprecated: subsection refs only represent the legacy subsection-centric task/socratic flow.
SubsectionRef = Tuple[int, int, int]
ParentRef = Tuple[int, int, int, int]
def _normalize_parent_refs(
parent_refs: Optional[List[ParentRef]],
) -> List[ParentRef]:
if not parent_refs:
return []
normalized: set[ParentRef] = set()
for chap, sec, sub, subsub in parent_refs:
chap_i, sec_i, sub_i, subsub_i = int(chap), int(sec), int(sub), int(subsub)
if sub_i <= 0:
normalized.add((chap_i, sec_i, 0, 0))
elif subsub_i <= 0:
normalized.add((chap_i, sec_i, sub_i, 0))
else:
normalized.add((chap_i, sec_i, sub_i, subsub_i))
return sorted(normalized)
def _parent_ref_scope(ref: ParentRef) -> DominantScope:
chap, sec, sub, subsub = ref
if sub <= 0:
return DominantScope(chapter_index=chap, section_index=sec, subsection_index=None, subsubsection_index=None, level="section")
if subsub <= 0:
return DominantScope(chapter_index=chap, section_index=sec, subsection_index=sub, subsubsection_index=None, level="subsection")
return DominantScope(chapter_index=chap, section_index=sec, subsection_index=sub, subsubsection_index=subsub, level="subsubsection")
def _normalize_subsection_refs(
subsection_refs: Optional[List[SubsectionRef]],
) -> List[SubsectionRef]:
# Deprecated compatibility helper for subsection-only retrieval.
if not subsection_refs:
return []
normalized = {
......@@ -784,6 +908,7 @@ def load_children_for_subsections(
pg_url: str,
subsection_refs: Optional[List[SubsectionRef]],
) -> List[Source]:
# Deprecated compatibility path for subsection-only task/socratic retrieval.
refs = _normalize_subsection_refs(subsection_refs)
if not refs:
return []
......@@ -794,8 +919,8 @@ def load_children_for_subsections(
sql = """
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.title, d.source_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
......@@ -820,15 +945,153 @@ def load_children_for_subsections(
return _retrivla_to_sources({"task_childs": children})
def load_sources_for_parent_refs(
pg_url: str,
parent_refs: Optional[List[ParentRef]],
) -> List[Source]:
refs = _normalize_parent_refs(parent_refs)
if not refs:
return []
groups: Dict[str, List[Retrieved]] = {
"sections": [],
"subsections": [],
"subsubsections": [],
"task_childs": [],
}
with psycopg.connect(pg_url, row_factory=dict_row) as conn:
with conn.cursor() as cur:
for ref in refs:
scope = _parent_ref_scope(ref)
if scope.level == "section":
cur.execute(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
WHERE d.doc_type = 'section'
AND d.chapter_index = %(cpt)s
AND d.section_index = %(sec)s
""",
{"cpt": scope.chapter_index, "sec": scope.section_index},
)
groups["sections"].extend([_row_to_retrieved(row, source_type="section") for row in cur.fetchall()])
cur.execute(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
WHERE d.doc_type = 'child'
AND d.chapter_index = %(cpt)s
AND d.section_index = %(sec)s
AND d.subsection_index = 0
AND d.subsubsection_index = 0
ORDER BY d.child_index
""",
{"cpt": scope.chapter_index, "sec": scope.section_index},
)
groups["task_childs"].extend([_row_to_retrieved(row) for row in cur.fetchall()])
continue
if scope.level == "subsection":
cur.execute(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
WHERE d.doc_type = 'subsection'
AND d.chapter_index = %(cpt)s
AND d.section_index = %(sec)s
AND d.subsection_index = %(sub)s
""",
{"cpt": scope.chapter_index, "sec": scope.section_index, "sub": scope.subsection_index},
)
groups["subsections"].extend([_row_to_retrieved(row, source_type="subsection") for row in cur.fetchall()])
cur.execute(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
WHERE d.doc_type = 'child'
AND d.chapter_index = %(cpt)s
AND d.section_index = %(sec)s
AND d.subsection_index = %(sub)s
AND d.subsubsection_index = 0
ORDER BY d.child_index
""",
{"cpt": scope.chapter_index, "sec": scope.section_index, "sub": scope.subsection_index},
)
groups["task_childs"].extend([_row_to_retrieved(row) for row in cur.fetchall()])
continue
cur.execute(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
WHERE d.doc_type = 'subsubsection'
AND d.chapter_index = %(cpt)s
AND d.section_index = %(sec)s
AND d.subsection_index = %(sub)s
AND d.subsubsection_index = %(subsub)s
""",
{"cpt": scope.chapter_index, "sec": scope.section_index, "sub": scope.subsection_index, "subsub": scope.subsubsection_index},
)
groups["subsubsections"].extend([_row_to_retrieved(row, source_type="subsubsection") for row in cur.fetchall()])
cur.execute(
"""
SELECT
d.uid, d.doc_type,
d.chapter_index, d.section_index, d.subsection_index, d.subsubsection_index, d.child_index,
d.chapter_title, d.section_title, d.subsection_title, d.subsubsection_title, d.title, d.source_type,
d.path, d.markdown,
1.0 AS score
FROM docs d
WHERE d.doc_type = 'child'
AND d.chapter_index = %(cpt)s
AND d.section_index = %(sec)s
AND d.subsection_index = %(sub)s
AND d.subsubsection_index = %(subsub)s
ORDER BY d.child_index
""",
{"cpt": scope.chapter_index, "sec": scope.section_index, "sub": scope.subsection_index, "subsub": scope.subsubsection_index},
)
groups["task_childs"].extend([_row_to_retrieved(row) for row in cur.fetchall()])
return merge_sources([], _retrivla_to_sources(groups))
def merge_sources(primary: List[Source], additional: List[Source]) -> List[Source]:
merged: List[Source] = []
index_by_key: Dict[Tuple[str, str, str, str, str, str], int] = {}
index_by_key: Dict[Tuple[str, str, str, str, str, str, str], int] = {}
def source_key(source: Source) -> Tuple[str, str, str, str, str, str]:
def source_key(source: Source) -> Tuple[str, str, str, str, str, str, str]:
return (
source.source_id.doc_type or "",
source.source_id.section_title or "",
source.source_id.subsection_title or "",
source.source_id.subsubsection_title or "",
source.source_id.title or "",
source.source_type or "",
source.markdown or "",
......@@ -857,10 +1120,12 @@ def retrieve_with_subsections(
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
subsubsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
expand_links: bool = True,
neighbor_expand: int = 0,
) -> List[Source]:
# Deprecated compatibility path for subsection-only retrieval composition.
# Oversampling improves recall with ivfflat when additional filters exclude
# close hits. We trim back to k after retrieval.
vector_k = max(k * 4, k + 16)
......@@ -872,6 +1137,7 @@ def retrieve_with_subsections(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......@@ -919,6 +1185,7 @@ class SourceID(BaseModel):
chapter_title: Optional[str] = None
section_title: Optional[str] = None
subsection_title: Optional[str] = None
subsubsection_title: Optional[str] = None
title: str
doc_type: str
......@@ -927,12 +1194,15 @@ class SourceID(BaseModel):
"chapter_title": self.chapter_title,
"section_title": self.section_title,
"subsection_title": self.subsection_title,
"subsubsection_title": self.subsubsection_title,
"title": self.title,
"doc_type": self.doc_type,
}
def to_string(self) -> str:
string_rep = self.title
if self.subsubsection_title:
string_rep = f"{self.subsubsection_title}|{string_rep}"
if self.subsection_title:
string_rep = f"{self.subsection_title}|{string_rep}"
if self.section_title:
......@@ -951,6 +1221,7 @@ def _retrivla_to_sources(retrievd: Dict[str, List[Retrieved]]) -> List[Source]:
chapter_title=retrival.metadata.get("chapter_title"),
section_title=retrival.metadata.get("section_title"),
subsection_title=retrival.metadata.get("subsection_title"),
subsubsection_title=retrival.metadata.get("subsubsection_title"),
title=retrival.metadata.get("title"),
doc_type=retrival.doc_type
),
......@@ -991,7 +1262,7 @@ def list_subsections(pg_url: str, section_idx: Optional[int] = None) -> List[Dic
section_index, subsection_index,
COALESCE(NULLIF(subsection_title, ''), '') AS subsection_title
FROM docs
WHERE subsection_index IS NOT NULL
WHERE subsection_index IS NOT NULL AND subsection_index > 0
ORDER BY section_index, subsection_index,
CASE WHEN subsection_title IS NULL OR subsection_title = '' THEN 1 ELSE 0 END,
subsection_title
......@@ -1003,7 +1274,7 @@ def list_subsections(pg_url: str, section_idx: Optional[int] = None) -> List[Dic
section_index, subsection_index,
COALESCE(NULLIF(subsection_title, ''), '') AS subsection_title
FROM docs
WHERE subsection_index IS NOT NULL AND section_index = %(sec)s
WHERE subsection_index IS NOT NULL AND subsection_index > 0 AND section_index = %(sec)s
ORDER BY section_index, subsection_index,
CASE WHEN subsection_title IS NULL OR subsection_title = '' THEN 1 ELSE 0 END,
subsection_title
......
......@@ -18,6 +18,8 @@ from app.deterministic_services.vector_store import (
merge_sources,
)
# Deprecated: this module keeps the legacy subsection-centric retrieval path for compatibility.
def retrieve(
pg_url: str,
......@@ -27,13 +29,15 @@ def retrieve(
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
subsubsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
expand_links: bool = False,
neighbor_expand: int = 0,
) -> List[Source]:
# Parameters kept for drop-in compatibility with child-level retrieve.
# Deprecated compatibility path. Parameters kept for drop-in compatibility with child-level retrieve.
_ = expand_links
_ = neighbor_expand
_ = subsubsection_index
qvec = Vector(embed_query(embedder, query))
......@@ -64,8 +68,8 @@ def retrieve(
sql = f"""
SELECT
uid, doc_type,
chapter_index, section_index, subsection_index, child_index,
chapter_title, section_title, subsection_title, title, source_type,
chapter_index, section_index, subsection_index, subsubsection_index, child_index,
chapter_title, section_title, subsection_title, subsubsection_title, title, source_type,
path, markdown,
1 - (embedding <=> %(qvec)s) AS score
FROM docs
......@@ -94,10 +98,12 @@ def retrieve_with_subsections(
chapter_index: Optional[int] = None,
section_index: Optional[int] = None,
subsection_index: Optional[int] = None,
subsubsection_index: Optional[int] = None,
source_type_filter: Optional[List[str]] = None,
expand_links: bool = False,
neighbor_expand: int = 0,
) -> List[Source]:
# Deprecated compatibility path for subsection-only retrieval composition.
vector_k = max(k * 4, k + 16)
vector_sources = retrieve(
pg_url=pg_url,
......@@ -107,6 +113,7 @@ def retrieve_with_subsections(
chapter_index=chapter_index,
section_index=section_index,
subsection_index=subsection_index,
subsubsection_index=subsubsection_index,
source_type_filter=source_type_filter,
expand_links=expand_links,
neighbor_expand=neighbor_expand,
......
......@@ -103,7 +103,6 @@ def convert_task_payload(payload: dict[str, Any], source_file_id: str, image_pat
return {
"id": source_file_id,
"titel": title,
"level": DEFAULT_LEVEL,
"slug": slug,
"description": intro,
"exercise_topic": _build_topic_entries(payload.get("subsections", [])),
......@@ -223,7 +222,7 @@ def convert_all(tasks_dir: Path = TASKS_DIR, dry_run: bool = False) -> tuple[lis
images_root = tasks_dir / "images"
used_names: set[str] = set()
for json_path in sorted(tasks_dir.glob("*.json")):
if json_path.name == "_subsection_map.json":
if json_path.name == "_topic-to-index-map.json":
continue
payload = load_json_task(json_path)
source_file_id = json_path.stem
......
......@@ -92,7 +92,6 @@ class SentenceTransformerJinaV5Test(unittest.TestCase):
api_key="gwdg-key",
model="e5-mistral-7b-instruct",
target_dim=2,
timeout=12.5,
)
)
result = embedder.embed_documents(["a", "b"])
......@@ -102,7 +101,6 @@ class SentenceTransformerJinaV5Test(unittest.TestCase):
{
"api_key": "gwdg-key",
"base_url": "https://chat-ai.academiccloud.de/v1",
"timeout": 12.5,
},
)
self.assertEqual(
......
......@@ -62,6 +62,7 @@ class TaskCatalogSocraticTest(unittest.TestCase):
"title": "Analysis",
"intro": "Intro",
"subsections": ["quadratische_gleichungen"],
"topic_refs": [],
"tasks": [
{
"id": "01",
......@@ -92,6 +93,27 @@ class TaskCatalogSocraticTest(unittest.TestCase):
}
],
)
self.assertEqual(catalog[0]["topics"], ["quadratische_gleichungen"])
def test_normalize_yaml_task_entry_excludes_images_from_hints(self) -> None:
task_entry = {
"aufgabe": [
{"type": "text", "text": "Bestimme f(x)."},
],
"hinweise": [
{"type": "text", "text": "Nutze den Graphen."},
{"type": "image", "src": "analysis/h1.png", "alt": "Hinweisgrafik."},
],
"loesung": [
{"type": "text", "text": "f(x)=x^2"},
],
}
normalized = task_catalog._normalize_yaml_task_entry(task_entry, 1)
self.assertIsNotNone(normalized)
self.assertEqual(normalized["hints"], ["Nutze den Graphen."])
self.assertEqual(normalized["images"], [])
def test_socratic_build_subsection_catalog_returns_response_shape(self) -> None:
with patch(
......@@ -135,13 +157,13 @@ class TaskCatalogSocraticTest(unittest.TestCase):
def test_build_subsection_catalog_includes_summary(self) -> None:
with patch(
"app.deterministic_services.task_catalog.load_subsection_map",
"app.deterministic_services.task_catalog.load_topic_map",
return_value={
"quadratische gleichungen": (1, 3, 3),
"mengen": (1, 1, 1),
"quadratische gleichungen": (1, 3, 3, 0),
"mengen": (1, 1, 1, 0),
},
), patch(
"app.deterministic_services.task_catalog.load_subsection_summaries",
"app.deterministic_services.task_catalog.load_topic_summaries",
return_value={
"mengen": "Mengen summary text",
"quadratische gleichungen": "Quadratische summary text",
......@@ -167,6 +189,51 @@ class TaskCatalogSocraticTest(unittest.TestCase):
],
)
def test_build_topic_catalog_supports_mixed_parent_levels(self) -> None:
with patch(
"app.deterministic_services.task_catalog.load_topic_map",
return_value={
"mengen": (1, 1, 0, 0),
"quadratische gleichungen": (1, 3, 2, 0),
"hornerschema": (1, 3, 3, 1),
},
), patch(
"app.deterministic_services.task_catalog.load_topic_summaries",
return_value={
"mengen": "Mengen summary text",
"quadratische gleichungen": "Quadratische summary text",
"hornerschema": "Hornerschema summary text",
},
):
catalog = task_catalog.build_topic_catalog()
self.assertEqual(
catalog,
[
{
"topic_key": "hornerschema",
"label": "Hornerschema",
"level": "subsubsection",
"refs": [[1, 3, 3, 1]],
"summary": "Hornerschema summary text",
},
{
"topic_key": "mengen",
"label": "Mengen",
"level": "section",
"refs": [[1, 1, 0, 0]],
"summary": "Mengen summary text",
},
{
"topic_key": "quadratische gleichungen",
"label": "Quadratische Gleichungen",
"level": "subsection",
"refs": [[1, 3, 2, 0]],
"summary": "Quadratische summary text",
},
],
)
def test_load_subsection_summaries_extracts_body(self) -> None:
temp_dir = Path(__file__).resolve().parent / "_tmp_subsection_summaries"
summary_root = temp_dir / "child_lvl" / "subsections"
......@@ -249,6 +316,42 @@ Zweite Zeile.
self.assertEqual(sheet["selected_subsection_key"], "quadratische gleichungen")
self.assertEqual(sheet["selected_subsection_refs"], [[1, 3, 3]])
def test_set_selected_task_stores_generic_parent_refs(self) -> None:
sheet: dict[str, object] = {}
task_file = {
"_file_id": "analysis_1",
"title": "Analysis",
"subsections": ["mengen"],
"topic_refs": [
{
"chapter_index": 1,
"section_index": 3,
"subsection_index": 2,
"subsubsection_index": 1,
"key": "hornerschema",
"label": "Hornerschema",
}
],
}
task_entry = {"id": "01", "statement": "Bestimme f(x).", "hints": [], "solution": "", "images": []}
with patch(
"app.deterministic_services.task_catalog.load_topic_map",
return_value={"mengen": (1, 1, 0, 0)},
):
task_catalog.set_selected_task(sheet, task_file, task_entry)
self.assertEqual(sheet["task_parent_refs"], [[1, 1, 0, 0], [1, 3, 2, 1]])
def test_get_selected_task_parent_refs_reads_normalized_refs(self) -> None:
sheet: dict[str, object] = {
"task_parent_refs": [[1, 1, 0, 0], [1, 3, 2, 1], [1, 3, 2, 1]],
}
refs = task_catalog.get_selected_task_parent_refs(sheet)
self.assertEqual(refs, [(1, 1, 0, 0), (1, 3, 2, 1)])
class TaskApiSocraticTest(unittest.TestCase):
def setUp(self) -> None:
......@@ -256,20 +359,23 @@ class TaskApiSocraticTest(unittest.TestCase):
app.include_router(tasks.router)
self.client = TestClient(app)
def test_list_tasks_includes_subsection_options(self) -> None:
def test_list_tasks_includes_topic_options(self) -> None:
payload = [
{
"file_id": "analysis_1",
"title": "Analysis",
"intro": "Intro",
"tasks": [],
"topics": ["quadratische gleichungen"],
"topic_options": [],
}
]
subsections = [
topics = [
{
"subsection_key": "quadratische gleichungen",
"topic_key": "quadratische gleichungen",
"label": "Quadratische Gleichungen",
"refs": [[1, 3, 3]],
"refs": [[1, 3, 3, 0]],
"level": "subsection",
}
]
......@@ -277,15 +383,15 @@ class TaskApiSocraticTest(unittest.TestCase):
"app.api.tasks.task_catalog.build_task_catalog",
return_value=payload,
), patch(
"app.api.tasks.task_catalog.build_subsection_catalog",
return_value=subsections,
"app.api.tasks.task_catalog.build_topic_catalog",
return_value=topics,
):
response = self.client.get("/api/tasks")
self.assertEqual(response.status_code, 200)
body = response.json()
self.assertEqual(body["orchestrator"], "socratic")
self.assertEqual(body["subsections"][0]["subsection_key"], "quadratische gleichungen")
self.assertEqual(body["topics"][0]["topic_key"], "quadratische gleichungen")
def test_task_asset_endpoint_serves_files_from_task_image_dir(self) -> None:
temp_dir = Path(tempfile.mkdtemp(prefix="task-assets-"))
......
......@@ -142,6 +142,7 @@ class TaskYamlMigrationTest(unittest.TestCase):
self.assertEqual(catalog[0]["file_id"], "analysis_1")
self.assertEqual(catalog[0]["title"], "Analysis")
self.assertEqual(catalog[0]["intro"], "Intro")
self.assertEqual(catalog[0]["topics"], ["quadratische-gleichungen"])
self.assertEqual(catalog[0]["tasks"][0]["task_id"], "01")
self.assertEqual(catalog[0]["tasks"][0]["statement"], "Bestimme f(x).")
self.assertEqual(
......
......@@ -24,6 +24,7 @@ def _mk_retrieved(
chapter_index: int,
section_index: int,
subsection_index: int | None,
subsubsection_index: int | None = 0,
child_index: int = 1,
) -> Retrieved:
return Retrieved(
......@@ -34,10 +35,12 @@ def _mk_retrieved(
"chapter_index": chapter_index,
"section_index": section_index,
"subsection_index": subsection_index,
"subsubsection_index": subsubsection_index,
"child_index": child_index,
"chapter_title": "C",
"section_title": "S",
"subsection_title": "SS",
"subsubsection_title": "SSS" if subsubsection_index else "",
"title": uid,
"source_type": "child",
"path": "",
......@@ -91,37 +94,77 @@ class VectorStorePipelineUnitTest(unittest.TestCase):
self.assertTrue(cfg.enable_scoped_child_search)
self.assertTrue(cfg.enable_context_docs)
def test_select_dominant_scope_prefers_count(self) -> None:
def test_select_dominant_scope_prefers_subsection_count(self) -> None:
children = [
_mk_retrieved("a", 0.9, 1, 1, 1),
_mk_retrieved("b", 0.8, 1, 1, 1),
_mk_retrieved("c", 0.95, 1, 1, 2),
]
scope = select_dominant_scope(children, level="subsection")
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "subsection")
self.assertEqual((scope.chapter_index, scope.section_index, scope.subsection_index), (1, 1, 1))
def test_select_dominant_scope_resolves_section_parent(self) -> None:
children = [
_mk_retrieved("a", 0.9, 1, 1, 0, 0),
_mk_retrieved("b", 0.8, 1, 1, 0, 0),
_mk_retrieved("c", 0.95, 1, 1, 2, 0),
]
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "section")
self.assertEqual((scope.chapter_index, scope.section_index), (1, 1))
self.assertIsNone(scope.subsection_index)
self.assertIsNone(scope.subsubsection_index)
def test_select_dominant_scope_resolves_subsubsection_parent(self) -> None:
children = [
_mk_retrieved("a", 0.9, 1, 1, 2, 1),
_mk_retrieved("b", 0.8, 1, 1, 2, 1),
_mk_retrieved("c", 0.95, 1, 1, 2, 0),
]
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "subsubsection")
self.assertEqual((scope.chapter_index, scope.section_index, scope.subsection_index, scope.subsubsection_index), (1, 1, 2, 1))
def test_select_dominant_scope_tiebreak_avg_score(self) -> None:
children = [
_mk_retrieved("a", 0.7, 1, 1, 1),
_mk_retrieved("b", 0.8, 1, 1, 2),
_mk_retrieved("a", 0.7, 1, 1, 1, 0),
_mk_retrieved("b", 0.8, 1, 1, 2, 0),
]
scope = select_dominant_scope(children, level="subsection")
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "subsection")
self.assertEqual((scope.chapter_index, scope.section_index, scope.subsection_index), (1, 1, 2))
def test_select_dominant_scope_tiebreak_lexicographic(self) -> None:
children = [
_mk_retrieved("a", 0.8, 2, 1, 1),
_mk_retrieved("b", 0.8, 1, 2, 3),
_mk_retrieved("a", 0.8, 2, 1, 1, 0),
_mk_retrieved("b", 0.8, 1, 2, 3, 0),
]
scope = select_dominant_scope(children, level="subsection")
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual((scope.chapter_index, scope.section_index, scope.subsection_index), (1, 2, 3))
def test_select_dominant_scope_keeps_exact_parent_levels_separate(self) -> None:
children = [
_mk_retrieved("section-hit", 0.91, 1, 1, 2, 0),
_mk_retrieved("subsub-hit", 0.92, 1, 1, 2, 1),
]
scope = select_dominant_scope(children)
self.assertIsNotNone(scope)
assert scope is not None
self.assertEqual(scope.level, "subsubsection")
self.assertEqual(scope.subsection_index, 2)
self.assertEqual(scope.subsubsection_index, 1)
def test_merge_retrieval_groups_dedup_max_score_and_trim_children(self) -> None:
groups = {
"children_direct": [
......@@ -146,10 +189,55 @@ class VectorStorePipelineUnitTest(unittest.TestCase):
self.assertEqual(merged["neighbors"], [])
def test_expand_neighbor_children_returns_empty_for_zero_expand(self) -> None:
children = [_mk_retrieved("u1", 0.8, 1, 1, 1, child_index=3)]
children = [_mk_retrieved("u1", 0.8, 1, 1, 1, 4, child_index=3)]
result = expand_neighbor_children("postgresql://unused", children, neighbor_expand=0)
self.assertEqual(result, [])
def test_source_id_to_string_includes_subsubsection_title(self) -> None:
source_id = SourceID(
chapter_title="Kapitel",
section_title="Section",
subsection_title="Subsection",
subsubsection_title="Subsubsection",
title="Child",
doc_type="child",
)
self.assertEqual(source_id.to_string(), "[Kapitel|Section|Subsection|Subsubsection|Child|child]")
def test_merge_sources_distinguishes_subsubsection_title(self) -> None:
first = Source(
source_id=SourceID(
chapter_title="Kapitel",
section_title="Section",
subsection_title="Subsection",
subsubsection_title="A",
title="Child A",
doc_type="child",
),
retrieved_as="children_direct",
source_type="child",
score=0.7,
markdown="same-md",
)
second = Source(
source_id=SourceID(
chapter_title="Kapitel",
section_title="Section",
subsection_title="Subsection",
subsubsection_title="B",
title="Child A",
doc_type="child",
),
retrieved_as="children_direct",
source_type="child",
score=0.6,
markdown="same-md",
)
merged = merge_sources([first], [second])
self.assertEqual(len(merged), 2)
if __name__ == "__main__":
unittest.main()
......@@ -15,8 +15,15 @@ export type TaskFile = {
title: string;
intro: string;
tasks: TaskItem[];
subsections?: string[];
subsection_options?: SubsectionOption[];
topics?: string[];
topic_options?: TopicOption[];
};
export type TopicOption = {
topic_key: string;
label: string;
refs: [number, number, number, number][];
summary: string;
};
export type SubsectionOption = {
......@@ -30,7 +37,7 @@ export type TasksResponse = {
orchestrator: string;
enabled: boolean;
task_files: TaskFile[];
subsections: SubsectionOption[];
topics: TopicOption[];
};
export type SocraticResponse = {
......
......@@ -9,9 +9,11 @@ export type RetrievedDoc = {
metadata: {
section_index?: number | null;
subsection_index?: number | null;
subsubsection_index?: number | null;
child_index?: number | null;
section_title?: string | null;
subsection_title?: string | null;
subsubsection_title?: string | null;
title?: string | null;
type?: string | null;
box_hint?: string | null;
......@@ -25,6 +27,7 @@ type DocPanelProps = {
taskChildren: RetrievedDoc[];
indirectChildren: RetrievedDoc[];
subsections: RetrievedDoc[];
subsubsections: RetrievedDoc[];
sections: RetrievedDoc[];
isLoading?: boolean;
error?: string | null;
......@@ -36,6 +39,7 @@ const buildTitle = (doc: RetrievedDoc) => {
const meta = doc.metadata;
return (
meta.title ||
meta.subsubsection_title ||
meta.subsection_title ||
meta.section_title ||
meta.path ||
......@@ -52,6 +56,9 @@ const buildSubtitle = (doc: RetrievedDoc) => {
if (meta.subsection_index !== null && meta.subsection_index !== undefined) {
parts.push(`ss${meta.subsection_index}`);
}
if (meta.subsubsection_index !== null && meta.subsubsection_index !== undefined) {
parts.push(`sss${meta.subsubsection_index}`);
}
if (meta.child_index !== null && meta.child_index !== undefined) {
parts.push(`c${meta.child_index}`);
}
......@@ -102,6 +109,7 @@ export default function DocPanel({
taskChildren,
indirectChildren,
subsections,
subsubsections,
sections,
isLoading,
error,
......@@ -113,6 +121,7 @@ export default function DocPanel({
taskChildren.length +
indirectChildren.length +
subsections.length +
subsubsections.length +
sections.length;
return (
......@@ -126,6 +135,7 @@ export default function DocPanel({
{renderGroup(t("taskChildren"), taskChildren, onCiteDoc, onInspectDoc)}
{renderGroup(t("directChildren"), directChildren, onCiteDoc, onInspectDoc)}
{renderGroup(t("indirectChildren"), indirectChildren, onCiteDoc, onInspectDoc)}
{renderGroup(t("subsubsectionSummary"), subsubsections, onCiteDoc, onInspectDoc)}
{renderGroup(t("subsectionSummary"), subsections, onCiteDoc, onInspectDoc)}
{renderGroup(t("sectionSummary"), sections, onCiteDoc, onInspectDoc)}
</div>
......
......@@ -31,6 +31,7 @@
directChildren: "Direct children",
taskChildren: "Task sources",
indirectChildren: "Indirect children",
subsubsectionSummary: "Subsubsection summary",
subsectionSummary: "Subsection summary",
sectionSummary: "Section summary",
untitled: "Untitled",
......@@ -130,6 +131,7 @@
directChildren: "Direkte Quellen",
taskChildren: "Aufgaben-Quellen",
indirectChildren: "Indirekte Quellen",
subsubsectionSummary: "Unterunterabschnitt-Zusammenfassung",
subsectionSummary: "Unterabschnitt-Zusammenfassung",
sectionSummary: "Abschnitt-Zusammenfassung",
untitled: "Ohne Titel",
......
......@@ -61,6 +61,7 @@ type ContextSource = {
chapter_title?: string | null;
section_title?: string | null;
subsection_title?: string | null;
subsubsection_title?: string | null;
title: string;
doc_type: string;
};
......@@ -75,6 +76,7 @@ const sourceIdToUid = (source: ContextSource, index: number) => {
source.source_id.chapter_title,
source.source_id.section_title,
source.source_id.subsection_title,
source.source_id.subsubsection_title,
source.source_id.title,
source.source_id.doc_type,
].filter(Boolean);
......@@ -86,6 +88,7 @@ const sourceIdToKey = (sourceId: ContextSource["source_id"]) => {
sourceId.chapter_title ?? "",
sourceId.section_title ?? "",
sourceId.subsection_title ?? "",
sourceId.subsubsection_title ?? "",
sourceId.title ?? "",
sourceId.doc_type ?? "",
];
......@@ -97,6 +100,7 @@ const sourceIdToPath = (source: ContextSource) => {
source.source_id.chapter_title,
source.source_id.section_title,
source.source_id.subsection_title,
source.source_id.subsubsection_title,
].filter(Boolean);
return parts.length ? parts.join(" / ") : null;
};
......@@ -109,6 +113,7 @@ const toRetrievedDoc = (source: ContextSource, index: number): RetrievedDoc => (
metadata: {
section_title: source.source_id.section_title ?? null,
subsection_title: source.source_id.subsection_title ?? null,
subsubsection_title: source.source_id.subsubsection_title ?? null,
title: source.source_id.title ?? null,
type: source.source_type ?? null,
path: sourceIdToPath(source),
......@@ -123,6 +128,7 @@ const applyRetrievedSources = (
setTaskChildren: (value: RetrievedDoc[]) => void;
setIndirectChildren: (value: RetrievedDoc[]) => void;
setSubsections: (value: RetrievedDoc[]) => void;
setSubsubsections: (value: RetrievedDoc[]) => void;
setSections: (value: RetrievedDoc[]) => void;
}
) => {
......@@ -130,6 +136,7 @@ const applyRetrievedSources = (
const nextTask: RetrievedDoc[] = [];
const nextIndirect: RetrievedDoc[] = [];
const nextSubsections: RetrievedDoc[] = [];
const nextSubsubsections: RetrievedDoc[] = [];
const nextSections: RetrievedDoc[] = [];
sources.forEach((source, index) => {
......@@ -147,6 +154,9 @@ const applyRetrievedSources = (
case "subsections":
nextSubsections.push(doc);
break;
case "subsubsections":
nextSubsubsections.push(doc);
break;
case "sections":
nextSections.push(doc);
break;
......@@ -162,6 +172,7 @@ const applyRetrievedSources = (
setters.setTaskChildren(nextTask);
setters.setIndirectChildren(nextIndirect);
setters.setSubsections(nextSubsections);
setters.setSubsubsections(nextSubsubsections);
setters.setSections(nextSections);
};
......@@ -210,6 +221,7 @@ export default function ChatPage() {
const [taskChildren, setTaskChildren] = useState<RetrievedDoc[]>([]);
const [indirectChildren, setIndirectChildren] = useState<RetrievedDoc[]>([]);
const [subsections, setSubsections] = useState<RetrievedDoc[]>([]);
const [subsubsections, setSubsubsections] = useState<RetrievedDoc[]>([]);
const [sections, setSections] = useState<RetrievedDoc[]>([]);
const [retrievalLoading, setRetrievalLoading] = useState(false);
const [retrievalError, setRetrievalError] = useState<string | null>(null);
......@@ -420,6 +432,7 @@ export default function ChatPage() {
setTaskChildren,
setIndirectChildren,
setSubsections,
setSubsubsections,
setSections,
});
socraticBootstrapRef.current = bootstrapKey;
......@@ -492,6 +505,7 @@ export default function ChatPage() {
...taskChildren,
...indirectChildren,
...subsections,
...subsubsections,
...sections,
];
......@@ -522,7 +536,7 @@ export default function ChatPage() {
});
return { bySourceKey, bySlug };
}, [directChildren, taskChildren, indirectChildren, subsections, sections]);
}, [directChildren, taskChildren, indirectChildren, subsections, subsubsections, sections]);
const nextTaskRef = useMemo(() => {
if (!selectedTaskFile || !selectedTaskRef) {
......@@ -734,6 +748,7 @@ export default function ChatPage() {
setTaskChildren,
setIndirectChildren,
setSubsections,
setSubsubsections,
setSections,
});
} catch (error) {
......@@ -806,6 +821,7 @@ export default function ChatPage() {
setTaskChildren([]);
setIndirectChildren([]);
setSubsections([]);
setSubsubsections([]);
setSections([]);
setRetrievalLoading(false);
setRetrievalError(null);
......@@ -1281,6 +1297,7 @@ export default function ChatPage() {
taskChildren={taskChildren}
indirectChildren={indirectChildren}
subsections={subsections}
subsubsections={subsubsections}
sections={sections}
isLoading={retrievalLoading}
error={retrievalError}
......
......@@ -20,6 +20,7 @@ import {
type SelectedSubsectionRef,
type SelectedTaskRef,
type SubsectionOption,
type TopicOption,
type TaskImage,
type TaskFile,
} from "../api/taskApi";
......@@ -44,6 +45,7 @@ export type SelectedSubsection = SelectedSubsectionRef & {
export type TaskSelectionState = {
taskFiles: TaskFile[];
topics: TopicOption[];
selectedTaskRef: SelectedTaskRef | null;
selectedTask: SelectedTask | null;
selectedTaskFile: TaskFile | null;
......@@ -88,7 +90,8 @@ export const getDefaultTaskId = (tasks: Array<{ task_id: string }>): string =>
tasks.find((task) => task.task_id === "01")?.task_id || tasks[0]?.task_id || "";
const isSelectableTaskFile = (file: TaskFile): boolean =>
Array.isArray(file.subsections) && file.subsections.length > 0;
(Array.isArray(file.topics) && file.topics.length > 0) ||
(Array.isArray(file.topic_options) && file.topic_options.length > 0);
const isTaskCoupledOrchestrator = (value: OrchestratorName): boolean =>
value === "task" || value === "socratic";
......@@ -107,6 +110,7 @@ export function TutorSessionProvider({ children }: PropsWithChildren) {
const [orchestratorError, setOrchestratorError] = useState<string | null>(null);
const [isTasksInitialized, setIsTasksInitialized] = useState(false);
const [taskFiles, setTaskFiles] = useState<TaskFile[]>([]);
const [topics, setTopics] = useState<TopicOption[]>([]);
const [subsections, setSubsections] = useState<SubsectionOption[]>([]);
const [selectedTaskRef, setSelectedTaskRef] = useState<SelectedTaskRef | null>(null);
const [selectedSubsectionRef, setSelectedSubsectionRef] =
......@@ -199,6 +203,7 @@ export function TutorSessionProvider({ children }: PropsWithChildren) {
try {
const isSocratic = orchestrator === "socratic";
let files: TaskFile[] = [];
let topicsPayload: TopicOption[] = [];
let subsectionsPayload: SubsectionOption[] = [];
if (isSocratic) {
......@@ -207,12 +212,13 @@ export function TutorSessionProvider({ children }: PropsWithChildren) {
} else {
const payload = await fetchTasks();
files = payload.task_files || [];
subsectionsPayload = payload.subsections || [];
topicsPayload = payload.topics || [];
}
const selectableFiles = files.filter((file) => isSelectableTaskFile(file));
setTaskFiles(files);
setTopics(topicsPayload);
setSubsections(subsectionsPayload);
setSelectedTaskRef((prev) => {
if (isSocratic) {
......@@ -257,6 +263,7 @@ export function TutorSessionProvider({ children }: PropsWithChildren) {
} catch (error) {
setTasksError(t("failedLoadTasks"));
setTaskFiles([]);
setTopics([]);
setSubsections([]);
setSelectedTaskRef(null);
setSelectedSubsectionRef(null);
......@@ -384,6 +391,7 @@ export function TutorSessionProvider({ children }: PropsWithChildren) {
isOrchestratorSelectable,
orchestratorError,
taskFiles,
topics,
subsections,
selectedTaskRef,
selectedTask,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment