Commit 3694c8fa authored by Kantz's avatar Kantz
Browse files

exclusion von subsections entfernt

parent 03d6bb5f
...@@ -302,8 +302,6 @@ def retrieve( ...@@ -302,8 +302,6 @@ def retrieve(
source_type_filter: Optional[List[str]] = None, source_type_filter: Optional[List[str]] = None,
expand_links: bool = True, expand_links: bool = True,
neighbor_expand: int = 0, neighbor_expand: int = 0,
exclude_subsection_refs: Optional[List["SubsectionRef"]] = None,
exclude_child_subsection_refs: Optional[List["SubsectionRef"]] = None,
) -> List[Source]: ) -> List[Source]:
qvec = Vector(embed_query(embedder, query)) qvec = Vector(embed_query(embedder, query))
...@@ -322,18 +320,6 @@ def retrieve( ...@@ -322,18 +320,6 @@ def retrieve(
where.append("source_type = ANY(%(source_type_filter)s)") where.append("source_type = ANY(%(source_type_filter)s)")
params["source_type_filter"] = source_type_filter params["source_type_filter"] = source_type_filter
excluded_child_subsections = _normalize_subsection_refs(exclude_child_subsection_refs)
if excluded_child_subsections:
sec_arr = [sec for sec, _ in excluded_child_subsections]
sub_arr = [sub for _, sub in excluded_child_subsections]
where.append(
"NOT EXISTS ("
"SELECT 1 FROM unnest(%(exclude_sec_arr)s::int[], %(exclude_sub_arr)s::int[]) AS ex(sec, sub) "
"WHERE ex.sec = section_index AND ex.sub = subsection_index)"
)
params["exclude_sec_arr"] = sec_arr
params["exclude_sub_arr"] = sub_arr
where_sql = " AND ".join(where) where_sql = " AND ".join(where)
sql = f""" sql = f"""
...@@ -355,7 +341,6 @@ def retrieve( ...@@ -355,7 +341,6 @@ def retrieve(
sections_docs: List[Retrieved] = [] sections_docs: List[Retrieved] = []
children_direct: List[Retrieved] = [] children_direct: List[Retrieved] = []
children_expanded: List[Retrieved] = [] children_expanded: List[Retrieved] = []
excluded_subsections = set(_normalize_subsection_refs(exclude_subsection_refs))
with psycopg.connect(pg_url, row_factory=dict_row) as conn: with psycopg.connect(pg_url, row_factory=dict_row) as conn:
register_vector(conn) register_vector(conn)
...@@ -379,25 +364,25 @@ def retrieve( ...@@ -379,25 +364,25 @@ def retrieve(
sec_sub_counts.items(), key=lambda x: x[1])[0] sec_sub_counts.items(), key=lambda x: x[1])[0]
most_common_sec, most_common_sub = most_common_sec_sub most_common_sec, most_common_sub = most_common_sec_sub
if (most_common_sec, most_common_sub) not in excluded_subsections: cur.execute(
cur.execute( """
""" SELECT
SELECT d.uid, d.doc_type,
d.uid, d.doc_type, d.section_index, d.subsection_index, d.child_index,
d.section_index, d.subsection_index, d.child_index, d.section_title, d.subsection_title, d.title, d.source_type,
d.section_title, d.subsection_title, d.title, d.source_type, d.path, d.markdown,
d.path, d.markdown, 1.0 AS score
1.0 AS score FROM docs d
FROM docs d WHERE d.doc_type = ANY(%(sub_doc_types)s)
WHERE d.doc_type = ANY(%(sub_doc_types)s) AND d.section_index = %(sec)s
AND d.section_index = %(sec)s AND d.subsection_index = %(sub)s
AND d.subsection_index = %(sub)s """,
""", {"sec": most_common_sec, "sub": most_common_sub,
{"sec": most_common_sec, "sub": most_common_sub, "sub_doc_types": ["subsection", "chapter"]},
"sub_doc_types": ["subsection", "chapter"]}, )
) subsections = [_row_to_retrieved(
subsections = [_row_to_retrieved( row, source_type="subsection") for row in cur.fetchall()]
row, source_type="subsection") for row in cur.fetchall()]
cur.execute( cur.execute(
""" """
...@@ -592,8 +577,6 @@ def retrieve_with_subsections( ...@@ -592,8 +577,6 @@ def retrieve_with_subsections(
source_type_filter=source_type_filter, source_type_filter=source_type_filter,
expand_links=expand_links, expand_links=expand_links,
neighbor_expand=neighbor_expand, neighbor_expand=neighbor_expand,
exclude_subsection_refs=subsection_refs,
exclude_child_subsection_refs=subsection_refs,
) )
vector_sources = vector_sources[:k] vector_sources = vector_sources[:k]
subsection_children = load_children_for_subsections( subsection_children = load_children_for_subsections(
......
...@@ -29,7 +29,6 @@ def retrieve( ...@@ -29,7 +29,6 @@ def retrieve(
source_type_filter: Optional[List[str]] = None, source_type_filter: Optional[List[str]] = None,
expand_links: bool = False, expand_links: bool = False,
neighbor_expand: int = 0, neighbor_expand: int = 0,
exclude_subsection_refs: Optional[List[SubsectionRef]] = None,
) -> List[Source]: ) -> List[Source]:
# Parameters kept for drop-in compatibility with child-level retrieve. # Parameters kept for drop-in compatibility with child-level retrieve.
_ = expand_links _ = expand_links
...@@ -56,19 +55,6 @@ def retrieve( ...@@ -56,19 +55,6 @@ def retrieve(
where.append("source_type = ANY(%(source_type_filter)s)") where.append("source_type = ANY(%(source_type_filter)s)")
params["source_type_filter"] = source_type_filter params["source_type_filter"] = source_type_filter
excluded = set((int(sec), int(sub)) for sec, sub in (exclude_subsection_refs or []))
if excluded:
exclude_clauses: List[str] = []
for idx, (sec, sub) in enumerate(sorted(excluded)):
sec_key = f"exclude_sec_{idx}"
sub_key = f"exclude_sub_{idx}"
exclude_clauses.append(
f"(section_index = %({sec_key})s AND subsection_index = %({sub_key})s)"
)
params[sec_key] = sec
params[sub_key] = sub
where.append(f"NOT ({' OR '.join(exclude_clauses)})")
where_sql = " AND ".join(where) where_sql = " AND ".join(where)
sql = f""" sql = f"""
SELECT SELECT
...@@ -117,7 +103,6 @@ def retrieve_with_subsections( ...@@ -117,7 +103,6 @@ def retrieve_with_subsections(
source_type_filter=source_type_filter, source_type_filter=source_type_filter,
expand_links=expand_links, expand_links=expand_links,
neighbor_expand=neighbor_expand, neighbor_expand=neighbor_expand,
exclude_subsection_refs=subsection_refs,
) )
vector_sources = vector_sources[:k] vector_sources = vector_sources[:k]
subsection_children = load_children_for_subsections( subsection_children = load_children_for_subsections(
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment