Skip to content
GitLab
Commit 83fe60f6 authored by minhnguyengp1's avatar minhnguyengp1
Browse files

baseline done! v2

parent 86b7f2a0
Show whitespace changes
Inline Side-by-side
agents/agent_decision.py
View file @ 83fe60f6
...@@ -32,23 +32,38 @@ def create_agent_graph(): ...@@ -32,23 +32,38 @@ def create_agent_graph():
new_state = state.copy() new_state = state.copy()
rag_agent = ResearchAssistantRAG(config) rag_agent = ResearchAssistantRAG(config)
messages: List[BaseMessage] = state["messages"] # messages: List[BaseMessage] = state["messages"]
query: Union[str, Dict] = state["current_input"] # query: Union[str, Dict] = state["current_input"]
rag_context_limit: int = config.rag.context_limit # rag_context_limit: int = config.rag.context_limit
#
print(f"DEBUG messages: {messages}") # print(f"DEBUG messages: {messages}")
#
buffer_memory = WindowBufferMemory(context_limit=rag_context_limit) # buffer_memory = WindowBufferMemory(context_limit=rag_context_limit)
chat_history: List[Dict[str, str]] = buffer_memory.build_context(messages) # chat_history: List[Dict[str, str]] = buffer_memory.build_context(messages)
#
print("DEBUG chat_history input:", chat_history) # print("DEBUG chat_history input:", chat_history)
print("DEBUG type of chat_history:", type(chat_history)) # print("DEBUG type of chat_history:", type(chat_history))
if chat_history: messages = state["messages"]
for i, msg in enumerate(chat_history): query = state["current_input"]
print(f"DEBUG chat_history[{i}]: {msg} (type: {type(msg)})") rag_context_limit = config.rag.context_limit
response: Dict[str, Any] = rag_agent.process_query(query, chat_history=chat_history) print(f"checkdd")
recent_context = ""
for msg in messages[-rag_context_limit:]: # limit controlled from config
if isinstance(msg, HumanMessage):
# print("######### DEBUG 1:", msg)
recent_context += f"User: {msg.content}\n"
elif isinstance(msg, AIMessage):
# print("######### DEBUG 2:", msg)
recent_context += f"Assistant: {msg.content}\n"
# if chat_history:
# for i, msg in enumerate(chat_history):
# print(f"DEBUG chat_history[{i}]: {msg} (type: {type(msg)})")
print("Before processing query:", recent_context)
response: Dict[str, Any] = rag_agent.process_query(query, chat_history=recent_context)
### CHECKPOINT ### CHECKPOINT
print("Final response from RAG agent:", response) print("Final response from RAG agent:", response)
......
agents/rag_agent/__init__.py
View file @ 83fe60f6
...@@ -5,118 +5,160 @@ import uuid ...@@ -5,118 +5,160 @@ import uuid
from pathlib import Path from pathlib import Path
import json import json
from .content_processor import ContentProcessor
from .doc_parser import DocParser
from .query_expander import QueryExpander from .query_expander import QueryExpander
from .vector_store import QdrantRetriever
from .document_processor import DocumentProcessor
from .query_processor import QueryProcessor from .query_processor import QueryProcessor
from .reranker import Reranker
from .response_generator import ResponseGenerator from .response_generator import ResponseGenerator
from .data_ingestion import DataIngestion
from .multi_query_generator import MultiQueryGenerator from .multi_query_generator import MultiQueryGenerator
from .query_rewriter import QueryRewriter from .query_rewriter import QueryRewriter
from .vectorstore_qdrant import VectorStore
class ResearchAssistantRAG: class ResearchAssistantRAG:
def __init__(self, config): def __init__(self, config):
self.config = config self.config = config
self._initialize() # self._initialize()
def _initialize(self):
try:
self.logger = logging.getLogger(f"{self.__module__}") self.logger = logging.getLogger(f"{self.__module__}")
self.logger.info("Initializing Research Assistant RAG system") self.logger.info("Initializing Research Assistant RAG system")
self.doc_parser = DocParser()
self.content_processor = ContentProcessor(config)
self.parsed_content_dir = self.config.rag.parsed_content_dir
self.vector_store = VectorStore(config)
self.reranker = Reranker(config)
self.response_generator = ResponseGenerator(config)
self.parsed_content_dir = self.config.rag.parsed_content_dir
self.llm = self.config.rag.llm
self.logger.info(f"Using LLM: {type(self.llm).__name__}")
self.embedding_model = self.config.rag.embedding_model
self.logger.info(f"Using embedding model: {type(self.embedding_model).__name__}")
self.sparse_embedder = self.config.rag.sparse_embedder
self.query_expander = QueryExpander(self.config)
self.query_rewriter = QueryRewriter(self.llm)
self.query_processor = QueryProcessor(self.config, self.embedding_model, self.sparse_embedder)
self.document_processor = DocumentProcessor(self.config, self.embedding_model)
self.query_generator = MultiQueryGenerator(llm=self.llm, max_queries=self.config.rag.max_queries or 5)
self.retriever = QdrantRetriever(self.config) def ingest_file(self, document_path: str) -> Dict[str, Any]:
"""
Ingest a single file into the RAG system.
self.response_generator = ResponseGenerator(self.config, self.llm) Args:
document_path: Path to the file to ingest
total_docs = self.retriever.count_documents() Returns:
self.logger.info(f"Vector store contains {total_docs} documents") Dictionary with ingestion results
if total_docs == 0: """
self.logger.warning("No documents in vector store. Results may be limited.") start_time = time.time()
self.logger.info(f"Ingesting file: {document_path}")
self.top_k = getattr(self.config.rag, "top_k", 5)
self.similarity_threshold = getattr(self.config.rag, "similarity_threshold", 0.0)
self.data_ingestion = DataIngestion() try:
# Step 1: Parse document
self.logger.info("1. Parsing document and extracting images...")
parsed_document, images = self.doc_parser.parse_document(document_path, self.parsed_content_dir)
self.logger.info(f" Parsed document and extracted {len(images)} images")
# Step 2: Summarize images
self.logger.info("2. Summarizing images...")
image_summaries = self.content_processor.summarize_images(images)
self.logger.info(f" Generated {len(image_summaries)} image summaries")
# Step 3: Format document with image summaries
self.logger.info("3. Formatting document with image summaries...")
formatted_document = self.content_processor.format_document_with_images(parsed_document, image_summaries)
# Save parsed_document to data/parsed
parsed_dir = Path("data/parsed")
parsed_dir.mkdir(exist_ok=True, parents=True)
parsed_path = parsed_dir / (Path(document_path).stem + "_parsed.json")
with open(parsed_path, "w", encoding="utf-8") as f:
import json
json.dump(formatted_document, f, ensure_ascii=False, indent=2, default=str)
# Step 4: Chunk document into semantic sections
self.logger.info("4. Chunking document into semantic sections...")
document_chunks: List[str] = self.content_processor.chunk_document(formatted_document)
self.logger.info(f" Document split into {len(document_chunks)} chunks")
with open("document_chunks.json", "w", encoding="utf-8") as f:
json.dump(document_chunks, f, ensure_ascii=False, indent=2)
# Step 5: Create vector store and document store
self.logger.info("5. Creating vector store knowledge base...")
self.vector_store.create_vectorstore(
document_chunks=document_chunks,
document_path=document_path
)
self.logger.info("Research Assistant RAG system successfully initialized") return {
"success": True,
"documents_ingested": 1,
"chunks_processed": len(document_chunks),
"processing_time": time.time() - start_time
}
except Exception as e: except Exception as e:
self.logger.error(f"Initialization error: {e}") self.logger.error(f"Error ingesting file: {e}")
import traceback return {
self.logger.error(traceback.format_exc()) "success": False,
raise "error": str(e),
"processing_time": time.time() - start_time
}
def process_query(self, query: str, chat_history: Optional[List[Dict[str, str]]] = None) -> Dict[str, Any]: def process_query(self, query: str, chat_history: Optional[List[Dict[str, str]]] = None) -> Dict[str, Any]:
self.logger.info(f"RAG Agent processing query: {query}") """
Process a query with the RAG system.
# Process query and return result, passing chat_history Args:
result = self.query(query, chat_history) query: The query string
chat_history: Optional chat history for context
return result
def query(self, query: str, chat_history: Optional[List[Dict[str, str]]] = None) -> Dict[str, Any]: Returns:
exec_count = 1 Response dictionary
self.logger.info(f"Processing query: {query}") """
start_time = time.time() start_time = time.time()
self.logger.info(f"RAG Agent processing query: {query}")
# Process query and return result, passing chat_history
try: try:
query_embedding: Dict[str, Any] # Step 1: Expand query - REWRITE QUERY
filters: Dict[str, Any] # self.logger.info(f"1. Expanding query: '{query}'")
# expansion_result = self.query_expander.expand_query(query)
if chat_history: # expanded_query = expansion_result["expanded_query"]
query = self.query_rewriter.rewrite(query, chat_history) # self.logger.info(f" Original: '{query}'")
# self.logger.info(f" Expanded: '{expanded_query}'")
query_embedding, filters = self.query_processor.process_query(query) # query = expanded_query
### DEBUG # Step 2: Retrieval
# dense_vector = query_embedding.get("dense") self.logger.info(f"2. Retrieving relevant documents for the query: '{query}'")
# sparse_vector = query_embedding.get("sparse") vectorstore, docstore = self.vector_store.load_vectorstore()
# retrieved_documents = self.vector_store.retrieve_relevant_chunks(
# if dense_vector: query=query,
# print("DEBUG dense_vector[:5]:", dense_vector[:5]) vectorstore=vectorstore,
# print("DEBUG dense_vector length:", len(dense_vector)) docstore=docstore,
# )
# if sparse_vector:
# print("DEBUG sparse_vector type:", type(sparse_vector))
# print("DEBUG sparse_vector indices[:5]:", getattr(sparse_vector, "indices", [])[:5])
# print("DEBUG sparse_vector values[:5]:", getattr(sparse_vector, "values", [])[:5])
#
# print("DEBUG has as_object:", hasattr(sparse_vector, "as_object"))
###
# TODO: Enhance retrieval by incorporating chat history context into the query embedding or retrieval strategy
search_results = self._run_vector_search(query_embedding, filters, query)
if self.similarity_threshold > 0:
search_results = [item for item in search_results if item.get('score', 0) >= self.similarity_threshold]
self.logger.info(f"{len(search_results)} results passed similarity threshold")
print(f"DEBUG chat_history at {exec_count}th time: \n{chat_history}")
exec_count += 1
self.logger.info(f" Retrieved {len(retrieved_documents)} relevant document chunks")
print(f"type of retrieved_documents = {type(retrieved_documents)}")
print(f"retrieved_documents = {retrieved_documents}")
# Step 3: Rerank the retrieved documents if we have a reranker and enough documents
self.logger.info(f"3. Reranking the retrieved documents")
if self.reranker and len(retrieved_documents) > 1:
# reranked_documents, reranked_top_k_picture_paths = self.reranker.rerank(query, retrieved_documents,
# self.parsed_content_dir)
reranked_documents = self.reranker.rerank(query, retrieved_documents,
self.parsed_content_dir)
self.logger.info(f" Reranked retrieved documents and chose top {len(reranked_documents)}")
# self.logger.info(f" Found {len(reranked_top_k_picture_paths)} referenced images")
else:
self.logger.info(f" Could not rerank the retrieved documents, falling back to original scores")
reranked_documents = retrieved_documents
reranked_top_k_picture_paths = []
# Step 4: Generate response
self.logger.info("4. Generating response...")
response = self.response_generator.generate_response( response = self.response_generator.generate_response(
query=query, query=query,
retrieved_docs=search_results, retrieved_docs=reranked_documents,
# picture_paths=reranked_top_k_picture_paths,
chat_history=chat_history chat_history=chat_history
) )
# Add timing information
processing_time = time.time() - start_time processing_time = time.time() - start_time
response["processing_time"] = processing_time response["processing_time"] = processing_time
...@@ -126,7 +168,7 @@ class ResearchAssistantRAG: ...@@ -126,7 +168,7 @@ class ResearchAssistantRAG:
self.logger.error(f"Error processing query: {e}") self.logger.error(f"Error processing query: {e}")
import traceback import traceback
self.logger.error(traceback.format_exc()) self.logger.error(traceback.format_exc())
# Return error response
return { return {
"response": f"I encountered an error while processing your query: {str(e)}", "response": f"I encountered an error while processing your query: {str(e)}",
"sources": [], "sources": [],
...@@ -134,83 +176,6 @@ class ResearchAssistantRAG: ...@@ -134,83 +176,6 @@ class ResearchAssistantRAG:
"processing_time": time.time() - start_time "processing_time": time.time() - start_time
} }
def ingest_documents(self, documents: List[Dict[str, Any]]) -> Dict[str, Any]:
self.logger.info(f"Ingesting {len(documents)} documents")
start_time = time.time()
###
for i, doc in enumerate(documents):
print(f"Document {i} keys: {list(doc.keys())}")
###
try:
processed_dir = Path("data/processed")
processed_dir.mkdir(exist_ok=True, parents=True)
all_chunks: List[Dict[str, Any]] = []
chunk_ids: List[str] = []
for document in documents:
content = document.get("content", "")
metadata = document.get("metadata", {})
elements = document.get("elements", [])
if "id" not in metadata:
metadata["id"] = str(uuid.uuid4())
chunks_from_file = self.document_processor.process_document(content, metadata, elements)
if chunks_from_file:
all_chunks.extend(chunks_from_file)
for chunk in chunks_from_file:
chunk_ids.append(chunk["id"])
doc_path = processed_dir / f"{chunk['id']}.json"
with open(doc_path, 'w', encoding='utf-8') as f:
json_safe_chunk = self._ensure_json_serializable(chunk)
json.dump(json_safe_chunk, f, indent=2)
if not all_chunks:
return {
"success": False,
"error": "No documents were successfully processed",
"processing_time": time.time() - start_time
}
# Here don’t recompute the embedding again, simply take it from all_chunks
upsert_batch: List[Dict[str, Any]] = []
for chunk in all_chunks:
chunk_record = {
"id": chunk["id"],
"content": chunk["content"],
"embedding": chunk["embedding"],
"metadata": chunk["metadata"]
}
upsert_batch.append(chunk_record)
self.retriever.upsert_chunks(upsert_batch)
return {
"success": True,
"documents_ingested": len(documents),
"chunks_processed": len(all_chunks),
# todo: change to "chunk_ids": chunk_ids
"document_ids": chunk_ids,
"processing_time": time.time() - start_time
}
except Exception as e:
self.logger.error(f"Error ingesting documents: {e}")
import traceback
self.logger.error(traceback.format_exc())
return {
"success": False,
"error": str(e),
"documents_processed": 0,
"chunks_created": 0,
"chunks_inserted": 0,
"processing_time": time.time() - start_time
}
def _ensure_json_serializable(self, obj): def _ensure_json_serializable(self, obj):
if isinstance(obj, dict): if isinstance(obj, dict):
...@@ -223,113 +188,8 @@ class ResearchAssistantRAG: ...@@ -223,113 +188,8 @@ class ResearchAssistantRAG:
return str(obj) return str(obj)
def ingest_file(self, file_path: str) -> Dict[str, Any]:
start_time = time.time()
self.logger.info(f"Ingesting file: {file_path}")
try:
ingestion_result = self.data_ingestion.ingest_file(file_path)
if not ingestion_result["success"]:
return {
"success": False,
"error": ingestion_result.get("error", "Unknown error during file ingestion"),
"processing_time": time.time() - start_time
}
documents = []
if "document" in ingestion_result:
documents = [ingestion_result["document"]]
elif "documents" in ingestion_result:
documents = ingestion_result["documents"]
if documents:
return self.ingest_documents(documents)
else:
return {
"success": False,
"error": "No valid documents found in file",
"processing_time": time.time() - start_time
}
except Exception as e:
self.logger.error(f"Error ingesting file: {e}")
return {
"success": False,
"error": str(e),
"processing_time": time.time() - start_time
}
def ingest_directory(self, directory_path: str, file_extension: Optional[str] = None) -> Dict[str, Any]: def ingest_directory(self, directory_path: str, file_extension: Optional[str] = None) -> Dict[str, Any]:
pass pass
def _run_vector_search(self, query_embedding: Dict[str, Any], filters: Dict[str, Any], query: str):
self.logger.info(f"Running vector search with filters: {filters}")
search_results = self.retriever.retrieve(
query_embedding=query_embedding,
filters=filters,
)
self.logger.info(f"Retrieved {len(search_results)} results from vector search")
return search_results
# def query(self, query: str, chat_history: Optional[List[Dict[str, str]]] = None) -> Dict[str, Any]:
# exec_count = 1
# self.logger.info(f"Processing query: {query}")
# start_time = time.time()
#
# try:
# # query_embedding: Dict[str, Any]
# # filters: Dict[str, Any]
# # query_embedding, filters = self.query_processor.process_query(query)
#
# # # TODO: Enhance retrieval by incorporating chat history context into the query embedding or retrieval strategy
# # search_results = self._run_vector_search(query_embedding, filters, query)
#
# # Step 1: Generate multiple sub-queries from the user query and chat history
# sub_queries = self.query_generator.generate_queries(
# user_question=query,
# chat_history=chat_history,
# )
#
# print("sub_queries: ", sub_queries)
#
# all_results = []
# for sq in sub_queries:
# sq_embedding, sq_filters = self.query_processor.process_query(sq)
# results = self._run_vector_search(sq_embedding, sq_filters, sq)
# all_results.extend(results)
#
# # 🧼 Lọc trùng theo nội dung
# seen = set()
# deduped_results = []
# for item in all_results:
# content = item.get("content")
# if content and content not in seen:
# deduped_results.append(item)
# seen.add(content)
#
# search_results = deduped_results
#
# print(f"search_results: {search_results}")
#
# if self.similarity_threshold > 0:
# search_results = [item for item in search_results if item.get('score', 0) >= self.similarity_threshold]
# self.logger.info(f"{len(search_results)} results passed similarity threshold")
#
# print(f"DEBUG chat_history at {exec_count}th time: \n{chat_history}")
# exec_count += 1
#
# response = self.response_generator.generate_response(
# query=query,
# retrieved_docs=search_results,
# chat_history=chat_history
# )
#
# processing_time = time.time() - start_time
# response["processing_time"] = processing_time
#
# return response
agents/rag_agent/data_ingestion.py
View file @ 83fe60f6
import os
import json
import logging
from pathlib import Path
import pandas as pd
from typing import List, Dict, Any, Optional, Union
import os
os.environ["PATH"] += os.pathsep + "/opt/homebrew/bin"
from unstructured.partition.pdf import partition_pdf
from unstructured.chunking.title import chunk_by_title
import json
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class DataIngestion:
def __init__(self):
self.stats = {
"files_processed": 0,
"documents_ingested": 0,
"errors": 0
}
logger.info("DataIngestion initialized")
def ingest_directory(self, directory_path: str, file_extension: Optional[str] = None) -> Dict[str, Any]:
logger.info(f"Processing directory: {directory_path}")
try:
directory = Path(directory_path)
if not directory.exists() or not directory.is_dir():
raise ValueError(f"Directory does not exist: {directory_path}")
if file_extension:
files = list(directory.glob(f"*{file_extension}"))
else:
files = [f for f in directory.iterdir() if f.is_file()]
logger.info(f"Found {len(files)} files to process")
for file_path in files:
try:
self.ingest_file(str(file_path))
self.stats["files_processed"] += 1
except Exception as e:
logger.error(f"Error processing file {file_path}: {e}")
self.stats["errors"] += 1
return self.stats
except Exception as e:
logger.error(f"Error processing directory: {e}")
return self.stats
def ingest_file(self, file_path: str) -> Dict[str, Any]:
print(f"Processing file (file_path): {file_path}")
logger.info(f"Processing file: {file_path}")
file_path = Path(file_path)
if not file_path.exists():
raise FileNotFoundError(f"File not found: {file_path}")
if file_path.suffix.lower() == '.txt':
return self._ingest_text_file(file_path)
elif file_path.suffix.lower() == '.pdf':
return self._ingest_pdf_file(file_path)
else:
logger.warning(f"Unsupported file format: {file_path.suffix}")
return {"success": False, "error": "Unsupported file format"}
def _ingest_text_file(self, file_path: Path) -> Dict[str, Any]:
try:
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
metadata = {
"source": file_path.name,
"file_type": "txt"
}
document = {
"content": content,
"metadata": metadata
}
logger.info(f"Successfully ingested text file: {file_path}")
self.stats["documents_ingested"] += 1
return {"success": True, "document": document}
except Exception as e:
logger.error(f"Error ingesting text file: {e}")
return {"success": False, "error": str(e)}
def _ingest_pdf_file(self, file_path: Path) -> Dict[str, Any]:
logger.info(f"Processing PDF with: {file_path}")
try:
elements = partition_pdf(
file_path,
extract_images_in_pdf=False,
extract_tables=True,
infer_table_structure=True,
languages=["de"],
ocr_languages = "deu"
# chunking_strategy="by_title"
)
##
with open("debug_elements.txt", "w", encoding="utf-8") as f:
for i, element in enumerate(elements):
f.write(f"\n=== Element {i} ===\n")
f.write(f"Category: {getattr(element, 'category', 'N/A')}\n")
f.write(f"Text: {str(element)}\n")
f.write(f"Metadata: {getattr(element, 'metadata', {})}\n")
##
content_parts = []
tables = []
images = []
metadata_parts = {}
for element in elements:
if hasattr(element, "category"):
element_text = str(element)
print(f"Element text: {element_text}")
element_category = element.category
print(f"Element category: {element_category}")
if element_category == "Title":
content_parts.append(f"\n## {element_text}\n")
elif element_category == "NarrativeText":
content_parts.append(element_text)
elif element_category == "ListItem":
content_parts.append(f"- {element_text}")
elif element_category == "Table":
formatted_table_text = self._format_table_as_markdown(element_text)
content_parts.append(f"\n{formatted_table_text}\n")
if hasattr(element, "metadata") and element.metadata:
table_metadata = element.metadata.__dict__ if hasattr(element.metadata,
"__dict__") else element.metadata
tables.append({
"text": formatted_table_text,
"raw_text": element_text,
"metadata": table_metadata
})
elif element_category == "Image":
content_parts.append(f"\n[IMAGE: {element_text}]\n")
if hasattr(element, "metadata") and element.metadata:
images.append({
"text": element_text,
"metadata": element.metadata.__dict__ if hasattr(element.metadata,
"__dict__") else element.metadata
})
else:
content_parts.append(element_text)
if hasattr(element, "metadata") and element.metadata:
metadata = element.metadata.__dict__ if hasattr(element.metadata,
"__dict__") else element.metadata
for key, value in metadata.items():
if key not in metadata_parts:
metadata_parts[key] = value
content = "\n".join(content_parts)
metadata = {
"source": file_path.name,
"file_type": "pdf",
"has_tables": len(tables) > 0,
"table_count": len(tables),
"has_images": len(images) > 0,
"image_count": len(images)
}
metadata.update(metadata_parts)
document = {
"content": content,
"metadata": metadata,
"tables": tables,
"images": images,
"elements": [{"category": getattr(e, "category", "Unknown"), "text": str(e)} for e in elements]
}
logger.info(f"Successfully ingested PDF file using unstructured.io: {file_path}")
self.stats["documents_ingested"] += 1
return {"success": True, "document": document}
except Exception as e:
logger.error(f"Error ingesting PDF file with unstructured.io: {e}")
return {"success": False, "error": str(e)}
def _format_table_as_markdown(self, table_text: str) -> str:
try:
lines = table_text.strip().split('\n')
if not lines:
return table_text
if '|' in table_text:
rows = [line.strip() for line in lines]
if not rows[0].startswith('|'):
rows[0] = '| ' + rows[0] + ' |'
if len(rows) > 1 and not rows[1].startswith('|---'):
cols = rows[0].count('|') - 1
separator = '|' + '|'.join(['---' for _ in range(cols)]) + '|'
rows.insert(1, separator)
for i in range(len(rows)):
if i > 1 and not rows[i].startswith('|'):
rows[i] = '| ' + rows[i] + ' |'
return '\n'.join(rows)
else:
rows = []
for line in lines:
if line.strip():
rows.append(line.split())
if not rows:
return table_text
num_cols = len(rows[0]) if rows else 0
if num_cols == 0:
return table_text
for i in range(len(rows)):
while len(rows[i]) < num_cols:
rows[i].append('')
rows[i] = rows[i][:num_cols]
header = '| ' + ' | '.join(rows[0]) + ' |'
separator = '|' + '|'.join(['---' for _ in range(num_cols)]) + '|'
formatted_rows = [header, separator]
for row in rows[1:]:
formatted_rows.append('| ' + ' | '.join(row) + ' |')
return '\n'.join(formatted_rows)
except Exception as e:
logger.warning(f"Error formatting table as markdown: {e}")
return f"Table:\n{table_text}"
agents/rag_agent/document_processor.py
View file @ 83fe60f6
import re
import uuid
import logging
from typing import List, Dict, Any, Optional, Tuple, Union
import os
from pathlib import Path
import hashlib
from datetime import datetime
import nltk
from nltk.tokenize import sent_tokenize
import json
# from sparse_embedder import GermanBM25SparseEmbedder
from fastembed import SparseTextEmbedding
try:
nltk.data.find('tokenizers/punkt')
except LookupError:
nltk.download('punkt', quiet=True)
def make_serializable(metadata: Dict[str, Any]) -> Dict[str, Any]:
serializable_metadata = {}
for key, value in metadata.items():
try:
json.dumps(value)
serializable_metadata[key] = value
except TypeError:
if isinstance(value, Header):
serializable_metadata[key] = str(value) # Convert Header to string
else:
serializable_metadata[key] = str(value) # Convert other non-serializable objects to string
return serializable_metadata
class DocumentProcessor:
def __init__(self, config, embedding_model):
self.logger = logging.getLogger(__name__)
self.embedding_model = embedding_model
self.chunk_size = config.rag.chunk_size
self.chunk_overlap = config.rag.chunk_overlap
self.processed_docs_dir = Path(config.rag.processed_docs_dir)
self.processed_docs_dir.mkdir(parents=True, exist_ok=True)
self.chunking_strategy = getattr(config.rag, "chunking_strategy", "sliding_window")
# self.sparse_embedder = GermanBM25SparseEmbedder()
self.sparse_embedder = SparseTextEmbedding("Qdrant/bm25")
def process_document(self, content: str, metadata: Dict[str, Any], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
try:
doc_id_base = hashlib.md5(content.encode()).hexdigest()
doc_id = str(uuid.UUID(doc_id_base[:32]))
enhanced_metadata = metadata.copy()
enhanced_metadata['processing_timestamp'] = datetime.now().isoformat()
if self.chunking_strategy == "semantic":
chunks = self._create_semantic_chunks(content, elements)
elif self.chunking_strategy == "sliding_window":
chunks = self._create_sliding_window_chunks(content)
else:
chunks = self._create_sliding_window_chunks(content)
processed_chunks = []
for i, chunk_info in enumerate(chunks):
if isinstance(chunk_info, tuple):
chunk_text, section, level = chunk_info[0], chunk_info[1], chunk_info[2] if len(chunk_info) > 2 else "standard"
else:
chunk_text, section, level = chunk_info, "general", "standard"
# Generate chunk ID as a UUID with a suffix
chunk_id = str(uuid.UUID(doc_id_base[:24] + f"{i:08}"))
print(f"DEBUGG chunk_text: {chunk_text}")
print(f"DEBUGG section: {section}")
# Generate embedding
dense_embedding = self.embedding_model.embed_documents([chunk_text])[0]
sparse_embeddings = list(
self.sparse_embedder.passage_embed([chunk_text])) # Lấy sparse embedding từ passage_embed
# Đảm bảo rằng sparse_embeddings không rỗng và lấy phần tử đầu tiên
if sparse_embeddings:
sparse_embedding = sparse_embeddings[0]
# In ra các chỉ số và giá trị của sparse_embedding
print(f"DEBUGG sparse_embedding indices (first 5): {sparse_embedding.indices[:5]}")
print(f"DEBUGG sparse_embedding values (first 5): {sparse_embedding.values[:5]}")
else:
sparse_embedding = {}
# Print embeddings to debug
print(f"DEBUGG dense_embedding (first 5 elements): {dense_embedding[:5]}") # Print first 5 elements for brevity
chunk_metadata = enhanced_metadata.copy()
chunk_metadata["chunk_number"] = i
chunk_metadata["total_chunks"] = len(chunks)
chunk_metadata["chunking_strategy"] = self.chunking_strategy
chunk_metadata["word_count"] = len(chunk_text.split())
processed_chunks.append({
"id": chunk_id,
"content": chunk_text,
"embedding": {
"dense": dense_embedding,
"sparse": sparse_embedding
},
"metadata": make_serializable(chunk_metadata)
})
return processed_chunks
except Exception as e:
self.logger.error(f"Error processing document: {e}")
raise
def _create_semantic_chunks(self, content: str, elements: List[Dict[str, Any]]) -> List[Tuple[str, str, str]]:
"""
Create semantic chunks from document based on 'elements' with category-based sectioning.
Falls back to paragraph-based chunking if elements are missing.
Returns:
List of (chunk_text, section_name, chunk_type) tuples
"""
chunks = []
current_section = None
section_text = []
encountered_title = False
for element in elements:
category = element.get("category", "")
element_text = element.get('text', '').strip()
# if not element_text:
# continue
if category == 'Title':
if section_text:
if not encountered_title:
chunks.append((' '.join(section_text), "preamble", "section"))
else:
chunks.append((' '.join(section_text), current_section, "section"))
current_section = element_text
section_text = [element_text]
encountered_title = True
else:
section_text.append(element_text)
if section_text:
if not encountered_title:
chunks.append((' '.join(section_text), "preamble", "section"))
else:
chunks.append((' '.join(section_text), current_section, "section"))
if not chunks and content:
paragraphs = re.split(r'\n\s*\n', content)
for para in paragraphs:
if para.strip():
chunks.append((para.strip(), "paragraph", "standard"))
print("Chunks:", chunks)
# TODO: Split section into paragraphs if it's too large
return chunks
def _create_sliding_window_chunks(self, text: str) -> List[str]:
sentences = sent_tokenize(text)
chunks = []
if len(sentences) <= 3:
return [text]
stride = max(1, (self.chunk_size - self.chunk_overlap) // 20)
for i in range(0, len(sentences), stride):
window_size = min(i + max(3, self.chunk_size // 20), len(sentences))
window_text = " ".join(sentences[i:window_size])
chunks.append(window_text)
return chunks
class Header:
def __init__(self, text):
self.text = text
def __str__(self):
return self.text
\ No newline at end of file
agents/rag_agent/local_embedding_model.py 0 → 100644
View file @ 83fe60f6
from sentence_transformers import SentenceTransformer
class LocalEmbeddingModel:
def __init__(self, model_name='paraphrase-multilingual-MiniLM-L12-v2'):
self.model = SentenceTransformer(model_name)
def embed_documents(self, texts):
return self.model.encode(texts, convert_to_numpy=True)
def embed_query(self, text):
return self.model.encode([text], convert_to_numpy=True)[0]
agents/rag_agent/response_generator.py
View file @ 83fe60f6
import logging import logging
from typing import List, Dict, Any, Optional from typing import List, Dict, Any, Optional
import re
class ResponseGenerator: class ResponseGenerator:
def __init__(self, config, llm): def __init__(self, config):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.llm = llm self.llm = config.rag.llm
self.max_context_length = config.rag.max_context_length self.max_context_length = config.rag.max_context_length
default_instructions = """
Bei der Formatierung der Antwort:
1. Verwenden Sie bei Bedarf klare Abschnitte mit Überschriften
2. Stellen Sie tabellarische Daten in korrektem Markdown-Tabellenformat dar:
| Kopf1 | Kopf2 | Kopf3 |
|-------|-------|-------|
| Daten1| Daten2| Daten3|
3. Für geordnete Listen verwenden Sie nummerierte Punkte
4. Für ungeordnete Listen verwenden Sie Aufzählungszeichen
5. Zitieren Sie Ihre Quellen, wenn Sie spezifische Informationen bereitstellen
6. Verwenden Sie präzise, wissenschaftliche Sprache
7. Antworten Sie auf Deutsch
"""
self.response_format_instructions = getattr(config.rag, "response_format_instructions", default_instructions)
self.include_sources = getattr(config.rag, "include_sources", True) self.include_sources = getattr(config.rag, "include_sources", True)
def generate_response(self, query: str, retrieved_docs: List[Dict[str, Any]], def _build_prompt(
chat_history: Optional[List[Dict[str, str]]] = None) -> Dict[str, Any]: self,
query: str,
try: context: str,
# tai cai nay chat_history: Optional[List[Dict[str, str]]] = None
if not retrieved_docs: ) -> str:
return self._generate_no_documents_response(query) """
Build the prompt for the language model.
formatted_context = self._format_context(retrieved_docs)
prompt = self._build_prompt(query, formatted_context, chat_history)
print(f"DEBUG prompt: \n{prompt}")
response = self.llm.invoke(prompt)
sources = self._extract_sources(retrieved_docs) if self.include_sources else [] Args:
query: User query
context: Formatted context from retrieved documents
chat_history: Optional chat history
result = { Returns:
"response": response, Complete prompt string
"sources": sources, """
"confidence": self._calculate_confidence(retrieved_docs)
}
return result
except Exception as e: table_instructions = """
self.logger.error(f"Error generating response: {e}") Einige der abgerufenen Informationen werden in Tabellenform präsentiert. Beim Verwenden von Informationen aus Tabellen:
return { 1. Stelle tabellarische Daten mit korrekter Markdown-Tabellenformatierung und Kopfzeilen dar, zum Beispiel:
"response": "Entschuldigung, es ist ein Fehler aufgetreten. Bitte versuchen Sie es erneut.", | Spalte1 | Spalte2 | Spalte3 |
"sources": [], |---------|---------|---------|
"confidence": 0.0 | Wert1 | Wert2 | Wert3 |
} 2. Formatiere die Tabellenstruktur so um, dass sie leichter zu lesen und zu verstehen ist.
3. Wenn beim Umformatieren der Tabelle eine neue Komponente hinzugefügt wird, erwähne dies ausdrücklich.
4. Interpretiere die tabellarischen Daten in deiner Antwort klar und verständlich.
5. Beziehe dich beim Präsentieren spezifischer Datenpunkte auf die jeweilige Tabelle.
6. Fasse, wenn sinnvoll, Trends oder Muster aus den Tabellen zusammen.
7. Falls nur Referenznummern genannt werden und du dazugehörige Werte wie den Titel einer wissenschaftlichen Arbeit oder Autoren aus dem Kontext holen kannst, ersetze die Referenznummern durch die tatsächlichen Werte.
"""
def _format_context(self, documents: List[Dict[str, Any]]) -> str: response_format_instructions = """Anweisungen:
context_parts = [] 1. Beantworte die Frage ausschließlich auf Basis der im Kontext bereitgestellten Informationen.
total_length = 0 2. Falls der Kontext keine relevanten Informationen zur Beantwortung der Frage enthält, sage: "Ich habe nicht genügend Informationen, um diese Frage auf Basis des bereitgestellten Kontexts zu beantworten."
3. Nutze kein Vorwissen, das nicht im Kontext enthalten ist.
5. Sei präzise und korrekt.
6. Gib eine gut strukturierte Antwort mit Überschriften, Unterüberschriften und ggf. tabellarischer Darstellung im Markdown-Format basierend auf dem abgerufenen Wissen. Halte Überschriften und Unterüberschriften möglichst kurz.
7. Führe nur solche Abschnitte auf, die in einer Chatbot-Antwort sinnvoll sind. Nenne z. B. keine Referenzen explizit.
8. Wenn Werte enthalten sind, gib exakt die im Kontext vorhandenen Werte wieder. Erfinde keine Werte.
9. Wiederhole die Frage nicht in der Antwort."""
# Build the prompt
prompt = f"""
Du bist ein wissenschaftlicher Chatbot, der folgende Aufgaben erfüllt:
- Zusammenfassung wissenschaftlicher Artikel
- Suche nach aktuellen und vergangenen Forschungsprojekten
- Navigation zu Artikeln: Bereitstellung von Links zu Veröffentlichungen, Anzeige meistzitierter oder neuester Artikel Antworte auf Deutsch.
for i, doc in enumerate(documents): Hier sind die letzten Nachrichten aus unserem Gespräch:
doc_text = doc.get("content", "").strip() {chat_history}
metadata = doc.get("metadata", {})
if not metadata and isinstance(doc, dict): Der Nutzer hat folgende Frage gestellt:
metadata = doc {query}
source = metadata.get("source", "Unknown Source") Ich habe die folgenden Informationen abgerufen, um bei der Beantwortung der Frage zu helfen:
{context}
is_table = metadata.get("is_table", False) {table_instructions}
content_type = metadata.get("content_type", "")
table_index = metadata.get("table_index", None)
if is_table or content_type == "table": {response_format_instructions}
clean_table_text = self._ensure_markdown_table_format(doc_text)
table_name = f"Table {table_index}" if table_index is not None else "Table"
formatted_doc = ( Bitte beantworte die Frage des Nutzers auf Grundlage der bereitgestellten Informationen umfassend, aber prägnant.
f"[Dokument {i + 1} - TABELLE] Folgende Informationen aus {table_name}:\n\n" Falls die Informationen keine Antwort enthalten, weise bitte auf die Grenzen der verfügbaren Informationen hin.
f"{clean_table_text}\n\n"
f"(Quelle: {source})"
)
else:
formatted_doc = f"[Dokument {i + 1}] {doc_text} (Quelle: {source})"
if total_length + len(formatted_doc) > self.max_context_length: Gib keinen Quell-Link an, der nicht im Kontext enthalten ist. Erfinde keinen Quell-Link.
break
context_parts.append(formatted_doc) Assistant Response:"""
total_length += len(formatted_doc)
return "\n\n".join(context_parts) return prompt
def _ensure_markdown_table_format(self, table_text: str) -> str: def generate_response(
if "|" in table_text: self,
lines = table_text.strip().split("\n") query: str,
retrieved_docs: List[Dict[str, Any]],
# picture_paths: List[str],
chat_history: Optional[List[Dict[str, str]]] = None,
) -> Dict[str, Any]:
"""
Generate a response based on retrieved documents.
has_separator = False Args:
for i, line in enumerate(lines): query: User query
if i > 0 and re.match(r"^\s*\|[\s\-\|]+\|\s*$", line): retrieved_docs: List of retrieved document dictionaries
has_separator = True chat_history: Optional chat history
break
if not has_separator and len(lines) > 1: Returns:
first_row = lines[0] Dict containing response text and source information
col_count = first_row.count("|") - 1 """
if col_count <= 0: try:
col_count = first_row.count("\t") + 1 # Try tab count
separator = "|" + "|".join(["---"] * col_count) + "|" # Extract content from documents for context
doc_texts = [doc["content"] for doc in retrieved_docs]
lines.insert(1, separator) # Combine retrieved documents into a single context
context = "\n\n===DOCUMENT SECTION===\n\n".join(doc_texts)
for i in range(len(lines)): # Build the prompt
if not lines[i].startswith("|"): prompt = self._build_prompt(query, context, chat_history)
lines[i] = "| " + lines[i]
if not lines[i].endswith("|"):
lines[i] = lines[i] + " |"
return "\n".join(lines) # Generate response
response = self.llm.invoke(prompt)
return table_text # Extract sources for citation
sources = self._extract_sources(retrieved_docs) if hasattr(self,
'include_sources') and self.include_sources else []
lines = table_text.strip().split("\n") # Calculate confidence
if not lines: confidence = self._calculate_confidence(retrieved_docs)
return table_text
rows = [] # Add sources to response
for line in lines: if hasattr(self, 'include_sources') and self.include_sources:
if line.strip(): response_with_source = response.content + "\n\n##### Source documents:"
if "\t" in line: for current_source in sources:
rows.append(line.split("\t")) source_path = current_source['path']
source_title = current_source['title']
response_with_source += f"\n- [{source_title}]({source_path})"
else: else:
rows.append(re.split(r"\s{2,}", line.strip())) response_with_source = response.content
if not rows:
return table_text
max_cols = max(len(row) for row in rows)
for row in rows:
while len(row) < max_cols:
row.append("")
md_table_lines = []
md_table_lines.append("| " + " | ".join(rows[0]) + " |") # # Add picture paths to response
# response_with_source_and_picture_paths = response_with_source + "\n\n##### Reference images:"
# for picture_path in picture_paths:
# response_with_source_and_picture_paths += f"\n- [{picture_path.split('/')[-1]}]({picture_path})"
md_table_lines.append("| " + " | ".join(["---"] * max_cols) + " |") # Format final response
result = {
for row in rows[1:]: # "response": response_with_source_and_picture_paths,
md_table_lines.append("| " + " | ".join(row) + " |") "response": response_with_source,
"sources": sources,
"confidence": confidence
}
return "\n".join(md_table_lines) return result
def _build_prompt(self, query: str, context: str, except Exception as e:
chat_history: Optional[List[Dict[str, str]]] = None) -> str: self.logger.error(f"Error generating response: {e}")
project_instructions = """ return {
Du bist ein wissenschaftlicher Chatbot, der folgende Aufgaben erfüllt: "response": "I apologize, but I encountered an error while generating a response. Please try rephrasing your question.",
- Zusammenfassung wissenschaftlicher Artikel "sources": [],
- Suche nach aktuellen und vergangenen Forschungsprojekten "confidence": 0.0
- Navigation zu Artikeln: Bereitstellung von Links zu Veröffentlichungen, Anzeige meistzitierter oder neuester Artikel }
Antworte auf Deutsch.
"""
table_instructions = "" def _extract_sources(self, documents: List[Dict[str, Any]]) -> List[Dict[str, str]]:
if "TABELLE" in context:
table_instructions = """
Einige Informationen liegen in Tabellenformat vor. Verwende:
1. Markdown-Tabellen mit Kopfzeilen
2. Erhalten Sie die Struktur bei der Darstellung
3. Interpretieren und referenzieren Sie die Tabellen deutlich
""" """
Extract source information from retrieved documents for citation.
chat_str = "" Args:
if chat_history and isinstance(chat_history, list): documents: List of retrieved document dictionaries
chat_lines = []
for msg in chat_history:
if isinstance(msg, dict):
role = "User" if msg.get("type") == "human" else "Assistant"
content = msg.get("content", "")
chat_lines.append(f"{role}: {content}")
else:
chat_lines.append(str(msg))
chat_str = "\n".join(chat_lines)
print(f"chat_str: {chat_str}")
prompt = f"""
{project_instructions}
Bisherige Konversation:
{chat_str}
Benutzeranfrage:
{query}
Verfügbare Informationen:
{context}
{table_instructions}
{self.response_format_instructions}
Wissenschaftliche Chatbot-Antwort:"""
return prompt
def _extract_sources(self, documents: List[Dict[str, Any]]) -> List[Dict[str, str]]:
Returns:
List of source information dictionaries
"""
sources = [] sources = []
seen_sources = set() seen_sources = set() # Track unique sources to avoid duplicates
for doc in documents: for doc in documents:
metadata = doc.get("metadata", {}) # Extract source and source_path
if not metadata and isinstance(doc, dict): source = doc.get("source")
metadata = doc source_path = doc.get("source_path")
source = metadata.get("source", "Unknown Source")
is_table = metadata.get("is_table", False) # Skip if no source information is available
content_type = metadata.get("content_type", "") if not source:
table_index = metadata.get("table_index", None) continue
source_key = f"{source}_{is_table}_{table_index}" if (is_table or content_type == "table") else source # Create a unique identifier for this source
source_id = f"{source}|{source_path}"
if source_key in seen_sources: # Skip if we've already included this source
if source_id in seen_sources:
continue continue
# Add to our sources list
source_info = { source_info = {
"title": source, "title": source,
"section": metadata.get("section", ""), "path": source_path,
"publication_date": metadata.get("publication_date", "") "score": doc.get("combined_score", doc.get("rerank_score", doc.get("score", 0.0)))
} }
if is_table or content_type == "table":
table_descriptor = f"Table {table_index}" if table_index is not None else "Table"
source_info["title"] = f"{source} ({table_descriptor})"
source_info["content_type"] = "table"
sources.append(source_info) sources.append(source_info)
seen_sources.add(source_key) seen_sources.add(source_id)
if len(sources) >= 5:
break
return sources # Sort sources by score from highest to lowest
sources.sort(key=lambda x: x.get("score", 0), reverse=True)
def _generate_no_documents_response(self, query: str) -> Dict[str, Any]: # Format the final sources list, removing the scores which were just used for sorting
prompt = f""" formatted_sources = []
Du bist ein wissenschaftlicher Chatbot. Der Benutzer fragt: for source in sources:
\"{query}\" formatted_source = {
"title": source["title"],
Leider liegen keine relevanten Dokumente vor. Bitte gib eine allgemeine Antwort und schlage gegebenenfalls Suchstrategien vor. "path": source["path"]
Wissenschaftliche Chatbot-Antwort:"""
response = self.llm.invoke(prompt)
return {
"response": response,
"sources": [],
"confidence": 0.0
} }
formatted_sources.append(formatted_source)
return formatted_sources
def _calculate_confidence(self, documents: List[Dict[str, Any]]) -> float: def _calculate_confidence(self, documents: List[Dict[str, Any]]) -> float:
"""
Calculate confidence score based on retrieved documents.
Args:
documents: Retrieved documents
Returns:
Confidence score between 0 and 1
"""
if not documents: if not documents:
return 0.0 return 0.0
# Use combined score (both reranker and cosine similarity) if available, otherwise use original score
if "combined_score" in documents[0]: if "combined_score" in documents[0]:
scores = [doc.get("combined_score", 0) for doc in documents[:3]] scores = [doc.get("combined_score", 0) for doc in documents[:3]]
elif "rerank_score" in documents[0]:
scores = [doc.get("rerank_score", 0) for doc in documents[:3]]
else: else:
scores = [doc.get("score", 0) for doc in documents[:3]] scores = [doc.get("score", 0) for doc in documents[:3]]
# Average of top 3 document scores or fewer if less than 3
return sum(scores) / len(scores) if scores else 0.0 return sum(scores) / len(scores) if scores else 0.0
\ No newline at end of file
agents/rag_agent/vector_store.py
View file @ 83fe60f6
from typing import List, Dict, Any, Optional, Union # from typing import List, Dict, Any, Optional, Union
import logging # import logging
from qdrant_client import QdrantClient # from qdrant_client import QdrantClient
from qdrant_client.http import models as qdrant_models # from qdrant_client.http import models as qdrant_models
from qdrant_client.http.models import ContextQuery, ContextPair, SearchParams, SparseVectorParams # from qdrant_client.http.models import ContextQuery, ContextPair, SearchParams, SparseVectorParams
from qdrant_client.http.exceptions import UnexpectedResponse # from qdrant_client.http.exceptions import UnexpectedResponse
from qdrant_client.http.models import Distance, VectorParams, PointStruct, Prefetch, FusionQuery, Fusion # from qdrant_client.http.models import Distance, VectorParams, PointStruct, Prefetch, FusionQuery, Fusion
from sklearn.feature_extraction.text import TfidfVectorizer # from sklearn.feature_extraction.text import TfidfVectorizer
#
class QdrantClientManager: # class QdrantClientManager:
_instance = None # _instance = None
_client = None # _client = None
#
@classmethod # @classmethod
def get_client(cls, config): # def get_client(cls, config):
if cls._client is None: # if cls._client is None:
if config.rag.use_local: # if config.rag.use_local:
logging.info(f"Connecting to local Qdrant at: {config.rag.url}") # logging.info(f"Connecting to local Qdrant at: {config.rag.url}")
cls._client = QdrantClient(url=config.rag.url) # Dùng URL, không phải path # cls._client = QdrantClient(url=config.rag.url) # Dùng URL, không phải path
else: # else:
logging.info(f"Connecting to remote Qdrant at: {config.rag.url}") # logging.info(f"Connecting to remote Qdrant at: {config.rag.url}")
cls._client = QdrantClient(url=config.rag.url, api_key=config.rag.api_key) # cls._client = QdrantClient(url=config.rag.url, api_key=config.rag.api_key)
#
logging.info("Initialized Qdrant client singleton") # logging.info("Initialized Qdrant client singleton")
#
return cls._client # return cls._client
#
#
class QdrantRetriever: # class QdrantRetriever:
#
def __init__(self, config): # def __init__(self, config):
self.logger = logging.getLogger(__name__) # self.logger = logging.getLogger(__name__)
self.collection_name = config.rag.collection_name # self.collection_name = config.rag.collection_name
self.embedding_dim = config.rag.embedding_dim # self.embedding_dim = config.rag.embedding_dim
self.distance_metric = config.rag.distance_metric # self.distance_metric = config.rag.distance_metric
self.tfidf_vectorizer = TfidfVectorizer() # self.tfidf_vectorizer = TfidfVectorizer()
self.sparse_vocab = {} # self.sparse_vocab = {}
self.is_fitted = False # self.is_fitted = False
#
self.client = QdrantClientManager.get_client(config) # self.client = QdrantClientManager.get_client(config)
#
self._ensure_collection() # self._ensure_collection()
#
def _ensure_collection(self): # def _ensure_collection(self):
collection_info = self.client.get_collections() # collection_info = self.client.get_collections()
print(f"Collection info: {collection_info}") # print(f"Collection info: {collection_info}")
collection_names = [collection.name for collection in collection_info.collections] # collection_names = [collection.name for collection in collection_info.collections]
print("Collection names:", collection_names) # print("Collection names:", collection_names)
#
# self.client.delete_collection(collection_name=self.collection_name) # # self.client.delete_collection(collection_name=self.collection_name)
#
if self.collection_name not in collection_names: # if self.collection_name not in collection_names:
self.logger.info(f"Creating new collection {self.collection_name}") # self.logger.info(f"Creating new collection {self.collection_name}")
try: # try:
self.client.create_collection( # self.client.create_collection(
collection_name=self.collection_name, # collection_name=self.collection_name,
vectors_config={ # vectors_config={
"dense": VectorParams( # "dense": VectorParams(
size=self.embedding_dim, # size=self.embedding_dim,
distance=self.distance_metric, # =Distance.COSINE # distance=self.distance_metric, # =Distance.COSINE
) # )
}, # },
sparse_vectors_config={ # sparse_vectors_config={
"sparse": SparseVectorParams( # "sparse": SparseVectorParams(
modifier=qdrant_models.Modifier.IDF # modifier=qdrant_models.Modifier.IDF
) # )
} # }
# vectors_config=VectorParams( # # vectors_config=VectorParams(
# size=self.embedding_dim, # # size=self.embedding_dim,
# distance=Distance.COSINE # # distance=Distance.COSINE
# ), # # ),
# optimizers_config=qdrant_models.OptimizersConfigDiff( # # optimizers_config=qdrant_models.OptimizersConfigDiff(
# indexing_threshold=10000, # # indexing_threshold=10000,
# ), # # ),
) # )
self.logger.info(f"Created new collection: {self.collection_name}") # self.logger.info(f"Created new collection: {self.collection_name}")
except Exception as e: # except Exception as e:
self.logger.error(f"Error creating collection: {e}") # self.logger.error(f"Error creating collection: {e}")
raise e # raise e
else: # else:
self.logger.info(f"Collection {self.collection_name} already exists") # self.logger.info(f"Collection {self.collection_name} already exists")
#
def upsert_chunks(self, chunks: List[Dict[str, Any]]): # def upsert_chunks(self, chunks: List[Dict[str, Any]]):
try: # try:
print(f"DEBUG Upserting chunks: {chunks}") # print(f"DEBUG Upserting chunks: {chunks}")
points: List[PointStruct] = [] # points: List[PointStruct] = []
#
for chunk in chunks: # for chunk in chunks:
payload = chunk["metadata"].copy() # payload = chunk["metadata"].copy()
#
payload["content"] = chunk["content"] # payload["content"] = chunk["content"]
#
print(f"DEBUG Upserting chunk: {chunk['id']}") # print(f"DEBUG Upserting chunk: {chunk['id']}")
print(f"DEBUG Embedding keys: {chunk['embedding']}") # print(f"DEBUG Embedding keys: {chunk['embedding']}")
print(f"DEBUG Payload: {payload}") # print(f"DEBUG Payload: {payload}")
#
points.append( # points.append(
PointStruct( # PointStruct(
id=chunk["id"], # id=chunk["id"],
# vector=chunk["embedding"], # # vector=chunk["embedding"],
vector={ # vector={
"dense": chunk["embedding"]["dense"], # "dense": chunk["embedding"]["dense"],
"sparse": chunk["embedding"]["sparse"].as_object() # "sparse": chunk["embedding"]["sparse"].as_object()
}, # },
#Giả định chunk["embedding"] là dict kiểu: # #Giả định chunk["embedding"] là dict kiểu:
# { # # {
# "dense": [...], # list[float] # # "dense": [...], # list[float]
# "sparse": {...} # dict[int -> float] # # "sparse": {...} # dict[int -> float]
# } # # }
payload=payload # payload=payload
) # )
) # )
#
batch_size = 100 # batch_size = 100
# Nghĩa là mỗi lần gửi lên database sẽ gửi tối đa 100 points (vector + metadata). # # Nghĩa là mỗi lần gửi lên database sẽ gửi tối đa 100 points (vector + metadata).
for i in range(0, len(points), batch_size): # for i in range(0, len(points), batch_size):
batch = points[i:i + batch_size] # batch = points[i:i + batch_size]
self.client.upsert( # self.client.upsert(
collection_name=self.collection_name, # collection_name=self.collection_name,
points=batch, # points=batch,
wait=True # wait=True
) # )
self.logger.info(f"Successfully upserted {len(chunks)} chunks into '{self.collection_name}'") # self.logger.info(f"Successfully upserted {len(chunks)} chunks into '{self.collection_name}'")
except Exception as e: # except Exception as e:
self.logger.error(f"Error upserting documents: {e}") # self.logger.error(f"Error upserting documents: {e}")
raise # raise
#
#
def count_documents(self) -> int: # def count_documents(self) -> int:
try: # try:
collection_info = self.client.get_collection(self.collection_name) # collection_info = self.client.get_collection(self.collection_name)
count = collection_info.vectors_count # count = collection_info.vectors_count
self.logger.info(f"Collection {self.collection_name} has {count} documents") # self.logger.info(f"Collection {self.collection_name} has {count} documents")
return count # return count
except Exception as e: # except Exception as e:
self.logger.error(f"Error getting collection info: {str(e)}") # self.logger.error(f"Error getting collection info: {str(e)}")
return 0 # return 0
#
#
def fit_sparse_vectorizer(self, corpus: List[str]): # def fit_sparse_vectorizer(self, corpus: List[str]):
try: # try:
if not self.is_fitted: # if not self.is_fitted:
self.tfidf_vectorizer.fit(corpus) # self.tfidf_vectorizer.fit(corpus)
self.is_fitted = True # self.is_fitted = True
self.logger.info("TF-IDF Vectorizer fitted successfully") # self.logger.info("TF-IDF Vectorizer fitted successfully")
else: # else:
self.logger.info("TF-IDF Vectorizer is already fitted") # self.logger.info("TF-IDF Vectorizer is already fitted")
except Exception as e: # except Exception as e:
self.logger.error(f"Error fitting TF-IDF Vectorizer: {e}") # self.logger.error(f"Error fitting TF-IDF Vectorizer: {e}")
#
#
def text_to_sparse_vector(self, query_text: str) -> Dict[str, List]: # def text_to_sparse_vector(self, query_text: str) -> Dict[str, List]:
if not self.is_fitted: # if not self.is_fitted:
self.logger.warning("TF-IDF Vectorizer is not fitted yet. Skipping transformation") # self.logger.warning("TF-IDF Vectorizer is not fitted yet. Skipping transformation")
return {} # return {}
#
vec = self.tfidf_vectorizer.transform([query_text]) # vec = self.tfidf_vectorizer.transform([query_text])
indices = vec.indices.tolist() # indices = vec.indices.tolist()
values = vec.data.tolist() # values = vec.data.tolist()
return { # return {
"indices": indices, # "indices": indices,
"values": values # "values": values
} # }
#
def retrieve( # def retrieve(
self, # self,
query_embedding: Dict[str, Any], # query_embedding: Dict[str, Any],
filters: Optional[Dict[str, Any]] = None, # filters: Optional[Dict[str, Any]] = None,
top_k: int = 5, # top_k: int = 5,
include_metadata: bool = True, # include_metadata: bool = True,
) -> List[Dict[str, Any]]: # ) -> List[Dict[str, Any]]:
""" # """
Perform hybrid retrieval using both dense and sparse embeddings. # Perform hybrid retrieval using both dense and sparse embeddings.
#
Args: # Args:
query_embedding: Dict with keys "dense" (List[float]) and "sparse" (SparseEmbedding) # query_embedding: Dict with keys "dense" (List[float]) and "sparse" (SparseEmbedding)
filters: Optional metadata filters # filters: Optional metadata filters
top_k: Number of documents to retrieve # top_k: Number of documents to retrieve
include_metadata: Whether to include payload in results # include_metadata: Whether to include payload in results
#
Returns: # Returns:
List of retrieved documents with score # List of retrieved documents with score
""" # """
dense_vector = query_embedding.get("dense") # dense_vector = query_embedding.get("dense")
sparse_vector = query_embedding.get("sparse") # sparse_vector = query_embedding.get("sparse")
#
self.logger.info(f""" # self.logger.info(f"""
[RETRIEVE PARAMS] # [RETRIEVE PARAMS]
#
dense_dim={len(dense_vector) if dense_vector else 'None'}, # dense_dim={len(dense_vector) if dense_vector else 'None'},
filters={filters}, # filters={filters},
top_k={top_k}, # top_k={top_k},
include_metadata={include_metadata} # include_metadata={include_metadata}
""") # """)
#
if sparse_vector: # if sparse_vector:
self.logger.info("Sparse vector detected → using hybrid search") # self.logger.info("Sparse vector detected → using hybrid search")
#
prefetch = [ # prefetch = [
qdrant_models.Prefetch(query=dense_vector, using="dense", limit=top_k), # qdrant_models.Prefetch(query=dense_vector, using="dense", limit=top_k),
qdrant_models.Prefetch( # qdrant_models.Prefetch(
query=qdrant_models.SparseVector( # query=qdrant_models.SparseVector(
indices=sparse_vector.indices.tolist(), # indices=sparse_vector.indices.tolist(),
values=sparse_vector.values.tolist() # values=sparse_vector.values.tolist()
), # ),
using="sparse", # using="sparse",
limit=top_k, # limit=top_k,
), # ),
] # ]
#
try: # try:
response = self.client.query_points( # response = self.client.query_points(
collection_name=self.collection_name, # collection_name=self.collection_name,
prefetch=prefetch, # prefetch=prefetch,
query=qdrant_models.FusionQuery(fusion=qdrant_models.Fusion.RRF), # query=qdrant_models.FusionQuery(fusion=qdrant_models.Fusion.RRF),
with_payload=include_metadata, # with_payload=include_metadata,
query_filter=None, # TODO: build filter from `filters` if needed # query_filter=None, # TODO: build filter from `filters` if needed
) # )
retrieved_points = response.points # retrieved_points = response.points
#
results = [] # results = []
for point in retrieved_points: # for point in retrieved_points:
print("DEBUG point:", point) # print("DEBUG point:", point)
#
result_item = dict(point.payload or {}) # result_item = dict(point.payload or {})
result_item["score"] = point.score # result_item["score"] = point.score
results.append(result_item) # results.append(result_item)
#
self.logger.info(f"Retrieved {len(results)} results (hybrid search)") # self.logger.info(f"Retrieved {len(results)} results (hybrid search)")
return results # return results
#
except Exception as e: # except Exception as e:
self.logger.error(f"Error during hybrid retrieval: {e}", exc_info=True) # self.logger.error(f"Error during hybrid retrieval: {e}", exc_info=True)
return [] # return []
#
else: # else:
self.logger.info("No sparse vector → fallback to dense-only search") # self.logger.info("No sparse vector → fallback to dense-only search")
#
try: # try:
response = self.client.query_points( # response = self.client.query_points(
collection_name=self.collection_name, # collection_name=self.collection_name,
prefetch=[ # prefetch=[
qdrant_models.Prefetch(query=dense_vector, using="dense", limit=top_k) # qdrant_models.Prefetch(query=dense_vector, using="dense", limit=top_k)
], # ],
query=qdrant_models.FusionQuery(fusion=qdrant_models.Fusion.RRF), # query=qdrant_models.FusionQuery(fusion=qdrant_models.Fusion.RRF),
with_payload=include_metadata, # with_payload=include_metadata,
query_filter=None, # query_filter=None,
) # )
retrieved_points = response.points # retrieved_points = response.points
#
results = [] # results = []
for point in retrieved_points: # for point in retrieved_points:
result_item = dict(point.payload or {}) # result_item = dict(point.payload or {})
result_item["score"] = point.score # result_item["score"] = point.score
results.append(result_item) # results.append(result_item)
#
self.logger.info(f"Retrieved {len(results)} results (dense-only search)") # self.logger.info(f"Retrieved {len(results)} results (dense-only search)")
return results # return results
#
except Exception as e: # except Exception as e:
self.logger.error(f"Error during dense-only retrieval: {e}", exc_info=True) # self.logger.error(f"Error during dense-only retrieval: {e}", exc_info=True)
return [] # return []
#
config.py
View file @ 83fe60f6
...@@ -4,6 +4,8 @@ from fastembed import SparseTextEmbedding ...@@ -4,6 +4,8 @@ from fastembed import SparseTextEmbedding
from langchain_openai import ChatOpenAI, OpenAIEmbeddings from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from openai import OpenAI from openai import OpenAI
from agents.rag_agent.local_embedding_model import LocalEmbeddingModel
load_dotenv() load_dotenv()
class AgentDecisoinConfig: class AgentDecisoinConfig:
...@@ -21,41 +23,51 @@ class RAGConfig: ...@@ -21,41 +23,51 @@ class RAGConfig:
self.distance_metric = "Cosine" self.distance_metric = "Cosine"
self.use_local = True self.use_local = True
self.vector_local_path = "./data/qdrant_db" # Add this with a default value self.vector_local_path = "./data/qdrant_db" # Add this with a default value
self.doc_local_path = "./data/docs_db"
self.parsed_content_dir = "./data/parsed_docs"
self.url = os.getenv("QDRANT_URL") self.url = os.getenv("QDRANT_URL")
self.api_key = os.getenv("QDRANT_API_KEY") self.api_key = os.getenv("QDRANT_API_KEY")
self.collection_name = "research_assistant_rag" self.collection_name = "pro_assistant_rag"
self.chunk_size = 512 self.chunk_size = 512
self.chunk_overlap = 50 self.chunk_overlap = 50
self.processed_docs_dir = "./data/processed" self.processed_docs_dir = "./data/processed"
self.embedding_model = OpenAIEmbeddings( self.embedding_model = OpenAIEmbeddings(
model=os.getenv("embedding_model_name"), model=os.getenv("embedding_model_name"),
api_key=os.getenv("openai_api_key") api_key=os.getenv("openai_api_key")
) )
self.sparse_embedder = SparseTextEmbedding("Qdrant/bm25") self.embedding_for_chunk_model = LocalEmbeddingModel('paraphrase-multilingual-MiniLM-L12-v2')
self.summarizer_model = ChatOpenAI(
model=os.getenv("model_name"),
api_key=os.getenv("openai_api_key"),
temperature=0.5
)
self.chunker_model = ChatOpenAI(
model=os.getenv("model_name"),
api_key=os.getenv("openai_api_key"),
temperature=0.0
)
self.llm = ChatOpenAI( self.llm = ChatOpenAI(
model=os.getenv("model_name"), model=os.getenv("model_name"),
api_key=os.getenv("openai_api_key"), api_key=os.getenv("openai_api_key"),
temperature=0.3 temperature=0.3
) )
self.sparse_embedder = SparseTextEmbedding("Qdrant/bm25")
self.max_queries = 3 self.max_queries = 3
self.top_k = 5 self.top_k = 5
self.similarity_threshold = 0.75 self.similarity_threshold = 0.75
self.huggingface_token = os.getenv("HUGGINGFACE_TOKEN") self.huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
self.vector_search_type = 'similarity' # or 'mmr' self.vector_search_type = 'similarity' # or 'mmr'
self.reranker_model = "cross-encoder/ms-marco-TinyBERT-L-6"
self.chunking_strategy = "semantic" self.reranker_top_k = 3
# self.chunking_strategy = "semantic"
self.max_context_length = 8192 self.max_context_length = 8192
self.include_sources = True
self.response_format_instructions = """Instructions: self.response_format_instructions = """Instructions:
1. Answer the query based ONLY on the information provided in the context. 1. Answer the query based ONLY on the information provided in the context.
2. If the context doesn't contain relevant information to answer the query, state: "I don't have enough information to answer this question based on the provided context." 2. If the context doesn't contain relevant information to answer the query, state: "I don't have enough information to answer this question based on the provided context."
3. Do not use prior knowledge not contained in the context. 3. Do not use prior knowledge not contained in the context.
5. Be concise and accurate. 5. Be concise and accurate.
6. Provide a well-structured response based on retrieved knowledge.""" 6. Provide a well-structured response based on retrieved knowledge."""
self.include_sources = True
self.metrics_save_path = "./logs/rag_metrics.json"
self.min_retrieval_confidence = 0.8 # the auto routing from RAG agent to WEB_SEARCH agent is dependent on this value self.min_retrieval_confidence = 0.8 # the auto routing from RAG agent to WEB_SEARCH agent is dependent on this value
self.context_limit = 20 # include last 20 messsages (10 Q&A pairs) in history self.context_limit = 20 # include last 20 messsages (10 Q&A pairs) in history
......
debug_elements.txt
View file @ 83fe60f6
This source diff could not be displayed because it is too large. You can view the blob instead.
requirements.txt
View file @ 83fe60f6
fastapi fastapi~=0.115.12
uvicorn==0.34.0 uvicorn==0.34.0
dotenv dotenv~=0.9.9
qdrant-client==1.13.3 qdrant-client==1.13.3
nltk==3.9.1 nltk==3.9.1
pandas==2.2.3 pandas==2.2.3
...@@ -15,7 +15,7 @@ langchain-core==0.3.44 ...@@ -15,7 +15,7 @@ langchain-core==0.3.44
langchain-openai==0.3.8 langchain-openai==0.3.8
langchain-text-splitters==0.3.6 langchain-text-splitters==0.3.6
langchain-qdrant langchain-qdrant
langgraph langgraph~=0.3.9
langgraph-checkpoint langgraph-checkpoint
langgraph-prebuilt langgraph-prebuilt
langgraph-sdk langgraph-sdk
...@@ -24,7 +24,13 @@ pikepdf==9.5.2 ...@@ -24,7 +24,13 @@ pikepdf==9.5.2
pillow==11.1.0 pillow==11.1.0
pipreqs==0.5.0 pipreqs==0.5.0
pdf2image==1.17.0 pdf2image==1.17.0
scikit-learn scikit-learn~=1.6.1
streamlit streamlit~=1.45.0
requests requests~=2.32.3
fastembed fastembed~=0.6.1
\ No newline at end of file sentence-transformers~=4.1.0
pydantic~=2.11.3
numpy~=1.26.4
docling~=2.31.0
python-dotenv~=1.1.0
openai~=1.76.0
\ No newline at end of file
test.json
View file @ 83fe60f6
{ [
"content": "Prof. Dr. Michael Krone Augmented/Virtual Reality Engineering Studienbereich Informatik Fakult¨at Vermessung, Informatik und Mathematik HFT Stuttgart\n\n[IMAGE: H O C h S C a U | eC flr Tecla mil Stuttga rt]\n\nInformationsvisualisualisierung und Visual Analytics WS 24/25 Übungsblatt 1 Abgabe: Dienstag, 15 Oktober 2024, 23:55\nÜbungsblatt 1\n\n## 1. Visualisierung des Auto-Datensatzes (10 Punkte)\n\nLaden Sie den Auto-Datensatz (CarPrice_deutsch.xlsx) von Moodle herunter. Sie können die Datei in MS Excel öffnen. Sie enthält 204 verschiedene Fahrzeugmodelle mit 26 verschiedenen Attributen (kategorisch, ordinal und quantitativ).\nSchauen Sie sich die Daten in Excel an und erstellen Sie zwei verschiedene Diagramme aus den Daten. Erstellen Sie ein kurzes Dokument (z.B. mit Word) mit Abbildungen der beiden Visualisierungen, einer kurzen Beschreibung was zu sehen ist. Welche Daten-Attribute und Diagramm-Typen haben Sie gewählt? Warum? Welche Fragen können mit Ihrer gewählten Visualisierung beantwortet werden? Laden Sie als Abgabe das Dokument in Moodle als PDF hoch.\n→ Sie können auch neue Daten aus dem Auto-Datensatz erstellen und diese Visualisieren (d.h. abgeleitete Daten) – Sie dürfen hier kreativ sein!\n→ Wenn Sie möchten können Sie auch ein anderes Tool als Excel verwenden (z.B. Tableau https://www.tableau.com/de-de/academic/students, kostenlose Version für Studierende bzw. Testversion). Geben Sie in diesem Fall an, welches Tool sie verwendet haben.\n\n## 2. Erste Website mit JavaScript (10 Punkte)\n\nErstellen Sie eine Website, um eine Zap-Hep-Zahlenfolge auszudrucken: Wenn eine Zahl durch 3 teilbar ist, drucken Sie ZAP, wenn eine 3 in der Zahl enthalten ist, drucken Sie HEP, wenn beides zutrifft, drucken Sie ZAP HEP, ansonsten drucken Sie die nächste Zahl. Beispiel für Zahlen von 1 bis 14:\n1\n2\nZAP HEP\n4\n5\n\n## ZAP\n\n7\n8\n\n## ZAP\n\n10\n11\n\n## ZAP\n\n\n## HEP\n\n14\nVerwenden Sie eine einzelne HTML-Datei, die Ihren HTML- und JavaScript-Code enthält (Tipp: siehe Folien “Tutorial 0 (Web-Dev-Intro)” in Moodle). Schreiben Sie die JavaScript-Funktion so, dass eine beliebige lange Zahlenreihe ausgegeben werden kann (d.h. die Länge der Zahlenreihe soll per Variable in ihrem JavaScript-Code gesetzt werden können). Laden Sie Ihre HTML-Datei in Moodle hoch.\nTipp: Es gibt mehrere Möglichkeiten, aus einem JavaScript-Programm Text auf eine Website zu setzen, die einfachste ist mittels document.write( ... \"Text\", Zahlen, etc. In diesem Tutorial werden noch weitere Möglichkeiten beschrieben: https://www.w3schools. com/js/js_output.asp ... );.\nTipp: Sie benötigen für diese Aufgaben noch nicht die D3-Library (dürfen diese aber verwenden, wenn Sie wollen).\n2", {
"metadata": { 'id': 'a597b10d-1a8b-4c9c-95a0-016d145c3d0f',
"source": "test.pdf", 'content': '## Eidesstattliche Versicherung\nIch erkläre hiermit, dass ich die vorliegende Arbeit selbstständig und ohne Verwendung anderer als der angegebenen Hilfsmittel verfasst habe. Ich habe sämtliche verwendeten Quellen erwähnt und gemäß den gängigen wissenschaftlichen Regeln zitiert. Die Arbeit hat in gleicher oder ähnlicher Form noch keiner anderen Prüfungsbehörde vorgelegen.\n\npicture_counter_1 no image summary\n\nKirchheim unter Teck, 30.06.2023\n## Danksagung\nIch möchte mich von ganzem Herzen bei allen Personen bedanken, die mich während meiner Bachelorarbeit unterstützt haben, ohne die wäre diese Thesis nicht möglich gewesen.\n\nEin besonderer Dank gebührt meinem externen Betreuer Thomas Heine von dem Ingenieurbüro Heine+Jud in Stuttgart der mich tatkräftig unterstützt hat und mit wertvollen Hinweisen bzw. Anregungen erfolgreich unterstützt hat. Ich bedanke mich herzlichst für die Zeit die man für mich genommen hat, um meine Thesis zu lesen und zu kommentieren. Ebenfalls dank ich meinem Betreuer Prof. Dr. Karl Georg Degen von der Hochschule der Technik in Stuttgart, der mich ebenfalls unterstützt hat und mir Hinweise und Anregungen gab. Die konstruktive Kritik und Hilfestelllungen haben mir geholfen, meine Arbeit und mein Repertoire zu erweitern und zu verbessern.\n\nEin weiterer großer Dank geht an alle meine Arbeitskollegen von Heine+Jud und meinem zweiten Chef Axel Jud, die mich während dieser herausfordernden Zeit motiviert und ermutigt haben. Ich bedanke mich zudem besonders an meine Messgehilfen Tobias Gassner, Christian Reutter, Sebastian Gerner und Lena Robert, die mich bei meinen Messungen tatkräftig unterstützt haben.\n\nEbenfalls geht auch ein Dank raus an meine Familie und meinem Freund, die mir moralischen Halt gegeben haben und mich immer motiviert haben, meine beste Leistung vorzulegen und nie aufzugeben. Hervorgehoben bedanke ich mich bei meinen Eltern die mir mein Studium durch ihre Unterstützung ermöglicht haben und fortwährend ein offenes Ohr für mich hatten.\n\nSchließlich möchte ich mich auch bei meinen Kommilitonen und Mitstudierenden bedanken, die mich während meines Studiums inspiriert und unterstützt haben. Ich bin dankbar für die wunderbaren Erinnerungen und Freundschaften, die ich während dieser Zeit geknüpft habe.\n\nNoch einmal vielen herzlichen Dank an alle, die mir geholfen haben, mein Bachelorstudium abzuschließen. Eure Unterstützung und Ermutigung haben mir sehr geholfen, diese Arbeit erfolgreich und ohne Einbüße abzuschließen.',
"file_type": "pdf", 'score': 0.8333334,
"has_tables": false, 'source': 'GermanThesis.pdf',
"table_count": 0, 'source_path': 'http://localhost:8000/data/raw/GermanThesis.pdf'
"has_images": true, },
"image_count": 1, {
"detection_class_prob": 0.8481391668319702, 'id': 'df34e512-b476-484e-b06a-bd9f11177812',
"coordinates": 'content': 'picture_counter_0 no image summary\n## Studiengang Bauphysik\n## Verladegeräusche in Speditionen\nBachelorarbeit zur Erlangung des akademischen Grades Bachelor of Engineering\n\nvorgelegt von\n\nSelin Karagöz\n\nMatrikelnummer Eingereicht am\n\n610260\n\n30.06.2023\n\nErstgutachter Zweitgutachter externer Betreuer\n\nProf. Dr. Karl Georg Degen Prof. Dr.-Ing. Berndt Zeitler Dipl.-Ing (FH) Bauphysik Thomas Heine\n\nHeine und Jud, Forststraße 9, 70174 Stuttgart\n## Kurzfassung\nDie Logistik spielt eine entscheidende Rolle in der globalen Wirtschaft, doch sie geht auch mit einer unvermeidlichen Lärmbelastung einher. Diese Arbeit konzentriert sich auf die Analyse und Bewertung von Verladetätigkeiten in Speditionen mit verschiedenen Flurfördergeräten und Anhängern. Ziel ist es, Emissionsdatenblätter für Verladetätigkeiten mit bestimmten Flurfördergeräten und Anhängern zu erstellen, die als Grundlage für Schallimmissionsprognosen dienen können. Weiterhin werden mögliche Lärmminderungsmaßnahmen aufgezeigt.\n\nIm Rahmen der Untersuchung wurde der Schalldruck im Fernfeld gemessen, wobei das Hüllflächen-Verfahren zur Ermittlung der Schallleistung angewendet wurde. Zusätzlich wurde die Richtcharakteristik entlang einer Kreisbahn im Nahfeld analysiert. Es wurde zudem eine Vergleichsanalyse der simulierten Messergebnisse mit den tatsächlichen Messungen durchgeführt, um die Genauigkeit der berechneten Kenngrößen zu überprüfen und Abweichungen zu interpretieren. Die Ergebnisse zeigen, dass die Beladung mit einem Gabelstapler und einem Gabelhubwagen ähnliche Schallleistungspegel aufweisen. Allerdings gibt es Unterschiede in der Impulshaftigkeit der Geräuschentwicklung. Der Gabelhubwagen erzeugt mehr impulsartige Schallereignisse, während der Gabelstapler eine kontinuierlichere Geräuschabstrahlung aufweist. Es gibt gewisse Variationen der Schallleistungspegel je nach Flurfördergerät und Anhängertyp, jedoch zeigt sich kein signifikanter Unterschied zwischen den Beladevorgängen der Flurfördergeräte in den verschiedenen Anhängern, was auf eine gewisse Ähnlichkeit hinweist (siehe Tabelle 1). Die Schallabstrahlung der Flurfördergeräte erfolgt weitestgehend in alle Richtungen, ohne eine ausgeprägte Richtcharakteristik aufzuweisen.\n\n| | Anzahl | Max-Pegel LAFmax | Schallleistungspegel | Schallleistungspegel | Schallleistungspegel | Schallleistungspegel | Schallleistungspegel |\n|-----------------|----------------------|--------------------|------------------------|------------------------|------------------------|------------------------|------------------------|\n| Flurfördergerät | [-] Einzelereignisse | [dB(A)] | LWA,5s [dB(A)] | LWAmax [dB(A)] | LWA,1h [dB(A)] | KI [dB(A)] | LWAT,1h [dB(A)] |\n| Gabelstapler | 297 | 69,3 | 100,7 | 108,8 | 72,1 | 5,7 | 77,8 |\n| STABW [dB] | | 6,5 | 4,9 | 6,5 | 4,9 | 2,9 | - |\n| Gabelhubwagen | 99 | 68,6 | 100,7 | 108,1 | 72,1 | 6,2 | 79,5 |\n| STABW [dB] | | 6,4 | 5,1 | 6,4 | 5,1 | 2,7 | - |',
\ No newline at end of file 'score': 0.7,
'source': 'GermanThesis.pdf',
'source_path': 'http://localhost:8000/data/raw/GermanThesis.pdf'
},
{
'id': '4ea40601-31ee-4519-8534-9d3255d34896',
'content': '## Ziel dieser Thesis\nUm eine umfassendere und praxisnähere Schallimmissionsprognose durchführen zu können, bedarf es weiterer Untersuchungen. Das Ziel dieser Thesis besteht darin, detailliert die Verladetätigkeiten in Speditionen zu analysieren und zu bewerten. Die messtechnisch ermittelten Daten werden ausgewertet und durch eine geeignete Simulation verifiziert. Die notwendigen Rechengrundlagen werden dazu erläutert, die zur Analyse und Bewertung von Verladegeräusche benötigt werden. Aus dem gewonnenen Wissen werden Emissionsansätze ausgearbeitet, die als Prognosegrundlage für weiterführende Schallimmissionsprognosen zur Verfügung steht. Zusätzlich wird die 2D-Richtcharakteristik der Verladegeräusche durch eine Untersuchung entlang einer Kreisbahn im Nah-\nfeld ermittelt. Die Thesis stellt eine wichtige Grundlage für die Entwicklung von geeigneten Maßnahmen zur Reduktion der Lärmbelastung in Speditionen dar, weshalb mögliche Minderungsmaßnahmen aufgezeigt werden, um die Lärmbelastung zu reduzieren. Die Thesis bietet somit eine umfassende Einführung in die Grundlagen und Rechengrundlagen der Lärmemissionsmessung und -bewertung.\n## 2 Grundlagen\nDas folgende Kapitel dient zur Einführung in die Thesis und bildet somit das Grundgerüst. Es werden die allgemeinen Grundlagen der Logistik und Grundlagen zur Ermittlung der notwendigen akustischen Kenngrößen aufgezeigt. Ziel dieses Kapitel ist es ein grundlegendes Wissen darzubieten, um ein vollständiges Verständnis der Thesis zu vermitteln.\n## 2.1 Logistik\nEiner der wichtigsten Komponente der globalen Wirtschaft ist die Logistik. Sie verbindet die Gesellschaft durch den Handel und Verbrauch von Gütern. Als Grundgerüst der Logistik, steht der Güteraustausch zwischen Unternehmen oder dem privaten Haushalt. Im täglichen Leben machen wir permanent Gebrauch von Logistik, sei es für den Transport von Personen oder von Gütern. Vernetzt werden die Unternehmen über die Lieferund Logistikkette, welche den Austausch von Gütern ermöglichen. Die Kette beginnt mit der Güterbereitstellung, der Güterverwendung und der Güterverteilung. Unternehmen stellen Güter über die Produktionskette her, die dann bereitgestellt werden. Diese bereitgestellten Güter werden entweder als Sachgüter wie Smartphones verwendet oder als Konsumgüter wie Nahrungsmittel verbraucht. Damit Unternehmen die Güter an den Kunden verteilen können wird die Logistik benötigt. Die Logistik stellt das Bindeglied zwischen Bereitstellung und Verwendung der Güter dar. Ein sogenannter Güterfluss ist entstanden [5].',
'score': 0.5833334,
'source': 'GermanThesis.pdf',
'source_path': 'http://localhost:8000/data/raw/GermanThesis.pdf'
},
{
'id': '7741197d-0799-4eb7-96c7-e8dc421277a1',
'content': '## Inhaltsverzeichnis\n| Kurzfassung .................................................................................................................. I | Kurzfassung .................................................................................................................. I |\n|------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| Eidesstattliche Versicherung ....................................................................................... III | Eidesstattliche Versicherung ....................................................................................... III |\n| Danksagung................................................................................................................ IV | Danksagung................................................................................................................ IV |\n| Inhaltsverzeichnis ........................................................................................................ 1 | Inhaltsverzeichnis ........................................................................................................ 1 |\n| 1 | Einleitung......................................................................................................... 3 |\n| 2 | Grundlagen ...................................................................................................... 6 |\n| 2.1 | Logistik .......................................................................................................................6 |\n| 2.2 | Ermittlung der akustischen Kenngrößen...................................................................9 |\n| 2.3 | Statistik ....................................................................................................................21 |\n| 3 | Aktuelle Prognosegrundlage ............................................................................24 |\n| 3.1 | Schallimmissionsprognosen für Gewerbe - Technischer Bericht des HLUG...........24 |\n| 3.2 | Be- und Entladevorgängen mit Palettenhubwagen und beladener Palette bei Lkw in Logistikzentren.....................................................................................................27 |\n| 3.3 | Geräuschemissionen durch Ladevorgänge in Ladezonen von Discountern............28 |\n| 3.4 | Fazit..........................................................................................................................30 |\n| 4 | Beschreibung des Betriebs und Untersuchungsobjekte.....................................31 |\n| 4.1 | Beschreibung des Betriebs ......................................................................................31 |\n| 4.2 | Beschreibung der Untersuchungsobjekte ...............................................................33 |\n| 5 | Beschreibung der Untersuchungsmethodik ......................................................38 |\n| 5.1 | Prozessablauf der untersuchten Verladetätigkeit...................................................38 |\n| 5.2 | Messequipment und Messaufbau...........................................................................39 |\n| 5.3 | Vorgehensweise bei der Messwerterfassung..........................................................41 |\n| 5.4 | Messpositionen........................................................................................................44 |\n| 6 | Messauswertung und Statistik .........................................................................46 |\n| 6.1 | Verladegeräusche....................................................................................................46 |\n| 6.2 | Statistik ....................................................................................................................48 |\n| 6.3 | Modellierung der Messsituation .............................................................................50 |\n| 6.4 | Richtcharakteristik...................................................................................................51 |\n| 6.5 | Validierung...............................................................................................................55 |',
'score': 0.25,
'source': 'GermanThesis.pdf',
'source_path': 'http://localhost:8000/data/raw/GermanThesis.pdf'
},
{
'id': '6b1a3175-2ce0-48ac-98de-e975b02ee937',
'content': '| 7 | Ergebnisse und Interpretation..........................................................................56 | |\n|-----------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|-------|\n| 7.1 | Verladegeräusche....................................................................................................56 | |\n| 7.2 | Statistik ....................................................................................................................60 | |\n| 7.3 | Richtcharakteristik...................................................................................................66 | |\n| 7.4 | Validierung...............................................................................................................68 | |\n| 7.5 | Mögliche Minderungsmaßnahmen .........................................................................70 | |\n| 7.6 | Fehlerdiskussion ......................................................................................................72 | |\n| 8 | Abschließende Bewertung und Ausblick...........................................................75 | |\n| 9 | Literatur............................................................................................................ | I |\n| Formelverzeichnis..................................................................................................... | Formelverzeichnis..................................................................................................... | VIII |\n| Abbildungsverzeichnis ................................................................................................ | Abbildungsverzeichnis ................................................................................................ | IX |\n| Tabellenverzeichnis ..................................................................................................... | Tabellenverzeichnis ..................................................................................................... | X |\n| Diagrammverzeichnis.................................................................................................. | Diagrammverzeichnis.................................................................................................. | XI |\n| Anhänge.................................................................................................................... | Anhänge.................................................................................................................... | XII |\n| Anhang 1: Emissionsdatenblätter........................................................................................XII | Anhang 1: Emissionsdatenblätter........................................................................................XII | |\n| Anhang 2: Lagepläne...........................................................................................................XIX | Anhang 2: Lagepläne...........................................................................................................XIX | |\n| Anhang 3: Messprotokolle.................................................................................................XXII | Anhang 3: Messprotokolle.................................................................................................XXII | |\n| Anhang 4: Messergebnisse der Beladevorgänge............................................................. | Anhang 4: Messergebnisse der Beladevorgänge............................................................. | XXIV |\n| Anhang 5: Rechenmodell SoundPlan 9.0 - Gabelhubwagen 27.04.23............................. | Anhang 5: Rechenmodell SoundPlan 9.0 - Gabelhubwagen 27.04.23............................. | XLIII |\n| Anhang 6: Rechenmodell SoundPlan 9.0 - Gabelstapler 03.05.2023...............................XLVI | Anhang 6: Rechenmodell SoundPlan 9.0 - Gabelstapler 03.05.2023...............................XLVI | |\n## 1 Einleitung\nLaute Klänge und Töne, wie etwa lautes Geschrei oder ein donnerndes Geräusch, werden von uns als störend oder bedrohlich wahrgenommen und werden auch als Lärm bezeichnet. Diese können sich negativ auf unser körperliches, seelischen und unser soziales Wohlbefinden auswirken. [1] Durch die ermittelte Lärmbelastung können Schlussfolgerungen über die Belästigung gezogen und somit Menschen vor schädlichem und störendem Lärm geschützt werden. Lärm kann an verschieden Orten auftreten und daher unterscheidet man in der Gesetzgebung nach Straßen-, Schienen-, Flug-, Gewerbe-, Bau-, Freizeit- und Sportlärm. Hinter jeder dieser Verursacher steht eine spezifische Gesetzgebung mit beigeordneten Regelungen und Zuständigkeiten. Um einen präventiven Schallschutz zu gewährleisten und ein angenehmes akustisches Stadtklima zu erreichen, ist der Schallimmissionsschutz von großer Bedeutung. Ziel ist es, alle störenden Geräusche zu reduzieren, besonders in Bereichen mit zunehmender Mobilität und Warenverkehr. [2] Der stetig wachsende Wohnraummangel zwingt die Wohnbebauung näher an Straßen- und Schienenwege, sowie näher an das Gewerbe. [3] Dadurch wächst das Potential von dem daraus entstehenden Lärm belästigt zu werden. Gemäß einer Umfrage aus dem Jahr 2020 des Bundesministeriums für Umwelt, Naturschutz, nukleare Sicherheit und Verbraucherschutz, empfinden 51 % der Bevölkerung in Deutschland eine Belästigung durch Lärm aus Industrie-/Gewerbelärm (siehe Abbildung 1). [4]\n\nAbbildung 1: Ausmaß der Lärmbelästigung der Bevölkerung in Deutschland 2020 [4]\n\npicture_counter_2 no image summary',
'score': 0.2,
'source': 'GermanThesis.pdf',
'source_path': 'http://localhost:8000/data/raw/GermanThesis.pdf'
}
]
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal