Commit db448d99 authored by minhnguyengp1's avatar minhnguyengp1
Browse files

init

parent c8f22990
import logging
import re
import uuid
from typing import List, Dict, Any, Optional, Tuple, Union
from datetime import datetime
class QueryProcessor:
def __init__(self, config, embedding_model):
self.logger = logging.getLogger(__name__)
self.embedding_model = embedding_model
self.synonyms = {
"zusammenfassung": ["summary", "zusammenfassen", "kurzfassung"],
"artikel": ["article", "studie", "paper"],
"projekt": ["project", "forschung", "studie"],
"aktuellen": ["aktuellen", "neuesten", "recent", "latest"],
}
self.intent_patterns = {
"summary": re.compile(r"\b(zusammenfassung|summary)\b", re.IGNORECASE),
"project_search": re.compile(r"\b(projekt|forschung|study|project)\b", re.IGNORECASE),
"article_navigation": re.compile(r"\b(link|artikel|paper|artikel)\b", re.IGNORECASE),
}
def process_query(self, query: str) -> Tuple[List[float], Dict[str, Any]]:
try:
query_id = str(uuid.uuid4())
expanded_query = self._expand_query(query)
query_intent = self._detect_query_intent(query)
query_embedding = self.embedding_model.embed_query(expanded_query)
if query_embedding is None:
raise ValueError("Embedding model returned None for query.")
query_embedding = query_embedding.tolist() if hasattr(query_embedding, "tolist") else query_embedding
filters: Dict[str, Any] = {
"query_id": query_id,
"timestamp": datetime.now().isoformat(),
"query_intent": query_intent,
}
filters = {k: v for k, v in filters.items() if v is not None}
self.logger.info(f"Processed query with filters: {filters}")
return query_embedding, filters
except Exception as e:
self.logger.error(f"Error processing query: {e}")
fallback_embedding = self.embedding_model.embed_query(query)
if fallback_embedding is None:
fallback_embedding = []
fallback_embedding = fallback_embedding.tolist() if hasattr(fallback_embedding,
"tolist") else fallback_embedding
fallback_filters = {
"query_id": str(uuid.uuid4()),
"timestamp": datetime.now().isoformat(),
"is_fallback": True
}
return fallback_embedding, fallback_filters
def _expand_query(self, query: str) -> str:
parts = [query]
low = query.lower()
for key, syns in self.synonyms.items():
if key in low:
parts.extend(syns)
return " ".join(dict.fromkeys(parts))
def _detect_query_intent(self, query: str) -> str:
for intent, pattern in self.intent_patterns.items():
if pattern.search(query):
return intent
return "general_information"
import logging
from typing import List, Dict, Any, Optional
import re
class ResponseGenerator:
def __init__(self, config, llm):
self.logger = logging.getLogger(__name__)
self.llm = llm
self.max_context_length = config.rag.max_context_length
default_instructions = """
Bei der Formatierung der Antwort:
1. Verwenden Sie bei Bedarf klare Abschnitte mit Überschriften
2. Stellen Sie tabellarische Daten in korrektem Markdown-Tabellenformat dar:
| Kopf1 | Kopf2 | Kopf3 |
|-------|-------|-------|
| Daten1| Daten2| Daten3|
3. Für geordnete Listen verwenden Sie nummerierte Punkte
4. Für ungeordnete Listen verwenden Sie Aufzählungszeichen
5. Zitieren Sie Ihre Quellen, wenn Sie spezifische Informationen bereitstellen
6. Verwenden Sie präzise, wissenschaftliche Sprache
7. Antworten Sie auf Deutsch
"""
self.response_format_instructions = getattr(config.rag, "response_format_instructions", default_instructions)
self.include_sources = getattr(config.rag, "include_sources", True)
def generate_response(self, query: str, retrieved_docs: List[Dict[str, Any]],
chat_history: Optional[List[Dict[str, str]]] = None) -> Dict[str, Any]:
try:
if not retrieved_docs:
return self._generate_no_documents_response(query)
formatted_context = self._format_context(retrieved_docs)
prompt = self._build_prompt(query, formatted_context, chat_history)
response = self.llm.invoke(prompt)
sources = self._extract_sources(retrieved_docs) if self.include_sources else []
result = {
"response": response,
"sources": sources,
"confidence": self._calculate_confidence(retrieved_docs)
}
return result
except Exception as e:
self.logger.error(f"Error generating response: {e}")
return {
"response": "Entschuldigung, es ist ein Fehler aufgetreten. Bitte versuchen Sie es erneut.",
"sources": [],
"confidence": 0.0
}
def _format_context(self, documents: List[Dict[str, Any]]) -> str:
context_parts = []
total_length = 0
for i, doc in enumerate(documents):
doc_text = doc.get("content", "").strip()
metadata = doc.get("metadata", {})
if not metadata and isinstance(doc, dict):
metadata = doc
source = metadata.get("source", "Unknown Source")
is_table = metadata.get("is_table", False)
content_type = metadata.get("content_type", "")
table_index = metadata.get("table_index", None)
if is_table or content_type == "table":
clean_table_text = self._ensure_markdown_table_format(doc_text)
table_name = f"Table {table_index}" if table_index is not None else "Table"
formatted_doc = (
f"[Dokument {i + 1} - TABELLE] Folgende Informationen aus {table_name}:\n\n"
f"{clean_table_text}\n\n"
f"(Quelle: {source})"
)
else:
formatted_doc = f"[Dokument {i + 1}] {doc_text} (Quelle: {source})"
if total_length + len(formatted_doc) > self.max_context_length:
break
context_parts.append(formatted_doc)
total_length += len(formatted_doc)
return "\n\n".join(context_parts)
def _ensure_markdown_table_format(self, table_text: str) -> str:
if "|" in table_text:
lines = table_text.strip().split("\n")
has_separator = False
for i, line in enumerate(lines):
if i > 0 and re.match(r"^\s*\|[\s\-\|]+\|\s*$", line):
has_separator = True
break
if not has_separator and len(lines) > 1:
first_row = lines[0]
col_count = first_row.count("|") - 1
if col_count <= 0:
col_count = first_row.count("\t") + 1 # Try tab count
separator = "|" + "|".join(["---"] * col_count) + "|"
lines.insert(1, separator)
for i in range(len(lines)):
if not lines[i].startswith("|"):
lines[i] = "| " + lines[i]
if not lines[i].endswith("|"):
lines[i] = lines[i] + " |"
return "\n".join(lines)
return table_text
lines = table_text.strip().split("\n")
if not lines:
return table_text
rows = []
for line in lines:
if line.strip():
if "\t" in line:
rows.append(line.split("\t"))
else:
rows.append(re.split(r"\s{2,}", line.strip()))
if not rows:
return table_text
max_cols = max(len(row) for row in rows)
for row in rows:
while len(row) < max_cols:
row.append("")
md_table_lines = []
md_table_lines.append("| " + " | ".join(rows[0]) + " |")
md_table_lines.append("| " + " | ".join(["---"] * max_cols) + " |")
for row in rows[1:]:
md_table_lines.append("| " + " | ".join(row) + " |")
return "\n".join(md_table_lines)
def _build_prompt(self, query: str, context: str,
chat_history: Optional[List[Dict[str, str]]] = None) -> str:
project_instructions = """
Du bist ein wissenschaftlicher Chatbot, der folgende Aufgaben erfüllt:
- Zusammenfassung wissenschaftlicher Artikel
- Suche nach aktuellen und vergangenen Forschungsprojekten
- Navigation zu Artikeln: Bereitstellung von Links zu Veröffentlichungen, Anzeige meistzitierter oder neuester Artikel
Antworte auf Deutsch.
"""
table_instructions = ""
if "TABELLE" in context:
table_instructions = """
Einige Informationen liegen in Tabellenformat vor. Verwende:
1. Markdown-Tabellen mit Kopfzeilen
2. Erhalten Sie die Struktur bei der Darstellung
3. Interpretieren und referenzieren Sie die Tabellen deutlich
"""
prompt = f"""
{project_instructions}
Bisherige Konversation:
{chat_history}
Benutzeranfrage:
{query}
Verfügbare Informationen:
{context}
{table_instructions}
{self.response_format_instructions}
Wissenschaftliche Chatbot-Antwort:"""
return prompt
def _extract_sources(self, documents: List[Dict[str, Any]]) -> List[Dict[str, str]]:
sources = []
seen_sources = set()
for doc in documents:
metadata = doc.get("metadata", {})
if not metadata and isinstance(doc, dict):
metadata = doc
source = metadata.get("source", "Unknown Source")
is_table = metadata.get("is_table", False)
content_type = metadata.get("content_type", "")
table_index = metadata.get("table_index", None)
source_key = f"{source}_{is_table}_{table_index}" if (is_table or content_type == "table") else source
if source_key in seen_sources:
continue
source_info = {
"title": source,
"section": metadata.get("section", ""),
"publication_date": metadata.get("publication_date", "")
}
if is_table or content_type == "table":
table_descriptor = f"Table {table_index}" if table_index is not None else "Table"
source_info["title"] = f"{source} ({table_descriptor})"
source_info["content_type"] = "table"
sources.append(source_info)
seen_sources.add(source_key)
if len(sources) >= 5:
break
return sources
def _generate_no_documents_response(self, query: str) -> Dict[str, Any]:
prompt = f"""
Du bist ein wissenschaftlicher Chatbot. Der Benutzer fragt:
\"{query}\"
Leider liegen keine relevanten Dokumente vor. Bitte gib eine allgemeine Antwort und schlage gegebenenfalls Suchstrategien vor.
Wissenschaftliche Chatbot-Antwort:"""
response = self.llm.invoke(prompt)
return {
"response": response,
"sources": [],
"confidence": 0.0
}
def _calculate_confidence(self, documents: List[Dict[str, Any]]) -> float:
if not documents:
return 0.0
if "combined_score" in documents[0]:
scores = [doc.get("combined_score", 0) for doc in documents[:3]]
else:
scores = [doc.get("score", 0) for doc in documents[:3]]
return sum(scores) / len(scores) if scores else 0.0
from typing import List, Dict, Any, Optional, Union
import logging
from qdrant_client import QdrantClient
from qdrant_client.http import models as qdrant_models
from qdrant_client.http.models import ContextQuery, ContextPair, SearchParams
from qdrant_client.http.exceptions import UnexpectedResponse
from qdrant_client.http.models import Distance, VectorParams, PointStruct, Prefetch, FusionQuery, Fusion
from sklearn.feature_extraction.text import TfidfVectorizer
class QdrantClientManager:
_instance = None
_client = None
@classmethod
def get_client(cls, config):
if cls._client is None:
if config.rag.use_local:
logging.info(f"Connecting to local Qdrant at: {config.rag.url}")
cls._client = QdrantClient(url=config.rag.url) # Dùng URL, không phải path
else:
logging.info(f"Connecting to remote Qdrant at: {config.rag.url}")
cls._client = QdrantClient(url=config.rag.url, api_key=config.rag.api_key)
logging.info("Initialized Qdrant client singleton")
return cls._client
class QdrantRetriever:
def __init__(self, config):
self.logger = logging.getLogger(__name__)
self.collection_name = config.rag.collection_name
self.embedding_dim = config.rag.embedding_dim
self.distance_metric = config.rag.distance_metric
self.tfidf_vectorizer = TfidfVectorizer()
self.sparse_vocab = {}
self.is_fitted = False
self.client = QdrantClientManager.get_client(config)
self._ensure_collection()
def _ensure_collection(self):
collection_info = self.client.get_collections()
print(f"Collection info: {collection_info}")
collection_names = [collection.name for collection in collection_info.collections]
print("Collection names:", collection_names)
if self.collection_name not in collection_names:
self.logger.info(f"Creating new collection {self.collection_name}")
try:
self.client.create_collection(
collection_name=self.collection_name,
vectors_config=VectorParams(
size=self.embedding_dim,
distance=Distance.COSINE
),
optimizers_config=qdrant_models.OptimizersConfigDiff(
indexing_threshold=10000,
),
)
self.logger.info(f"Collection {self.collection_name} created")
except Exception as e:
self.logger.error(f"Error creating collection: {e}")
raise e
else:
self.logger.info(f"Collection {self.collection_name} already exists")
def upsert_chunks(self, chunks: List[Dict[str, Any]]):
try:
points: List[PointStruct] = []
for chunk in chunks:
payload = chunk["metadata"].copy()
payload["content"] = chunk["content"]
points.append(
qdrant_models.PointStruct(
id=chunk["id"],
vector=chunk["embedding"],
payload=payload
)
)
batch_size = 100
for i in range(0, len(points), batch_size):
batch = points[i:i + batch_size]
self.client.upsert(
collection_name=self.collection_name,
points=batch,
wait=True
)
self.logger.info(f"Successfully upserted {len(chunks)} chunks into '{self.collection_name}'")
except Exception as e:
self.logger.error(f"Error upserting documents: {e}")
raise
def count_documents(self) -> int:
try:
collection_info = self.client.get_collection(self.collection_name)
count = collection_info.vectors_count
self.logger.info(f"Collection {self.collection_name} has {count} documents")
return count
except Exception as e:
self.logger.error(f"Error getting collection info: {str(e)}")
return 0
def fit_sparse_vectorizer(self, corpus: List[str]):
try:
if not self.is_fitted:
self.tfidf_vectorizer.fit(corpus)
self.is_fitted = True
self.logger.info("TF-IDF Vectorizer fitted successfully")
else:
self.logger.info("TF-IDF Vectorizer is already fitted")
except Exception as e:
self.logger.error(f"Error fitting TF-IDF Vectorizer: {e}")
def text_to_sparse_vector(self, query_text: str) -> Dict[str, List]:
if not self.is_fitted:
self.logger.warning("TF-IDF Vectorizer is not fitted yet. Skipping transformation")
return {}
vec = self.tfidf_vectorizer.transform([query_text])
indices = vec.indices.tolist()
values = vec.data.tolist()
return {
"indices": indices,
"values": values
}
from qdrant_client.http import models as qdrant_models
from qdrant_client import QdrantClient
from qdrant_client.http.models import SearchParams, ContextPair, ContextQuery, FusionQuery, Fusion, Prefetch
def retrieve(
self,
query_vector: List[float],
filters: Optional[Dict] = None,
top_k: int = 5,
include_metadata: bool = True,
query_text: Optional[str] = None,
use_sparse: bool = False
) -> List[Dict]:
# 1. Log params
self.logger.info(f"""
[RETRIEVE PARAMS]
query_vector dim={len(query_vector)}, filters={filters},
top_k={top_k}, include_metadata={include_metadata},
query_text={query_text[:50] if query_text else None},
use_sparse={use_sparse}
""")
sparse_vector = {}
if use_sparse and query_text:
self.logger.info("Creating sparse vector from query_text.")
sparse_vector = self.text_to_sparse_vector(query_text)
self.logger.debug(f"Sparse vector: {sparse_vector}")
filter_obj = None
if filters:
try:
conditions = []
for key, value in filters.items():
filter_obj = qdrant_models.Filter(should=conditions)
except Exception:
self.logger.error("Error creating filter", exc_info=True)
if not sparse_vector.get("indices"):
self.logger.info("Dense-only fallback: using client.search()")
try:
hits = self.client.search(
collection_name=self.collection_name,
query_vector=query_vector,
limit=top_k,
with_payload=include_metadata,
query_filter=filter_obj,
)
results = [
{**(hit.payload or {}), "score": hit.score}
for hit in hits
]
self.logger.info(f"Found {len(results)} docs (dense-only)")
return results
except Exception as e:
self.logger.error(f"Error in dense-only search: {e}", exc_info=True)
return []
self.logger.info("Using hybrid search with query_points()")
prefetch = [
Prefetch(query=query_vector, using="dense", limit=top_k),
Prefetch(
query=qdrant_models.SparseVector(
indices=sparse_vector["indices"],
values=sparse_vector["values"],
),
using="sparse",
limit=top_k,
),
]
try:
hits = self.client.query_points(
collection_name=self.collection_name,
prefetch=prefetch,
query=FusionQuery(fusion=Fusion.RRF),
with_payload=include_metadata,
query_filter=filter_obj,
)
results = [
{**(hit.payload or {}), "score": hit.score}
for hit in hits
]
self.logger.info(f"Found {len(results)} docs (hybrid)")
return results
except Exception as e:
self.logger.error(f"Error querying points: {e}", exc_info=True)
return []
import os
import uuid
from fastapi import FastAPI, UploadFile, File, Form, HTTPException, Request, Response
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
import uvicorn
from agents.agent_decision import process_query
UPLOAD_FOLDER = "uploads/backend"
os.makedirs(UPLOAD_FOLDER, exist_ok=True)
app = FastAPI(title="Transfer-Bot", version="1.0")
app.mount("/uploads", StaticFiles(directory="uploads"), name="uploads")
class QueryRequest(BaseModel):
query: str
@app.post("/chat")
def chat(request: QueryRequest,
response: Response,
request_obj: Request
):
session_id = request_obj.cookies.get("session_id", str(uuid.uuid4()))
try:
response_data = process_query(request.query)
response_text = response_data['response']
response.set_cookie(key="session_id", value=session_id)
result = {
"response": response_text,
"agent": response_data.get("agent_name", "default_agent")
}
if "result_file" in response_data:
result["result_file_url"] = f"/uploads/{response_data['result_file']}"
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/upload")
async def upload_file(response: Response, request_obj: Request, file: UploadFile = File(...), text: str = Form("")):
file_path = os.path.join(UPLOAD_FOLDER, file.filename)
with open(file_path, "wb") as f:
f.write(await file.read())
session_id = request_obj.cookies.get("session_id", str(uuid.uuid4()))
try:
query = {"text": text, "file_path": file_path}
response_data = process_query(query)
response_text = response_data['messages'][-1].content
response.set_cookie(key="session_id", value=session_id)
result = {
"response": response_text,
"agent": response_data.get("agent_name", "default_agent")
}
if "result_file" in response_data:
result["result_file_url"] = f"/uploads/{response_data['result_file']}"
try:
os.remove(file_path)
except Exception as e:
print(f"Failed to delete temp file: {str(e)}")
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000, log_config="logging_config.yaml"
)
import streamlit as st
import requests
API_URL = "http://localhost:8000"
st.set_page_config(page_title="Transfer-Bot")
st.title("Transfer-Bot Frontend")
if "chat_history" not in st.session_state:
st.session_state.chat_history = []
tab1, tab2 = st.tabs(["Chat", "Upload mit Text"])
with tab1:
st.subheader("Chat mit dem Bot")
for role, msg in st.session_state.chat_history:
if role == "user":
st.markdown(
f"""
<div style="text-align: right;">
<div style="display: inline-block; background-color: #DCF8C6; color: black; padding: 10px 15px; border-radius: 15px; margin: 5px 0; max-width: 70%;">
{msg}
</div>
</div>
""",
unsafe_allow_html=True
)
else:
st.markdown(
f"""
<div style="text-align: left;">
<div style="display: inline-block; background-color: #F1F0F0; color: black; padding: 10px 15px; border-radius: 15px; margin: 5px 0; max-width: 70%;">
{msg}
</div>
</div>
""",
unsafe_allow_html=True
)
user_query = st.chat_input("Stell mir deine Frage...")
if user_query:
st.session_state.chat_history.append(("user", user_query))
st.session_state.pending_query = user_query
st.rerun()
if "pending_query" in st.session_state:
with st.spinner("Wird verarbeitet..."):
try:
response = requests.post(f"{API_URL}/chat", json={"query": st.session_state.pending_query})
data = response.json()
bot_reply = data["response"]
except Exception as e:
bot_reply = f"Error: {str(e)}"
st.session_state.chat_history.append(("bot", bot_reply))
del st.session_state.pending_query
st.rerun()
with tab2:
st.subheader("Lade eine Datei und Text hoch")
upload_file = st.file_uploader("Such dir eine Datei aus")
upload_text = st.text_area("Schreib einen Text dazu")
if st.button("Senden", key="upload_button") and upload_file:
with st.spinner("Wird gesendet..."):
try:
files = {"file": (upload_file.name, upload_file.getvalue())}
data = {"text": upload_text}
response = requests.post(f"{API_URL}/upload", data=data, files=files)
data = response.json()
st.success("Antwort:")
st.markdown(data["response"])
st.caption(f"Agent: {data.get('agent', 'unknown')}")
if "result_file_url" in data:
st.markdown(f"[Ergebnisdatei anzeigen]({API_URL}{data['result_file_url']})",
unsafe_allow_html=True)
except Exception as e:
st.error(f"Error: {str(e)}")
import os
from dotenv import load_dotenv
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from openai import OpenAI
load_dotenv()
# class AgentDecisoinConfig:
# # def __init__(self):
# # self.llm = AzureChatOpenAI(
# # deployment_name=os.getenv("deployment_name"),
# # model_name=os.getenv("model_name"),
# # azure_endpoint=os.getenv("azure_endpoint"),
# # openai_api_key=os.getenv("openai_api_key"),
# # openai_api_version=os.getenv("openai_api_version"),
# # temperature=0.1
# # )
# def __init__(self):
# self.client = OpenAI(api_key=os.getenv("openai_api_key"))
#
# self.llm = self.get_openai_llm()
#
# def get_openai_llm(self):
# return self.client.chat.completions.create(
# model="gpt-3.5-turbo",
# messages=[{"role": "user", "content": "Hello"}],
# temperature=0.1
# )
class RAGConfig:
def __init__(self):
self.vector_db_type = "qdrant"
self.embedding_dim = 1536
self.distance_metric = "Cosine"
self.use_local = True
self.local_path = "./data/qdrant_db"
self.url = os.getenv("QDRANT_URL")
self.api_key = os.getenv("QDRANT_API_KEY")
self.collection_name = "transfer_rag"
self.chunk_size = 512
self.chunk_overlap = 50
self.processed_docs_dir = "./data/processed"
self.embedding_model = OpenAIEmbeddings(
model=os.getenv("embedding_model_name"),
api_key=os.getenv("openai_api_key")
)
self.llm = ChatOpenAI(
model=os.getenv("model_name"),
api_key=os.getenv("openai_api_key"),
temperature=0.3
)
self.top_k = 5
self.similarity_threshold = 0.75
self.huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
self.chunking_strategy = "hybrid"
self.max_context_length = 8192
self.response_format_instructions = """Instructions:
1. Answer the query based ONLY on the information provided in the context.
2. If the context doesn't contain relevant information to answer the query, state: "I don't have enough information to answer this question based on the provided context."
3. Do not use prior knowledge not contained in the context.
5. Be concise and accurate.
6. Provide a well-structured response based on retrieved knowledge."""
self.include_sources = True
self.metrics_save_path = "./logs/rag_metrics.json"
self.min_retrieval_confidence = 0.8
self.context_limit = 20
class APIConfig:
def __init__(self):
self.host = "0.0.0.0"
self.port = 8000
self.debug = True
self.rate_limit = 10
self.max_image_upload_size = 5
class Config:
def __init__(self):
# self.agent_decision = AgentDecisoinConfig()
self.rag = RAGConfig()
self.api = APIConfig()
self.max_conversation_history = 40
# config = Config()
\ No newline at end of file
File added
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment