Remove unnessary files from git

.dockerignore

@@ -30,7 +30,6 @@ htmlcov/
 archived/
 data/
 eval/
-ollama-models/
 raw_documents/
 qdrant_data/
 
@@ -41,4 +40,3 @@ qdrant_data/
 **/.DS_Store
 
 .env
-../../Library/Application Support/JetBrains/PyCharm2024.3/scratches/requirements.dev.txt

archived/chat_gui.py

-# import streamlit as st
-# import requests
-#
-# API_URL = "http://localhost:8000"
-# RAG_ENDPOINT = f"{API_URL}/rag"
-#
-# st.set_page_config(page_title="TransferBot")
-# st.title("TransferBot Frontend")
-#
-# if "chat_history" not in st.session_state:
-#     st.session_state.chat_history = []
-#
-#
-# def get_serialized_history():
-#     return [
-#         {"role": "user" if role == "user" else "assistant", "content": msg}
-#         for role, msg in st.session_state.chat_history
-#     ]
-#
-#
-# tab1, = st.tabs(["Chat"])
-#
-# with tab1:
-#     st.subheader("Chat mit dem Bot")
-#
-#     for role, msg in st.session_state.chat_history:
-#         if role == "user":
-#             st.markdown(
-#                 f"""
-#                 <div style="text-align: right;">
-#                     <div style="display: inline-block; background-color: #DCF8C6; color: black; padding: 10px 15px; border-radius: 15px; margin: 5px 0; max-width: 70%;">
-#                         {msg}
-#                     </div>
-#                 </div>
-#                 """,
-#                 unsafe_allow_html=True
-#             )
-#         else:
-#             st.markdown(
-#                 f"""
-#                 <div style="text-align: left;">
-#                     <div style="display: inline-block; background-color: #F1F0F0; color: black; padding: 10px 15px; border-radius: 15px; margin: 5px 0; max-width: 70%;">
-#                         {msg}
-#                     </div>
-#                 </div>
-#                 """,
-#                 unsafe_allow_html=True
-#             )
-#
-#     user_query = st.chat_input("Stell mir deine Frage...")
-#
-#     if user_query:
-#         st.session_state.chat_history.append(("user", user_query))
-#         st.session_state.pending_query = user_query
-#         st.rerun()
-#
-#     if "pending_query" in st.session_state:
-#         with st.spinner("Wird verarbeitet..."):
-#             data = {}
-#             try:
-#                 payload = {
-#                     "messages": get_serialized_history(),
-#                     "session_id": st.session_state.get("session_id", None),
-#                 }
-#                 response = requests.post(f"{API_URL}/chat", json=payload)
-#                 data = response.json()
-#                 bot_reply = data.get("message", {}).get("content", "Keine Antwort erhalten.")
-#
-#                 # if "messages" in data:
-#                 #     st.session_state.chat_history = [
-#                 #         (m["role"], m["content"])
-#                 #         for m in data["messages"]
-#                 #     ]
-#                 # else:
-#                 st.session_state.chat_history.append(("assistant", bot_reply))
-#             except Exception as e:
-#                 bot_reply = f"Error: {str(e)}"
-#                 st.session_state.chat_history.append(("assistant", bot_reply))
-#
-#             del st.session_state.pending_query
-#             st.rerun()

archived/query_expander.py

-import logging
-from typing import Dict, Any
-
-
-class QueryExpander:
-    def __init__(self, config):
-        self.logger = logging.getLogger(__name__)
-        self.config = config
-        self.model = config.llm
-
-    def expand_query(self, original_query: str) -> Dict[str, Any]:
-        self.logger.info(f"Expanding query: {original_query}")
-        expanded_query = self.expand_with_synonyms_and_related_terms(original_query)
-        expanded_query = self.expand_with_semantics(expanded_query)
-
-        return {
-            "original_query": original_query,
-            "expanded_query": expanded_query
-        }
-
-    def expand_with_synonyms_and_related_terms(self, query: str) -> str:
-        synonym_dict = {
-            "geography": ["physical geography", "human geography", "geospatial studies", "cartography"],
-            "ecosystem": ["biome", "environment", "habitat", "ecological system"]
-        }
-
-        expanded_query = query.lower()
-
-        for key, synonyms in synonym_dict.items():
-            if key.lower() in expanded_query:
-                expanded_query += " OR " + " OR ".join(synonyms)
-
-        self.logger.info(f"Expanded query after synonym addition: {expanded_query}")
-
-        return expanded_query
-
-    def expand_with_semantics(self, query: str) -> str:
-        prompt = f"""
-        As a research assistant, expand the following query with relevant scientific terminology,
-        synonyms, related concepts, and keywords that would help in retrieving relevant academic papers and research articles:
-
-        User Query: {query}
-
-        Expand the query only if you feel like it is required, otherwise keep the user query intact.
-        Be specific to the field of study mentioned, do not add other irrelevant domains.
-        If the user query asks about answering in tabular format, include that in the expanded query and do not answer in tabular format yourself.
-        Provide only the expanded query without explanations.
-        """
-        expansion = self.model.invoke(prompt)
-
-        return expansion

archived/sparse_embedder.py

-# import nltk
-# from sklearn.feature_extraction.text import TfidfVectorizer
-# from typing import List, Dict
-#
-# nltk.download('stopwords', quiet=True)
-# from nltk.corpus import stopwords
-#
-# german_stopwords = stopwords.words('german')
-#
-# class GermanBM25SparseEmbedder:
-#     def __init__(self):
-#         self.vectorizer = TfidfVectorizer(
-#             stop_words=german_stopwords,
-#             ngram_range=(1, 2),
-#             max_features=20000,
-#             norm='l2'
-#         )
-#         self.fitted = False
-#         self.feature_names = []
-#
-#     def fit(self, corpus: List[str]):
-#         self.vectorizer.fit(corpus)
-#         self.feature_names = self.vectorizer.get_feature_names_out()
-#         self.fitted = True
-#
-#     def encode(self, texts: List[str]) -> List[Dict[int, float]]:
-#         if not self.fitted:
-#             raise ValueError("Call `fit()` before `encode()`.")
-#         tfidf_matrix = self.vectorizer.transform(texts)
-#         return [
-#             {int(i): float(v) for i, v in zip(row.indices, row.data)}
-#             for row in tfidf_matrix
-#         ]
\ No newline at end of file