updated filestructure

bot_history.py

@@ -18,10 +18,10 @@ intents.members = True
 
 client = discord.Client(intents=intents)
 
-DATA_DIR = Path("logs")
+DATA_DIR = Path("ingest")
 DATA_DIR.mkdir(exist_ok=True)
-MESSAGES_DIR = DATA_DIR / "messages"
-MESSAGES_DIR.mkdir(exist_ok=True)
+META_DIR = DATA_DIR / ".meta"
+META_DIR.mkdir(exist_ok=True)
 
 URL_REGEX = r'(https?://\S+)'
 
@@ -40,6 +40,10 @@ async def save_json_file_async(data: dict, dest: Path) -> None:
     loop = asyncio.get_running_loop()
     await loop.run_in_executor(None, _atomic_write_json, data, dest)
 
+def save_text(data: str, dest: Path) -> None:
+    with open(dest, "w", encoding="utf-8") as f:
+        f.write(data)
+
 # === Bot startup: fetch history ===
 @client.event
 async def on_ready():
@@ -52,30 +56,33 @@ async def on_ready():
                 print(f'  -> Channel: #{channel.name}')
                 async for message in channel.history(limit=None):
                     ts_safe = _safe_filename(message.created_at.isoformat())
-                    message_file = MESSAGES_DIR / f"{message.id}_{ts_safe}.json"
+                    file_name=f"{message.id}_{ts_safe}"
+                    meta_file = META_DIR / f"{file_name}.txt.json"
+                    message_file = DATA_DIR / f"{file_name}.txt"
 
-                    if message_file.exists():
-                        with open(message_file, 'r', encoding='utf-8') as f:
-                            message_data = json.load(f)
+                    if meta_file.exists():
+                        with open(meta_file, 'r', encoding='utf-8') as f:
+                            meta_data = json.load(f)
                     else:
-                        message_data = {
+                        meta_data = {
                             "message_id": str(message.id),
                             "user_id": str(message.author.id),
                             "text": message.content,
                             "reactions_history": [],
                             "links": re.findall(URL_REGEX, message.content)
                         }
+                        save_text(message.content,message_file)
 
                     # Ensure reactions_history exists
-                    message_data.setdefault("reactions_history", [])
+                    meta_data.setdefault("reactions_history", [])
 
                     current_emojis = [str(reaction.emoji) for reaction in message.reactions]
-                    message_data["reactions_history"].append({
+                    meta_data["reactions_history"].append({
                         "timestamp": str(message.created_at.isoformat()),
                         "emojis": current_emojis
                     })
 
-                    await save_json_file_async(message_data, message_file)
+                    await save_json_file_async(meta_data, meta_file)
             except discord.Forbidden:
                 print(f'  ⚠️ No permission to read #{channel.name}')
             except discord.HTTPException as e:
@@ -90,7 +97,9 @@ async def on_message(message: discord.Message):
         return
 
     ts_safe = _safe_filename(message.created_at.isoformat())
-    message_file = MESSAGES_DIR / f"{message.id}_{ts_safe}.json"
+    file_name=f"{message.id}_{ts_safe}"
+    meta_file = META_DIR /f"{file_name}.txt.json"
+    message_file = DATA_DIR / f"{file_name}.txt"
 
     message_data = {
         "message_id": str(message.id),
@@ -99,19 +108,19 @@ async def on_message(message: discord.Message):
         "reactions_history": [],
         "links": re.findall(URL_REGEX, message.content)
     }
-
-    await save_json_file_async(message_data, message_file)
+    save_text(message.content,message_file)
+    await save_json_file_async(message_data, meta_file)
     print(f'💬 {message.author} said: {message.content} (saved to {message_file})')
 
 # === Reaction events ===
 async def update_reaction_history(message_id: str, emoji: str, user_id: str, action: str, timestamp: str):
     # Find the message file (support historic messages)
-    message_files = list(MESSAGES_DIR.glob(f"{message_id}_*.json"))
-    if not message_files:
+    meta_file = list(META_DIR.glob(f"{message_id}_*.json"))
+    if not meta_file:
         return
 
-    message_file = message_files[0]
-    with open(message_file, 'r', encoding='utf-8') as f:
+    meta_file = meta_file[0]
+    with open(meta_file, 'r', encoding='utf-8') as f:
         message_data = json.load(f)
 
     message_data.setdefault("reactions_history", [])
@@ -125,7 +134,7 @@ async def update_reaction_history(message_id: str, emoji: str, user_id: str, act
 
     message_data["reactions_history"].append(entry)
 
-    await save_json_file_async(message_data, message_file)
+    await save_json_file_async(message_data, meta_file)
 
 @client.event
 async def on_raw_reaction_add(payload: discord.RawReactionActionEvent):