Commit 0d7d7152 authored by Pranav's avatar Pranav
Browse files

Implemented code changes.

Created two different JSON files.
1)First JSON file is Meta Data JSON file.
2)Second JSON file is Description/Content of JSON file mapped using the chatID and messageID
parent a74bc803
--extra-index-url https://download.pytorch.org/whl/cpu
telethon telethon
python-dotenv python-dotenv
transformers==4.44.2 transformers==4.44.2
torch==2.3.1 torch==2.5.1+cpu
accelerate==0.33.0 accelerate==0.33.0
...@@ -96,9 +96,21 @@ def write_csv_row(row_dict): ...@@ -96,9 +96,21 @@ def write_csv_row(row_dict):
writer.writerow(row_dict) writer.writerow(row_dict)
def write_json_entry(entry): def safe_serialize_reactions(message):
"""Append a single JSON entry to a file.""" r = getattr(message, "reactions", None)
if not r or not getattr(r, "results", None):
return ""
parts = []
for rc in r.results:
label = render_reaction_label(rc.reaction)
parts.append(f"{label}:{rc.count}")
return ";".join(parts)
def write_json_entry_per_post(entry, message_id, sender_id):
"""Create a separate JSON file for each post."""
try: try:
JSON_PATH = DATA_DIR / f"output/post_{message_id}_{sender_id}.json"
JSON_PATH.parent.mkdir(parents=True, exist_ok=True) # <-- ADD THIS LINE ✅
existing_data = [] existing_data = []
if JSON_PATH.exists(): if JSON_PATH.exists():
with open(JSON_PATH, "r", encoding="utf-8-sig") as f: with open(JSON_PATH, "r", encoding="utf-8-sig") as f:
...@@ -110,27 +122,34 @@ def write_json_entry(entry): ...@@ -110,27 +122,34 @@ def write_json_entry(entry):
existing_data.append(entry) existing_data.append(entry)
with open(JSON_PATH, "w", encoding="utf-8") as f: with open(JSON_PATH, "w", encoding="utf-8") as f:
json.dump(existing_data, f, ensure_ascii=False, indent=4) json.dump(existing_data, f, ensure_ascii=False, indent=4)
print(f"JSON file saved: {JSON_PATH}")
except Exception as e: except Exception as e:
print(f"JSON write error: {e}") print(f"JSON write error: {e}")
print(f"Error writing JSON for post {message_id}: {e}")
def write_text_file_per_post(message,sender_id):
"""Save only the text of a message to a separate .txt file."""
def safe_serialize_reactions(message): try:
r = getattr(message, "reactions", None) FILE_PATH = DATA_DIR / f"output/post_{message.id}_{sender_id}.txt"
if not r or not getattr(r, "results", None): FILE_PATH.parent.mkdir(parents=True, exist_ok=True)
return "" text_content = message.text or ""
parts = [] if not text_content.strip():
for rc in r.results: return # skip empty or non-text messages
label = render_reaction_label(rc.reaction) with open(FILE_PATH, "w", encoding="utf-8-sig") as f:
parts.append(f"{label}:{rc.count}") f.write(text_content.strip())
return ";".join(parts)
print(f"Text file saved: {FILE_PATH}")
except Exception as e:
print(f"Error writing text file for post {message.id}: {e}")
async def process_message_for_csv(message): async def process_message_for_csv(message):
sentiment = await asyncio.to_thread(sentiment_analyzer, message.text or "") sentiment = await asyncio.to_thread(sentiment_analyzer, message.text or "")
sender_id = getattr(message, "sender_id", None)
row = { row = {
"chat_id": message.chat_id, "chat_id": message.chat_id,
"message_id": message.id, "message_id": message.id,
"sender_id": getattr(message, "sender_id", None), "sender_id": sender_id,
"timestamp": message.date.isoformat() if getattr(message, "date", None) else None, "timestamp": message.date.isoformat() if getattr(message, "date", None) else None,
"text": (message.text[:3000] if message.text else ""), "text": (message.text[:3000] if message.text else ""),
"has_media": bool(message.media), "has_media": bool(message.media),
...@@ -143,7 +162,8 @@ async def process_message_for_csv(message): ...@@ -143,7 +162,8 @@ async def process_message_for_csv(message):
} }
try: try:
#write_csv_row(row) #write_csv_row(row)
write_json_entry(row) write_json_entry_per_post(row, message.id, sender_id)
write_text_file_per_post(message,sender_id)
except Exception as e: except Exception as e:
print(f"CSV write error: {e}") print(f"CSV write error: {e}")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment