Commit 6e34c730 authored by Pranav's avatar Pranav
Browse files

Updated the python code to support the intended API Format

parent 445ae311
Pipeline #12258 failed with stages
in 22 seconds
...@@ -114,19 +114,12 @@ def safe_serialize_reactions(message): ...@@ -114,19 +114,12 @@ def safe_serialize_reactions(message):
def write_json_entry_per_post(entry, message_id, sender_id): def write_json_entry_per_post(entry, message_id, sender_id):
"""Create a separate JSON file for each post.""" """Create a separate JSON file for each post."""
try: try:
JSON_PATH = DATA_DIR / f"output/post_{message_id}_{sender_id}.json" JSON_PATH = DATA_DIR / f"output/telegram/.meta/post_{message_id}_{sender_id}.txt.json"
JSON_PATH.parent.mkdir(parents=True, exist_ok=True) # <-- ADD THIS LINE ✅ JSON_PATH.parent.mkdir(parents=True, exist_ok=True) # <-- ADD THIS LINE ✅
existing_data = []
if JSON_PATH.exists():
with open(JSON_PATH, "r", encoding="utf-8-sig") as f:
try:
existing_data = json.load(f)
except json.JSONDecodeError:
existing_data = []
existing_data.append(entry)
with open(JSON_PATH, "w", encoding="utf-8") as f: with open(JSON_PATH, "w", encoding="utf-8") as f:
json.dump(existing_data, f, ensure_ascii=False, indent=4) json.dump(entry, f, ensure_ascii=False, indent=4)
print(f"JSON file saved: {JSON_PATH}") print(f"JSON file saved: {JSON_PATH}")
except Exception as e: except Exception as e:
print(f"JSON write error: {e}") print(f"JSON write error: {e}")
...@@ -136,7 +129,7 @@ def write_text_file_per_post(message,sender_id): ...@@ -136,7 +129,7 @@ def write_text_file_per_post(message,sender_id):
"""Save only the text of a message to a separate .txt file.""" """Save only the text of a message to a separate .txt file."""
try: try:
FILE_PATH = DATA_DIR / f"output/post_{message.id}_{sender_id}.txt" FILE_PATH = DATA_DIR / f"output/telegram/post_{message.id}_{sender_id}.txt"
FILE_PATH.parent.mkdir(parents=True, exist_ok=True) FILE_PATH.parent.mkdir(parents=True, exist_ok=True)
text_content = message.text or "" text_content = message.text or ""
if not text_content.strip(): if not text_content.strip():
...@@ -151,7 +144,13 @@ def write_text_file_per_post(message,sender_id): ...@@ -151,7 +144,13 @@ def write_text_file_per_post(message,sender_id):
async def process_message_for_json(message): async def process_message_for_json(message):
sentiment = await asyncio.to_thread(sentiment_analyzer, message.text or "") sentiment = await asyncio.to_thread(sentiment_analyzer, message.text or "")
sender_id = getattr(message, "sender_id", None) sender_id = getattr(message, "sender_id", None)
print("Generating json data")
row = { row = {
"title": "Telegram News",
"backlink": str(DATA_DIR / f"output/telegram/post_{message.id}_{sender_id}.txt"),
"language": "en",
"classification": ["worldnews"],
"properties": {
"chat_id": message.chat_id, "chat_id": message.chat_id,
"message_id": message.id, "message_id": message.id,
"sender_id": sender_id, "sender_id": sender_id,
...@@ -160,15 +159,23 @@ async def process_message_for_json(message): ...@@ -160,15 +159,23 @@ async def process_message_for_json(message):
"has_media": bool(message.media), "has_media": bool(message.media),
"media_type": type(message.media).__name__ if message.media else "", "media_type": type(message.media).__name__ if message.media else "",
"file_name": getattr(message.file, "name", "") if getattr(message, "file", None) else "", "file_name": getattr(message.file, "name", "") if getattr(message, "file", None) else "",
"file_size_kb": round(getattr(message.file, "size", 0) / 1024, 2) if getattr(message, "file", None) else "", "file_size_kb": (round(getattr(message.file, "size", 0) / 1024, 2)
if getattr(message, "file", None)
else ""
),
"views": getattr(message, "views", None), "views": getattr(message, "views", None),
"reactions": safe_serialize_reactions(message), "reactions": safe_serialize_reactions(message),
"sentiment": sentiment[0]["label"] if sentiment else "" "sentiment": sentiment[0]["label"] if sentiment else ""
}
} }
print("Done generating")
try: try:
#write_csv_row(row) #write_csv_row(row)
write_json_entry_per_post(row, message.id, sender_id) print("Saving files")
write_text_file_per_post(message,sender_id) write_text_file_per_post(message,sender_id)
write_json_entry_per_post(row, message.id, sender_id)
except Exception as e: except Exception as e:
print(f"CSV write error: {e}") print(f"CSV write error: {e}")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment