Skip to content
GitLab
Commit 86ebcf8d authored by Pranav's avatar Pranav
Browse files

Changes in the program to include data for all the messages for a single channel into a single file

parent 6e34c730
Hide whitespace changes
Inline Side-by-side
Desktop/Telegram_Bot/devops@vm33.fkc.hft-stuttgart.de 0 → 100644
View file @ 86ebcf8d
API_ID=39098285
API_HASH=d188a6fed1c3e9e0ce2b9c847e25b356
SESSION=session_name
INVITE_LINK=
DATA_DIR=/app/data
Desktop/Telegram_Bot/services/telegram_listener.py
View file @ 86ebcf8d
# python
import asyncio import asyncio
import csv
import os import os
import json import json
from pathlib import Path from pathlib import Path
from dotenv import load_dotenv,find_dotenv
from datetime import datetime from datetime import datetime
from dotenv import load_dotenv, find_dotenv
from telethon import TelegramClient, events from telethon import TelegramClient, events
from telethon.errors import UserAlreadyParticipantError from telethon.errors import UserAlreadyParticipantError
from telethon.tl.functions.messages import ImportChatInviteRequest from telethon.tl.functions.messages import ImportChatInviteRequest
from telethon.tl.functions.channels import JoinChannelRequest from telethon.tl.functions.channels import JoinChannelRequest
from telethon.tl.types import InputChannel, Channel, ReactionEmoji, ReactionCustomEmoji, ReactionPaid from telethon.tl.types import Channel, ReactionEmoji, ReactionCustomEmoji, ReactionPaid
from transformers import pipeline from transformers import pipeline
# ======================================================
# ENV SETUP
# ======================================================
load_dotenv(find_dotenv()) load_dotenv(find_dotenv())
# Please create a .env file. Add the environment variables over there. And never the env file on github or gitlab.
API_ID = int(os.getenv("API_ID")) API_ID = int(os.getenv("API_ID"))
API_HASH = os.getenv("API_HASH") API_HASH = os.getenv("API_HASH")
SESSION = os.getenv("SESSION") SESSION = os.getenv("SESSION")
INVITE_LINK = os.getenv("INVITE_LINK") INVITE_LINK = os.getenv("INVITE_LINK")
DATA_DIR = Path(os.getenv("DATA_DIR", "/app/data")) DATA_DIR = Path(os.getenv("DATA_DIR", "/app/data"))
DATA_DIR.mkdir(parents=True, exist_ok=True) OUTPUT_DIR = DATA_DIR / "output/telegram"
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") META_DIR = OUTPUT_DIR / ".meta"
JSON_PATH = DATA_DIR / f"messages_{timestamp}.json" META_DIR.mkdir(parents=True, exist_ok=True)
sentiment_analyzer = pipeline("sentiment-analysis") sentiment_analyzer = pipeline("sentiment-analysis")
# These are the headers for the Json file. # ======================================================
HEADER = ["chat_id", "message_id", "sender_id", "timestamp", "text", # RUNTIME STATE (PER CHANNEL)
"has_media", "media_type", "file_name", "file_size_kb", # ======================================================
"views", "reactions", "sentiment"]
async def list_joined_channels(client): CHANNEL_FILE_CONTEXT = {} # chat_id -> {txt, json}
channels = []
async for dialog in client.iter_dialogs():
ent = dialog.entity
if isinstance(ent, Channel):
channels.append(ent)
return channels
# ======================================================
# UTILITIES
# ======================================================
async def process_messages_for_all_channels(client, process_message, limit): def is_empty(val):
channels = await list_joined_channels(client) return val is None or not str(val).strip()
print(f"No INVITE_URL provided. Listening to all chats. Found {len(channels)} channels.")
for ch in channels: def safe_channel_name(chat):
try: return (chat.username or chat.title or "telegram").replace(" ", "_")
# Pull last N messages for this channel
msgs = [m async for m in client.iter_messages(ch, limit=limit)]
for m in reversed(msgs):
await process_message(m)
except Exception as e:
print(f"Failed to fetch history for {getattr(ch, 'title', ch.id)}: {e}")
# Register one global handler for new messages (all chats) def get_channel_file_paths(chat):
@client.on(events.NewMessage()) """
async def handle_all(event): Create deterministic filenames ONCE per channel:
await process_message(event.message) <name>_<id>_<timestamp>.txt
<name>_<id>_<timestamp>.txt.json
"""
if chat.id in CHANNEL_FILE_CONTEXT:
return CHANNEL_FILE_CONTEXT[chat.id]
base_name = f"{safe_channel_name(chat)}_{chat.id}"
@client.on(events.MessageEdited()) txt_path = OUTPUT_DIR / f"{base_name}.txt"
async def handle_edit(event): json_path = META_DIR / f"{base_name}.txt.json"
message = event.message
await process_message(message)
return channels # optional, if you need the list
async def process_messages_for_one_channel(client, process_message, target, limit):
msgs = [msg async for msg in client.iter_messages(target, limit=limit)]
for m in reversed(msgs):
await process_message(m)
@client.on(events.NewMessage(chats=target))
async def handle_new(event):
message = event.message
await process_message(message)
@client.on(events.MessageEdited(chats=target))
async def handle_edit(event):
message = event.message
await process_message(message)
CHANNEL_FILE_CONTEXT[chat.id] = {
"txt": txt_path,
"json": json_path
}
def write_csv_row(row_dict): return CHANNEL_FILE_CONTEXT[chat.id]
# Ensure file exists and has header; then append
CSV_PATH = Path(r"{CSV_FILE_PATH}")
file_exists = os.path.exists(CSV_PATH)
with open(CSV_PATH, mode="a", newline="", encoding="utf-8-sig") as f:
writer = csv.DictWriter(f, fieldnames=HEADER)
if not file_exists:
writer.writeheader()
writer.writerow(row_dict)
def render_reaction_label(r):
if isinstance(r, ReactionEmoji):
return r.emoticon
if isinstance(r, ReactionCustomEmoji):
return f"custom_emoji:{r.document_id}"
if isinstance(r, ReactionPaid):
return "paid_reaction"
return str(r)
def safe_serialize_reactions(message): def safe_serialize_reactions(message):
r = getattr(message, "reactions", None) r = getattr(message, "reactions", None)
if not r or not getattr(r, "results", None): if not r or not getattr(r, "results", None):
return "" return ""
parts = [] return ";".join(
for rc in r.results: f"{render_reaction_label(rc.reaction)}:{rc.count}"
label = render_reaction_label(rc.reaction) for rc in r.results
parts.append(f"{label}:{rc.count}") )
return ";".join(parts)
# ======================================================
def write_json_entry_per_post(entry, message_id, sender_id): # FILE WRITERS
"""Create a separate JSON file for each post.""" # ======================================================
try:
JSON_PATH = DATA_DIR / f"output/telegram/.meta/post_{message_id}_{sender_id}.txt.json" def append_text_to_channel(message, chat):
JSON_PATH.parent.mkdir(parents=True, exist_ok=True) # <-- ADD THIS LINE ✅ if not message.text or not message.text.strip():
return
with open(JSON_PATH, "w", encoding="utf-8") as f:
json.dump(entry, f, ensure_ascii=False, indent=4) paths = get_channel_file_paths(chat)
txt_path = paths["txt"]
print(f"JSON file saved: {JSON_PATH}")
except Exception as e: with open(txt_path, "a", encoding="utf-8") as f:
print(f"JSON write error: {e}") f.write(
print(f"Error writing JSON for post {message_id}: {e}") "\n----------------------------------------\n"
f"Message ID : {message.id}\n"
def write_text_file_per_post(message,sender_id): f"Timestamp : {message.date.isoformat() if message.date else ''}\n"
"""Save only the text of a message to a separate .txt file.""" f"Sender ID : {message.sender_id}\n\n"
f"{message.text.strip()}\n"
try: )
FILE_PATH = DATA_DIR / f"output/telegram/post_{message.id}_{sender_id}.txt"
FILE_PATH.parent.mkdir(parents=True, exist_ok=True) def append_json_to_channel(message_entry, chat):
text_content = message.text or "" paths = get_channel_file_paths(chat)
if not text_content.strip(): json_path = paths["json"]
return # skip empty or non-text messages
with open(FILE_PATH, "w", encoding="utf-8-sig") as f: if json_path.exists():
f.write(text_content.strip()) with open(json_path, "r", encoding="utf-8") as f:
data = json.load(f)
print(f"Text file saved: {FILE_PATH}") else:
except Exception as e: data = {
print(f"Error writing text file for post {message.id}: {e}") "title": f"Telegram {chat.title}",
"backlink": str(paths["txt"]),
async def process_message_for_json(message): "language": "en",
sentiment = await asyncio.to_thread(sentiment_analyzer, message.text or "") "classification": [chat.username or chat.title],
sender_id = getattr(message, "sender_id", None) "properties": {
print("Generating json data") "chat_id": chat.id,
row = { "messages": []
"title": "Telegram News", }
"backlink": str(DATA_DIR / f"output/telegram/post_{message.id}_{sender_id}.txt"), }
"language": "en",
"classification": ["worldnews"], data["properties"]["messages"].append(message_entry)
"properties": {
"chat_id": message.chat_id, with open(json_path, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
# ======================================================
# MESSAGE PROCESSING
# ======================================================
async def process_message_for_storage(message):
chat = await message.get_chat()
sentiment = await asyncio.to_thread(
sentiment_analyzer,
message.text or ""
)
message_entry = {
"message_id": message.id, "message_id": message.id,
"sender_id": sender_id, "sender_id": message.sender_id,
"timestamp": message.date.isoformat() if getattr(message, "date", None) else None, "timestamp": message.date.isoformat() if message.date else None,
"text": (message.text[:3000] if message.text else ""),
"has_media": bool(message.media), "has_media": bool(message.media),
"media_type": type(message.media).__name__ if message.media else "", "media_type": type(message.media).__name__ if message.media else "",
"file_name": getattr(message.file, "name", "") if getattr(message, "file", None) else "", "file_name": message.file.name if getattr(message, "file", None) else None,
"file_size_kb": (round(getattr(message.file, "size", 0) / 1024, 2) "file_size_kb": (
if getattr(message, "file", None) round(message.file.size / 1024, 2)
else "" if getattr(message, "file", None)
), else None
),
"views": getattr(message, "views", None), "views": getattr(message, "views", None),
"reactions": safe_serialize_reactions(message), "reactions": safe_serialize_reactions(message),
"sentiment": sentiment[0]["label"] if sentiment else "" "sentiment": sentiment[0]["label"] if sentiment else None
} }
} append_text_to_channel(message, chat)
append_json_to_channel(message_entry, chat)
print("Done generating") # ======================================================
try: # CHANNEL HANDLING
#write_csv_row(row) # ======================================================
print("Saving files")
write_text_file_per_post(message,sender_id)
write_json_entry_per_post(row, message.id, sender_id)
except Exception as e:
print(f"CSV write error: {e}")
def render_reaction_label(r): async def list_joined_channels(client):
if isinstance(r, ReactionEmoji): channels = []
return r.emoticon # e.g., "👍" async for dialog in client.iter_dialogs():
if isinstance(r, ReactionCustomEmoji): if isinstance(dialog.entity, Channel):
# We can later resolve document_id to a sticker/emoji file if needed channels.append(dialog.entity)
return f"custom_emoji:{r.document_id}" return channels
if isinstance(r, ReactionPaid):
# Paid reactions don’t have an emoji string async def ensure_joined(client, invite_url):
return "paid_reaction" if is_empty(invite_url):
# Fallback for future types
return str(r)
def _is_empty(s: str) -> bool:
return s is None or not str(s).strip()
async def ensure_joined(client, invite_url: str):
if _is_empty(invite_url):
return None return None
# Handles both invite hash links and public @ links
if "/+" in invite_url: if "/+" in invite_url:
# Private invite hash style: https://t.me/+<hash>
invite_hash = invite_url.rsplit("/", 1)[-1].replace("+", "") invite_hash = invite_url.rsplit("/", 1)[-1].replace("+", "")
try: try:
res = await client(ImportChatInviteRequest(invite_hash)) res = await client(ImportChatInviteRequest(invite_hash))
entity = res.chats[0] if res.chats else res.updates[0].chat return res.chats[0]
except UserAlreadyParticipantError: except UserAlreadyParticipantError:
entity = await client.get_entity(invite_url) return await client.get_entity(invite_url)
else: else:
# Public link or t.me/username
entity = await client.get_entity(invite_url) entity = await client.get_entity(invite_url)
# If it's a channel and we’re not a participant, try joining
try: try:
await client(JoinChannelRequest(entity)) await client(JoinChannelRequest(entity))
except Exception: except Exception:
pass pass
return entity return entity
async def process_message(message): # ======================================================
# MESSAGE STREAMS
# 1. Basic Info # ======================================================
print("--- New Message ---")
print(f"Message ID: {message.id}") async def process_single_channel(client, target, limit):
print(f"Timestamp: {message.date.strftime('%Y-%m-%d %H:%M:%S')}") history = [m async for m in client.iter_messages(target, limit=limit)]
for m in reversed(history):
# 2. Text Content await process_message_for_storage(m)
if message.text:
print(f"Text: {message.text[:100]}...") # Trimmed to first 100 chars @client.on(events.NewMessage(chats=target))
async def on_new(event):
# 3. Media/File Info await process_message_for_storage(event.message)
if message.media:
print("Media detected.") @client.on(events.MessageEdited(chats=target))
if message.file: async def on_edit(event):
print(f" - File Name: {message.file.name}") await process_message_for_storage(event.message)
print(f" - File Size: {round(message.file.size / 1024, 2)} KB")
async def process_all_channels(client, limit):
# Example: Download media (uncomment if needed) channels = await list_joined_channels(client)
# await client.download_media(message.media, file="downloads/")
for ch in channels:
# 4. Reaction Info try:
if message.reactions: history = [m async for m in client.iter_messages(ch, limit=limit)]
print("Reactions:") for m in reversed(history):
for reaction_count in message.reactions.results: await process_message_for_storage(m)
label = render_reaction_label(reaction_count.reaction) except Exception as e:
print(f" - {label}: {reaction_count.count}") print(f"History fetch failed for {ch.title}: {e}")
print("------\n") @client.on(events.NewMessage())
await process_message_for_json(message) async def on_new(event):
await process_message_for_storage(event.message)
@client.on(events.MessageEdited())
async def on_edit(event):
await process_message_for_storage(event.message)
# ======================================================
# MAIN
# ======================================================
async def main(): async def main():
client = TelegramClient(SESSION, API_ID, API_HASH) client = TelegramClient(SESSION, API_ID, API_HASH)
await client.start() await client.start()
# Resolve and ensure we are re in the channel/group
target = await ensure_joined(client, INVITE_LINK) target = await ensure_joined(client, INVITE_LINK)
if target: if target:
print("Listening to single channel ") print(f"Listening to channel: {target.title}")
await process_messages_for_one_channel(client, process_message, target, limit= 5) await process_single_channel(client, target, limit=50)
else: else:
print("No INVITE_URL provided. Listening to all chats.") print("Listening to all joined channels")
await process_messages_for_all_channels(client, process_message, limit=5) await process_all_channels(client, limit=50)
print("Listening for messages...") print("Listening for messages...")
await client.run_until_disconnected() await client.run_until_disconnected()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Dies ist die Gitlab-Instanz des Transferportals der Hochschule für Technik Stuttgart. Hier geht es zurück zum Portal