add metadata file

app.py

@@ -2,6 +2,7 @@ import praw
 import json
 import os
 from dotenv import load_dotenv
+from datetime import datetime, timezone
 
 # --- Load environment variables ---
 load_dotenv()
@@ -14,12 +15,12 @@ reddit = praw.Reddit(
 )
 
 # --- Choose subreddit and fetch posts ---
-subreddit_name = "news"
+subreddit_name = "worldnews"
 subreddit = reddit.subreddit(subreddit_name)
 
 data = []
 
-for post in subreddit.hot(limit=3):  # get top 3 hot posts
+for post in subreddit.hot(limit=5):  # get top 5 hot posts
     post.comments.replace_more(limit=0)  # remove 'MoreComments' objects
 
     post_data = {
@@ -33,7 +34,7 @@ for post in subreddit.hot(limit=3):  # get top 3 hot posts
     }
 
     # Collect top-level comments
-    for comment in post.comments.list()[:10]:  # limit to first 10 comments
+    for comment in post.comments.list()[:15]:  # limit to first 15 comments
         post_data["comments"].append({
             "id": comment.id,
             "body": comment.body,
@@ -46,12 +47,26 @@ for post in subreddit.hot(limit=3):  # get top 3 hot posts
 # --- Convert to JSON string ---
 json_output = json.dumps(data, indent=4)
 
-# --- Print or save ---
-print(json_output)
+# --- Save Reddit data to .txt file ---
+txt_filename = "reddit_data.txt"
+with open(txt_filename, "w", encoding="utf-8") as f:
+    f.write(json_output)
 
-# Optionally, save to file:
-# with open("reddit_data.json", "w") as f:
-#     json.dump(data, f, indent=4)
+print(f"Saved Reddit data to {txt_filename}")
 
-with open("/app/output/reddit_data.json", "w") as f:
-    json.dump(data, f, indent=4)
+# --- Create metadata JSON ---
+metadata = {
+    "title": "Reddit " + subreddit_name,
+    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_data.txt",  # Replace for local ingest path
+    "language": "en",
+    "classifications": [subreddit_name],
+    "properties": {
+        "creation_date": int(datetime.now(timezone.utc).timestamp() * 1000)
+    }
+}
+
+metadata_filename = "reddit_data.txt.json"
+with open(metadata_filename, "w", encoding="utf-8") as f:
+    json.dump(metadata, f, indent=4)
+
+print(f"Saved metadata to {metadata_filename}")
\ No newline at end of file

reddit_data2.txt.json

+{
+    "title": "Reddit World Nws",
+    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_data.txt",
+    "language": "en",
+    "classifications": [
+        "worldnews"
+    ],
+    "properties": {
+        "creation_date": 1762722971028
+    }
+}
\ No newline at end of file

reddit_data3.txt.json

+{
+    "title": "Reddit politics",
+    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_data.txt",
+    "language": "en",
+    "classifications": [
+        "politics"
+    ],
+    "properties": {
+        "creation_date": 1762727111202
+    }
+}
\ No newline at end of file

reddit_data_metadata.json

+{
+    "title": "Reddit",
+    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_data.txt",
+    "language": "en",
+    "classifications": [
+        "news"
+    ],
+    "properties": {
+        "creation_date": 1762722101114
+    }
+}
\ No newline at end of file