add mutiple subreddit fetch and error handling

app.py

 import praw
 import json
 import os
+import logging
 from dotenv import load_dotenv
 from datetime import datetime, timezone
 
+# --- Setup logging ---
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
 # --- Load environment variables ---
 load_dotenv()
 
+# --- Validate environment variables ---
+try:
+    subreddits_env = os.getenv("SUBREDDITS")
+    if not subreddits_env:
+        raise ValueError("Environment variable SUBREDDITS is missing.")
+    subreddit_list = [s.strip() for s in subreddits_env.split(",")]
+
+    POST_LIMIT = int(os.getenv("POST_LIMIT", 10))
+    COMMENT_LIMIT = int(os.getenv("COMMENT_LIMIT", 10))
+
+    if POST_LIMIT < 1 or COMMENT_LIMIT < 0:
+        raise ValueError("POST_LIMIT and COMMENT_LIMIT must be positive integers.")
+
+    OUTPUT_DIR = os.getenv("OUTPUT_DIR", ".")
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+    # Create .meta folder inside OUTPUT_DIR
+    META_DIR = os.path.join(OUTPUT_DIR, ".meta")
+    os.makedirs(META_DIR, exist_ok=True)
+
+except Exception as e:
+    logger.error(f"Invalid configuration: {e}")
+    raise SystemExit(1)
+
 # --- Reddit API credentials ---
-reddit = praw.Reddit(
+try:
+    reddit = praw.Reddit(
         client_id=os.getenv("REDDIT_CLIENT_ID"),
         client_secret=os.getenv("REDDIT_CLIENT_SECRET"),
         user_agent="MyRedditScraper/1.0"
-)
+    )
+    reddit.user.me()  # Test authentication
+except Exception as e:
+    logger.error(f"Failed to authenticate with Reddit API: {e}")
+    raise SystemExit(1)
 
-# --- Choose subreddit and fetch posts ---
-subreddit_name = "worldnews"
-subreddit = reddit.subreddit(subreddit_name)
+# --- Process each subreddit ---
+for subreddit_name in subreddit_list:
+    logger.info(f"Processing subreddit: {subreddit_name}")
+    data = []
 
-data = []
+    try:
+        subreddit = reddit.subreddit(subreddit_name)
+    except Exception as e:
+        logger.error(f"Cannot access subreddit '{subreddit_name}': {e}")
+        continue
 
-for post in subreddit.hot(limit=5):  # get top 5 hot posts
-    post.comments.replace_more(limit=0)  # remove 'MoreComments' objects
+    # Fetch posts
+    try:
+        posts = subreddit.hot(limit=POST_LIMIT)
+    except Exception as e:
+        logger.error(f"Failed to retrieve posts for r/{subreddit_name}: {e}")
+        continue
 
+    # Process posts
+    for post in posts:
+        try:
+            post.comments.replace_more(limit=0)
+        except Exception as e:
+            logger.warning(f"Comments for post {post.id} not fully loaded: {e}")
+
+        try:
             post_data = {
                 "id": post.id,
                 "title": post.title,
@@ -32,41 +89,64 @@ for post in subreddit.hot(limit=5):  # get top 5 hot posts
                 "created_utc": post.created_utc,
                 "comments": []
             }
+        except Exception as e:
+            logger.error(f"Failed to read post data for {post.id}: {e}")
+            continue
+
+        # Fetch comments
+        try:
+            comments = post.comments.list()[:COMMENT_LIMIT]
+        except Exception as e:
+            logger.warning(f"Failed to load comments for post {post.id}: {e}")
+            comments = []
 
-    # Collect top-level comments
-    for comment in post.comments.list()[:15]:  # limit to first 15 comments
+        for comment in comments:
+            try:
                 post_data["comments"].append({
                     "id": comment.id,
                     "body": comment.body,
                     "score": comment.score,
                     "created_utc": comment.created_utc
                 })
+            except Exception as e:
+                logger.warning(f"Failed to parse a comment in post {post.id}: {e}")
 
         data.append(post_data)
 
-# --- Convert to JSON string ---
-json_output = json.dumps(data, indent=4)
+    # Wrap in top-level field
+    wrapped_output = {
+        "subreddit": subreddit_name,
+        "posts": data
+    }
 
-# --- Save Reddit data to .txt file ---
-txt_filename = "reddit_data.txt"
-with open(txt_filename, "w", encoding="utf-8") as f:
-    f.write(json_output)
+    # --- Generate Unix timestamp in milliseconds for unique file names ---
+    timestamp_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
 
-print(f"Saved Reddit data to {txt_filename}")
+    # Output data file
+    data_filename = os.path.join(OUTPUT_DIR, f"reddit_{subreddit_name}_{timestamp_ms}.txt")
+    try:
+        with open(data_filename, "w", encoding="utf-8") as f:
+            json.dump(wrapped_output, f, indent=4)
+        logger.info(f"Saved Reddit data → {data_filename}")
+    except Exception as e:
+        logger.error(f"Failed to save data file for r/{subreddit_name}: {e}")
+        continue
 
-# --- Create metadata JSON ---
-metadata = {
-    "title": "Reddit " + subreddit_name,
-    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_data.txt",  # Replace for local ingest path
+    # Output metadata file in .meta folder
+    metadata_filename = os.path.join(META_DIR, f"reddit_{subreddit_name}_{timestamp_ms}.txt.json")
+    metadata = {
+        "title": f"Reddit {subreddit_name}",
+        "backlink": data_filename,
         "language": "en",
         "classifications": [subreddit_name],
         "properties": {
-        "creation_date": int(datetime.now(timezone.utc).timestamp() * 1000)
+            "creation_date": timestamp_ms
+        }
     }
-}
 
-metadata_filename = "reddit_data.txt.json"
-with open(metadata_filename, "w", encoding="utf-8") as f:
+    try:
+        with open(metadata_filename, "w", encoding="utf-8") as f:
             json.dump(metadata, f, indent=4)
-
-print(f"Saved metadata to {metadata_filename}")
\ No newline at end of file
+        logger.info(f"Saved metadata → {metadata_filename}")
+    except Exception as e:
+        logger.error(f"Failed to save metadata for r/{subreddit_name}: {e}")
\ No newline at end of file

reddit_data.txt.json → output-example/.meta/reddit_worldnews_1764111620240.txt.json

 {
     "title": "Reddit worldnews",
-    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_data.txt",
+    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_worldnews_1764111620240.txt",
     "language": "en",
     "classifications": [
         "worldnews"
     ],
     "properties": {
-        "creation_date": 1764001787548
+        "creation_date": 1764111620240
     }
 }
\ No newline at end of file