add mutiple subreddit fetch and error handling

app.py

 import praw
 import json
 import os
+import logging
 from dotenv import load_dotenv
 from datetime import datetime, timezone
 
+# --- Setup logging ---
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
 # --- Load environment variables ---
 load_dotenv()
 
+# --- Validate environment variables ---
+try:
+    subreddits_env = os.getenv("SUBREDDITS")
+    if not subreddits_env:
+        raise ValueError("Environment variable SUBREDDITS is missing.")
+    subreddit_list = [s.strip() for s in subreddits_env.split(",")]
+
+    POST_LIMIT = int(os.getenv("POST_LIMIT", 10))
+    COMMENT_LIMIT = int(os.getenv("COMMENT_LIMIT", 10))
+
+    if POST_LIMIT < 1 or COMMENT_LIMIT < 0:
+        raise ValueError("POST_LIMIT and COMMENT_LIMIT must be positive integers.")
+
+    OUTPUT_DIR = os.getenv("OUTPUT_DIR", ".")
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+    # Create .meta folder inside OUTPUT_DIR
+    META_DIR = os.path.join(OUTPUT_DIR, ".meta")
+    os.makedirs(META_DIR, exist_ok=True)
+
+except Exception as e:
+    logger.error(f"Invalid configuration: {e}")
+    raise SystemExit(1)
+
 # --- Reddit API credentials ---
-reddit = praw.Reddit(
-    client_id=os.getenv("REDDIT_CLIENT_ID"),
-    client_secret=os.getenv("REDDIT_CLIENT_SECRET"),
-    user_agent="MyRedditScraper/1.0"
-)
+try:
+    reddit = praw.Reddit(
+        client_id=os.getenv("REDDIT_CLIENT_ID"),
+        client_secret=os.getenv("REDDIT_CLIENT_SECRET"),
+        user_agent="MyRedditScraper/1.0"
+    )
+    reddit.user.me()  # Test authentication
+except Exception as e:
+    logger.error(f"Failed to authenticate with Reddit API: {e}")
+    raise SystemExit(1)
 
-# --- Choose subreddit and fetch posts ---
-subreddit_name = "worldnews"
-subreddit = reddit.subreddit(subreddit_name)
+# --- Process each subreddit ---
+for subreddit_name in subreddit_list:
+    logger.info(f"Processing subreddit: {subreddit_name}")
+    data = []
 
-data = []
+    try:
+        subreddit = reddit.subreddit(subreddit_name)
+    except Exception as e:
+        logger.error(f"Cannot access subreddit '{subreddit_name}': {e}")
+        continue
 
-for post in subreddit.hot(limit=5):  # get top 5 hot posts
-    post.comments.replace_more(limit=0)  # remove 'MoreComments' objects
+    # Fetch posts
+    try:
+        posts = subreddit.hot(limit=POST_LIMIT)
+    except Exception as e:
+        logger.error(f"Failed to retrieve posts for r/{subreddit_name}: {e}")
+        continue
 
-    post_data = {
-        "id": post.id,
-        "title": post.title,
-        "score": post.score,
-        "url": post.url,
-        "num_comments": post.num_comments,
-        "created_utc": post.created_utc,
-        "comments": []
-    }
+    # Process posts
+    for post in posts:
+        try:
+            post.comments.replace_more(limit=0)
+        except Exception as e:
+            logger.warning(f"Comments for post {post.id} not fully loaded: {e}")
+
+        try:
+            post_data = {
+                "id": post.id,
+                "title": post.title,
+                "score": post.score,
+                "url": post.url,
+                "num_comments": post.num_comments,
+                "created_utc": post.created_utc,
+                "comments": []
+            }
+        except Exception as e:
+            logger.error(f"Failed to read post data for {post.id}: {e}")
+            continue
+
+        # Fetch comments
+        try:
+            comments = post.comments.list()[:COMMENT_LIMIT]
+        except Exception as e:
+            logger.warning(f"Failed to load comments for post {post.id}: {e}")
+            comments = []
+
+        for comment in comments:
+            try:
+                post_data["comments"].append({
+                    "id": comment.id,
+                    "body": comment.body,
+                    "score": comment.score,
+                    "created_utc": comment.created_utc
+                })
+            except Exception as e:
+                logger.warning(f"Failed to parse a comment in post {post.id}: {e}")
 
-    # Collect top-level comments
-    for comment in post.comments.list()[:15]:  # limit to first 15 comments
-        post_data["comments"].append({
-            "id": comment.id,
-            "body": comment.body,
-            "score": comment.score,
-            "created_utc": comment.created_utc
-        })
-    
-    data.append(post_data)
-
-# --- Convert to JSON string ---
-json_output = json.dumps(data, indent=4)
-
-# --- Save Reddit data to .txt file ---
-txt_filename = "reddit_data.txt"
-with open(txt_filename, "w", encoding="utf-8") as f:
-    f.write(json_output)
-
-print(f"Saved Reddit data to {txt_filename}")
-
-# --- Create metadata JSON ---
-metadata = {
-    "title": "Reddit " + subreddit_name,
-    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_data.txt",  # Replace for local ingest path
-    "language": "en",
-    "classifications": [subreddit_name],
-    "properties": {
-        "creation_date": int(datetime.now(timezone.utc).timestamp() * 1000)
+        data.append(post_data)
+
+    # Wrap in top-level field
+    wrapped_output = {
+        "subreddit": subreddit_name,
+        "posts": data
     }
-}
 
-metadata_filename = "reddit_data.txt.json"
-with open(metadata_filename, "w", encoding="utf-8") as f:
-    json.dump(metadata, f, indent=4)
+    # --- Generate Unix timestamp in milliseconds for unique file names ---
+    timestamp_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
+
+    # Output data file
+    data_filename = os.path.join(OUTPUT_DIR, f"reddit_{subreddit_name}_{timestamp_ms}.txt")
+    try:
+        with open(data_filename, "w", encoding="utf-8") as f:
+            json.dump(wrapped_output, f, indent=4)
+        logger.info(f"Saved Reddit data → {data_filename}")
+    except Exception as e:
+        logger.error(f"Failed to save data file for r/{subreddit_name}: {e}")
+        continue
+
+    # Output metadata file in .meta folder
+    metadata_filename = os.path.join(META_DIR, f"reddit_{subreddit_name}_{timestamp_ms}.txt.json")
+    metadata = {
+        "title": f"Reddit {subreddit_name}",
+        "backlink": data_filename,
+        "language": "en",
+        "classifications": [subreddit_name],
+        "properties": {
+            "creation_date": timestamp_ms
+        }
+    }
 
-print(f"Saved metadata to {metadata_filename}")
\ No newline at end of file
+    try:
+        with open(metadata_filename, "w", encoding="utf-8") as f:
+            json.dump(metadata, f, indent=4)
+        logger.info(f"Saved metadata → {metadata_filename}")
+    except Exception as e:
+        logger.error(f"Failed to save metadata for r/{subreddit_name}: {e}")
\ No newline at end of file

reddit_data.txt.json → output-example/.meta/reddit_worldnews_1764111620240.txt.json

 {
     "title": "Reddit worldnews",
-    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_data.txt",
+    "backlink": "C:\\Users\\USER\\Desktop\\Semester-1\\ST-SOP_Software-Project\\BigData4Biz\\ingest\\reddit_worldnews_1764111620240.txt",
     "language": "en",
     "classifications": [
         "worldnews"
     ],
     "properties": {
-        "creation_date": 1764001787548
+        "creation_date": 1764111620240
     }
 }
\ No newline at end of file