Update data_fetch.py

data_fetch.py

-import requests
+import os
 import json
-
+import uuid
+import requests
 from document import Document
 from document_writer import DocumentWriter
 
+
 def fetch_issues(owner, repo):
-    '''Return a list of issues from OWNER/REPO.
-    '''
+    '''Return a list of issues from OWNER/REPO.'''
     # GitHub API endpoint
     url = f"https://api.github.com/repos/{owner}/{repo}/issues"
     headers = {
         "Accept": "application/vnd.github+json",
         # "Authorization": "Bearer YOUR_GITHUB_TOKEN"  # optional
+        "User-Agent": "BigData4Biz-Ingestion-Plugin/1.0"
     }
-    # Fetch issues
-    response = requests.get(url, headers=headers)
-    if response.status_code == 200:
+
+    # Fetch issues with error handling
+    try:
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()  # Raises an HTTPError for bad status codes
         return response.json()
-    else:
-        raise f"Failed to fetch issues from {owner}/{repo}: status {response.status_code}"
+    except requests.exceptions.RequestException as e:
+        raise Exception(f"Failed to fetch issues from {owner}/{repo}: {e}")
+
 
 def serialize_issue(issue, owner, repo):
-    '''Serializes an issue object from the Github API to a Document object.
-    '''
-    id = issue.get("id")
+    '''Serializes an issue object from the Github API to a Document object.'''
+    issue_id = issue.get("id")
+    issue_number = issue.get("number")
 
     doc = Document()
-    doc.set_name(f"{owner}/{repo}/issues/{id}")
-    doc.set_title(issue.get("title"))
+    # Use issue number instead of ID for better readability
+    doc.set_name(f"{owner}/{repo}/issues/{issue_number}.txt")
+    doc.set_title(issue.get("title", "Untitled Issue"))
+
+    # Enhanced attributes with user information
+    user_info = issue.get("user", {})
     doc.set_attributes({
-        "id": issue.get("id"),
-        "number": issue.get("number"),
+        "id": issue_id,
+        "number": issue_number,
         "state": issue.get("state"),
         "creation_date": issue.get("created_at"),
         "updated_at": issue.get("updated_at"),
-        #"user_login": issue["user"]["login"] if "user" in issue and issue["user"] else None,
+        "user_login": user_info.get("login"),
+        "user_id": user_info.get("id"),
         "comments": issue.get("comments"),
-        "html_url": issue.get("html_url")
+        "html_url": issue.get("html_url"),
+        "labels": [label.get("name") for label in issue.get("labels", [])],
+        "assignees": [assignee.get("login") for assignee in issue.get("assignees", [])]
     })
-    doc.set_main_content(issue.get("body") or "")
+
+    # Build comprehensive content
+    content_parts = []
+
+    # Issue header
+    content_parts.append(f"Issue #{issue_number}: {issue.get('title', '')}")
+    content_parts.append("=" * 50)
+
+    # Basic info
+    content_parts.append(f"State: {issue.get('state', '')}")
+    content_parts.append(f"Created: {issue.get('created_at', '')}")
+    content_parts.append(f"Updated: {issue.get('updated_at', '')}")
+    content_parts.append(f"Author: {user_info.get('login', 'Unknown')}")
+    content_parts.append(f"Comments: {issue.get('comments', 0)}")
+
+    # Labels
+    labels = [label.get("name") for label in issue.get("labels", [])]
+    if labels:
+        content_parts.append(f"Labels: {', '.join(labels)}")
+
+    # Assignees
+    assignees = [assignee.get("login") for assignee in issue.get("assignees", [])]
+    if assignees:
+        content_parts.append(f"Assignees: {', '.join(assignees)}")
+
+    # Body content
+    content_parts.append("\nDescription:")
+    content_parts.append("=" * 30)
+    content_parts.append(issue.get("body") or "No description provided.")
+
+    # URL reference
+    content_parts.append(f"\nGitHub URL: {issue.get('html_url', '')}")
+
+    doc.set_main_content("\n".join(content_parts))
     return doc
 
+
+def create_user_config(platform_config, source_path, plugin_id):
+    """
+    Create user-config.json file for BigData4Biz ingestion protocol.
+
+    Args:
+        platform_config: Dictionary with platform configuration
+        source_path: Path to the data source directory
+        plugin_id: Unique UUID for the plugin
+    """
+    config = {
+        "platform": platform_config,
+        "plugin_id": plugin_id,
+        "source_paths": [source_path]
+    }
+
+    with open("user-config.json", "w", encoding='utf-8') as f:
+        json.dump(config, f, indent=2, ensure_ascii=False)
+
+    print(f"Configuration file created: user-config.json")
+    print(f"Plugin ID: {plugin_id}")
+
+
 def main():
     # Repository details
     owner = "EbookFoundation"
     repo = "free-programming-books"
 
-    issues = fetch_issues(owner, repo)
-    documents = []
-    for issue in issues:
-        documents.append(serialize_issue(issue, owner, repo))
+    # Platform configuration - UPDATE THESE WITH YOUR ACTUAL CREDENTIALS
+    platform_config = {
+        "url": "YOUR_PLATFORM_URL_HERE",  # e.g., "https://your-platform.example.com"
+        "username": "YOUR_USERNAME_HERE",
+        "tenant": "YOUR_TENANT_HERE",
+        "password": "YOUR_PASSWORD_HERE"
+    }
+
+    print(f"Fetching issues from {owner}/{repo}...")
+
+    try:
+        # Fetch issues from GitHub
+        issues = fetch_issues(owner, repo)
+        documents = []
+
+        for issue in issues:
+            documents.append(serialize_issue(issue, owner, repo))
+
+        print(f"Collected {len(documents)} issues")
+
+        # Initialize document writer
+        output_dir = "output"
+        writer = DocumentWriter(output_dir)
+
+        # Write all documents
+        print("Writing documents and metadata...")
+        for doc in documents:
+            writer.write_document(doc)
+
+        # Create configuration file
+        create_user_config(platform_config, output_dir, writer.get_plugin_id())
+
+        print(f"\nSuccessfully processed {len(documents)} issues")
+        print(f" Output directory: {output_dir}")
+        print(f" Plugin ID: {writer.get_plugin_id()}")
+        print(f"Configuration: user-config.json")
+
+        # Show sample of what was processed
+        if documents:
+            print(f"\nSample issues processed:")
+            for i, doc in enumerate(documents[:3]):  # Show first 3
+                print(f"  {i + 1}. #{doc.get_attributes().get('number')}: {doc.get_title()}")
+            if len(documents) > 3:
+                print(f"  ... and {len(documents) - 3} more")
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
 
-    print(f"Collected {len(documents)} documents")
+    return 0
 
-    writer = DocumentWriter("output")
-    for doc in documents:
-        writer.write_document(doc)
 
-if __name__=="__main__":
-    main()
+if __name__ == "__main__":
+    exit(main())