Use document writer to output issues in right format

data_fetch.py

 import requests
 import json
-import pandas as pd
+
+from document import Document
+from document_writer import DocumentWriter
 
 def fetch_issues(owner, repo):
     '''Return a list of issues from OWNER/REPO.
@@ -18,21 +20,26 @@ def fetch_issues(owner, repo):
     else:
         raise f"Failed to fetch issues from {owner}/{repo}: status {response.status_code}"
 
-def serialize_issue(issue):
-    '''Serializes an issue object from the Github API to a dictionary
-    of key fields.
+def serialize_issue(issue, owner, repo):
+    '''Serializes an issue object from the Github API to a Document object.
     '''
-    return {
+    id = issue.get("id")
+
+    doc = Document()
+    doc.set_name(f"{owner}/{repo}/issues/{id}")
+    doc.set_title(issue.get("title"))
+    doc.set_attributes({
         "id": issue.get("id"),
         "number": issue.get("number"),
-        "title": issue.get("title"),
         "state": issue.get("state"),
-        "created_at": issue.get("created_at"),
+        "creation_date": issue.get("created_at"),
         "updated_at": issue.get("updated_at"),
-        "user_login": issue["user"]["login"] if "user" in issue and issue["user"] else None,
+        #"user_login": issue["user"]["login"] if "user" in issue and issue["user"] else None,
         "comments": issue.get("comments"),
         "html_url": issue.get("html_url")
-    }
+    })
+    doc.set_main_content(issue.get("body") or "")
+    return doc
 
 def main():
     # Repository details
@@ -40,14 +47,15 @@ def main():
     repo = "free-programming-books"
 
     issues = fetch_issues(owner, repo)
-    data = []
+    documents = []
     for issue in issues:
-        data.append(serialize_issue(issue))
+        documents.append(serialize_issue(issue, owner, repo))
+
+    print(f"Collected {len(documents)} documents")
 
-    # Convert to DataFrame
-    df = pd.DataFrame(data)
-    print(df.head())
-    df.to_json("github_data.json")
+    writer = DocumentWriter("output")
+    for doc in documents:
+        writer.write_document(doc)
 
 if __name__=="__main__":
     main()