document_writer

document_writer.py

 import os
 import json
+import uuid
+from datetime import datetime
 from document import Document
 
+
 class DocumentWriter():
     def __init__(self, target_directory) -> None:
         self.target = target_directory
+        self.plugin_id = str(uuid.uuid4())  # Generate unique plugin ID
         os.makedirs(self.target, exist_ok=True)
         os.makedirs(os.path.join(self.target, ".meta"), exist_ok=True)
 
     def write_document(self, document: Document) -> None:
+        """
+        Write the document to a file and create corresponding metadata file
+        with all required fields for BigData4Biz ingestion protocol.
+        """
         main_file_name = os.path.join(self.target, document.get_name())
         os.makedirs(os.path.dirname(main_file_name), exist_ok=True)
-        with open(main_file_name, "w") as f:
+
+        # Write main content with UTF-8 encoding
+        with open(main_file_name, "w", encoding='utf-8') as f:
             f.write(document.get_main_content())
 
         meta_file_name = os.path.join(self.target, ".meta", document.get_name() + ".json")
         os.makedirs(os.path.dirname(meta_file_name), exist_ok=True)
-        with open(meta_file_name, "w") as f:
-            f.write(json.dumps({
-                "title": document.get_title(),
-                "backlink": main_file_name,
-                "properties": document.get_attributes()
-            }))
+
+        # Get document attributes
+        attributes = document.get_attributes()
+
+        # Extract and convert dates to milliseconds
+        creation_date = self._iso_to_milliseconds(attributes.get("creation_date"))
+        updated_date = self._iso_to_milliseconds(attributes.get("updated_at"))
+
+        # Use updated_date for last_modify_date, fallback to creation_date
+        last_modify_date = updated_date if updated_date else creation_date
+
+        # Get external_link from attributes (GitHub HTML URL)
+        external_link = attributes.get("html_url", "")
+
+        # Create metadata with all required fields
+        metadata = {
+            "title": document.get_title(),
+            "backlink": main_file_name,
+            "properties": attributes,
+            # Required fields for BigData4Biz protocol
+            "external_link": external_link,
+            "creation_date": creation_date,
+            "last_modify_date": last_modify_date
+        }
+
+        # Write metadata file with UTF-8 encoding
+        with open(meta_file_name, "w", encoding='utf-8') as f:
+            json.dump(metadata, f, ensure_ascii=False, indent=2)
+
+    def _iso_to_milliseconds(self, iso_date_string):
+        """
+        Convert ISO date string to milliseconds since epoch.
+
+        Args:
+            iso_date_string: ISO format date string from GitHub API
+
+        Returns:
+            int: Milliseconds since epoch, or current time if parsing fails
+        """
+        if not iso_date_string:
+            return int(datetime.now().timestamp() * 1000)
+
+        try:
+            # Handle both 'Z' suffix and timezone offsets
+            if iso_date_string.endswith('Z'):
+                iso_date_string = iso_date_string[:-1] + '+00:00'
+
+            # Parse ISO format string to datetime object
+            dt = datetime.fromisoformat(iso_date_string)
+
+            # Convert to milliseconds since epoch
+            return int(dt.timestamp() * 1000)
+
+        except (ValueError, TypeError) as e:
+            print(f"Warning: Could not parse date '{iso_date_string}': {e}")
+            return int(datetime.now().timestamp() * 1000)
+
+    def get_plugin_id(self):
+        """
+        Get the generated plugin ID for configuration.
+
+        Returns:
+            str: Unique UUID for this plugin instance
+        """
+        return self.plugin_id
+
+    def write_documents(self, documents):
+        """
+        Write multiple documents to files.
+
+        Args:
+            documents: List of Document objects to write
+        """
+        for doc in documents:
+            self.write_document(doc)
+
+    def get_target_directory(self):
+        """
+        Get the target directory path.
+
+        Returns:
+            str: Path to the target directory
+        """
+        return self.target