bot main file

bot.py

+import os
+import json
+import uuid
+import requests
+from datetime import datetime
+
+# ------------------------------
+# Document class
+# ------------------------------
+class Document:
+    def __init__(self):
+        self.name = ""
+        self.title = ""
+        self.attributes = {}
+        self.main_content = ""
+
+    def set_name(self, name): self.name = name
+    def get_name(self): return self.name
+    def set_title(self, title): self.title = title
+    def get_title(self): return self.title
+    def set_attributes(self, attributes): self.attributes = attributes
+    def get_attributes(self): return self.attributes
+    def set_main_content(self, content): self.main_content = content
+    def get_main_content(self): return self.main_content
+
+# ------------------------------
+# DocumentWriter class
+# ------------------------------
+class DocumentWriter:
+    def __init__(self, target_directory):
+        self.target = target_directory
+        self.plugin_id = str(uuid.uuid4())
+        os.makedirs(self.target, exist_ok=True)
+        os.makedirs(os.path.join(self.target, ".meta"), exist_ok=True)
+
+    def write_document(self, doc: Document):
+        main_file = os.path.join(self.target, doc.get_name())
+        os.makedirs(os.path.dirname(main_file), exist_ok=True)
+        with open(main_file, "w", encoding="utf-8") as f:
+            f.write(doc.get_main_content())
+
+        meta_file = os.path.join(self.target, ".meta", doc.get_name() + ".json")
+        os.makedirs(os.path.dirname(meta_file), exist_ok=True)
+
+        attrs = doc.get_attributes()
+        creation_date = self._iso_to_milliseconds(attrs.get("creation_date"))
+        updated_date = self._iso_to_milliseconds(attrs.get("updated_at"))
+        metadata = {
+            "title": doc.get_title(),
+            "backlink": main_file,
+            "properties": attrs,
+            "external_link": attrs.get("html_url", ""),
+            "creation_date": creation_date,
+            "last_modify_date": updated_date if updated_date else creation_date
+        }
+
+        with open(meta_file, "w", encoding="utf-8") as f:
+            json.dump(metadata, f, ensure_ascii=False, indent=2)
+
+    def _iso_to_milliseconds(self, iso_date):
+        if not iso_date:
+            return int(datetime.now().timestamp() * 1000)
+        if isinstance(iso_date, int):
+            return iso_date
+        try:
+            if iso_date.endswith("Z"): iso_date = iso_date[:-1] + "+00:00"
+            return int(datetime.fromisoformat(iso_date).timestamp() * 1000)
+        except:
+            return int(datetime.now().timestamp() * 1000)
+
+    def get_plugin_id(self):
+        return self.plugin_id
+
+# ------------------------------
+# GitHub API functions
+# ------------------------------
+GITHUB_HEADERS = {
+    "Accept": "application/vnd.github+json",
+    "User-Agent": "BigData4Biz-Ingestion-Plugin/1.0"
+}
+
+def fetch_issues(owner, repo):
+    """
+    Fetch all issues (excluding pull requests) with pagination.
+    """
+    issues = []
+    page = 1
+    while True:
+        url = f"https://api.github.com/repos/{owner}/{repo}/issues?per_page=100&page={page}"
+        response = requests.get(url, headers=GITHUB_HEADERS, timeout=30)
+        response.raise_for_status()
+        data = [i for i in response.json() if "pull_request" not in i]
+        if not data:
+            break
+        issues.extend(data)
+        page += 1
+    return issues
+
+def serialize_issue(issue, owner, repo):
+    issue_number = issue.get("number", 0)
+    doc = Document()
+    doc.set_name(f"{owner}/{repo}/issues/{issue_number}.txt")
+    doc.set_title(issue.get("title", "Untitled Issue"))
+
+    user_info = issue.get("user", {})
+    doc.set_attributes({
+        "id": issue.get("id"),
+        "number": issue_number,
+        "state": issue.get("state"),
+        "creation_date": issue.get("created_at"),
+        "updated_at": issue.get("updated_at"),
+        "user_login": user_info.get("login"),
+        "user_id": user_info.get("id"),
+        "comments": issue.get("comments"),
+        "html_url": issue.get("html_url"),
+        "labels": [label.get("name") for label in issue.get("labels", [])],
+        "assignees": [assignee.get("login") for assignee in issue.get("assignees", [])]
+    })
+
+    content_parts = [
+        f"Issue #{issue_number}: {issue.get('title', '')}",
+        "=" * 50,
+        f"State: {issue.get('state', '')}",
+        f"Created: {issue.get('created_at', '')}",
+        f"Updated: {issue.get('updated_at', '')}",
+        f"Author: {user_info.get('login', 'Unknown')}",
+        f"Comments: {issue.get('comments', 0)}"
+    ]
+
+    labels = [label.get("name") for label in issue.get("labels", [])]
+    if labels: content_parts.append(f"Labels: {', '.join(labels)}")
+    assignees = [assignee.get("login") for assignee in issue.get("assignees", [])]
+    if assignees: content_parts.append(f"Assignees: {', '.join(assignees)}")
+
+    content_parts.extend([
+        "\nDescription:",
+        "=" * 30,
+        issue.get("body") or "No description provided.",
+        f"\nGitHub URL: {issue.get('html_url', '')}"
+    ])
+
+    doc.set_main_content("\n".join(content_parts))
+    return doc
+
+# ------------------------------
+# Configuration creation
+# ------------------------------
+def create_user_config(platform_config, source_path, plugin_id):
+    config = {
+        "platform": platform_config,
+        "plugin_id": plugin_id,
+        "source_paths": [source_path]
+    }
+    with open("user-config.json", "w", encoding="utf-8") as f:
+        json.dump(config, f, indent=2, ensure_ascii=False)
+    print(f"Configuration file created: user-config.json")
+    print(f"Plugin ID: {plugin_id}")
+
+# ------------------------------
+# Main execution
+# ------------------------------
+def main():
+    # Set the repo you want
+    owner = "straight-tamago"
+    repo = "misaka26"
+
+    platform_config = {
+        "url": "YOUR_PLATFORM_URL_HERE",
+        "username": "YOUR_USERNAME_HERE",
+        "tenant": "YOUR_TENANT_HERE",
+        "password": "YOUR_PASSWORD_HERE"
+    }
+
+    print(f"Fetching issues from {owner}/{repo}...")
+    try:
+        issues = fetch_issues(owner, repo)
+        print(f"Collected {len(issues)} issues")
+
+        output_dir = "output"
+        writer = DocumentWriter(output_dir)
+        documents = [serialize_issue(issue, owner, repo) for issue in issues]
+
+        print("Writing documents and metadata...")
+        for doc in documents:
+            writer.write_document(doc)
+
+        create_user_config(platform_config, output_dir, writer.get_plugin_id())
+
+        print(f"\nSuccessfully processed {len(documents)} issues")
+        print(f" Output directory: {output_dir}")
+        print(f" Plugin ID: {writer.get_plugin_id()}")
+        print(f"Configuration: user-config.json")
+
+        if documents:
+            print(f"\nSample issues processed:")
+            for i, doc in enumerate(documents[:3]):
+                print(f"  {i + 1}. #{doc.get_attributes().get('number')}: {doc.get_title()}")
+            if len(documents) > 3:
+                print(f"  ... and {len(documents) - 3} more")
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())