switch to sqlite for backups also

src/core/backup_database.py

@@ -0,0 +1,762 @@
+import sqlite3
+import logging
+import json
+import threading
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+
+logger = logging.getLogger(__name__)
+
+class BackupDatabase:
+    """Manages the SQLite database for local Discord backups."""
+    
+    def __init__(self, db_path: Path | str):
+        self.db_path = Path(db_path)
+        self._lock = threading.Lock()
+        self._init_db()
+
+    def _get_conn(self):
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def _init_db(self):
+        """Initializes the database schema."""
+        with self._lock:
+            conn = self._get_conn()
+            try:
+                # Guild Profile
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS guild_profile (
+                        id TEXT PRIMARY KEY,
+                        name TEXT,
+                        description TEXT,
+                        icon_file TEXT,
+                        icon_url TEXT,
+                        banner_file TEXT,
+                        banner_url TEXT,
+                        owner_id TEXT,
+                        last_backup TEXT,
+                        ignore_channels TEXT
+                    )
+                """)
+
+                # Roles
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS roles (
+                        id TEXT PRIMARY KEY,
+                        name TEXT,
+                        color INTEGER,
+                        position INTEGER,
+                        permissions TEXT,
+                        hoist INTEGER,
+                        mentionable INTEGER
+                    )
+                """)
+
+                # Channels
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS channels (
+                        id TEXT PRIMARY KEY,
+                        name TEXT,
+                        type INTEGER,
+                        position INTEGER,
+                        category_id TEXT,
+                        topic TEXT,
+                        nsfw INTEGER
+                    )
+                """)
+
+                # Channel Permissions
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS permissions (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        channel_id TEXT,
+                        target_id TEXT,
+                        target_type TEXT,
+                        allow INTEGER,
+                        deny INTEGER
+                    )
+                """)
+                conn.execute("CREATE INDEX IF NOT EXISTS idx_permissions_chan ON permissions(channel_id)")
+
+                # Users (Author cache)
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS users (
+                        id TEXT PRIMARY KEY,
+                        username TEXT,
+                        display_name TEXT,
+                        avatar_file TEXT,
+                        avatar_url TEXT,
+                        roles TEXT
+                    )
+                """)
+
+                # Messages
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS messages (
+                        id TEXT PRIMARY KEY,
+                        channel_id TEXT,
+                        author_id TEXT,
+                        content TEXT,
+                        timestamp TEXT,
+                        type INTEGER,
+                        message_reference TEXT,
+                        is_pinned INTEGER,
+                        extra_data TEXT
+                    )
+                """)
+                conn.execute("CREATE INDEX IF NOT EXISTS idx_messages_channel ON messages(channel_id)")
+                conn.execute("CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp)")
+
+                # Attachments
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS attachments (
+                        id TEXT PRIMARY KEY,
+                        message_id TEXT,
+                        filename TEXT,
+                        size INTEGER,
+                        url TEXT,
+                        content_type TEXT,
+                        local_hash TEXT
+                    )
+                """)
+                conn.execute("CREATE INDEX IF NOT EXISTS idx_attachments_msg ON attachments(message_id)")
+
+                # Embeds
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS embeds (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        message_id TEXT,
+                        title TEXT,
+                        description TEXT,
+                        url TEXT,
+                        color INTEGER,
+                        timestamp TEXT,
+                        thumbnail_url TEXT,
+                        image_url TEXT,
+                        author_name TEXT,
+                        author_url TEXT,
+                        author_icon_url TEXT,
+                        footer_text TEXT,
+                        footer_icon_url TEXT,
+                        fields TEXT
+                    )
+                """)
+                conn.execute("CREATE INDEX IF NOT EXISTS idx_embeds_msg ON embeds(message_id)")
+
+                # Reactions
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS reactions (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        message_id TEXT,
+                        emoji_id TEXT,
+                        emoji_name TEXT,
+                        count INTEGER
+                    )
+                """)
+                conn.execute("CREATE INDEX IF NOT EXISTS idx_reactions_msg ON reactions(message_id)")
+
+                # Message Stickers
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS message_stickers (
+                        message_id TEXT,
+                        sticker_id TEXT,
+                        name TEXT,
+                        url TEXT,
+                        format_type INTEGER,
+                        local_hash TEXT,
+                        PRIMARY KEY (message_id, sticker_id)
+                    )
+                """)
+                conn.execute("CREATE INDEX IF NOT EXISTS idx_message_stickers_msg ON message_stickers(message_id)")
+
+                # Threads
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS threads (
+                        id TEXT PRIMARY KEY,
+                        name TEXT,
+                        type INTEGER,
+                        parent_id TEXT,
+                        message_count INTEGER,
+                        member_count INTEGER,
+                        archived INTEGER,
+                        archive_timestamp TEXT,
+                        auto_archive_duration INTEGER,
+                        locked INTEGER,
+                        applied_tags TEXT
+                    )
+                """)
+                conn.execute("CREATE INDEX IF NOT EXISTS idx_threads_parent ON threads(parent_id)")
+
+                # Forum Tags (Definitions for a forum channel)
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS forum_tags (
+                        id TEXT PRIMARY KEY,
+                        forum_id TEXT,
+                        name TEXT,
+                        moderated INTEGER,
+                        emoji_id TEXT,
+                        emoji_name TEXT
+                    )
+                """)
+                conn.execute("CREATE INDEX IF NOT EXISTS idx_forum_tags_forum ON forum_tags(forum_id)")
+
+                # Media Pool (CAS)
+                # Maps content hashes to local storage paths
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS media_pool (
+                        hash TEXT PRIMARY KEY,
+                        local_path TEXT,
+                        size INTEGER,
+                        mime_type TEXT,
+                        first_seen_url TEXT
+                    )
+                """)
+                
+                # Server Assets (Emojis, Stickers, etc.)
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS server_assets (
+                        id TEXT PRIMARY KEY,
+                        name TEXT,
+                        type TEXT,
+                        filename TEXT,
+                        url TEXT,
+                        mime_type TEXT
+                    )
+                """)
+
+                conn.commit()
+            finally:
+                conn.close()
+
+    def set_guild_profile(self, data: Dict[str, Any]):
+        with self._lock:
+            conn = self._get_conn()
+            conn.execute("""
+                INSERT OR REPLACE INTO guild_profile (id, name, description, icon_file, icon_url, banner_file, banner_url, owner_id, last_backup, ignore_channels)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """, (
+                str(data.get("id")), data.get("name"), data.get("description"),
+                data.get("icon_file"), data.get("icon_url"), 
+                data.get("banner_file"), data.get("banner_url"), 
+                str(data.get("owner_id")),
+                data.get("last_backup"), json.dumps(data.get("ignore_channels", []))
+            ))
+            conn.commit()
+            conn.close()
+
+    def get_guild_profile(self) -> Optional[Dict[str, Any]]:
+        with self._lock:
+            conn = self._get_conn()
+            row = conn.execute("SELECT * FROM guild_profile LIMIT 1").fetchone()
+            conn.close()
+            if row:
+                data = dict(row)
+                if data.get("ignore_channels"):
+                    data["ignore_channels"] = json.loads(data["ignore_channels"])
+                else:
+                    data["ignore_channels"] = []
+                return data
+            return None
+
+    def save_roles(self, roles: List[Dict[str, Any]]):
+        with self._lock:
+            conn = self._get_conn()
+            # Ensure complex fields are strings if they aren't already
+            formatted = []
+            for r in roles:
+                formatted.append({
+                    "id": str(r["id"]),
+                    "name": r["name"],
+                    "color": r["color"],
+                    "position": r["position"],
+                    "permissions": str(r["permissions"]),
+                    "hoist": 1 if r["hoist"] else 0,
+                    "mentionable": 1 if r["mentionable"] else 0
+                })
+
+            conn.executemany("""
+                INSERT OR REPLACE INTO roles (id, name, color, position, permissions, hoist, mentionable)
+                VALUES (:id, :name, :color, :position, :permissions, :hoist, :mentionable)
+            """, formatted)
+            conn.commit()
+            conn.close()
+
+    def save_channels(self, channels: List[Dict[str, Any]]):
+        with self._lock:
+            conn = self._get_conn()
+            conn.executemany("""
+                INSERT OR REPLACE INTO channels (id, name, type, position, category_id, topic, nsfw)
+                VALUES (:id, :name, :type, :position, :category_id, :topic, :nsfw)
+            """, channels)
+            conn.commit()
+            conn.close()
+
+    def save_permissions(self, permissions: List[Dict[str, Any]]):
+        """Saves a batch of channel permission overwrites."""
+        with self._lock:
+            conn = self._get_conn()
+            try:
+                conn.executemany("""
+                    INSERT INTO permissions (channel_id, target_id, target_type, allow, deny)
+                    VALUES (:channel_id, :target_id, :target_type, :allow, :deny)
+                """, permissions)
+                conn.commit()
+            finally:
+                conn.close()
+
+    def save_users(self, users: List[Dict[str, Any]]):
+        """Saves users to the author cache."""
+        with self._lock:
+            conn = self._get_conn()
+            conn.executemany("""
+                INSERT OR REPLACE INTO users (id, username, display_name, avatar_file, avatar_url, roles)
+                VALUES (:id, :username, :display_name, :avatar_file, :avatar_url, :roles)
+            """, users)
+            conn.commit()
+            conn.close()
+
+    def save_server_assets(self, assets: List[Dict[str, Any]]):
+        """Saves a batch of server assets (emojis, stickers) to the database."""
+        with self._lock:
+            conn = self._get_conn()
+            formatted = []
+            for a in assets:
+                formatted.append({
+                    "id": str(a["id"]),
+                    "name": a.get("name"),
+                    "type": a.get("type"),
+                    "filename": a.get("filename"),
+                    "url": a.get("url"),
+                    "mime_type": a.get("mime_type")
+                })
+
+            conn.executemany("""
+                INSERT OR REPLACE INTO server_assets (id, name, type, filename, url, mime_type)
+                VALUES (:id, :name, :type, :filename, :url, :mime_type)
+            """, formatted)
+            conn.commit()
+            conn.close()
+
+    def save_threads(self, threads: List[Dict[str, Any]]):
+        """Saves metadata for threads to the database."""
+        with self._lock:
+            conn = self._get_conn()
+            try:
+                conn.executemany("""
+                    INSERT OR REPLACE INTO threads (id, name, type, parent_id, message_count, member_count, archived, archive_timestamp, auto_archive_duration, locked, applied_tags)
+                    VALUES (:id, :name, :type, :parent_id, :message_count, :member_count, :archived, :archive_timestamp, :auto_archive_duration, :locked, :applied_tags)
+                """, threads)
+                conn.commit()
+            finally:
+                conn.close()
+
+    def save_forum_tags(self, tags: List[Dict[str, Any]]):
+        """Saves definitions for forum tags."""
+        with self._lock:
+            conn = self._get_conn()
+            try:
+                conn.executemany("""
+                    INSERT OR REPLACE INTO forum_tags (id, forum_id, name, moderated, emoji_id, emoji_name)
+                    VALUES (:id, :forum_id, :name, :moderated, :emoji_id, :emoji_name)
+                """, tags)
+                conn.commit()
+            finally:
+                conn.close()
+
+    def save_messages_batch(self, messages: List[Dict[str, Any]]):
+        """Batch inserts messages and their attachments."""
+        with self._lock:
+            conn = self._get_conn()
+            try:
+                # Insert messages
+                conn.executemany("""
+                    INSERT OR REPLACE INTO messages (id, channel_id, author_id, content, timestamp, type, message_reference, is_pinned, extra_data)
+                    VALUES (:id, :channel_id, :author_id, :content, :timestamp, :type, :message_reference, :is_pinned, :extra_data)
+                """, messages)
+                
+                # Extract attachments, reactions, and stickers
+                all_attachments = []
+                all_reactions = []
+                all_stickers = []
+                
+                for msg in messages:
+                    # Attachments
+                    if "attachments" in msg:
+                        for att in msg["attachments"]:
+                            att["message_id"] = msg["id"]
+                            all_attachments.append(att)
+                    
+                    # Reactions
+                    if "reactions" in msg and msg["reactions"]:
+                        for rea in msg["reactions"]:
+                            all_reactions.append({
+                                "message_id": msg["id"],
+                                "emoji_id": str(rea["emoji_id"]) if rea.get("emoji_id") else None,
+                                "emoji_name": rea.get("emoji_name"),
+                                "count": rea.get("count", 0)
+                            })
+                    
+                    # Stickers
+                    if "stickers" in msg and msg["stickers"]:
+                        for st in msg["stickers"]:
+                            all_stickers.append({
+                                "message_id": msg["id"],
+                                "sticker_id": str(st["id"]),
+                                "name": st.get("name"),
+                                "url": st.get("url"),
+                                "format_type": st.get("format_type"),
+                                "local_hash": st.get("local_hash")
+                            })
+                            
+                if all_attachments:
+                    conn.executemany("""
+                        INSERT OR REPLACE INTO attachments (id, message_id, filename, size, url, content_type, local_hash)
+                        VALUES (:id, :message_id, :filename, :size, :url, :content_type, :local_hash)
+                    """, all_attachments)
+
+                # Save Embeds (Normalized with JSON Fields)
+                for msg in messages:
+                    if "embeds" in msg and msg["embeds"]:
+                        for emb in msg["embeds"]:
+                            # Insert top-level embed
+                            conn.execute("""
+                                INSERT INTO embeds (
+                                    message_id, title, description, url, color, timestamp,
+                                    thumbnail_url, image_url, author_name, author_url, 
+                                    author_icon_url, footer_text, footer_icon_url, fields
+                                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                            """, (
+                                msg["id"], emb.get("title"), emb.get("description"), emb.get("url"),
+                                emb.get("color"), emb.get("timestamp"),
+                                emb.get("thumbnail", {}).get("url") if isinstance(emb.get("thumbnail"), dict) else None,
+                                emb.get("image", {}).get("url") if isinstance(emb.get("image"), dict) else None,
+                                emb.get("author", {}).get("name") if isinstance(emb.get("author"), dict) else None,
+                                emb.get("author", {}).get("url") if isinstance(emb.get("author"), dict) else None,
+                                emb.get("author", {}).get("icon_url") if isinstance(emb.get("author"), dict) else None,
+                                emb.get("footer", {}).get("text") if isinstance(emb.get("footer"), dict) else None,
+                                emb.get("footer", {}).get("icon_url") if isinstance(emb.get("footer"), dict) else None,
+                                json.dumps(emb.get("fields", []))
+                            ))
+
+                if all_reactions:
+                    conn.executemany("""
+                        INSERT INTO reactions (message_id, emoji_id, emoji_name, count)
+                        VALUES (:message_id, :emoji_id, :emoji_name, :count)
+                    """, all_reactions)
+                
+                if all_stickers:
+                    conn.executemany("""
+                        INSERT OR REPLACE INTO message_stickers (message_id, sticker_id, name, url, format_type, local_hash)
+                        VALUES (:message_id, :sticker_id, :name, :url, :format_type, :local_hash)
+                    """, all_stickers)
+                
+                conn.commit()
+            finally:
+                conn.close()
+
+    def get_last_message_id(self, channel_id: str) -> Optional[str]:
+        with self._lock:
+            conn = self._get_conn()
+            row = conn.execute("SELECT id FROM messages WHERE channel_id = ? ORDER BY id DESC LIMIT 1", (str(channel_id),)).fetchone()
+            conn.close()
+            return row["id"] if row else None
+
+    def get_media_by_hash(self, file_hash: str) -> Optional[Dict[str, Any]]:
+        with self._lock:
+            conn = self._get_conn()
+            row = conn.execute("SELECT * FROM media_pool WHERE hash = ?", (file_hash,)).fetchone()
+            conn.close()
+            return dict(row) if row else None
+
+    def get_media_by_url(self, url: str) -> Optional[Dict[str, Any]]:
+        with self._lock:
+            conn = self._get_conn()
+            row = conn.execute("SELECT * FROM media_pool WHERE first_seen_url = ?", (url,)).fetchone()
+            conn.close()
+            return dict(row) if row else None
+
+    def add_media_to_pool(self, file_hash: str, local_path: str, size: int, mime_type: str, url: str):
+        with self._lock:
+            conn = self._get_conn()
+            conn.execute("""
+                INSERT OR REPLACE INTO media_pool (hash, local_path, size, mime_type, first_seen_url)
+                VALUES (?, ?, ?, ?, ?)
+            """, (file_hash, str(local_path), size, mime_type, url))
+            conn.commit()
+            conn.close()
+
+    def get_stats_by_channel(self) -> Dict[int, Dict[str, Any]]:
+        """Returns aggregate stats for all channels with backups."""
+        with self._lock:
+            conn = self._get_conn()
+            # Summary of messages per effective channel (including threads rolled up)
+            msg_rows = conn.execute("""
+                SELECT 
+                    COALESCE(t.parent_id, m.channel_id) as channel_id,
+                    COUNT(m.id) as msg_count
+                FROM messages m
+                LEFT JOIN threads t ON m.channel_id = t.id
+                GROUP BY channel_id
+            """).fetchall()
+            
+            # Thread counts per parent
+            thread_rows = conn.execute("""
+                SELECT parent_id, COUNT(*) as thread_count 
+                FROM threads 
+                GROUP BY parent_id
+            """).fetchall()
+            
+            # Summary of attachments per effective channel (including threads rolled up)
+            att_rows = conn.execute("""
+                SELECT 
+                    COALESCE(t.parent_id, m.channel_id) as channel_id,
+                    COUNT(a.id) as att_count,
+                    SUM(a.size) as total_size
+                FROM attachments a
+                JOIN messages m ON a.message_id = m.id
+                LEFT JOIN threads t ON m.channel_id = t.id
+                GROUP BY channel_id
+            """).fetchall()
+            
+            conn.close()
+            
+            stats = {}
+            for r in msg_rows:
+                cid_raw = r["channel_id"]
+                if cid_raw is None or cid_raw == "None": continue
+                cid = int(cid_raw)
+                stats[cid] = {
+                    "message_count": r["msg_count"],
+                    "thread_count": 0,
+                    "attachment_count": 0,
+                    "total_size": 0
+                }
+
+            for r in thread_rows:
+                cid_raw = r["parent_id"]
+                if cid_raw is None or cid_raw == "None": continue
+                cid = int(cid_raw)
+                if cid not in stats:
+                    stats[cid] = {"message_count": 0, "thread_count": 0, "attachment_count": 0, "total_size": 0}
+                stats[cid]["thread_count"] = r["thread_count"]
+            
+            for r in att_rows:
+                cid_raw = r["channel_id"]
+                if cid_raw is None or cid_raw == "None": continue
+                cid = int(cid_raw)
+                if cid not in stats:
+                    stats[cid] = {"message_count": 0, "thread_count": 0, "attachment_count": 0, "total_size": 0}
+                stats[cid]["attachment_count"] = r["att_count"]
+                stats[cid]["total_size"] = r["total_size"] or 0
+                
+            return stats
+
+    def get_all_roles(self) -> List[Dict[str, Any]]:
+        with self._lock:
+            conn = self._get_conn()
+            rows = conn.execute("SELECT * FROM roles ORDER BY position DESC").fetchall()
+            conn.close()
+            return [dict(r) for r in rows]
+
+    def get_all_channels(self) -> List[Dict[str, Any]]:
+        with self._lock:
+            conn = self._get_conn()
+            rows = conn.execute("SELECT * FROM channels ORDER BY position ASC").fetchall()
+            
+            # Fetch all permissions
+            chan_list = [dict(r) for r in rows]
+            if chan_list:
+                ids = [c["id"] for c in chan_list]
+                placeholders = ",".join(["?"] * len(ids))
+                perm_rows = conn.execute(f"SELECT * FROM permissions WHERE channel_id IN ({placeholders})", ids).fetchall()
+                
+                perms_by_chan = {}
+                for pr in perm_rows:
+                    cid = pr["channel_id"]
+                    if cid not in perms_by_chan: perms_by_chan[cid] = []
+                    perms_by_chan[cid].append({
+                        "id": pr["target_id"],
+                        "type": pr["target_type"],
+                        "allow": pr["allow"],
+                        "deny": pr["deny"]
+                    })
+                
+                for c in chan_list:
+                    c["overwrites"] = perms_by_chan.get(c["id"], [])
+                
+                # Fetch Forum Tags
+                tag_rows = conn.execute(f"SELECT * FROM forum_tags WHERE forum_id IN ({placeholders})", ids).fetchall()
+                tags_by_forum = {}
+                for tr in tag_rows:
+                    fid = tr["forum_id"]
+                    if fid not in tags_by_forum: tags_by_forum[fid] = []
+                    tags_by_forum[fid].append(dict(tr))
+                
+                for c in chan_list:
+                    c["available_tags"] = tags_by_forum.get(c["id"], [])
+            
+            conn.close()
+            return chan_list
+
+    def get_all_threads(self) -> List[Dict[str, Any]]:
+        """Returns metadata for all threads in the backup."""
+        with self._lock:
+            conn = self._get_conn()
+            rows = conn.execute("SELECT * FROM threads").fetchall()
+            conn.close()
+            return [dict(r) for r in rows]
+
+    def get_forum_tags(self, forum_id: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Returns forum tag definitions."""
+        with self._lock:
+            conn = self._get_conn()
+            if forum_id:
+                rows = conn.execute("SELECT * FROM forum_tags WHERE forum_id = ?", (str(forum_id),)).fetchall()
+            else:
+                rows = conn.execute("SELECT * FROM forum_tags").fetchall()
+            conn.close()
+            return [dict(r) for r in rows]
+
+    def get_threads_by_parent(self, parent_id: str) -> List[Dict[str, Any]]:
+        """Returns all threads belonging to a parent channel."""
+        with self._lock:
+            conn = self._get_conn()
+            rows = conn.execute("SELECT * FROM threads WHERE parent_id = ?", (str(parent_id),)).fetchall()
+            conn.close()
+            return [dict(r) for r in rows]
+
+    def get_thread(self, thread_id: str) -> Optional[Dict[str, Any]]:
+        """Retrieves a single thread's metadata."""
+        with self._lock:
+            conn = self._get_conn()
+            row = conn.execute("SELECT * FROM threads WHERE id = ?", (str(thread_id),)).fetchone()
+            conn.close()
+            return dict(row) if row else None
+
+    def get_all_users(self) -> List[Dict[str, Any]]:
+        with self._lock:
+            conn = self._get_conn()
+            rows = conn.execute("SELECT * FROM users").fetchall()
+            conn.close()
+            return [dict(r) for r in rows]
+
+    def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
+        with self._lock:
+            conn = self._get_conn()
+            row = conn.execute("SELECT * FROM users WHERE id = ?", (str(user_id),)).fetchone()
+            conn.close()
+            if row:
+                data = dict(row)
+                if data.get("roles"):
+                    data["roles"] = json.loads(data["roles"])
+                return data
+            return None
+
+    def get_server_assets(self, asset_type: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Returns all server assets, optionally filtered by type."""
+        with self._lock:
+            conn = self._get_conn()
+            if asset_type:
+                rows = conn.execute("SELECT * FROM server_assets WHERE type = ?", (asset_type,)).fetchall()
+            else:
+                rows = conn.execute("SELECT * FROM server_assets").fetchall()
+            conn.close()
+            return [dict(r) for r in rows]
+
+    def get_all_media(self) -> Dict[str, Dict[str, Any]]:
+        """Returns the entire media pool as a dictionary indexed by hash."""
+        with self._lock:
+            conn = self._get_conn()
+            rows = conn.execute("SELECT * FROM media_pool").fetchall()
+            conn.close()
+            return {r["hash"]: dict(r) for r in rows}
+
+    def get_messages_paged(self, channel_id: str, limit: int = 100, offset: int = 0, after_id: Optional[str] = None) -> List[Dict[str, Any]]:
+        with self._lock:
+            conn = self._get_conn()
+            query = "SELECT * FROM messages WHERE channel_id = ?"
+            params = [str(channel_id)]
+            
+            if after_id:
+                query += " AND id > ?"
+                params.append(str(after_id))
+            
+            query += " ORDER BY id ASC LIMIT ? OFFSET ?"
+            params.extend([limit, offset])
+            
+            rows = conn.execute(query, params).fetchall()
+            msg_list = [dict(r) for r in rows]
+            
+            if msg_list:
+                # Fetch attachments for these messages
+                msg_ids = [m["id"] for m in msg_list]
+                placeholders = ",".join(["?"] * len(msg_ids))
+                
+                # Attachments
+                att_rows = conn.execute(f"SELECT * FROM attachments WHERE message_id IN ({placeholders})", msg_ids).fetchall()
+                atts_by_msg = {}
+                for ar in att_rows:
+                    mid = ar["message_id"]
+                    if mid not in atts_by_msg: atts_by_msg[mid] = []
+                    atts_by_msg[mid].append(dict(ar))
+                
+                # Embeds
+                emb_rows = conn.execute(f"SELECT * FROM embeds WHERE message_id IN ({placeholders})", msg_ids).fetchall()
+                embs_by_msg = {}
+                for er in emb_rows:
+                    mid = er["message_id"]
+                    if mid not in embs_by_msg: embs_by_msg[mid] = []
+                    
+                    e_dict = {
+                        "title": er["title"],
+                        "description": er["description"],
+                        "url": er["url"],
+                        "color": er["color"],
+                        "timestamp": er["timestamp"],
+                        "thumbnail": {"url": er["thumbnail_url"]} if er["thumbnail_url"] else None,
+                        "image": {"url": er["image_url"]} if er["image_url"] else None,
+                        "author": {
+                            "name": er["author_name"],
+                            "url": er["author_url"],
+                            "icon_url": er["author_icon_url"]
+                        } if er["author_name"] else None,
+                        "footer": {
+                            "text": er["footer_text"],
+                            "icon_url": er["footer_icon_url"]
+                        } if er["footer_text"] else None,
+                        "fields": json.loads(er["fields"]) if er["fields"] else []
+                    }
+                    embs_by_msg[mid].append(e_dict)
+                
+                # Reactions
+                rea_rows = conn.execute(f"SELECT * FROM reactions WHERE message_id IN ({placeholders})", msg_ids).fetchall()
+                reas_by_msg = {}
+                for rr in rea_rows:
+                    mid = rr["message_id"]
+                    if mid not in reas_by_msg: reas_by_msg[mid] = []
+                    reas_by_msg[mid].append(dict(rr))
+                
+                # Stickers (Message-specific)
+                st_rows = conn.execute(f"SELECT * FROM message_stickers WHERE message_id IN ({placeholders})", msg_ids).fetchall()
+                sts_by_msg = {}
+                for sr in st_rows:
+                    mid = sr["message_id"]
+                    if mid not in sts_by_msg: sts_by_msg[mid] = []
+                    sts_by_msg[mid].append(dict(sr))
+
+                for m in msg_list:
+                    m_id = m["id"]
+                    m["attachments"] = atts_by_msg.get(m_id, [])
+                    m["embeds"] = embs_by_msg.get(m_id, [])
+                    m["reactions"] = reas_by_msg.get(m_id, [])
+                    m["stickers"] = sts_by_msg.get(m_id, [])
+            
+            conn.close()
+            return msg_list
+
+    def close(self):
+        # We don't keep long-lived connections in this model to avoid locking issues,
+        # but the method is here for parity with other DB classes.
+        pass

src/core/discord_reader.py

@@ -279,9 +279,14 @@ class DiscordReader:
                 try:
                     # In a forum, the starter message ID is the thread ID
                     starter = await thread.fetch_message(thread.id)
-                    # Bind the thread so migrate_messages handles it properly
+                    # Bind the thread if possible so downstream code has context
+                    try:
                         if not hasattr(starter, 'thread') or starter.thread is None:
                             starter.thread = thread
+                    except (AttributeError, TypeError):
+                        # Some versions of discord.py don't allow setting the thread property
+                        # or it might already be populated as a read-only property
+                        pass
                     yield starter
                 except Exception as e:
                     logger.debug(f"Could not fetch starter message for forum thread {thread.id}: {e}")

src/ui/backup_stats.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from pathlib import Path
 from datetime import datetime
 import asyncio
@@ -7,11 +8,15 @@ from textual.app import ComposeResult
 from textual.screen import Screen
 from textual.containers import Container, Vertical, VerticalScroll, Horizontal
 from textual.widgets import Button, Label, Rule, Tree
-from src.ui.widgets import RamDisplay
 from textual import work
 from rich.text import Text
 from rich.style import Style
 
+from src.ui.widgets import RamDisplay
+from src.core.backup_reader import BackupReader, ChannelType
+
+logger = logging.getLogger(__name__)
+
 class BackupStatsScreen(Screen[None]):
     """Full-screen view for displaying detailed backup statistics."""
 
@@ -254,28 +259,22 @@ class BackupStatsScreen(Screen[None]):
     @work(exclusive=True)
     async def load_data(self) -> None:
         try:
-            # Locate the correct backup directory passed from backup_ops
+            # Initialize BackupReader
-            if not self.target_dir.exists():
+            reader = BackupReader(self.target_dir)
-                raise FileNotFoundError(f"Config directory {self.target_dir} not found.")
+            await reader.start()
             
-            target_dir = self.target_dir
+            if not reader.guild:
-            if not (target_dir / "server_profile" / "profile.json").exists():
                 raise FileNotFoundError(f"No valid backup found in {self.target_dir}")
             
-            self.profile_path = target_dir / "server_profile"
-            self.backup_path = target_dir / "message_backup"
-
             # 1. Profile / Server Info
-            server_name = "Unknown Server"
+            guild = reader.guild
-            server_id = "0"
+            server_name = guild.name
+            server_id = str(guild.id)
+            
+            # Get last backup info from guild profile
+            profile = reader.db.get_guild_profile()
             last_backup_str = "Never"
-            profile_file = self.profile_path / "profile.json"
+            ts = profile.get("last_backup")
-            if profile_file.exists():
-                with open(profile_file, "r", encoding="utf-8") as f:
-                    data = json.load(f)
-                    server_name = data.get("name", server_name)
-                    server_id = data.get("id", server_id)
-                    ts = data.get("last_backup")
             if ts:
                 try:
                     dt = datetime.fromisoformat(ts)
@@ -287,145 +286,106 @@ class BackupStatsScreen(Screen[None]):
             self.query_one("#bs_id", Label).update(f"{server_id}")
             self.query_one("#bs_last_backup", Label).update(f"{last_backup_str}")
 
-            # 2. Assets / Entities
+            # 2. Assets / Entities (using properties)
-            member_count = 0
+            member_count = len(reader.members)
-            emoji_count = 0
+            emoji_count = len(reader.emojis)
-            sticker_count = 0
+            sticker_count = len(reader.stickers)
-            
+            role_count = len(reader.roles)
-            # Fetch members from users/user_info.json
-            if self.backup_path:
-                user_info_file = self.backup_path / "users" / "user_info.json"
-                if user_info_file.exists():
-                    try:
-                        with open(user_info_file, "r", encoding="utf-8") as f:
-                            data = json.load(f)
-                            if isinstance(data, list):
-                                member_count = len(data)
-                            elif isinstance(data, dict):
-                                member_count = len(data)
-                    except Exception:
-                        pass
-                        
-            assets_file = self.profile_path / "assets.json"
-            if assets_file.exists():
-                with open(assets_file, "r", encoding="utf-8") as f:
-                    data = json.load(f)
-                    emoji_count = len(data.get("emojis", []))
-                    sticker_count = len(data.get("stickers", []))
-                    
-            role_count = 0
-            roles_file = self.profile_path / "roles.json"
-            if roles_file.exists():
-                with open(roles_file, "r", encoding="utf-8") as f:
-                    role_count = len(json.load(f))
 
             self.query_one("#bs_val_members", Label).update(f"{member_count}")
             self.query_one("#bs_val_roles", Label).update(f"{role_count}")
             self.query_one("#bs_val_emojis", Label).update(f"{emoji_count}")
             self.query_one("#bs_val_stickers", Label).update(f"{sticker_count}")
 
-            # 3. Structure & Per-Channel Stats
+            # 3. Aggregate Stats from DB
-            structure_file = self.profile_path / "structure.json"
+            channel_stats = reader.db.get_stats_by_channel()
-            total_channels = 0
-            backed_up_channels = 0
             
-            total_msgs = 0
+            total_msgs = sum(s["message_count"] for s in channel_stats.values())
-            total_threads = 0
+            total_threads = sum(s["thread_count"] for s in channel_stats.values())
-            total_files = 0
+            total_files = sum(s["attachment_count"] for s in channel_stats.values())
-            total_size = 0
+            total_size = sum(s["total_size"] for s in channel_stats.values())
             
-            cat_nodes = [] # Collect category data
+            backed_up_channel_ids = set(channel_stats.keys())
             
-            if structure_file.exists():
+            # 4. Structure & Per-Channel Stats
-                with open(structure_file, "r", encoding="utf-8") as f:
+            categories = reader.categories
-                    structure = json.load(f)
+            all_channels = reader.channels
             
-                for cat in structure:
+            total_channels = len([c for c in all_channels if c.type != ChannelType.category])
-                    cat_name = cat.get("name", "Unknown Category").upper()
+            backed_up_channels = len([cid for cid in backed_up_channel_ids if any(c.id == cid for c in all_channels)])
+            
+            # Helper to map channels to categories
+            cat_map = {cat.id: {"cat": cat, "chans": []} for cat in categories}
+            cat_map[None] = {"cat": None, "chans": []} # Uncategorized
+            
+            for chan in all_channels:
+                if chan.type == ChannelType.category: continue
+                cid = chan.category_id
+                if cid not in cat_map: cid = None
+                cat_map[cid]["chans"].append(chan)
+
+            # Global update
+            self.query_one("#bs_val_msgs", Label).update(f"{total_msgs}")
+            self.query_one("#bs_val_threads", Label).update(f"{total_threads}")
+            self.query_one("#bs_val_files", Label).update(f"{total_files}")
+            self.query_one("#bs_val_size", Label).update(f"{self._format_size(total_size)}")
+            self.query_one("#bs_val_coverage", Label).update(f"{backed_up_channels} / {total_channels}")
+
+            # 5. Build Tree
+            for cat_id, info in cat_map.items():
+                cat = info["cat"]
+                chans = info["chans"]
+                if not chans: continue
+                
+                cat_name = cat.name.upper() if cat else "UNCATEGORIZED"
+                
+                # Aggregate for category
                 c_msgs = 0
                 c_thds = 0
                 c_files = 0
                 c_size = 0
                 
-                    chan_list = []
+                chan_nodes_data = []
+                for ch in chans:
+                    stats = channel_stats.get(ch.id, {"message_count": 0, "thread_count": 0, "attachment_count": 0, "total_size": 0})
+                    is_bu = ch.id in backed_up_channel_ids
                     
-                    for chan in cat.get("channels", []):
+                    chan_nodes_data.append({
-                        total_channels += 1
+                        "name": f"# {ch.name}",
-                        ch_id = chan.get("id")
+                        "msgs": stats["message_count"] if is_bu else "NA",
-                        ch_name = f"# {chan.get('name', 'unknown')}"
+                        "threads": stats["thread_count"] if is_bu else "NA",
-                        
+                        "files": stats["attachment_count"] if is_bu else "NA",
-                        m_count = 0
+                        "size": stats["total_size"] if is_bu else 0,
-                        t_count = 0
+                        "is_backed_up": is_bu
-                        f_count = 0
-                        s_bytes = 0
-                        is_backed_up = False
-                        
-                        msg_file = self.backup_path / str(ch_id) / "messages.json"
-                        if msg_file.exists():
-                            is_backed_up = True
-                            backed_up_channels += 1
-                            try:
-                                with open(msg_file, "r", encoding="utf-8") as mf:
-                                    mdata = json.load(mf)
-                                    m_count = mdata.get("messageCount", 0)
-                                    t_count = mdata.get("threadCount", 0)
-                                    f_count = mdata.get("numberOfAttachments", 0)
-                                    s_bytes = mdata.get("totalAttachmentSizeBytes", 0)
-                            except Exception:
-                                pass
-                                
-                        chan_list.append({
-                            "name": ch_name,
-                            "msgs": m_count if is_backed_up else "NA",
-                            "threads": t_count if is_backed_up else "NA",
-                            "files": f_count if is_backed_up else "NA",
-                            "size": s_bytes,
-                            "is_backed_up": is_backed_up
-                        })
-                        c_msgs += m_count
-                        c_thds += t_count
-                        c_files += f_count
-                        c_size += s_bytes
-                        
-                    total_msgs += c_msgs
-                    total_threads += c_thds
-                    total_files += c_files
-                    total_size += c_size
-                    
-                    cat_nodes.append({
-                        "name": cat_name,
-                        "channels": chan_list,
-                        "msgs": c_msgs,
-                        "threads": c_thds,
-                        "files": c_files,
-                        "size": c_size
                     })
                     
-            # 4. Global Stats
+                    c_msgs += stats["message_count"]
-            self.query_one("#bs_val_msgs", Label).update(f"{total_msgs}")
+                    c_thds += stats["thread_count"]
-            self.query_one("#bs_val_threads", Label).update(f"{total_threads}")
+                    c_files += stats["attachment_count"]
-            self.query_one("#bs_val_files", Label).update(f"{total_files}")
+                    c_size += stats["total_size"]
-            self.query_one("#bs_val_size", Label).update(f"{self._format_size(total_size)}")
                 
-            self.query_one("#bs_val_coverage", Label).update(f"{backed_up_channels} / {total_channels}")
+                cat_lbl = self._format_tree_row(cat_name, c_msgs, c_thds, c_files, self._format_size(c_size))
-
-            # 5. Build Tree
-            for cat in cat_nodes:
-                cat_lbl = self._format_tree_row(f"{cat['name']}", cat['msgs'], cat['threads'], cat['files'], self._format_size(cat['size']))
                 cat_lbl.stylize("bold yellow")
                 node = self.stats_tree.root.add(cat_lbl, expand=True)
                 
-                for ch in cat["channels"]:
+                for ch_data in chan_nodes_data:
-                    size_str = self._format_size(ch['size']) if ch['is_backed_up'] else "NA"
+                    size_str = self._format_size(ch_data['size']) if ch_data['is_backed_up'] else "NA"
-                    ch_lbl = self._format_tree_row(f"  {ch['name']}", ch['msgs'], ch['threads'], ch['files'], size_str)
+                    ch_lbl = self._format_tree_row(f"  {ch_data['name']}", ch_data['msgs'], ch_data['threads'], ch_data['files'], size_str)
                     
-                    if ch['is_backed_up']:
+                    if ch_data['is_backed_up']:
                         ch_lbl.stylize("bold white")
                     else:
-                        ch_lbl.stylize("dim white") # Textual 'dim' looks like a dull grey
+                        ch_lbl.stylize("dim white")
                     
                     node.add_leaf(ch_lbl)
 
         except Exception as e:
+            import traceback
+            logger.exception("Failed to load backup stats")
+            trace_str = traceback.format_exc()
+            logger.error(f"Full Traceback: {trace_str}")
             self.query_one("#bs_name", Label).update(f"[red]Error loading data[/red]")
             self.query_one("#bs_id", Label).update(f"[red]{e}[/red]")
+        finally:
+            if 'reader' in locals():
+                await reader.close()

src/ui/shuttle_ops.py

@@ -183,7 +183,6 @@ class OperationPane(Container):
             self.exporter = DiscordExporter(self.engine.discord_reader, base_dir=self._base_dir())
 
     def _get_backup_info(self) -> str | None:
-        import json
         if not self.config or not self.config.discord_server_id:
             return None
             
@@ -191,14 +190,16 @@ class OperationPane(Container):
         if not target_dir.exists():
             return None
             
-        profile_file = target_dir / "server_profile" / "profile.json"
+        db_file = target_dir / "backup.db"
-        if not profile_file.exists():
+        if not db_file.exists():
             return None
         try:
             from datetime import datetime
-            with open(profile_file, "r", encoding="utf-8") as f:
+            from src.core.backup_database import BackupDatabase
-                data = json.load(f)
+            db = BackupDatabase(db_file)
-                ts_str = data.get("last_backup")
+            profile = db.get_guild_profile()
+            if profile:
+                ts_str = profile.get("last_backup")
                 if ts_str:
                     dt = datetime.fromisoformat(ts_str)
                     return dt.strftime("%d-%b-%Y %H:%M")
@@ -1767,8 +1768,10 @@ class OperationPane(Container):
 
             any_found = False
             backed_up_ids = set()
+            if self.exporter.db:
+                channel_stats = self.exporter.db.get_stats_by_channel()
                 for chan in eligible_channels:
-                if (self.exporter.export_path / "message_backup" / str(chan.id) / "messages.json").exists():
+                    if chan.id in channel_stats:
                         any_found = True
                         backed_up_ids.add(chan.id)
 
@@ -1867,6 +1870,8 @@ class OperationPane(Container):
             modal_prog.write(f"[yellow]Starting backup for {total_chans} channels...[/yellow]")
 
             accumulated_msgs = 0
+            accumulated_threads = 0
+            accumulated_files = 0
 
             for i, chan in enumerate(selected_channels):
                 if not self.exporter.is_running:
@@ -1874,7 +1879,7 @@ class OperationPane(Container):
                     break
                 await asyncio.sleep(0.01) # Yield to UI thread to keep it responsive
                 
-                backup_exists = (self.exporter.export_path / "message_backup" / str(chan.id) / "messages.json").exists()
+                backup_exists = chan.id in backed_up_ids
                 is_sync = backup_exists and not force_overwrite
 
                 label = "Syncing Backup" if is_sync else "Backing up"
@@ -1884,20 +1889,17 @@ class OperationPane(Container):
                 logger.info(f"{label} for channel: #{chan.name} ({chan.id})")
 
                 _msg_log_counter = 0
-                async def update_msg_count(name, count, author_name=None, message_preview=None):
+                async def update_msg_count(name, count, author_name=None, message_preview=None, thread_count=0, file_count=0):
                     nonlocal _msg_log_counter
-                    modal_prog.update_stats(messages=str(count))
+                    modal_prog.update_stats(messages=str(count), threads=str(thread_count), files=str(file_count))
                     _msg_log_counter += 1
                     if author_name and message_preview and _msg_log_counter % 10 == 0:
                         modal_prog.write(f"[bold]{author_name}:[/bold] {message_preview}")
 
-                accumulated_msgs = await self.exporter.export_channel_messages(
+                accumulated_msgs, accumulated_threads, accumulated_files = await self.exporter.export_channel_messages(
                     chan.id, progress_callback=update_msg_count, force=force_overwrite,
-                    accumulated_count=accumulated_msgs, after_id=after_id
+                    accumulated_count=accumulated_msgs, accumulated_threads=accumulated_threads, accumulated_files=accumulated_files,
-                )
+                    after_id=after_id
-                accumulated_msgs = await self.exporter.export_threads(
-                    chan.id, progress_callback=update_msg_count, force=force_overwrite,
-                    accumulated_count=accumulated_msgs
                 )
 
                 modal_prog.write(f"[green]Completed: {chan.name}[/green]")
@@ -1981,9 +1983,14 @@ class OperationPane(Container):
                 ]
             ]
 
+            # Get channels that have messages in the database
+            backed_up_channel_ids = set()
+            if self.exporter.db:
+                backed_up_channel_ids = set(self.exporter.db.get_stats_by_channel().keys())
+            
             selected_channels = [
                 c for c in eligible_channels
-                if (self.exporter.export_path / "message_backup" / str(c.id) / "messages.json").exists()
+                if c.id in backed_up_channel_ids
             ]
 
             if not selected_channels:
@@ -1999,6 +2006,8 @@ class OperationPane(Container):
                 modal_prog.cancel_callback = lambda: setattr(self.exporter, "is_running", False)
                 
                 accumulated_msgs = 0
+                accumulated_threads = 0
+                accumulated_files = 0
                 
                 for i, chan in enumerate(selected_channels):
                     if not self.exporter.is_running:
@@ -2012,20 +2021,16 @@ class OperationPane(Container):
                     logger.info(f"Syncing backup for channel: #{chan.name} ({chan.id})")
 
                     _msg_log_counter = 0
-                    async def update_msg_count(name, count, author_name=None, message_preview=None):
+                    async def update_msg_count(name, count, author_name=None, message_preview=None, thread_count=0, file_count=0):
                         nonlocal _msg_log_counter
-                        modal_prog.update_stats(messages=str(count))
+                        modal_prog.update_stats(messages=str(count), threads=str(thread_count), files=str(file_count))
                         _msg_log_counter += 1
                         if author_name and message_preview and _msg_log_counter % 10 == 0:
                             modal_prog.write(f"[bold]{author_name}:[/bold] {message_preview}")
 
-                    accumulated_msgs = await self.exporter.export_channel_messages(
+                    accumulated_msgs, accumulated_threads, accumulated_files = await self.exporter.export_channel_messages(
                         chan.id, progress_callback=update_msg_count, force=False,
-                        accumulated_count=accumulated_msgs
+                        accumulated_count=accumulated_msgs, accumulated_threads=accumulated_threads, accumulated_files=accumulated_files
-                    )
-                    accumulated_msgs = await self.exporter.export_threads(
-                        chan.id, progress_callback=update_msg_count, force=False,
-                        accumulated_count=accumulated_msgs
                     )
                     modal_prog.write(f"[green]Synced: {chan.name}[/green]")