import os import json import logging import asyncio import discord from pathlib import Path from typing import Dict, Any, List, Optional, AsyncGenerator logger = logging.getLogger(__name__) class DiscordExporter: """Core logic for exporting Discord server data.""" def __init__(self, reader): self.reader = reader self.server_name = "" self.server_id = "" self.user_cache = {} async def setup(self): """Prepares the output directory and fetches server metadata.""" metadata = await self.reader.get_server_metadata() self.server_name = metadata.get("name", "Unknown Server") self.server_id = metadata.get("id", "0") # Create safe folder name import re safe_name = re.sub(r'[^a-zA-Z0-9_\-\.]', '_', self.server_name) self.export_path = Path(".") / f"EXPORT-{safe_name}-{self.server_id}" self.export_path.mkdir(parents=True, exist_ok=True) # Consolidate media into one folder self.media_path = self.export_path / "server_media" self.media_path.mkdir(exist_ok=True) logger.info(f"Export directory set to: {self.export_path}") logger.info(f"Targeting server: {self.server_name} ({self.server_id})") return metadata async def export_metadata(self): """Saves server metadata to a JSON file.""" metadata = await self.reader.get_server_metadata() # Add relative paths to local assets if self.reader.guild: if self.reader.guild.icon: ext = "gif" if self.reader.guild.icon.is_animated() else "png" metadata["icon"] = f"server_media/server_icon.{ext}" else: metadata["icon"] = None if self.reader.guild.banner: ext = "gif" if self.reader.guild.banner.is_animated() else "png" metadata["banner"] = f"server_media/server_banner.{ext}" else: metadata["banner"] = None output_file = self.export_path / "server_profile.json" with open(output_file, "w", encoding="utf-8") as f: json.dump(metadata, f, indent=4, ensure_ascii=False) return metadata async def export_roles(self): """Exports all roles to server_roles.json.""" roles = await self.reader.get_roles() role_data = [] for r in roles: role_data.append({ "id": str(r.id), "name": r.name, "color": str(r.color), "position": r.position, "permissions": r.permissions.value, "hoist": r.hoist, "mentionable": r.mentionable }) output_file = self.export_path / "server_roles.json" with open(output_file, "w", encoding="utf-8") as f: json.dump(role_data, f, indent=4, ensure_ascii=False) return role_data async def download_server_assets(self): """Downloads server icon and banner to media folder.""" metadata = await self.reader.get_server_metadata() # Download Server Icon if metadata.get("icon_url"): try: if self.reader.guild and self.reader.guild.icon: logger.info(f"Downloading server icon: {self.reader.guild.icon.url}") data = await self.reader.download_asset(self.reader.guild.icon) ext = "gif" if self.reader.guild.icon.is_animated() else "png" icon_path = self.media_path / f"server_icon.{ext}" with open(icon_path, "wb") as f: f.write(data) logger.info(f"Saved server icon to {icon_path}") else: logger.warning("Icon URL found in metadata but guild icon asset is missing.") except discord.Forbidden: logger.error("403 Forbidden: Missing Access to download server icon.") except Exception as e: logger.error(f"Failed to download server icon: {e}") else: logger.info("No server icon found to download.") # Download Server Banner if metadata.get("banner_url"): try: if self.reader.guild and self.reader.guild.banner: logger.info(f"Downloading server banner: {self.reader.guild.banner.url}") data = await self.reader.download_asset(self.reader.guild.banner) ext = "gif" if self.reader.guild.banner.is_animated() else "png" banner_path = self.media_path / f"server_banner.{ext}" with open(banner_path, "wb") as f: f.write(data) logger.info(f"Saved server banner to {banner_path}") except discord.Forbidden: logger.error("403 Forbidden: Missing Access to download server banner.") except Exception as e: logger.error(f"Failed to download server banner: {e}") else: logger.info("No server banner found to download.") async def export_assets(self): """Exports emojis, stickers, and server media to server_assets.json and server_media/.""" await self.download_server_assets() emojis = await self.reader.get_emojis() stickers = await self.reader.get_stickers() emoji_data = [] logger.info(f"Exporting {len(emojis)} emojis...") for e in emojis: ext = "gif" if e.animated else "png" filename = f"emoji_{e.name}_{e.id}.{ext}" emoji_path = self.media_path / filename try: data = await self.reader.download_emoji(e) with open(emoji_path, "wb") as f: f.write(data) emoji_data.append({ "id": str(e.id), "name": e.name, "animated": e.animated, "filename": filename }) except discord.Forbidden: logger.error(f"403 Forbidden: Missing Access to download emoji {e.name}") except Exception as ex: logger.error(f"Failed to download emoji {e.name}: {ex}") sticker_data = [] logger.info(f"Exporting {len(stickers)} stickers...") for s in stickers: ext = "png" if s.url: if ".json" in str(s.url): ext = "json" elif ".gif" in str(s.url): ext = "gif" elif ".webp" in str(s.url): ext = "webp" filename = f"sticker_{s.name}_{s.id}.{ext}" sticker_path = self.media_path / filename try: data = await self.reader.download_sticker(s) with open(sticker_path, "wb") as f: f.write(data) sticker_data.append({ "id": str(s.id), "name": s.name, "filename": filename }) except discord.Forbidden: logger.error(f"403 Forbidden: Missing Access to download sticker {s.name}") except Exception as ex: logger.error(f"Failed to download sticker {s.name}: {ex}") # Try to load existing customization to merge (if it exists) custom_file = self.export_path / "server_assets.json" customization = {"emojis": emoji_data, "stickers": sticker_data, "members": []} if custom_file.exists(): try: with open(custom_file, "r", encoding="utf-8") as f: old_data = json.load(f) customization["members"] = old_data.get("members", []) except Exception: pass with open(custom_file, "w", encoding="utf-8") as f: json.dump(customization, f, indent=4, ensure_ascii=False) return len(emoji_data), len(sticker_data) async def export_channels_structure(self): """Exports categories and channels hierarchy.""" categories = await self.reader.get_categories() channels = await self.reader.get_channels() structure = [] chan_count = 0 cat_count = len(categories) for cat in categories: cat_channels = [c for c in channels if c.category_id == cat.id] formatted_channels = [self._format_channel(c) for c in cat_channels] chan_count += len(formatted_channels) structure.append({ "type": "category", "id": str(cat.id), "name": cat.name, "position": cat.position, "channels": formatted_channels }) # Uncategorized uncategorized = [c for c in channels if not c.category_id] if uncategorized: formatted_uncat = [self._format_channel(c) for c in uncategorized] chan_count += len(formatted_uncat) structure.append({ "type": "category", "id": "uncategorized", "name": "Uncategorized", "channels": formatted_uncat }) # No need to increment cat_count for 'Uncategorized' usually, # but let's see if the user wants it. For now, cat_count is real Discord categories. output_file = self.export_path / "server_structure.json" with open(output_file, "w", encoding="utf-8") as f: json.dump(structure, f, indent=4, ensure_ascii=False) return structure, cat_count, chan_count def _format_channel(self, c): return { "id": str(c.id), "name": c.name, "type": str(c.type), "position": c.position, "topic": getattr(c, "topic", None), "nsfw": getattr(c, "nsfw", False) } async def export_channel_messages(self, channel_id: int, progress_callback=None): """Exports all messages from a channel, including attachments, pins, reactions.""" channel = await self.reader.get_channel(channel_id) if not channel: return 0 channel_name = channel.name safe_name = channel_name.replace(" ", "-").lower() # Detection for thread grouping is_thread = isinstance(channel, discord.Thread) backup_root = self.export_path / "message_backup" if is_thread: backup_dir = backup_root / "threads" avatar_rel_base = "../user_avatars" else: backup_dir = backup_root avatar_rel_base = "user_avatars" backup_dir.mkdir(parents=True, exist_ok=True) # Shared avatars directory (always at root of message_backup) avatar_dir = backup_root / "user_avatars" avatar_dir.mkdir(exist_ok=True) # Load existing user_info.json user_info_file = backup_root / "user_info.json" if not self.user_cache and user_info_file.exists(): try: with open(user_info_file, "r", encoding="utf-8") as f: u_list = json.load(f) self.user_cache = {u["id"]: u for u in u_list} except Exception: self.user_cache = {} base_filename = str(channel_id) json_file = backup_dir / f"{base_filename}.json" asset_dir = backup_dir / base_filename asset_dir.mkdir(exist_ok=True) messages = [] last_id = None # Load existing messages for incremental sync if json_file.exists(): try: with open(json_file, "r", encoding="utf-8") as f: old_data = json.load(f) messages = old_data.get("messages", []) if "lastMessageID" in old_data: last_id = int(old_data["lastMessageID"]) elif messages: last_id = int(messages[-1]["messageID"]) except Exception as e: logger.warning(f"Could not load existing backup for sync in {channel_name}: {e}") messages = [] count = len(messages) new_count = 0 thread_count = 0 thread_msg_count = 0 # 1. Fetch new messages - Handle Forbidden gracefully try: async for msg in self.reader.fetch_message_history(channel_id, after_id=last_id): msg_data = await self._format_message(msg, asset_dir, base_filename, avatar_dir, avatar_rel_base) messages.append(msg_data) new_count += 1 if progress_callback: await progress_callback(channel_name, count + new_count) except discord.Forbidden: logger.error(f"403 Forbidden: Missing Access to read messages in {channel_name} ({channel_id})") if not messages: return 0 except Exception as e: logger.error(f"Error fetching messages for {channel_name}: {e}") if not messages: return 0 # 2. Handle Threads and collect counts accurately all_threads = [] try: # Active threads: Use active_threads() coroutine for 2.6.4 if self.reader.guild: threads = await self.reader.guild.active_threads() all_threads.extend([t for t in threads if t.parent_id == channel_id]) # Archived threads: Use the consolidated archived_threads() iterator try: if hasattr(channel, "archived_threads"): async for thread in channel.archived_threads(limit=None): all_threads.append(thread) except discord.Forbidden: logger.warning(f"403 Forbidden: Cannot fetch archived threads in {channel_name}") except Exception as e: logger.warning(f"Error fetching archived threads: {e}") except Exception as e: logger.warning(f"Failed to fetch threads for count in {channel_name}: {e}") thread_count = len(all_threads) for t in all_threads: thread_msg_count += (t.message_count or 0) output_data = { "channelName": channel_name, "channelID": str(channel_id), "messageCount": len(messages), "threadCount": thread_count, "lastMessageID": str(messages[-1]["messageID"]) if messages else None, "threadMessagesCount": thread_msg_count, "lastBackup": discord.utils.utcnow().isoformat(), "messages": messages } if is_thread: output_data["parentID"] = str(channel.parent_id) # Save channel messages with open(json_file, "w", encoding="utf-8") as f: json.dump(output_data, f, indent=4, ensure_ascii=False) # Save/Update user_info.json with open(user_info_file, "w", encoding="utf-8") as f: json.dump(list(self.user_cache.values()), f, indent=4, ensure_ascii=False) return count async def _format_message(self, msg, asset_dir, asset_prefix, avatar_dir, avatar_rel_base): """Formats a single message to match the reference format.""" attachments = [] for a in msg.attachments: # mimic reference asset naming (suffixing hash/id) safe_name = a.filename short_id = str(a.id)[-5:] stored_name = f"{Path(safe_name).stem}-{short_id}{Path(safe_name).suffix}" try: # Check if exists, else download (basic cache) target = asset_dir / stored_name if not target.exists(): data = await a.read() with open(target, "wb") as f: f.write(data) attachments.append({ "id": str(a.id), "url": f"{asset_prefix}/{stored_name}", "fileName": a.filename, "fileSizeBytes": a.size }) except Exception as e: logger.error(f"Failed to download attachment {a.filename}: {e}") # Author info extraction and deduplication author = msg.author user_id = str(author.id) if user_id not in self.user_cache: avatar_url = None if author.avatar: try: av_name = f"{user_id}.png" av_target = avatar_dir / av_name if not av_target.exists(): await author.avatar.save(av_target) avatar_url = f"{avatar_rel_base}/{av_name}" except Exception as e: logger.error(f"Failed to save avatar for {author.name}: {e}") roles = [] if hasattr(author, "roles"): for r in author.roles: if r.is_default(): continue roles.append({ "id": str(r.id), "name": r.name, "color": str(r.color), "position": r.position }) self.user_cache[user_id] = { "userID": user_id, "username": author.name, "userNickname": getattr(author, "display_name", author.name), "userColor": str(author.color) if hasattr(author, "color") else None, "userIsBot": author.bot, "userRoles": roles, "userAvatar": f"user_avatars/{user_id}.png" if author.avatar else None } reactions = [] for r in msg.reactions: emoji_str = str(r.emoji) if not r.is_custom_emoji() else f"{r.emoji.name}:{r.emoji.id}" reactions.append({ "emoji": emoji_str, "count": r.count }) # Process Stickers (Download and Metadata) stickers = [] for s in msg.stickers: sticker_filename = f"sticker_{s.id}" # Extension mapping based on format ext = "png" if str(s.format).endswith("apng"): ext = "apng" elif str(s.format).endswith("lottie"): ext = "json" elif str(s.format).endswith("gif"): ext = "gif" sticker_filename += f".{ext}" sticker_path = asset_dir / sticker_filename try: if not sticker_path.exists(): # Handle Lottie stickers manually since discord.py Refuses to save them if str(s.format).endswith("lottie"): # Use the name-mangled internal session from the client session = self.reader.client.http._HTTPClient__session async with session.get(s.url) as resp: if resp.status == 200: with open(sticker_path, "wb") as f: f.write(await resp.read()) else: raise Exception(f"HTTP {resp.status}") else: await s.save(sticker_path) stickers.append({ "id": str(s.id), "name": s.name, "format": str(s.format).split(".")[-1], "localPath": f"{asset_prefix}/{sticker_filename}" }) except Exception as e: logger.error(f"Failed to download sticker {s.name} ({s.id}): {e}") # Fallback to minimal metadata if download fails stickers.append({ "id": str(s.id), "name": s.name, "format": str(s.format).split(".")[-1] }) # Determine message type (Override if it's a thread starter or forward) msg_type = str(msg.type).split(".")[-1].capitalize() if msg.thread: msg_type = "ThreadStarter" # Check for forwarded flags (newer discord.py feature) try: if hasattr(msg.flags, "forwarded") and msg.flags.forwarded: msg_type = "Forward" except Exception: pass msg_content = msg.content if msg_type == "Forward" and not msg_content: try: if hasattr(msg, "message_snapshots") and msg.message_snapshots: msg_content = msg.message_snapshots[0].content except Exception: pass data = { "messageID": str(msg.id), "type": msg_type, "timestamp": msg.created_at.isoformat(), "isPinned": msg.pinned, "content": msg_content, "userID": user_id, "attachments": attachments, "embeds": [e.to_dict() for e in msg.embeds], "stickers": stickers, "reactions": reactions } # Thread info for creation/starter messages if msg.thread: data["thread"] = { "id": str(msg.thread.id), "name": msg.thread.name, "messageCount": getattr(msg.thread, "message_count", 0), "archived": msg.thread.archived, "archiveDuration": msg.thread.auto_archive_duration, "locked": msg.thread.locked } # Add reply reference if exists if msg.reference and msg.reference.message_id: data["reference"] = { "messageId": str(msg.reference.message_id), "channelId": str(msg.reference.channel_id) } return data async def export_threads(self, channel_id: int): """Exports active and archived threads for a channel.""" channel = await self.reader.get_channel(channel_id) if not hasattr(channel, "threads") and not hasattr(channel, "public_archived_threads"): return 0 all_threads = [] try: # Active threads if self.reader.guild: threads = await self.reader.guild.active_threads() all_threads.extend([t for t in threads if t.parent_id == channel_id]) # Archived threads try: if hasattr(channel, "archived_threads"): async for thread in channel.archived_threads(limit=None): all_threads.append(thread) except discord.Forbidden: logger.warning(f"403 Forbidden: Cannot fetch archived threads in {channel.name}") except Exception as e: logger.warning(f"Error fetching archived threads: {e}") except Exception as e: logger.error(f"Failed to fetch threads for {channel.name}: {e}") thread_count = 0 if all_threads: logger.info(f"Found {len(all_threads)} threads in {channel.name}. Starting backup...") for thread in all_threads: await self.export_channel_messages(thread.id) thread_count += 1 return thread_count