diff --git a/README.md b/README.md index aac1b6e..486c5dc 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ ![Disco Reaper](images/fluxer-reaper.jpg) -### Modern Terminal Interface -The tool now features a unified, intuitive TUI (Terminal User Interface) - no more text commands +### Video Guide - [Youtube](https://www.youtube.com/watch?v=SwIPQDxLzqA) + | Features | Fluxer | Stoat | | :--- | :---: | :---: | diff --git a/src/core/backup_database.py b/src/core/backup_database.py index 88d0df1..40e0011 100644 --- a/src/core/backup_database.py +++ b/src/core/backup_database.py @@ -791,6 +791,83 @@ class BackupDatabase: return msg_list + def get_global_messages_paged(self, limit: int = 100, offset: int = 0, after_id: Optional[str] = None) -> List[Dict[str, Any]]: + """Fetches messages across ALL channels globally, ordered by timestamp/ID ascending.""" + with self._lock: + query = "SELECT * FROM messages" + params = [] + + if after_id: + query += " WHERE id > ?" + params.append(parse_snowflake(after_id)) + + query += " ORDER BY id ASC LIMIT ? OFFSET ?" + params.extend([limit, offset]) + + rows = self._conn.execute(query, params).fetchall() + msg_list = [dict(r) for r in rows] + + if msg_list: + msg_ids = [m["id"] for m in msg_list] + placeholders = ",".join(["?"] * len(msg_ids)) + + att_rows = self._conn.execute(f"SELECT * FROM attachments WHERE message_id IN ({placeholders})", msg_ids).fetchall() + atts_by_msg = {} + for ar in att_rows: + mid = ar["message_id"] + if mid not in atts_by_msg: atts_by_msg[mid] = [] + atts_by_msg[mid].append(dict(ar)) + + emb_rows = self._conn.execute(f"SELECT * FROM embeds WHERE message_id IN ({placeholders})", msg_ids).fetchall() + embs_by_msg = {} + for er in emb_rows: + mid = er["message_id"] + if mid not in embs_by_msg: embs_by_msg[mid] = [] + + e_dict = { + "title": er["title"], + "description": er["description"], + "url": er["url"], + "color": er["color"], + "timestamp": er["timestamp"], + "thumbnail": {"url": er["thumbnail_url"]} if er["thumbnail_url"] else None, + "image": {"url": er["image_url"]} if er["image_url"] else None, + "author": { + "name": er["author_name"], + "url": er["author_url"], + "icon_url": er["author_icon_url"] + } if er["author_name"] else None, + "footer": { + "text": er["footer_text"], + "icon_url": er["footer_icon_url"] + } if er["footer_text"] else None, + "fields": json.loads(er["fields"]) if er["fields"] else [] + } + embs_by_msg[mid].append(e_dict) + + rea_rows = self._conn.execute(f"SELECT * FROM reactions WHERE message_id IN ({placeholders})", msg_ids).fetchall() + reas_by_msg = {} + for rr in rea_rows: + mid = rr["message_id"] + if mid not in reas_by_msg: reas_by_msg[mid] = [] + reas_by_msg[mid].append(dict(rr)) + + st_rows = self._conn.execute(f"SELECT * FROM message_stickers WHERE message_id IN ({placeholders})", msg_ids).fetchall() + sts_by_msg = {} + for sr in st_rows: + mid = sr["message_id"] + if mid not in sts_by_msg: sts_by_msg[mid] = [] + sts_by_msg[mid].append(dict(sr)) + + for m in msg_list: + m_id = m["id"] + m["attachments"] = atts_by_msg.get(m_id, []) + m["embeds"] = embs_by_msg.get(m_id, []) + m["reactions"] = reas_by_msg.get(m_id, []) + m["stickers"] = sts_by_msg.get(m_id, []) + + return msg_list + def delete_channel_messages(self, channel_id: Union[str, int]): """Deletes all messages and related metadata for a specific channel and its threads.""" cid = parse_snowflake(channel_id) diff --git a/src/core/backup_reader.py b/src/core/backup_reader.py index 3ec7c92..c9d250d 100644 --- a/src/core/backup_reader.py +++ b/src/core/backup_reader.py @@ -1417,6 +1417,42 @@ class BackupReader: if len(msgs) < batch_size: break + async def fetch_global_message_history( + self, + limit: int = None, + after_id: int = None + ) -> AsyncGenerator["BackupMessage", None]: + """Yields BackupMessages globally from SQLite across all channels, natively ordered by timestamp/ID.""" + if not self.db: return + + offset = 0 + batch_size = 100 + count = 0 + + while True: + actual_limit = batch_size + if limit: + rem = limit - count + if rem <= 0: break + actual_limit = min(batch_size, rem) + + msgs = self.db.get_global_messages_paged( + limit=actual_limit, + offset=offset, + after_id=str(after_id) if after_id else None + ) + + if not msgs: + break + + for m in msgs: + yield self._hydrate_message(m) + count += 1 + + offset += len(msgs) + if len(msgs) < batch_size: + break + # ── download helpers ───────────────────────────────────────────────── async def download_emoji(self, emoji: BackupEmoji) -> bytes: diff --git a/src/core/database.py b/src/core/database.py index bfaf490..47578ea 100644 --- a/src/core/database.py +++ b/src/core/database.py @@ -451,6 +451,49 @@ class MigrationDatabase: return dict(row) return {"last_msg_id": None, "last_msg_ts": None, "msg_count": 0, "file_count": 0} + + def get_global_min_last_message_id(self, all_mapped_ids: List[str]) -> Optional[int]: + """ + Returns the minimum last_msg_id successfully migrated across all mapped channels/threads. + If any mapped entity has no progress record, it is treated as ID 0. + Returns None only if NO progress has been made across ANY entity. + """ + if not all_mapped_ids: + return None + + conn = self._get_conn() + placeholders = ",".join(["?"] * len(all_mapped_ids)) + + # 1. Get last message IDs from channel tracking + c_rows = conn.execute(f"SELECT channel_id, last_msg_id FROM channel_tracking WHERE channel_id IN ({placeholders})", all_mapped_ids).fetchall() + c_map = {r["channel_id"]: r["last_msg_id"] for r in c_rows} + + # 2. Get last message IDs from thread tracking + t_rows = conn.execute(f"SELECT thread_id, last_msg_id FROM thread_tracking WHERE thread_id IN ({placeholders})", all_mapped_ids).fetchall() + t_map = {r["thread_id"]: r["last_msg_id"] for r in t_rows} + + # Combine maps + progress_map = {**c_map, **t_map} + + # 3. Aggregate IDs + ids = [] + has_any_progress = False + for mid in all_mapped_ids: + last_id = progress_map.get(mid) + if not last_id: + ids.append(0) # Unmigrated entity + else: + try: + ids.append(int(last_id)) + has_any_progress = True + except (ValueError, TypeError): + ids.append(0) + + if not has_any_progress: + return None + + return min(ids) + # Thread methods similar to channel methods def set_thread_message_mapping(self, channel_id: str, thread_id: str, source_id: str, target_id: str, timestamp: str = None): conn = self._get_conn() @@ -496,6 +539,18 @@ class MigrationDatabase: return dict(row) return {"last_msg_id": None, "last_msg_ts": None, "msg_count": 0, "file_count": 0} + def get_all_channel_tracking_ids(self) -> Dict[str, str]: + """Returns a map of channel_id -> last_msg_id for all tracked channels.""" + conn = self._get_conn() + rows = conn.execute("SELECT channel_id, last_msg_id FROM channel_tracking WHERE last_msg_id IS NOT NULL").fetchall() + return {str(row["channel_id"]): str(row["last_msg_id"]) for row in rows} + + def get_all_thread_tracking_ids(self) -> Dict[str, str]: + """Returns a map of thread_id -> last_msg_id for all tracked threads.""" + conn = self._get_conn() + rows = conn.execute("SELECT thread_id, last_msg_id FROM thread_tracking WHERE last_msg_id IS NOT NULL").fetchall() + return {str(row["thread_id"]): str(row["last_msg_id"]) for row in rows} + def clear_channel_data(self, channel_id: str): """Purge all mappings and tracking data for a specific channel and its threads.""" conn = self._get_conn() diff --git a/src/core/state.py b/src/core/state.py index dcfd446..eb816c5 100644 --- a/src/core/state.py +++ b/src/core/state.py @@ -38,7 +38,14 @@ class MigrationState: if self.db: self.db.delete_server_mapping("channel", str(discord_id)) + def remove_target_channel_mapping(self, discord_id: int | str): + if self.db: + self.db.delete_server_mapping("channel", str(discord_id)) + def set_target_channel_id(self, discord_id: int | str, target_id: str, *args): + """Alias for set_channel_mapping to handle legacy calls.""" + self.set_channel_mapping(discord_id, target_id) + get_fluxer_channel_id = get_target_channel_id set_target_channel_mapping = set_channel_mapping @@ -58,6 +65,10 @@ class MigrationState: if self.db: self.db.delete_server_mapping("category", str(discord_id)) + def set_target_category_id(self, discord_id: int | str, target_id: str, *args): + """Alias for set_category_mapping to handle legacy calls.""" + self.set_category_mapping(discord_id, target_id) + get_fluxer_category_id = get_category_mapping get_target_category_id = get_category_mapping set_target_category_mapping = set_category_mapping @@ -78,6 +89,10 @@ class MigrationState: if self.db: self.db.delete_server_mapping("role", str(discord_id)) + def set_target_role_id(self, discord_id: int | str, target_id: str, *args): + """Alias for set_role_mapping to handle legacy calls.""" + self.set_role_mapping(discord_id, target_id) + get_fluxer_role_id = get_role_mapping get_target_role_id = get_role_mapping set_target_role_mapping = set_role_mapping @@ -212,8 +227,34 @@ class MigrationState: def get_last_message_id(self, target_channel_id: str) -> str | None: if self._ensure_db(): - return self.db.get_channel_tracking(str(target_channel_id)).get("last_msg_id") + tracking = self.db.get_channel_tracking(str(target_channel_id)) + return tracking.get("last_msg_id") if tracking else None return None + + + def get_global_min_last_message_id(self, all_mapped_ids: List[str]) -> int | None: + """Returns the absolute minimum last_msg_id among the given list of mapped target IDs (channels and threads).""" + if self._ensure_db(): + return self.db.get_global_min_last_message_id(all_mapped_ids) + return None + + def set_waterfall_last_id(self, last_id: str | int): + if self.db: + self.db.set_metadata("waterfall_last_id", str(last_id)) + + def get_waterfall_last_id(self) -> int | None: + if self.db: + val = self.db.get_metadata("waterfall_last_id") + return int(val) if val else None + return None + + def get_all_last_message_ids(self) -> Dict[str, str]: + """Returns a combined map of channel_id/thread_id -> last_msg_id.""" + if self._ensure_db(): + c_map = self.db.get_all_channel_tracking_ids() + t_map = self.db.get_all_thread_tracking_ids() + return {**c_map, **t_map} + return {} def get_thread_last_message_id(self, target_channel_id: str, thread_id: str) -> str | None: if self._ensure_db(): diff --git a/src/fluxer/migrate_message.py b/src/fluxer/migrate_message.py index 12e656c..5a2c0ea 100644 --- a/src/fluxer/migrate_message.py +++ b/src/fluxer/migrate_message.py @@ -4,6 +4,7 @@ import re import json import io from typing import Callable, Awaitable, Dict, Any, List +from pathlib import Path try: from lottie.objects import Animation @@ -27,22 +28,25 @@ def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, uid = int(match.group(1)) if anonymize_users and state: alias = state.get_user_alias(str(uid)) - return f"`@{alias}`" + return f"`@{alias}`" if alias else "`@Anonymized User`" + # 1. Try provided guild member = guild.get_member(uid) if member: return f"`@{member.display_name}`" - # 2. Try message's user_mentions + + # 2. Try provided user_mentions if user_mentions: - for u in user_mentions: - if u.id == uid: - return f"`@{getattr(u, 'display_name', u.name)}`" + m = next((u for u in user_mentions if u.id == uid), None) + if m: + return f"`@{m.display_name}`" + # 3. Try global cache via guild.client if hasattr(guild, 'client'): user = guild.client.get_user(uid) if user: return f"`@{user.name}`" - return f"`@Unknown User`" + return "`@Unknown User`" def replace_role(match): rid = int(match.group(1)) @@ -539,8 +543,11 @@ async def migrate_messages( except Exception as e: logger.error(f"Failed to download sticker {getattr(s, 'name', 'unknown')}: {e}") + # Check for existing mapping to avoid duplicates when resuming + if context.state.get_target_message_id(target_channel_id, str(msg.id)): + continue + try: - # Check if this message is a reply reply_to_fluxer_id = None if msg.reference and msg.reference.message_id: reply_to_fluxer_id = context.state.get_fluxer_message_id(target_channel_id, str(msg.reference.message_id)) @@ -557,23 +564,22 @@ async def migrate_messages( if thread_name and stats["messages"] == 0: content = f"> <<< THREAD: **{thread_name}** >>>\n{content}" - # Get or generate alias + # Always ensure alias is created/retrieved to populate user_alias table alias = context.state.get_user_alias(str(msg.author.id)) - - if context.config.anonymize_users: - author_name = alias - avatar_url = f"https://api.dicebear.com/9.x/fun-emoji/jpg?seed={alias}" + + anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False + if anonymize_users: + author_name = alias or "Anonymized User" + author_avatar_url = None else: author_name = msg.author.display_name - avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None - if avatar_url and not avatar_url.startswith("http"): - avatar_url = None + author_avatar_url = msg.author.avatar.url if hasattr(msg.author, 'avatar') and msg.author.avatar else None logger.debug(f"Fluxer: Calling send_message for Discord ID {msg.id}") fluxer_msg_id = await context.fluxer_writer.send_message( channel_id=target_channel_id, author_name=author_name, - author_avatar_url=avatar_url, + author_avatar_url=author_avatar_url, content=content, timestamp=int(msg.created_at.timestamp()), files=files if files else None, @@ -663,3 +669,261 @@ async def migrate_messages( pass return stats + + +async def analyze_global_migration(context: MigrationContext, after_message_id: int | None = None, inclusive: bool = False, progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None) -> Dict[str, int]: + """ + Scans the entire server history to count messages, threads, and attachments globally. + """ + stats = {"messages": 0, "threads": 0, "attachments": 0} + + # In global mode, thread messages are returned natively in timestamp order by global fetch if they're in the DB + # However we just count them if the fetcher yields them. + # Fetch global progress map to skip migrated messages efficiently + progress_map = context.state.get_all_last_message_ids() + + async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id): + if not context.is_running: + break + + # Determine target channel to check for existing mapping + if not msg.channel: + continue + + target_channel_id = context.state.get_target_channel_id(str(msg.channel.id)) + if not target_channel_id: + continue + + # Efficient skip: if message ID is <= last migrated ID for this channel/thread + # This is the primary resume mechanism: wait until we pass the last migrated ID for this channel + last_id = progress_map.get(str(msg.channel.id)) + if last_id and msg.id <= int(last_id): + continue + + if msg.type not in [ + context.discord_reader.MESSAGE_TYPE_DEFAULT, + context.discord_reader.MESSAGE_TYPE_REPLY, + context.discord_reader.MESSAGE_TYPE_THREAD_STARTER, + context.discord_reader.MESSAGE_TYPE_FORWARD, + context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND, + context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND, + context.discord_reader.MESSAGE_TYPE_POLL_RESULT, + context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION + ]: + continue + + stats["messages"] += 1 + stats["attachments"] += len(msg.attachments) + if hasattr(msg, 'thread') and msg.thread: + # We don't recursively traverse here, we just count the fact there is a thread + # The actual thread messages are also fetched by the global fetcher because they have their own timestamp/id + stats["threads"] += 1 + + if progress_callback and stats["messages"] % 100 == 0: + await progress_callback(stats) + + if progress_callback: + await progress_callback(stats) + + return stats + + +async def migrate_global_messages( + context: MigrationContext, + after_message_id: int | None = None, + inclusive: bool = False, + progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None +) -> Dict[str, Any]: + """ + Migrates messages across all channels chronologically. + """ + stats = { + "messages": 0, + "threads": 0, + "attachments": 0, + "last_message_content": "", + "last_message_author": "", + "first_message_url": None, + "last_message_url": None + } + + processed_threads = set() + logger.info("Starting Global Waterfall Migration for Fluxer...") + + # Keep track of active thread mapping natively to pass parent target IDs if needed + thread_to_target_channel = {} + + # Emojis and mapped users cache setup + emoji_map = context.state.emoji_map + db_media = context.discord_reader.db.get_all_media() if context.discord_reader.db else {} + target_server_id = getattr(context.fluxer_writer, "server_id", None) + + # Fetch global progress map to skip migrated messages efficiently + progress_map = context.state.get_all_last_message_ids() + + try: + async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id): + if not context.is_running: + logger.warning("Global migration interrupted by user") + break + + if msg.type not in [ + context.discord_reader.MESSAGE_TYPE_DEFAULT, + context.discord_reader.MESSAGE_TYPE_REPLY, + context.discord_reader.MESSAGE_TYPE_THREAD_STARTER, + context.discord_reader.MESSAGE_TYPE_FORWARD, + context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND, + context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND, + context.discord_reader.MESSAGE_TYPE_POLL_RESULT, + context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION + ]: + continue + + # Determine target channel + if not msg.channel: + continue + + target_channel_id = context.state.get_target_channel_id(str(msg.channel.id)) + if not target_channel_id: + continue + + # Efficient skip: if message ID is <= last migrated ID for this channel/thread + # This ensures we only resume a channel once we reach its last known progress point + last_id = progress_map.get(str(target_channel_id)) + if last_id and msg.id <= int(last_id): + continue + + # If it's a thread message, we need to handle it based on if it's the thread starter or a reply + parent_target_id = None + if hasattr(msg, 'thread') and msg.thread and msg.id == msg.thread.id: + processed_threads.add(msg.thread.id) + stats["threads"] += 1 + elif msg.channel.type in [11, 12]: # Thread channels + # It's a message IN a thread. + # In Fluxer, threads might just be linear messages or threaded replies depending on schema + # For basic migration we just send it to the parent mapped target channel. + # The parent mapped target channel ID should already be calculated correctly by get_target_channel_id (which returns mapped thread or parent channel) + pass + + # Formatting + files = [] + file_names = [] + + # Always ensure alias is created/retrieved to populate user_alias table + alias = context.state.get_user_alias(str(msg.author.id)) + + anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False + + if anonymize_users: + author_name = alias or "Anonymized User" + author_avatar_url = None + else: + author_name = msg.author.display_name + author_avatar_url = msg.author.avatar.url if hasattr(msg.author, 'avatar') and msg.author.avatar else None + + for att in msg.attachments: + media_info = db_media.get(att.local_hash) if db_media else None + local_path = None + if media_info: + local_path = Path(media_info["local_path"]) + elif hasattr(att, 'read'): + # Fallback + pass + + if local_path and local_path.exists(): + files.append(local_path) + file_names.append(att.filename) + + content = msg.content or "" + + # Stickers + for sticker in msg.stickers: + sticker_name = sticker.name + sticker_url = sticker.url + + # Check for uploaded media pool logic first + s_hash = sticker.local_hash + sticker_file = None + s_media = db_media.get(s_hash) if db_media and s_hash else None + if s_media: + s_path = Path(s_media["local_path"]) + if s_path.exists(): + sticker_file = s_path + + content += f"\n[Sticker: {sticker_name}]" + if sticker_file: + files.append(sticker_file) + file_names.append(f"sticker_{sticker_name}.png") + + content = clean_mentions( + content=content, + guild=context.discord_reader.guild, + user_mentions=msg.mentions, + role_mentions=msg.role_mentions, + channel_mentions=msg.channel_mentions, + emoji_map=emoji_map, + channel_map=context.state.channel_map, + state=context.state, + target_server_id=target_server_id, + channel_names=context.channel_names if hasattr(context, 'channel_names') else None, + anonymize_users=anonymize_users + ) + + if not content and not files: + logger.debug(f"Message {msg.id} empty after processing, skipping.") + continue + + timestamp_int = int(msg.created_at.timestamp()) + + if msg.reference and msg.reference.message_id: + # Resolve the author of the message being replied to + source_ref_msg = await context.discord_reader.get_message(msg.channel.id, msg.reference.message_id) + if source_ref_msg and source_ref_msg.author: + ref_author_id = str(source_ref_msg.author.id) + if anonymize_users: + ref_name = context.state.get_user_alias(ref_author_id) or "Anonymized User" + else: + ref_name = source_ref_msg.author.display_name + content = f"`@{ref_name}`\n{content}" + else: + # Fallback if author cannot be resolved (e.g. deleted/missing from backup) + tgt_reply = context.state.get_target_message_id(target_channel_id, msg.reference.message_id) + if tgt_reply: + content = f"[Reply to {tgt_reply}]\n{content}" + + try: + fluxer_msg_id = await context.fluxer_writer.send_message( + channel_id=target_channel_id, + author_name=author_name, + author_avatar_url=author_avatar_url, + content=content, + files=files, + timestamp=timestamp_int, + embeds=msg.embeds + ) + + if fluxer_msg_id: + context.state.set_target_message_mapping(target_channel_id, msg.id, fluxer_msg_id) + context.state.update_last_message_id(target_channel_id, msg.id) + context.state.set_waterfall_last_id(msg.id) + stats["attachments"] += len(files) if files else 0 + + stats["messages"] += 1 + stats["last_message_content"] = content + stats["last_message_author"] = author_name + + if not stats["first_message_url"]: + stats["first_message_url"] = msg.jump_url + stats["last_message_url"] = msg.jump_url + + if progress_callback: + await progress_callback(stats) + + except Exception as e: + logger.error(f"Failed to process global message {msg.id}: {e}") + + except (KeyboardInterrupt, asyncio.CancelledError): + context.is_running = False + pass + + return stats diff --git a/src/stoat/migrate_message.py b/src/stoat/migrate_message.py index c0861bd..130579f 100644 --- a/src/stoat/migrate_message.py +++ b/src/stoat/migrate_message.py @@ -4,6 +4,7 @@ import re import json import io from typing import Callable, Awaitable, Dict, Any, List +from pathlib import Path try: from lottie.objects import Animation @@ -27,22 +28,25 @@ def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, uid = int(match.group(1)) if anonymize_users and state: alias = state.get_user_alias(str(uid)) - return f"`@{alias}`" + return f"`@{alias}`" if alias else "`@Anonymized User`" + # 1. Try provided guild member = guild.get_member(uid) if member: return f"`@{member.display_name}`" - # 2. Try message's user_mentions + + # 2. Try provided user_mentions if user_mentions: - for u in user_mentions: - if u.id == uid: - return f"`@{getattr(u, 'display_name', u.name)}`" + m = next((u for u in user_mentions if u.id == uid), None) + if m: + return f"`@{m.display_name}`" + # 3. Try global cache via guild.client if hasattr(guild, 'client'): user = guild.client.get_user(uid) if user: return f"`@{user.name}`" - return f"`@Unknown User`" + return "`@Unknown User`" def replace_role(match): rid = int(match.group(1)) @@ -543,6 +547,10 @@ async def migrate_messages( logger.error(f"Failed to download sticker {getattr(s, 'name', 'unknown')}: {e}") try: + # Check for existing mapping to avoid duplicates when resuming + if context.state.get_target_message_id(target_channel_id, str(msg.id)): + continue + # Check if this message is a reply reply_to_stoat_id = None if msg.reference and msg.reference.message_id: @@ -560,22 +568,24 @@ async def migrate_messages( if thread_name and stats["messages"] == 0: content = f"> <<< THREAD: **{thread_name}** >>>\n{content}" - # Get or generate alias + # Always ensure alias is created/retrieved to populate user_alias table alias = context.state.get_user_alias(str(msg.author.id)) - - if context.config.anonymize_users: - author_name = alias - avatar_url = f"https://api.dicebear.com/9.x/fun-emoji/jpg?seed={alias}" + + anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False + if anonymize_users: + author_name = alias or "Anonymized User" + author_avatar_url = None else: author_name = msg.author.display_name - avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None - if avatar_url and not avatar_url.startswith("http"): - avatar_url = None + author_avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None + if author_avatar_url and not author_avatar_url.startswith("http"): + author_avatar_url = None + logger.debug(f"Stoat: Calling send_message for Discord ID {msg.id}") stoat_msg_id = await context.stoat_writer.send_message( channel_id=target_channel_id, author_name=author_name, - author_avatar_url=avatar_url, + author_avatar_url=author_avatar_url, content=content, timestamp=int(msg.created_at.timestamp()), files=files if files else None, @@ -664,3 +674,240 @@ async def migrate_messages( pass return stats + + +async def analyze_global_migration(context: MigrationContext, after_message_id: int | None = None, inclusive: bool = False, progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None) -> Dict[str, int]: + """ + Scans the entire server history to count messages, threads, and attachments globally. + """ + stats = {"messages": 0, "threads": 0, "attachments": 0} + + # Fetch global progress map to skip migrated messages efficiently + progress_map = context.state.get_all_last_message_ids() + + async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id): + if not context.is_running: + break + + # Determine target channel to check for existing mapping + if not msg.channel: + continue + + target_channel_id = context.state.get_target_channel_id(str(msg.channel.id)) + if not target_channel_id: + continue + + # Efficient skip: if message ID is <= last migrated ID for this channel/thread + # This is the primary resume mechanism: wait until we pass the last migrated ID for this channel + last_id = progress_map.get(str(msg.channel.id)) + if last_id and msg.id <= int(last_id): + continue + + if msg.type not in [ + context.discord_reader.MESSAGE_TYPE_DEFAULT, + context.discord_reader.MESSAGE_TYPE_REPLY, + context.discord_reader.MESSAGE_TYPE_THREAD_STARTER, + context.discord_reader.MESSAGE_TYPE_FORWARD, + context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND, + context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND, + context.discord_reader.MESSAGE_TYPE_POLL_RESULT, + context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION + ]: + continue + + stats["messages"] += 1 + stats["attachments"] += len(msg.attachments) + if hasattr(msg, 'thread') and msg.thread: + stats["threads"] += 1 + + if progress_callback and stats["messages"] % 100 == 0: + await progress_callback(stats) + + if progress_callback: + await progress_callback(stats) + + return stats + + +async def migrate_global_messages( + context: MigrationContext, + after_message_id: int | None = None, + inclusive: bool = False, + progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None +) -> Dict[str, Any]: + """ + Migrates messages across all channels chronologically to Stoat. + """ + stats = { + "messages": 0, + "threads": 0, + "attachments": 0, + "last_message_content": "", + "last_message_author": "", + "first_message_url": None, + "last_message_url": None + } + + processed_threads = set() + logger.info("Starting Global Waterfall Migration for Stoat...") + + emoji_map = context.state.emoji_map + db_media = context.discord_reader.db.get_all_media() if context.discord_reader.db else {} + # Fetch global progress map to skip migrated messages efficiently + progress_map = context.state.get_all_last_message_ids() + + try: + async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id): + if not context.is_running: + logger.warning("Global migration interrupted by user") + break + + if msg.type not in [ + context.discord_reader.MESSAGE_TYPE_DEFAULT, + context.discord_reader.MESSAGE_TYPE_REPLY, + context.discord_reader.MESSAGE_TYPE_THREAD_STARTER, + context.discord_reader.MESSAGE_TYPE_FORWARD, + context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND, + context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND, + context.discord_reader.MESSAGE_TYPE_POLL_RESULT, + context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION + ]: + continue + + # Determine target channel + if not msg.channel: + continue + + target_channel_id = context.state.get_target_channel_id(str(msg.channel.id)) + if not target_channel_id: + continue + + # Efficient skip: if message ID is <= last migrated ID for this channel/thread + # This ensures we only resume a channel once we reach its last known progress point + last_id = progress_map.get(str(target_channel_id)) + if last_id and msg.id <= int(last_id): + continue + + parent_target_id = None + if hasattr(msg, 'thread') and msg.thread and msg.id == msg.thread.id: + processed_threads.add(msg.thread.id) + stats["threads"] += 1 + elif msg.channel.type in [11, 12]: + pass + + # Formatting + files = [] + + # Always ensure alias is created/retrieved to populate user_alias table + alias = context.state.get_user_alias(str(msg.author.id)) + + anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False + + if anonymize_users: + author_name = alias or "Anonymized User" + author_avatar_url = None + else: + author_name = msg.author.display_name + author_avatar_url = msg.author.avatar.url if hasattr(msg.author, 'avatar') and msg.author.avatar else None + + for att in msg.attachments: + media_info = db_media.get(att.local_hash) if db_media else None + local_path = None + if media_info: + local_path = Path(media_info["local_path"]) + + if local_path and local_path.exists(): + try: + with open(local_path, "rb") as f: + files.append({"filename": att.filename, "data": f.read()}) + except Exception as fe: + logger.error(f"Failed to read file {local_path}: {fe}") + + content = msg.content or "" + + for sticker in msg.stickers: + sticker_name = sticker.name + s_hash = sticker.local_hash + sticker_file = None + s_media = db_media.get(s_hash) if db_media and s_hash else None + if s_media: + s_path = Path(s_media["local_path"]) + if s_path.exists(): + sticker_file = s_path + + content += f"\n[Sticker: {sticker_name}]" + if sticker_file: + files.append(sticker_file) + file_names.append(f"sticker_{sticker_name}.png") + + content = clean_mentions( + content=content, + guild=context.discord_reader.guild, + user_mentions=msg.mentions, + role_mentions=msg.role_mentions, + channel_mentions=msg.channel_mentions, + emoji_map=emoji_map, + channel_map=context.state.channel_map, + state=context.state, + target_server_id=target_server_id, + channel_names=context.channel_names if hasattr(context, 'channel_names') else None, + anonymize_users=anonymize_users + ) + + if not content and not files: + logger.debug(f"Message {msg.id} empty after processing, skipping.") + continue + + timestamp_int = int(msg.created_at.timestamp()) + + if msg.reference and msg.reference.message_id: + # Resolve the author of the message being replied to + source_ref_msg = await context.discord_reader.get_message(msg.channel_id, msg.reference.message_id) + if source_ref_msg and source_ref_msg.author: + ref_author_id = str(source_ref_msg.author.id) + if anonymize_users: + ref_name = context.state.get_user_alias(ref_author_id) or "Anonymized User" + else: + ref_name = source_ref_msg.author.display_name + content = f"`@{ref_name}`\n{content}" + else: + tgt_reply = context.state.get_target_message_id(target_channel_id, msg.reference.message_id) + if tgt_reply: + content = f"[Reply to {tgt_reply}]\n{content}" + + try: + stoat_msg_id = await context.stoat_writer.send_message( + channel_id=target_channel_id, + author_name=author_name, + author_avatar_url=author_avatar_url, + content=content, + files=files, + timestamp=timestamp_int, + embeds=msg.embeds + ) + + if stoat_msg_id: + context.state.set_target_message_mapping(target_channel_id, msg.id, stoat_msg_id) + context.state.update_last_message_id(target_channel_id, msg.id) + context.state.set_waterfall_last_id(msg.id) + stats["attachments"] += len(files) if files else 0 + + stats["messages"] += 1 + stats["last_message_content"] = content + stats["last_message_author"] = author_name + + if not stats["first_message_url"]: + stats["first_message_url"] = msg.jump_url + stats["last_message_url"] = msg.jump_url + + if progress_callback: + await progress_callback(stats) + + except Exception as e: + logger.error(f"Failed to process global message {msg.id}: {e}") + + except (KeyboardInterrupt, asyncio.CancelledError): + context.is_running = False + pass + + return stats diff --git a/src/ui/shuttle_ops.py b/src/ui/shuttle_ops.py index 362ecc7..fc17583 100644 --- a/src/ui/shuttle_ops.py +++ b/src/ui/shuttle_ops.py @@ -160,6 +160,7 @@ class OperationPane(Container): yield Button("Clone Server Template", id="op_clone", disabled=True, tooltip="Clone server roles, categories, and channels to the target community") yield Button("Sync Server Settings", id="op_sync", disabled=True, tooltip="Sync emojis, stickers, server name, and icon to the target community") yield Button("Migrate Message History", id="op_messages", disabled=True, variant="primary", tooltip="Migrate message history from Discord to the target platform") + yield Button("Waterfall Migration", id="op_waterfall", disabled=True, variant="primary", tooltip="Migrate all messages globally in chronological order to prevent broken links.\n(Available for Local Backups)") yield Rule(id="footer_rule") yield Button("Danger Zone ⚠", id="op_danger", variant="error", disabled=True, flat=True, tooltip="Dangerous operations:\ndelete channels, roles, emojis on target\n(use with caution)") @@ -394,7 +395,7 @@ class OperationPane(Container): lbl.update(f"{t_status}") # Buttons - for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_danger"): + for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_waterfall", "#op_danger"): for btn in self.query(bid): btn.disabled = not self.tokens_valid # ── validation ──────────────────────────────────────────────────────── @@ -415,7 +416,7 @@ class OperationPane(Container): # Disable all operation buttons while validation is in progress if self.view_mode == "shuttle": - for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_danger"): + for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_waterfall", "#op_danger"): for btn in self.query(bid): btn.disabled = True elif self.view_mode == "backup": for bid in ("#op_backup_msgs", "#op_backup_sync"): @@ -551,8 +552,13 @@ class OperationPane(Container): else: target_ok = v.get("target_token") and v.get("target_community") self.tokens_valid = bool(discord_ok and target_ok) - - + + # Post validation adjustments + if self.tokens_valid: + is_backup = (self.config.tool_mode == "backup_transfer") + for btn in self.query("#op_waterfall"): + btn.disabled = not is_backup + btn.display = is_backup self._update_info_labels() @@ -570,6 +576,8 @@ class OperationPane(Container): self._open_sync_menu() elif bid == "op_messages": self.run_migrate_messages() + elif bid == "op_waterfall": + self.run_waterfall_migration() elif bid == "op_danger": self._open_danger_menu() @@ -716,7 +724,6 @@ class OperationPane(Container): return elif choice == "btn_main_menu": modal.dismiss() - self.app.switch_screen("config_selection") return force_mode = (choice == "btn_start_id") @@ -798,7 +805,6 @@ class OperationPane(Container): return elif choice == "btn_main_menu": modal.dismiss() - self.app.switch_screen("config_selection") return force_mode = (choice == "btn_start_id") @@ -1099,6 +1105,11 @@ class OperationPane(Container): source_channel = next(c for c in d_channels if c.id == src_id) target_channel = next(c for c in f_channels if c.get("id") == tgt_id) + # 2. Analyze + modal = ProgressScreen(log_level=self.config.log_level) + self.app.push_screen(modal) + await asyncio.sleep(0.1) + # Determine after_id status (skip for pending channels) if pending_create_name: last_migrated = None @@ -1106,11 +1117,6 @@ class OperationPane(Container): else: last_migrated = self.engine.state.get_last_message_id(str(target_channel.get('id'))) has_previous = bool(last_migrated) - - # Analyze - modal = ProgressScreen(log_level=self.config.log_level) - self.app.push_screen(modal) - await asyncio.sleep(0.1) src_server = getattr(self.engine.discord_reader, 'guild', None) tgt_server_info = await self.engine.writer.validate() @@ -1243,7 +1249,6 @@ class OperationPane(Container): continue # Return to channel picker elif choice == "btn_main_menu": modal.dismiss() - self.app.switch_screen("config_selection") self.engine.is_running = False await self.engine.close_connections() return @@ -1294,6 +1299,14 @@ class OperationPane(Container): try: self.engine.is_running = True + + # Ensure state is initialized (database exists) + if self.target_platform == "stoat": + tid = self.config.stoat_server_id + else: + tid = self.config.fluxer_server_id + self.engine.ensure_state_initialized(str(tid or ""), platform_name) + stats_analysis = await migrate_mod.analyze_migration( self.engine, source_channel_id=source_channel.id, @@ -1399,6 +1412,239 @@ class OperationPane(Container): else: modal.write(f"[bold red]Error: {err}[/bold red]") modal.phase_report("Message Migration", "error", show_back=False) + logger.error(f"Migration Error: {traceback.format_exc()}") + finally: + self.engine.is_running = False + await self.engine.close_connections() + + @work(exclusive=True) + async def run_waterfall_migration(self) -> None: + if not self.tokens_valid: + return + + migrate_mod = fluxer_migrate if self.target_platform == "fluxer" else stoat_migrate + platform_name = self.target_platform.capitalize() + + modal = ProgressScreen(log_level=self.config.log_level) + self.app.push_screen(modal) + await asyncio.sleep(0.1) + + try: + modal.show_info("[bold cyan]Waterfall Migration Ready[/bold cyan]", "Checking mapping and missing channels...") + modal.set_status("Connecting to Servers...") + await self.engine.start_connections() + + modal.set_status("Synchronizing entity mappings...") + await self._perform_auto_matching() + + # 1. Missing channels check + if hasattr(self.engine.discord_reader, "get_all_channels"): + full_d = await self.engine.discord_reader.get_all_channels() + # Include TEXT (0), CATEGORY (4), and NEWS (5) + d_channels = [c for c in full_d if c.type in [0, 4, 5]] + else: + full_d = await self.engine.discord_reader.get_channels() + d_channels = [c for c in full_d if c.type in [0, 4, 5]] + missing_channels = [] + for d in d_channels: + if d.type == 4: + tgt_id = self.engine.state.get_target_category_id(str(d.id)) + else: + tgt_id = self.engine.state.get_target_channel_id(str(d.id)) + if not tgt_id: + missing_channels.append(d) + + if missing_channels: + modal.write(f"\n[bold yellow]Found {len(missing_channels)} backed-up channels/categories missing from target platform:[/bold yellow]") + for mc in missing_channels: + prefix = "[bold cyan]📁[/bold cyan] " if mc.type == 4 else "[bold white]#[/bold white] " + modal.write(f" {prefix}{mc.name}") + + choice = await modal.phase_wait_confirm( + show_continue=False, + show_id=True, + btn_start_label="Clone missing channels", + btn_id_label="Skip missing channels", + btn_start_variant="primary", + btn_start_tooltip=f"Automatically create {len(missing_channels)} entities on target", + btn_id_tooltip="Start migration without these channels" + ) + + if choice == "btn_back": + modal.dismiss() + await self.engine.close_connections() + return + elif choice == "btn_main_menu": + modal.dismiss() + await self.engine.close_connections() + return + + if choice == "btn_start_first": + modal.set_status("Cloning missing categories and channels...") + # Sort so categories (type 4) come first + missing_channels.sort(key=lambda x: 0 if x.type == 4 else 1) + + for mc in missing_channels: + try: + parent_target_id = None + if mc.type == 4: + modal.write(f"Creating category '[bold cyan]{mc.name}[/bold cyan]'...") + new_id = await self.engine.writer.create_channel(name=mc.name, type=4) + self.engine.state.set_target_category_mapping(str(mc.id), new_id) + modal.write(f"[green]Created Category {mc.name} ({new_id})[/green]") + else: + if hasattr(mc, 'category_id') and mc.category_id: + parent_target_id = self.engine.state.get_target_category_id(str(mc.category_id)) + + modal.write(f"Creating channel '#{mc.name}'...") + new_id = await self.engine.writer.create_channel(name=mc.name, parent_id=parent_target_id) + self.engine.state.set_target_channel_id(str(mc.id), new_id, self.engine.target_platform) + modal.write(f"[green]Created Channel {mc.name} ({new_id})[/green]") + except Exception as e: + logger.error(f"Failed to create {mc.name}: {e}\n{traceback.format_exc()}") + modal.write(f"[red]Failed to create {mc.name}: {e}[/red]") + elif choice == "btn_id": + # Skip missing channels: remove them from the active list + missing_ids = {str(c.id) for c in missing_channels} + d_channels = [c for c in d_channels if str(c.id) not in missing_ids] + + # 2. Resumption check + all_mapped_tgt_ids = [] + # Check regular text channels (exclude categories for resume check) + for c in d_channels: + if c.type == 4: continue + did = str(c.id) + tid = self.engine.state.get_target_channel_id(did) + if tid: all_mapped_tgt_ids.append(tid) + + # Also check threads (filtering to only include those belonging to active channels) + active_channel_ids = {str(c.id) for c in d_channels} + if hasattr(self.engine.discord_reader, "get_active_threads"): + threads = await self.engine.discord_reader.get_active_threads() + for t in threads: + pid = str(getattr(t, 'parent_id', getattr(t, 'channel_id', None))) + if pid not in active_channel_ids: continue + tid = self.engine.state.get_target_channel_id(str(t.id)) + if tid: all_mapped_tgt_ids.append(tid) + + # 2.5 Filter by actual content (Only for BackupReader) + # If a channel has NO messages in the backup, it will always be at 0 progress. + # We exclude those from the global MIN calculation to avoid pulling it to 0. + if hasattr(self.engine.discord_reader, "get_backed_up_channel_ids"): + backed_up_src_ids = await self.engine.discord_reader.get_backed_up_channel_ids() + backed_up_src_ids_str = {str(sid) for sid in backed_up_src_ids} + + filtered_tgt_ids = [] + # Find which target IDs belong to source channels that HAVE messages + for c in d_channels: # (d_channels is already filtered for skipped) + if str(c.id) in backed_up_src_ids_str: + tid = self.engine.state.get_target_channel_id(str(c.id)) + if tid: filtered_tgt_ids.append(tid) + + # Also check threads + if hasattr(self.engine.discord_reader, "threads"): + for t in self.engine.discord_reader.threads: + if str(t.id) in backed_up_src_ids_str: + tid = self.engine.state.get_target_channel_id(str(t.id)) + if tid: filtered_tgt_ids.append(tid) + + if filtered_tgt_ids: + all_mapped_tgt_ids = filtered_tgt_ids + + # 2.6 Resume Point: Prioritize Global waterfall tracker, fallback to channel minimums + min_last_id = self.engine.state.get_waterfall_last_id() + if min_last_id is None: + min_last_id = self.engine.state.get_global_min_last_message_id(all_mapped_tgt_ids) + + modal.write(f"\n[bold cyan]Waterfall Migration Resume Point:[/bold cyan]") + if min_last_id is not None: + modal.write(f"Minimum unmigrated message ID found: [green]{min_last_id}[/green]") + else: + modal.write("No previous migration state found. Starting from the beginning.") + + choice = await modal.phase_wait_confirm( + show_continue=min_last_id is not None, + show_id=False, + btn_start_label="Start From Beginning", + btn_start_tooltip="Safe, skips duplicates automatically", + btn_start_variant="default" if min_last_id is not None else "primary", + btn_continue_label=f"Continue from ID {min_last_id if min_last_id is not None else 0}" if min_last_id is not None else "Continue Migration", + btn_continue_tooltip="Fastest" + ) + + if choice == "btn_back": + modal.dismiss() + await self.engine.close_connections() + return + elif choice == "btn_main_menu": + modal.dismiss() + await self.engine.close_connections() + return + + after_id = None + if choice == "btn_continue" and min_last_id is not None: + after_id = int(min_last_id) + + # Phase 3: Progress + modal.cancel_callback = lambda: setattr(self.engine, "is_running", False) + modal.phase_progress() + modal.set_status("Migrating messages Globally...") + + self.engine.is_running = True + + # Ensure state is initialized (database exists) + if self.target_platform == "stoat": + tid = self.config.stoat_server_id + else: + tid = self.config.fluxer_server_id + self.engine.ensure_state_initialized(str(tid or ""), platform_name) + + modal.write("Scanning global footprint for totals ...") + stats_analysis = await migrate_mod.analyze_global_migration(self.engine, after_message_id=after_id) + total_messages = stats_analysis["messages"] + + modal.write(f"[bold cyan]Global Migration Started:[/bold cyan] {total_messages} total messages to process.") + modal.update_stats(messages=f"0/{total_messages}", threads=str(stats_analysis["threads"]), files=str(stats_analysis["attachments"])) + + async def update_msg(current_stats): + c_msgs = current_stats["messages"] + c_threads = current_stats["threads"] + c_files = current_stats["attachments"] + + msg_stat = f"{c_msgs}/{total_messages}" if total_messages > 0 else str(c_msgs) + modal.set_progress(c_msgs, total_messages or 100) + modal.update_stats(messages=msg_stat, threads=str(c_threads), files=str(c_files)) + + content = current_stats.get("last_message_content", "") + author = current_stats.get("last_message_author", "Unknown") + if content: + disp_content = (content[:100] + '...') if len(content) > 100 else content + modal.write(f"[bold]{author}:[/bold] {disp_content}") + + result = await migrate_mod.migrate_global_messages( + self.engine, + after_message_id=after_id, + inclusive=False, + progress_callback=update_msg, + ) + + if self.engine.is_running: + modal.write(f"[bold green]Success! {result['messages']} messages migrated globally.[/bold green]") + modal.phase_report("Waterfall Migration", show_back=False) + else: + modal.write(f"[bold yellow]Interrupted! {result['messages']} messages migrated.[/bold yellow]") + modal.phase_report("Waterfall Migration", "stopped", show_back=False) + + lines = [f"Migrated Server Globally → {platform_name}:"] + lines.append(f"{result['messages']} messages, {result['attachments']} attachments, {result['threads']} threads") + await log_audit_event(self.engine, "Waterfall Migration", "\n".join(lines)) + + except Exception as e: + err = str(e) + modal.write(f"[bold red]Error: {err}[/bold red]") + modal.phase_report("Waterfall Migration", "error", show_back=False) + + logger.error(traceback.format_exc()) finally: self.engine.is_running = False await self.engine.close_connections() @@ -1485,7 +1731,6 @@ class OperationPane(Container): return elif choice == "btn_main_menu": modal.dismiss() - self.app.switch_screen("config_selection") return modal.cancel_callback = lambda: setattr(self.engine, "is_running", False) @@ -1644,6 +1889,39 @@ class OperationPane(Container): logger.warning(f"Auto-matching: failed to fetch target data: {e}") return # Cannot match without target data + # 1.5 Cleanup deleted entities from mapping database + # This prevents "Ghost" mappings to channels/roles that were deleted on target + valid_chan_ids = {str(c.get("id")) for c in target_chans_raw} + valid_cat_ids = {str(c.get("id")) for c in target_chans_raw if c.get("type") == 4} + valid_role_ids = set(target_roles_map.values()) + valid_emoji_ids = set(target_emojis_map.values()) + + # Channels + for src_id, tgt_id in self.engine.state.channel_map.items(): + if str(tgt_id) not in valid_chan_ids: + logger.info(f"Auto-matching: clearing deleted channel mapping {src_id} -> {tgt_id}") + self.engine.state.remove_target_channel_mapping(src_id) + + # Categories + for src_id, tgt_id in self.engine.state.category_map.items(): + if str(tgt_id) not in valid_cat_ids: + logger.info(f"Auto-matching: clearing deleted category mapping {src_id} -> {tgt_id}") + self.engine.state.remove_category_mapping(src_id) + + # Roles + for src_id, tgt_id in self.engine.state.role_map.items(): + if str(tgt_id) not in valid_role_ids: + logger.info(f"Auto-matching: clearing deleted role mapping {src_id} -> {tgt_id}") + self.engine.state.remove_role_mapping(src_id) + + # Emojis + for src_id, tgt_id in self.engine.state.emoji_map.items(): + if str(tgt_id) not in valid_emoji_ids: + # Emojis might be URLs in some platforms, but we check if they are IDs first + if isinstance(tgt_id, str) and tgt_id.isdigit(): + logger.info(f"Auto-matching: clearing deleted emoji mapping {src_id} -> {tgt_id}") + self.engine.state.remove_emoji_mapping(src_id) + # 2. Match entities try: # Roles @@ -1687,6 +1965,7 @@ class OperationPane(Container): logger.info(f"Auto-matched Sticker: {s.name} -> {target_stickers_map[name_l]}") self.engine.state.set_target_sticker_mapping(s.id, target_stickers_map[name_l]) except Exception as e: + logger.error(f"Auto-matching error: {e}\n{traceback.format_exc()}") logger.warning(f"Auto-matching error: {e}") return { @@ -1951,7 +2230,6 @@ class OperationPane(Container): after_id = verified_id elif choice == "btn_main_menu": modal_prog.dismiss() - self.app.switch_screen("config_selection") return # If we are here, proceeding either via Start First or Start from ID (after_id) @@ -2062,7 +2340,7 @@ class OperationPane(Container): self.engine.is_running = False await self.engine.close_connections() if choice == "btn_main_menu": - self.app.switch_screen("config_selection") + pass return modal_prog.cancel_callback = lambda: setattr(self.engine, "is_running", False)