Merge pull request #9 from rambros3d/waterfall

Waterfall Mode

README.md

@@ -9,8 +9,8 @@
 
 ![Disco Reaper](images/fluxer-reaper.jpg)
 
-### Modern Terminal Interface
+### Video Guide - [Youtube](https://www.youtube.com/watch?v=SwIPQDxLzqA)
-The tool now features a unified, intuitive TUI (Terminal User Interface) - no more text commands
+
 
 | Features | Fluxer | Stoat |
 | :--- | :---: | :---: |

src/core/backup_database.py

@@ -791,6 +791,83 @@ class BackupDatabase:
             
             return msg_list
 
+    def get_global_messages_paged(self, limit: int = 100, offset: int = 0, after_id: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Fetches messages across ALL channels globally, ordered by timestamp/ID ascending."""
+        with self._lock:
+            query = "SELECT * FROM messages"
+            params = []
+            
+            if after_id:
+                query += " WHERE id > ?"
+                params.append(parse_snowflake(after_id))
+            
+            query += " ORDER BY id ASC LIMIT ? OFFSET ?"
+            params.extend([limit, offset])
+            
+            rows = self._conn.execute(query, params).fetchall()
+            msg_list = [dict(r) for r in rows]
+            
+            if msg_list:
+                msg_ids = [m["id"] for m in msg_list]
+                placeholders = ",".join(["?"] * len(msg_ids))
+                
+                att_rows = self._conn.execute(f"SELECT * FROM attachments WHERE message_id IN ({placeholders})", msg_ids).fetchall()
+                atts_by_msg = {}
+                for ar in att_rows:
+                    mid = ar["message_id"]
+                    if mid not in atts_by_msg: atts_by_msg[mid] = []
+                    atts_by_msg[mid].append(dict(ar))
+                
+                emb_rows = self._conn.execute(f"SELECT * FROM embeds WHERE message_id IN ({placeholders})", msg_ids).fetchall()
+                embs_by_msg = {}
+                for er in emb_rows:
+                    mid = er["message_id"]
+                    if mid not in embs_by_msg: embs_by_msg[mid] = []
+                    
+                    e_dict = {
+                        "title": er["title"],
+                        "description": er["description"],
+                        "url": er["url"],
+                        "color": er["color"],
+                        "timestamp": er["timestamp"],
+                        "thumbnail": {"url": er["thumbnail_url"]} if er["thumbnail_url"] else None,
+                        "image": {"url": er["image_url"]} if er["image_url"] else None,
+                        "author": {
+                            "name": er["author_name"],
+                            "url": er["author_url"],
+                            "icon_url": er["author_icon_url"]
+                        } if er["author_name"] else None,
+                        "footer": {
+                            "text": er["footer_text"],
+                            "icon_url": er["footer_icon_url"]
+                        } if er["footer_text"] else None,
+                        "fields": json.loads(er["fields"]) if er["fields"] else []
+                    }
+                    embs_by_msg[mid].append(e_dict)
+                
+                rea_rows = self._conn.execute(f"SELECT * FROM reactions WHERE message_id IN ({placeholders})", msg_ids).fetchall()
+                reas_by_msg = {}
+                for rr in rea_rows:
+                    mid = rr["message_id"]
+                    if mid not in reas_by_msg: reas_by_msg[mid] = []
+                    reas_by_msg[mid].append(dict(rr))
+                
+                st_rows = self._conn.execute(f"SELECT * FROM message_stickers WHERE message_id IN ({placeholders})", msg_ids).fetchall()
+                sts_by_msg = {}
+                for sr in st_rows:
+                    mid = sr["message_id"]
+                    if mid not in sts_by_msg: sts_by_msg[mid] = []
+                    sts_by_msg[mid].append(dict(sr))
+
+                for m in msg_list:
+                    m_id = m["id"]
+                    m["attachments"] = atts_by_msg.get(m_id, [])
+                    m["embeds"] = embs_by_msg.get(m_id, [])
+                    m["reactions"] = reas_by_msg.get(m_id, [])
+                    m["stickers"] = sts_by_msg.get(m_id, [])
+            
+            return msg_list
+
     def delete_channel_messages(self, channel_id: Union[str, int]):
         """Deletes all messages and related metadata for a specific channel and its threads."""
         cid = parse_snowflake(channel_id)

src/core/backup_reader.py

@@ -1417,6 +1417,42 @@ class BackupReader:
             if len(msgs) < batch_size:
                 break
 
+    async def fetch_global_message_history(
+        self,
+        limit: int = None,
+        after_id: int = None
+    ) -> AsyncGenerator["BackupMessage", None]:
+        """Yields BackupMessages globally from SQLite across all channels, natively ordered by timestamp/ID."""
+        if not self.db: return
+        
+        offset = 0
+        batch_size = 100
+        count = 0
+        
+        while True:
+            actual_limit = batch_size
+            if limit:
+                rem = limit - count
+                if rem <= 0: break
+                actual_limit = min(batch_size, rem)
+            
+            msgs = self.db.get_global_messages_paged(
+                limit=actual_limit, 
+                offset=offset, 
+                after_id=str(after_id) if after_id else None
+            )
+            
+            if not msgs:
+                break
+                
+            for m in msgs:
+                yield self._hydrate_message(m)
+                count += 1
+                
+            offset += len(msgs)
+            if len(msgs) < batch_size:
+                break
+
     # ── download helpers ─────────────────────────────────────────────────
 
     async def download_emoji(self, emoji: BackupEmoji) -> bytes:

src/core/database.py

@@ -451,6 +451,49 @@ class MigrationDatabase:
             return dict(row)
         return {"last_msg_id": None, "last_msg_ts": None, "msg_count": 0, "file_count": 0}
 
+
+    def get_global_min_last_message_id(self, all_mapped_ids: List[str]) -> Optional[int]:
+        """
+        Returns the minimum last_msg_id successfully migrated across all mapped channels/threads.
+        If any mapped entity has no progress record, it is treated as ID 0.
+        Returns None only if NO progress has been made across ANY entity.
+        """
+        if not all_mapped_ids:
+            return None
+            
+        conn = self._get_conn()
+        placeholders = ",".join(["?"] * len(all_mapped_ids))
+        
+        # 1. Get last message IDs from channel tracking
+        c_rows = conn.execute(f"SELECT channel_id, last_msg_id FROM channel_tracking WHERE channel_id IN ({placeholders})", all_mapped_ids).fetchall()
+        c_map = {r["channel_id"]: r["last_msg_id"] for r in c_rows}
+        
+        # 2. Get last message IDs from thread tracking
+        t_rows = conn.execute(f"SELECT thread_id, last_msg_id FROM thread_tracking WHERE thread_id IN ({placeholders})", all_mapped_ids).fetchall()
+        t_map = {r["thread_id"]: r["last_msg_id"] for r in t_rows}
+        
+        # Combine maps
+        progress_map = {**c_map, **t_map}
+        
+        # 3. Aggregate IDs
+        ids = []
+        has_any_progress = False
+        for mid in all_mapped_ids:
+            last_id = progress_map.get(mid)
+            if not last_id:
+                ids.append(0) # Unmigrated entity
+            else:
+                try:
+                    ids.append(int(last_id))
+                    has_any_progress = True
+                except (ValueError, TypeError):
+                    ids.append(0)
+                    
+        if not has_any_progress:
+            return None
+            
+        return min(ids)
+
     # Thread methods similar to channel methods
     def set_thread_message_mapping(self, channel_id: str, thread_id: str, source_id: str, target_id: str, timestamp: str = None):
         conn = self._get_conn()
@@ -496,6 +539,18 @@ class MigrationDatabase:
             return dict(row)
         return {"last_msg_id": None, "last_msg_ts": None, "msg_count": 0, "file_count": 0}
 
+    def get_all_channel_tracking_ids(self) -> Dict[str, str]:
+        """Returns a map of channel_id -> last_msg_id for all tracked channels."""
+        conn = self._get_conn()
+        rows = conn.execute("SELECT channel_id, last_msg_id FROM channel_tracking WHERE last_msg_id IS NOT NULL").fetchall()
+        return {str(row["channel_id"]): str(row["last_msg_id"]) for row in rows}
+
+    def get_all_thread_tracking_ids(self) -> Dict[str, str]:
+        """Returns a map of thread_id -> last_msg_id for all tracked threads."""
+        conn = self._get_conn()
+        rows = conn.execute("SELECT thread_id, last_msg_id FROM thread_tracking WHERE last_msg_id IS NOT NULL").fetchall()
+        return {str(row["thread_id"]): str(row["last_msg_id"]) for row in rows}
+
     def clear_channel_data(self, channel_id: str):
         """Purge all mappings and tracking data for a specific channel and its threads."""
         conn = self._get_conn()

src/core/state.py

@@ -38,6 +38,13 @@ class MigrationState:
         if self.db:
             self.db.delete_server_mapping("channel", str(discord_id))
 
+    def remove_target_channel_mapping(self, discord_id: int | str):
+        if self.db:
+            self.db.delete_server_mapping("channel", str(discord_id))
+    
+    def set_target_channel_id(self, discord_id: int | str, target_id: str, *args):
+        """Alias for set_channel_mapping to handle legacy calls."""
+        self.set_channel_mapping(discord_id, target_id)
         
     get_fluxer_channel_id = get_target_channel_id
     set_target_channel_mapping = set_channel_mapping
@@ -58,6 +65,10 @@ class MigrationState:
         if self.db:
             self.db.delete_server_mapping("category", str(discord_id))
     
+    def set_target_category_id(self, discord_id: int | str, target_id: str, *args):
+        """Alias for set_category_mapping to handle legacy calls."""
+        self.set_category_mapping(discord_id, target_id)
+        
     get_fluxer_category_id = get_category_mapping
     get_target_category_id = get_category_mapping
     set_target_category_mapping = set_category_mapping
@@ -78,6 +89,10 @@ class MigrationState:
         if self.db:
             self.db.delete_server_mapping("role", str(discord_id))
     
+    def set_target_role_id(self, discord_id: int | str, target_id: str, *args):
+        """Alias for set_role_mapping to handle legacy calls."""
+        self.set_role_mapping(discord_id, target_id)
+        
     get_fluxer_role_id = get_role_mapping
     get_target_role_id = get_role_mapping
     set_target_role_mapping = set_role_mapping
@@ -212,9 +227,35 @@ class MigrationState:
         
     def get_last_message_id(self, target_channel_id: str) -> str | None:
         if self._ensure_db():
-            return self.db.get_channel_tracking(str(target_channel_id)).get("last_msg_id")
+            tracking = self.db.get_channel_tracking(str(target_channel_id))
+            return tracking.get("last_msg_id") if tracking else None
         return None
         
+
+    def get_global_min_last_message_id(self, all_mapped_ids: List[str]) -> int | None:
+        """Returns the absolute minimum last_msg_id among the given list of mapped target IDs (channels and threads)."""
+        if self._ensure_db():
+            return self.db.get_global_min_last_message_id(all_mapped_ids)
+        return None
+
+    def set_waterfall_last_id(self, last_id: str | int):
+        if self.db:
+            self.db.set_metadata("waterfall_last_id", str(last_id))
+            
+    def get_waterfall_last_id(self) -> int | None:
+        if self.db:
+            val = self.db.get_metadata("waterfall_last_id")
+            return int(val) if val else None
+        return None
+        
+    def get_all_last_message_ids(self) -> Dict[str, str]:
+        """Returns a combined map of channel_id/thread_id -> last_msg_id."""
+        if self._ensure_db():
+            c_map = self.db.get_all_channel_tracking_ids()
+            t_map = self.db.get_all_thread_tracking_ids()
+            return {**c_map, **t_map}
+        return {}
+
     def get_thread_last_message_id(self, target_channel_id: str, thread_id: str) -> str | None:
         if self._ensure_db():
             return self.db.get_thread_tracking(str(target_channel_id), str(thread_id)).get("last_msg_id")

src/fluxer/migrate_message.py

@@ -4,6 +4,7 @@ import re
 import json
 import io
 from typing import Callable, Awaitable, Dict, Any, List
+from pathlib import Path
 
 try:
     from lottie.objects import Animation
@@ -27,22 +28,25 @@ def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None,
         uid = int(match.group(1))
         if anonymize_users and state:
             alias = state.get_user_alias(str(uid))
-            return f"`@{alias}`"
+            return f"`@{alias}`" if alias else "`@Anonymized User`"
+        
         # 1. Try provided guild
         member = guild.get_member(uid)
         if member:
             return f"`@{member.display_name}`"
-        # 2. Try message's user_mentions
+        
+        # 2. Try provided user_mentions
         if user_mentions:
-            for u in user_mentions:
+            m = next((u for u in user_mentions if u.id == uid), None)
-                if u.id == uid:
+            if m:
-                    return f"`@{getattr(u, 'display_name', u.name)}`"
+                return f"`@{m.display_name}`"
+        
         # 3. Try global cache via guild.client
         if hasattr(guild, 'client'):
             user = guild.client.get_user(uid)
             if user:
                 return f"`@{user.name}`"
-        return f"`@Unknown User`"
+        return "`@Unknown User`"
         
     def replace_role(match):
         rid = int(match.group(1))
@@ -539,8 +543,11 @@ async def migrate_messages(
                     except Exception as e:
                         logger.error(f"Failed to download sticker {getattr(s, 'name', 'unknown')}: {e}")
                 
+            # Check for existing mapping to avoid duplicates when resuming
+            if context.state.get_target_message_id(target_channel_id, str(msg.id)):
+                continue
+                
             try:
-                # Check if this message is a reply
                 reply_to_fluxer_id = None
                 if msg.reference and msg.reference.message_id:
                     reply_to_fluxer_id = context.state.get_fluxer_message_id(target_channel_id, str(msg.reference.message_id))
@@ -557,23 +564,22 @@ async def migrate_messages(
                 if thread_name and stats["messages"] == 0:
                     content = f"> <<< THREAD: **{thread_name}** >>>\n{content}"
                 
-                # Get or generate alias
+                # Always ensure alias is created/retrieved to populate user_alias table
                 alias = context.state.get_user_alias(str(msg.author.id))
                 
-                if context.config.anonymize_users:
+                anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False
-                    author_name = alias
+                if anonymize_users:
-                    avatar_url = f"https://api.dicebear.com/9.x/fun-emoji/jpg?seed={alias}"
+                    author_name = alias or "Anonymized User"
+                    author_avatar_url = None
                 else:
                     author_name = msg.author.display_name
-                    avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
+                    author_avatar_url = msg.author.avatar.url if hasattr(msg.author, 'avatar') and msg.author.avatar else None
-                    if avatar_url and not avatar_url.startswith("http"):
-                        avatar_url = None
 
                 logger.debug(f"Fluxer: Calling send_message for Discord ID {msg.id}")
                 fluxer_msg_id = await context.fluxer_writer.send_message(
                     channel_id=target_channel_id,
                     author_name=author_name,
-                    author_avatar_url=avatar_url,
+                    author_avatar_url=author_avatar_url,
                     content=content,
                     timestamp=int(msg.created_at.timestamp()),
                     files=files if files else None,
@@ -663,3 +669,261 @@ async def migrate_messages(
         pass
     
     return stats
+
+
+async def analyze_global_migration(context: MigrationContext, after_message_id: int | None = None, inclusive: bool = False, progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None) -> Dict[str, int]:
+    """
+    Scans the entire server history to count messages, threads, and attachments globally.
+    """
+    stats = {"messages": 0, "threads": 0, "attachments": 0}
+    
+    # In global mode, thread messages are returned natively in timestamp order by global fetch if they're in the DB
+    # However we just count them if the fetcher yields them.
+    # Fetch global progress map to skip migrated messages efficiently
+    progress_map = context.state.get_all_last_message_ids()
+    
+    async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id):
+        if not context.is_running:
+            break
+            
+        # Determine target channel to check for existing mapping
+        if not msg.channel:
+            continue
+            
+        target_channel_id = context.state.get_target_channel_id(str(msg.channel.id))
+        if not target_channel_id:
+            continue
+
+        # Efficient skip: if message ID is <= last migrated ID for this channel/thread
+        # This is the primary resume mechanism: wait until we pass the last migrated ID for this channel
+        last_id = progress_map.get(str(msg.channel.id))
+        if last_id and msg.id <= int(last_id):
+            continue
+            
+        if msg.type not in [
+            context.discord_reader.MESSAGE_TYPE_DEFAULT,
+            context.discord_reader.MESSAGE_TYPE_REPLY,
+            context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
+            context.discord_reader.MESSAGE_TYPE_FORWARD,
+            context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
+            context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
+            context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
+            context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
+        ]:
+            continue
+            
+        stats["messages"] += 1
+        stats["attachments"] += len(msg.attachments)
+        if hasattr(msg, 'thread') and msg.thread:
+            # We don't recursively traverse here, we just count the fact there is a thread
+            # The actual thread messages are also fetched by the global fetcher because they have their own timestamp/id
+            stats["threads"] += 1
+            
+        if progress_callback and stats["messages"] % 100 == 0:
+            await progress_callback(stats)
+            
+    if progress_callback:
+        await progress_callback(stats)
+        
+    return stats
+
+
+async def migrate_global_messages(
+    context: MigrationContext,
+    after_message_id: int | None = None,
+    inclusive: bool = False,
+    progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None
+) -> Dict[str, Any]:
+    """
+    Migrates messages across all channels chronologically.
+    """
+    stats = {
+        "messages": 0,
+        "threads": 0,
+        "attachments": 0,
+        "last_message_content": "",
+        "last_message_author": "",
+        "first_message_url": None,
+        "last_message_url": None
+    }
+    
+    processed_threads = set()
+    logger.info("Starting Global Waterfall Migration for Fluxer...")
+    
+    # Keep track of active thread mapping natively to pass parent target IDs if needed
+    thread_to_target_channel = {}
+    
+    # Emojis and mapped users cache setup
+    emoji_map = context.state.emoji_map
+    db_media = context.discord_reader.db.get_all_media() if context.discord_reader.db else {}
+    target_server_id = getattr(context.fluxer_writer, "server_id", None)
+    
+    # Fetch global progress map to skip migrated messages efficiently
+    progress_map = context.state.get_all_last_message_ids()
+
+    try:
+        async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id):
+            if not context.is_running:
+                logger.warning("Global migration interrupted by user")
+                break
+                
+            if msg.type not in [
+                context.discord_reader.MESSAGE_TYPE_DEFAULT,
+                context.discord_reader.MESSAGE_TYPE_REPLY,
+                context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
+                context.discord_reader.MESSAGE_TYPE_FORWARD,
+                context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
+                context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
+                context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
+                context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
+            ]:
+                continue
+                
+            # Determine target channel
+            if not msg.channel:
+                continue
+                
+            target_channel_id = context.state.get_target_channel_id(str(msg.channel.id))
+            if not target_channel_id:
+                continue
+
+            # Efficient skip: if message ID is <= last migrated ID for this channel/thread
+            # This ensures we only resume a channel once we reach its last known progress point
+            last_id = progress_map.get(str(target_channel_id))
+            if last_id and msg.id <= int(last_id):
+                continue
+                
+            # If it's a thread message, we need to handle it based on if it's the thread starter or a reply
+            parent_target_id = None
+            if hasattr(msg, 'thread') and msg.thread and msg.id == msg.thread.id:
+                processed_threads.add(msg.thread.id)
+                stats["threads"] += 1
+            elif msg.channel.type in [11, 12]:  # Thread channels
+                # It's a message IN a thread.
+                # In Fluxer, threads might just be linear messages or threaded replies depending on schema
+                # For basic migration we just send it to the parent mapped target channel.
+                # The parent mapped target channel ID should already be calculated correctly by get_target_channel_id (which returns mapped thread or parent channel)
+                pass
+
+            # Formatting
+            files = []
+            file_names = []
+            
+            # Always ensure alias is created/retrieved to populate user_alias table
+            alias = context.state.get_user_alias(str(msg.author.id))
+            
+            anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False
+            
+            if anonymize_users:
+                author_name = alias or "Anonymized User"
+                author_avatar_url = None
+            else:
+                author_name = msg.author.display_name
+                author_avatar_url = msg.author.avatar.url if hasattr(msg.author, 'avatar') and msg.author.avatar else None
+
+            for att in msg.attachments:
+                media_info = db_media.get(att.local_hash) if db_media else None
+                local_path = None
+                if media_info:
+                    local_path = Path(media_info["local_path"])
+                elif hasattr(att, 'read'):
+                    # Fallback
+                    pass 
+
+                if local_path and local_path.exists():
+                    files.append(local_path)
+                    file_names.append(att.filename)
+
+            content = msg.content or ""
+            
+            # Stickers
+            for sticker in msg.stickers:
+                sticker_name = sticker.name
+                sticker_url = sticker.url
+                
+                # Check for uploaded media pool logic first
+                s_hash = sticker.local_hash
+                sticker_file = None
+                s_media = db_media.get(s_hash) if db_media and s_hash else None
+                if s_media:
+                    s_path = Path(s_media["local_path"])
+                    if s_path.exists():
+                        sticker_file = s_path
+                
+                content += f"\n[Sticker: {sticker_name}]"
+                if sticker_file:
+                    files.append(sticker_file)
+                    file_names.append(f"sticker_{sticker_name}.png")
+
+            content = clean_mentions(
+                content=content,
+                guild=context.discord_reader.guild,
+                user_mentions=msg.mentions,
+                role_mentions=msg.role_mentions,
+                channel_mentions=msg.channel_mentions,
+                emoji_map=emoji_map,
+                channel_map=context.state.channel_map,
+                state=context.state,
+                target_server_id=target_server_id,
+                channel_names=context.channel_names if hasattr(context, 'channel_names') else None,
+                anonymize_users=anonymize_users
+            )
+
+            if not content and not files:
+                logger.debug(f"Message {msg.id} empty after processing, skipping.")
+                continue
+                
+            timestamp_int = int(msg.created_at.timestamp())
+
+            if msg.reference and msg.reference.message_id:
+                # Resolve the author of the message being replied to
+                source_ref_msg = await context.discord_reader.get_message(msg.channel.id, msg.reference.message_id)
+                if source_ref_msg and source_ref_msg.author:
+                    ref_author_id = str(source_ref_msg.author.id)
+                    if anonymize_users:
+                        ref_name = context.state.get_user_alias(ref_author_id) or "Anonymized User"
+                    else:
+                        ref_name = source_ref_msg.author.display_name
+                    content = f"`@{ref_name}`\n{content}"
+                else:
+                    # Fallback if author cannot be resolved (e.g. deleted/missing from backup)
+                    tgt_reply = context.state.get_target_message_id(target_channel_id, msg.reference.message_id)
+                    if tgt_reply:
+                        content = f"[Reply to {tgt_reply}]\n{content}"
+
+            try:
+                fluxer_msg_id = await context.fluxer_writer.send_message(
+                    channel_id=target_channel_id,
+                    author_name=author_name,
+                    author_avatar_url=author_avatar_url,
+                    content=content,
+                    files=files,
+                    timestamp=timestamp_int,
+                    embeds=msg.embeds
+                )
+                
+                if fluxer_msg_id:
+                    context.state.set_target_message_mapping(target_channel_id, msg.id, fluxer_msg_id)
+                    context.state.update_last_message_id(target_channel_id, msg.id)
+                    context.state.set_waterfall_last_id(msg.id)
+                    stats["attachments"] += len(files) if files else 0
+
+                stats["messages"] += 1
+                stats["last_message_content"] = content
+                stats["last_message_author"] = author_name
+
+                if not stats["first_message_url"]:
+                    stats["first_message_url"] = msg.jump_url
+                stats["last_message_url"] = msg.jump_url
+                
+                if progress_callback:
+                    await progress_callback(stats)
+                    
+            except Exception as e:
+                logger.error(f"Failed to process global message {msg.id}: {e}")
+                
+    except (KeyboardInterrupt, asyncio.CancelledError):
+        context.is_running = False
+        pass
+        
+    return stats

src/stoat/migrate_message.py

@@ -4,6 +4,7 @@ import re
 import json
 import io
 from typing import Callable, Awaitable, Dict, Any, List
+from pathlib import Path
 
 try:
     from lottie.objects import Animation
@@ -27,22 +28,25 @@ def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None,
         uid = int(match.group(1))
         if anonymize_users and state:
             alias = state.get_user_alias(str(uid))
-            return f"`@{alias}`"
+            return f"`@{alias}`" if alias else "`@Anonymized User`"
+        
         # 1. Try provided guild
         member = guild.get_member(uid)
         if member:
             return f"`@{member.display_name}`"
-        # 2. Try message's user_mentions
+        
+        # 2. Try provided user_mentions
         if user_mentions:
-            for u in user_mentions:
+            m = next((u for u in user_mentions if u.id == uid), None)
-                if u.id == uid:
+            if m:
-                    return f"`@{getattr(u, 'display_name', u.name)}`"
+                return f"`@{m.display_name}`"
+        
         # 3. Try global cache via guild.client
         if hasattr(guild, 'client'):
             user = guild.client.get_user(uid)
             if user:
                 return f"`@{user.name}`"
-        return f"`@Unknown User`"
+        return "`@Unknown User`"
         
     def replace_role(match):
         rid = int(match.group(1))
@@ -543,6 +547,10 @@ async def migrate_messages(
                         logger.error(f"Failed to download sticker {getattr(s, 'name', 'unknown')}: {e}")
                 
             try:
+                # Check for existing mapping to avoid duplicates when resuming
+                if context.state.get_target_message_id(target_channel_id, str(msg.id)):
+                    continue
+
                 # Check if this message is a reply
                 reply_to_stoat_id = None
                 if msg.reference and msg.reference.message_id:
@@ -560,22 +568,24 @@ async def migrate_messages(
                 if thread_name and stats["messages"] == 0:
                     content = f"> <<< THREAD: **{thread_name}** >>>\n{content}"
                 
-                # Get or generate alias
+                # Always ensure alias is created/retrieved to populate user_alias table
                 alias = context.state.get_user_alias(str(msg.author.id))
                 
-                if context.config.anonymize_users:
+                anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False
-                    author_name = alias
+                if anonymize_users:
-                    avatar_url = f"https://api.dicebear.com/9.x/fun-emoji/jpg?seed={alias}"
+                    author_name = alias or "Anonymized User"
+                    author_avatar_url = None
                 else:
                     author_name = msg.author.display_name
-                    avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
+                    author_avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
-                    if avatar_url and not avatar_url.startswith("http"):
+                    if author_avatar_url and not author_avatar_url.startswith("http"):
-                        avatar_url = None
+                        author_avatar_url = None
 
+                logger.debug(f"Stoat: Calling send_message for Discord ID {msg.id}")
                 stoat_msg_id = await context.stoat_writer.send_message(
                     channel_id=target_channel_id,
                     author_name=author_name,
-                    author_avatar_url=avatar_url,
+                    author_avatar_url=author_avatar_url,
                     content=content,
                     timestamp=int(msg.created_at.timestamp()),
                     files=files if files else None,
@@ -664,3 +674,240 @@ async def migrate_messages(
         pass
         
     return stats
+
+
+async def analyze_global_migration(context: MigrationContext, after_message_id: int | None = None, inclusive: bool = False, progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None) -> Dict[str, int]:
+    """
+    Scans the entire server history to count messages, threads, and attachments globally.
+    """
+    stats = {"messages": 0, "threads": 0, "attachments": 0}
+    
+    # Fetch global progress map to skip migrated messages efficiently
+    progress_map = context.state.get_all_last_message_ids()
+
+    async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id):
+        if not context.is_running:
+            break
+            
+        # Determine target channel to check for existing mapping
+        if not msg.channel:
+            continue
+            
+        target_channel_id = context.state.get_target_channel_id(str(msg.channel.id))
+        if not target_channel_id:
+            continue
+
+        # Efficient skip: if message ID is <= last migrated ID for this channel/thread
+        # This is the primary resume mechanism: wait until we pass the last migrated ID for this channel
+        last_id = progress_map.get(str(msg.channel.id))
+        if last_id and msg.id <= int(last_id):
+            continue
+
+        if msg.type not in [
+            context.discord_reader.MESSAGE_TYPE_DEFAULT,
+            context.discord_reader.MESSAGE_TYPE_REPLY,
+            context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
+            context.discord_reader.MESSAGE_TYPE_FORWARD,
+            context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
+            context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
+            context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
+            context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
+        ]:
+            continue
+            
+        stats["messages"] += 1
+        stats["attachments"] += len(msg.attachments)
+        if hasattr(msg, 'thread') and msg.thread:
+            stats["threads"] += 1
+            
+        if progress_callback and stats["messages"] % 100 == 0:
+            await progress_callback(stats)
+            
+    if progress_callback:
+        await progress_callback(stats)
+        
+    return stats
+
+
+async def migrate_global_messages(
+    context: MigrationContext,
+    after_message_id: int | None = None,
+    inclusive: bool = False,
+    progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None
+) -> Dict[str, Any]:
+    """
+    Migrates messages across all channels chronologically to Stoat.
+    """
+    stats = {
+        "messages": 0,
+        "threads": 0,
+        "attachments": 0,
+        "last_message_content": "",
+        "last_message_author": "",
+        "first_message_url": None,
+        "last_message_url": None
+    }
+    
+    processed_threads = set()
+    logger.info("Starting Global Waterfall Migration for Stoat...")
+    
+    emoji_map = context.state.emoji_map
+    db_media = context.discord_reader.db.get_all_media() if context.discord_reader.db else {}
+    # Fetch global progress map to skip migrated messages efficiently
+    progress_map = context.state.get_all_last_message_ids()
+    
+    try:
+        async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id):
+            if not context.is_running:
+                logger.warning("Global migration interrupted by user")
+                break
+                
+            if msg.type not in [
+                context.discord_reader.MESSAGE_TYPE_DEFAULT,
+                context.discord_reader.MESSAGE_TYPE_REPLY,
+                context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
+                context.discord_reader.MESSAGE_TYPE_FORWARD,
+                context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
+                context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
+                context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
+                context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
+            ]:
+                continue
+                
+            # Determine target channel
+            if not msg.channel:
+                continue
+                
+            target_channel_id = context.state.get_target_channel_id(str(msg.channel.id))
+            if not target_channel_id:
+                continue
+
+            # Efficient skip: if message ID is <= last migrated ID for this channel/thread
+            # This ensures we only resume a channel once we reach its last known progress point
+            last_id = progress_map.get(str(target_channel_id))
+            if last_id and msg.id <= int(last_id):
+                continue
+                
+            parent_target_id = None
+            if hasattr(msg, 'thread') and msg.thread and msg.id == msg.thread.id:
+                processed_threads.add(msg.thread.id)
+                stats["threads"] += 1
+            elif msg.channel.type in [11, 12]:
+                pass
+
+            # Formatting
+            files = []
+            
+            # Always ensure alias is created/retrieved to populate user_alias table
+            alias = context.state.get_user_alias(str(msg.author.id))
+            
+            anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False
+            
+            if anonymize_users:
+                author_name = alias or "Anonymized User"
+                author_avatar_url = None
+            else:
+                author_name = msg.author.display_name
+                author_avatar_url = msg.author.avatar.url if hasattr(msg.author, 'avatar') and msg.author.avatar else None
+
+            for att in msg.attachments:
+                media_info = db_media.get(att.local_hash) if db_media else None
+                local_path = None
+                if media_info:
+                    local_path = Path(media_info["local_path"])
+
+                if local_path and local_path.exists():
+                    try:
+                        with open(local_path, "rb") as f:
+                            files.append({"filename": att.filename, "data": f.read()})
+                    except Exception as fe:
+                        logger.error(f"Failed to read file {local_path}: {fe}")
+
+            content = msg.content or ""
+            
+            for sticker in msg.stickers:
+                sticker_name = sticker.name
+                s_hash = sticker.local_hash
+                sticker_file = None
+                s_media = db_media.get(s_hash) if db_media and s_hash else None
+                if s_media:
+                    s_path = Path(s_media["local_path"])
+                    if s_path.exists():
+                        sticker_file = s_path
+                
+                content += f"\n[Sticker: {sticker_name}]"
+                if sticker_file:
+                    files.append(sticker_file)
+                    file_names.append(f"sticker_{sticker_name}.png")
+
+            content = clean_mentions(
+                content=content,
+                guild=context.discord_reader.guild,
+                user_mentions=msg.mentions,
+                role_mentions=msg.role_mentions,
+                channel_mentions=msg.channel_mentions,
+                emoji_map=emoji_map,
+                channel_map=context.state.channel_map,
+                state=context.state,
+                target_server_id=target_server_id,
+                channel_names=context.channel_names if hasattr(context, 'channel_names') else None,
+                anonymize_users=anonymize_users
+            )
+
+            if not content and not files:
+                logger.debug(f"Message {msg.id} empty after processing, skipping.")
+                continue
+                
+            timestamp_int = int(msg.created_at.timestamp())
+
+            if msg.reference and msg.reference.message_id:
+                # Resolve the author of the message being replied to
+                source_ref_msg = await context.discord_reader.get_message(msg.channel_id, msg.reference.message_id)
+                if source_ref_msg and source_ref_msg.author:
+                    ref_author_id = str(source_ref_msg.author.id)
+                    if anonymize_users:
+                        ref_name = context.state.get_user_alias(ref_author_id) or "Anonymized User"
+                    else:
+                        ref_name = source_ref_msg.author.display_name
+                    content = f"`@{ref_name}`\n{content}"
+                else:
+                    tgt_reply = context.state.get_target_message_id(target_channel_id, msg.reference.message_id)
+                    if tgt_reply:
+                        content = f"[Reply to {tgt_reply}]\n{content}"
+
+            try:
+                stoat_msg_id = await context.stoat_writer.send_message(
+                    channel_id=target_channel_id,
+                    author_name=author_name,
+                    author_avatar_url=author_avatar_url,
+                    content=content,
+                    files=files,
+                    timestamp=timestamp_int,
+                    embeds=msg.embeds
+                )
+                
+                if stoat_msg_id:
+                    context.state.set_target_message_mapping(target_channel_id, msg.id, stoat_msg_id)
+                    context.state.update_last_message_id(target_channel_id, msg.id)
+                    context.state.set_waterfall_last_id(msg.id)
+                    stats["attachments"] += len(files) if files else 0
+
+                stats["messages"] += 1
+                stats["last_message_content"] = content
+                stats["last_message_author"] = author_name
+
+                if not stats["first_message_url"]:
+                    stats["first_message_url"] = msg.jump_url
+                stats["last_message_url"] = msg.jump_url
+                
+                if progress_callback:
+                    await progress_callback(stats)
+                    
+            except Exception as e:
+                logger.error(f"Failed to process global message {msg.id}: {e}")
+                
+    except (KeyboardInterrupt, asyncio.CancelledError):
+        context.is_running = False
+        pass
+        
+    return stats

src/ui/shuttle_ops.py

@@ -160,6 +160,7 @@ class OperationPane(Container):
                     yield Button("Clone Server Template", id="op_clone", disabled=True, tooltip="Clone server roles, categories, and channels to the target community")
                     yield Button("Sync Server Settings", id="op_sync", disabled=True, tooltip="Sync emojis, stickers, server name, and icon to the target community")
                     yield Button("Migrate Message History", id="op_messages", disabled=True, variant="primary", tooltip="Migrate message history from Discord to the target platform")
+                    yield Button("Waterfall Migration", id="op_waterfall", disabled=True, variant="primary", tooltip="Migrate all messages globally in chronological order to prevent broken links.\n(Available for Local Backups)")
                     yield Rule(id="footer_rule")
                     yield Button("Danger Zone ⚠", id="op_danger", variant="error", disabled=True, flat=True, tooltip="Dangerous operations:\ndelete channels, roles, emojis on target\n(use with caution)")
 
@@ -394,7 +395,7 @@ class OperationPane(Container):
                 lbl.update(f"{t_status}")
 
             # Buttons
-            for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_danger"):
+            for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_waterfall", "#op_danger"):
                 for btn in self.query(bid): btn.disabled = not self.tokens_valid
 
     # ── validation ────────────────────────────────────────────────────────
@@ -415,7 +416,7 @@ class OperationPane(Container):
             
             # Disable all operation buttons while validation is in progress
             if self.view_mode == "shuttle":
-                for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_danger"):
+                for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_waterfall", "#op_danger"):
                     for btn in self.query(bid): btn.disabled = True
             elif self.view_mode == "backup":
                 for bid in ("#op_backup_msgs", "#op_backup_sync"):
@@ -552,7 +553,12 @@ class OperationPane(Container):
             target_ok = v.get("target_token") and v.get("target_community")
             self.tokens_valid = bool(discord_ok and target_ok)
             
-
+            # Post validation adjustments
+            if self.tokens_valid:
+                is_backup = (self.config.tool_mode == "backup_transfer")
+                for btn in self.query("#op_waterfall"):
+                    btn.disabled = not is_backup
+                    btn.display = is_backup
 
         self._update_info_labels()
 
@@ -570,6 +576,8 @@ class OperationPane(Container):
             self._open_sync_menu()
         elif bid == "op_messages":
             self.run_migrate_messages()
+        elif bid == "op_waterfall":
+            self.run_waterfall_migration()
         elif bid == "op_danger":
             self._open_danger_menu()
             
@@ -716,7 +724,6 @@ class OperationPane(Container):
                 return
             elif choice == "btn_main_menu":
                 modal.dismiss()
-                self.app.switch_screen("config_selection")
                 return
 
             force_mode = (choice == "btn_start_id")
@@ -798,7 +805,6 @@ class OperationPane(Container):
                 return
             elif choice == "btn_main_menu":
                 modal.dismiss()
-                self.app.switch_screen("config_selection")
                 return
 
             force_mode = (choice == "btn_start_id")
@@ -1099,6 +1105,11 @@ class OperationPane(Container):
                     source_channel = next(c for c in d_channels if c.id == src_id)
                     target_channel = next(c for c in f_channels if c.get("id") == tgt_id)
 
+                # 2. Analyze
+                modal = ProgressScreen(log_level=self.config.log_level)
+                self.app.push_screen(modal)
+                await asyncio.sleep(0.1)
+
                 # Determine after_id status (skip for pending channels)
                 if pending_create_name:
                     last_migrated = None
@@ -1107,11 +1118,6 @@ class OperationPane(Container):
                     last_migrated = self.engine.state.get_last_message_id(str(target_channel.get('id')))
                     has_previous = bool(last_migrated)
 
-                # Analyze
-                modal = ProgressScreen(log_level=self.config.log_level)
-                self.app.push_screen(modal)
-                await asyncio.sleep(0.1)
-
                 src_server = getattr(self.engine.discord_reader, 'guild', None)
                 tgt_server_info = await self.engine.writer.validate()
                 tgt_server_name = tgt_server_info.get("community_name", "target community")
@@ -1243,7 +1249,6 @@ class OperationPane(Container):
                     continue # Return to channel picker
                 elif choice == "btn_main_menu":
                     modal.dismiss()
-                    self.app.switch_screen("config_selection")
                     self.engine.is_running = False
                     await self.engine.close_connections()
                     return
@@ -1294,6 +1299,14 @@ class OperationPane(Container):
 
                     try:
                         self.engine.is_running = True
+                        
+                        # Ensure state is initialized (database exists)
+                        if self.target_platform == "stoat":
+                            tid = self.config.stoat_server_id
+                        else:
+                            tid = self.config.fluxer_server_id
+                        self.engine.ensure_state_initialized(str(tid or ""), platform_name)
+
                         stats_analysis = await migrate_mod.analyze_migration(
                             self.engine,
                             source_channel_id=source_channel.id,
@@ -1399,6 +1412,239 @@ class OperationPane(Container):
             else:
                 modal.write(f"[bold red]Error: {err}[/bold red]")
             modal.phase_report("Message Migration", "error", show_back=False)
+            logger.error(f"Migration Error: {traceback.format_exc()}")
+        finally:
+            self.engine.is_running = False
+            await self.engine.close_connections()
+
+    @work(exclusive=True)
+    async def run_waterfall_migration(self) -> None:
+        if not self.tokens_valid:
+            return
+
+        migrate_mod = fluxer_migrate if self.target_platform == "fluxer" else stoat_migrate
+        platform_name = self.target_platform.capitalize()
+
+        modal = ProgressScreen(log_level=self.config.log_level)
+        self.app.push_screen(modal)
+        await asyncio.sleep(0.1)
+
+        try:
+            modal.show_info("[bold cyan]Waterfall Migration Ready[/bold cyan]", "Checking mapping and missing channels...")
+            modal.set_status("Connecting to Servers...")
+            await self.engine.start_connections()
+            
+            modal.set_status("Synchronizing entity mappings...")
+            await self._perform_auto_matching()
+
+            # 1. Missing channels check
+            if hasattr(self.engine.discord_reader, "get_all_channels"):
+                full_d = await self.engine.discord_reader.get_all_channels()
+                # Include TEXT (0), CATEGORY (4), and NEWS (5)
+                d_channels = [c for c in full_d if c.type in [0, 4, 5]]
+            else:
+                full_d = await self.engine.discord_reader.get_channels()
+                d_channels = [c for c in full_d if c.type in [0, 4, 5]]
+            missing_channels = []
+            for d in d_channels:
+                if d.type == 4:
+                    tgt_id = self.engine.state.get_target_category_id(str(d.id))
+                else:
+                    tgt_id = self.engine.state.get_target_channel_id(str(d.id))
+                if not tgt_id:
+                    missing_channels.append(d)
+
+            if missing_channels:
+                modal.write(f"\n[bold yellow]Found {len(missing_channels)} backed-up channels/categories missing from target platform:[/bold yellow]")
+                for mc in missing_channels:
+                    prefix = "[bold cyan]📁[/bold cyan] " if mc.type == 4 else "[bold white]#[/bold white] "
+                    modal.write(f"  {prefix}{mc.name}")
+                
+                choice = await modal.phase_wait_confirm(
+                    show_continue=False,
+                    show_id=True,
+                    btn_start_label="Clone missing channels",
+                    btn_id_label="Skip missing channels",
+                    btn_start_variant="primary",
+                    btn_start_tooltip=f"Automatically create {len(missing_channels)} entities on target",
+                    btn_id_tooltip="Start migration without these channels"
+                )
+                
+                if choice == "btn_back":
+                    modal.dismiss()
+                    await self.engine.close_connections()
+                    return
+                elif choice == "btn_main_menu":
+                    modal.dismiss()
+                    await self.engine.close_connections()
+                    return
+                
+                if choice == "btn_start_first":
+                    modal.set_status("Cloning missing categories and channels...")
+                    # Sort so categories (type 4) come first
+                    missing_channels.sort(key=lambda x: 0 if x.type == 4 else 1)
+                    
+                    for mc in missing_channels:
+                        try:
+                            parent_target_id = None
+                            if mc.type == 4:
+                                modal.write(f"Creating category '[bold cyan]{mc.name}[/bold cyan]'...")
+                                new_id = await self.engine.writer.create_channel(name=mc.name, type=4)
+                                self.engine.state.set_target_category_mapping(str(mc.id), new_id)
+                                modal.write(f"[green]Created Category {mc.name} ({new_id})[/green]")
+                            else:
+                                if hasattr(mc, 'category_id') and mc.category_id:
+                                    parent_target_id = self.engine.state.get_target_category_id(str(mc.category_id))
+                                
+                                modal.write(f"Creating channel '#{mc.name}'...")
+                                new_id = await self.engine.writer.create_channel(name=mc.name, parent_id=parent_target_id)
+                                self.engine.state.set_target_channel_id(str(mc.id), new_id, self.engine.target_platform)
+                                modal.write(f"[green]Created Channel {mc.name} ({new_id})[/green]")
+                        except Exception as e:
+                            logger.error(f"Failed to create {mc.name}: {e}\n{traceback.format_exc()}")
+                            modal.write(f"[red]Failed to create {mc.name}: {e}[/red]")
+                elif choice == "btn_id":
+                    # Skip missing channels: remove them from the active list
+                    missing_ids = {str(c.id) for c in missing_channels}
+                    d_channels = [c for c in d_channels if str(c.id) not in missing_ids]
+            
+            # 2. Resumption check
+            all_mapped_tgt_ids = []
+            # Check regular text channels (exclude categories for resume check)
+            for c in d_channels:
+                if c.type == 4: continue
+                did = str(c.id)
+                tid = self.engine.state.get_target_channel_id(did)
+                if tid: all_mapped_tgt_ids.append(tid)
+            
+            # Also check threads (filtering to only include those belonging to active channels)
+            active_channel_ids = {str(c.id) for c in d_channels}
+            if hasattr(self.engine.discord_reader, "get_active_threads"):
+                threads = await self.engine.discord_reader.get_active_threads()
+                for t in threads:
+                    pid = str(getattr(t, 'parent_id', getattr(t, 'channel_id', None)))
+                    if pid not in active_channel_ids: continue
+                    tid = self.engine.state.get_target_channel_id(str(t.id))
+                    if tid: all_mapped_tgt_ids.append(tid)
+            
+            # 2.5 Filter by actual content (Only for BackupReader)
+            # If a channel has NO messages in the backup, it will always be at 0 progress.
+            # We exclude those from the global MIN calculation to avoid pulling it to 0.
+            if hasattr(self.engine.discord_reader, "get_backed_up_channel_ids"):
+                backed_up_src_ids = await self.engine.discord_reader.get_backed_up_channel_ids()
+                backed_up_src_ids_str = {str(sid) for sid in backed_up_src_ids}
+                
+                filtered_tgt_ids = []
+                # Find which target IDs belong to source channels that HAVE messages
+                for c in d_channels: # (d_channels is already filtered for skipped)
+                    if str(c.id) in backed_up_src_ids_str:
+                        tid = self.engine.state.get_target_channel_id(str(c.id))
+                        if tid: filtered_tgt_ids.append(tid)
+                
+                # Also check threads
+                if hasattr(self.engine.discord_reader, "threads"):
+                    for t in self.engine.discord_reader.threads:
+                        if str(t.id) in backed_up_src_ids_str:
+                            tid = self.engine.state.get_target_channel_id(str(t.id))
+                            if tid: filtered_tgt_ids.append(tid)
+                
+                if filtered_tgt_ids:
+                    all_mapped_tgt_ids = filtered_tgt_ids
+
+            # 2.6 Resume Point: Prioritize Global waterfall tracker, fallback to channel minimums
+            min_last_id = self.engine.state.get_waterfall_last_id()
+            if min_last_id is None:
+                min_last_id = self.engine.state.get_global_min_last_message_id(all_mapped_tgt_ids)
+            
+            modal.write(f"\n[bold cyan]Waterfall Migration Resume Point:[/bold cyan]")
+            if min_last_id is not None:
+                modal.write(f"Minimum unmigrated message ID found: [green]{min_last_id}[/green]")
+            else:
+                modal.write("No previous migration state found. Starting from the beginning.")
+            
+            choice = await modal.phase_wait_confirm(
+                show_continue=min_last_id is not None,
+                show_id=False,
+                btn_start_label="Start From Beginning",
+                btn_start_tooltip="Safe, skips duplicates automatically",
+                btn_start_variant="default" if min_last_id is not None else "primary",
+                btn_continue_label=f"Continue from ID {min_last_id if min_last_id is not None else 0}" if min_last_id is not None else "Continue Migration",
+                btn_continue_tooltip="Fastest"
+            )
+            
+            if choice == "btn_back":
+                modal.dismiss()
+                await self.engine.close_connections()
+                return
+            elif choice == "btn_main_menu":
+                modal.dismiss()
+                await self.engine.close_connections()
+                return
+                
+            after_id = None
+            if choice == "btn_continue" and min_last_id is not None:
+                after_id = int(min_last_id)
+            
+            # Phase 3: Progress
+            modal.cancel_callback = lambda: setattr(self.engine, "is_running", False)
+            modal.phase_progress()
+            modal.set_status("Migrating messages Globally...")
+            
+            self.engine.is_running = True
+            
+            # Ensure state is initialized (database exists)
+            if self.target_platform == "stoat":
+                tid = self.config.stoat_server_id
+            else:
+                tid = self.config.fluxer_server_id
+            self.engine.ensure_state_initialized(str(tid or ""), platform_name)
+
+            modal.write("Scanning global footprint for totals ...")
+            stats_analysis = await migrate_mod.analyze_global_migration(self.engine, after_message_id=after_id)
+            total_messages = stats_analysis["messages"]
+            
+            modal.write(f"[bold cyan]Global Migration Started:[/bold cyan] {total_messages} total messages to process.")
+            modal.update_stats(messages=f"0/{total_messages}", threads=str(stats_analysis["threads"]), files=str(stats_analysis["attachments"]))
+            
+            async def update_msg(current_stats):
+                c_msgs = current_stats["messages"]
+                c_threads = current_stats["threads"]
+                c_files = current_stats["attachments"]
+                
+                msg_stat = f"{c_msgs}/{total_messages}" if total_messages > 0 else str(c_msgs)
+                modal.set_progress(c_msgs, total_messages or 100)
+                modal.update_stats(messages=msg_stat, threads=str(c_threads), files=str(c_files))
+                
+                content = current_stats.get("last_message_content", "")
+                author = current_stats.get("last_message_author", "Unknown")
+                if content:
+                    disp_content = (content[:100] + '...') if len(content) > 100 else content
+                    modal.write(f"[bold]{author}:[/bold] {disp_content}")
+
+            result = await migrate_mod.migrate_global_messages(
+                self.engine,
+                after_message_id=after_id,
+                inclusive=False,
+                progress_callback=update_msg,
+            )
+
+            if self.engine.is_running:
+                modal.write(f"[bold green]Success! {result['messages']} messages migrated globally.[/bold green]")
+                modal.phase_report("Waterfall Migration", show_back=False)
+            else:
+                modal.write(f"[bold yellow]Interrupted! {result['messages']} messages migrated.[/bold yellow]")
+                modal.phase_report("Waterfall Migration", "stopped", show_back=False)
+                
+            lines = [f"Migrated Server Globally → {platform_name}:"]
+            lines.append(f"{result['messages']} messages, {result['attachments']} attachments, {result['threads']} threads")
+            await log_audit_event(self.engine, "Waterfall Migration", "\n".join(lines))
+
+        except Exception as e:
+            err = str(e)
+            modal.write(f"[bold red]Error: {err}[/bold red]")
+            modal.phase_report("Waterfall Migration", "error", show_back=False)
+            
+            logger.error(traceback.format_exc())
         finally:
             self.engine.is_running = False
             await self.engine.close_connections()
@@ -1485,7 +1731,6 @@ class OperationPane(Container):
                 return
             elif choice == "btn_main_menu":
                 modal.dismiss()
-                self.app.switch_screen("config_selection")
                 return
 
             modal.cancel_callback = lambda: setattr(self.engine, "is_running", False)
@@ -1644,6 +1889,39 @@ class OperationPane(Container):
             logger.warning(f"Auto-matching: failed to fetch target data: {e}")
             return # Cannot match without target data
 
+        # 1.5 Cleanup deleted entities from mapping database
+        # This prevents "Ghost" mappings to channels/roles that were deleted on target
+        valid_chan_ids = {str(c.get("id")) for c in target_chans_raw}
+        valid_cat_ids = {str(c.get("id")) for c in target_chans_raw if c.get("type") == 4}
+        valid_role_ids = set(target_roles_map.values())
+        valid_emoji_ids = set(target_emojis_map.values())
+
+        # Channels
+        for src_id, tgt_id in self.engine.state.channel_map.items():
+            if str(tgt_id) not in valid_chan_ids:
+                logger.info(f"Auto-matching: clearing deleted channel mapping {src_id} -> {tgt_id}")
+                self.engine.state.remove_target_channel_mapping(src_id)
+
+        # Categories
+        for src_id, tgt_id in self.engine.state.category_map.items():
+            if str(tgt_id) not in valid_cat_ids:
+                logger.info(f"Auto-matching: clearing deleted category mapping {src_id} -> {tgt_id}")
+                self.engine.state.remove_category_mapping(src_id)
+
+        # Roles
+        for src_id, tgt_id in self.engine.state.role_map.items():
+            if str(tgt_id) not in valid_role_ids:
+                logger.info(f"Auto-matching: clearing deleted role mapping {src_id} -> {tgt_id}")
+                self.engine.state.remove_role_mapping(src_id)
+
+        # Emojis
+        for src_id, tgt_id in self.engine.state.emoji_map.items():
+            if str(tgt_id) not in valid_emoji_ids:
+                # Emojis might be URLs in some platforms, but we check if they are IDs first
+                if isinstance(tgt_id, str) and tgt_id.isdigit():
+                    logger.info(f"Auto-matching: clearing deleted emoji mapping {src_id} -> {tgt_id}")
+                    self.engine.state.remove_emoji_mapping(src_id)
+
         # 2. Match entities
         try:
             # Roles
@@ -1687,6 +1965,7 @@ class OperationPane(Container):
                         logger.info(f"Auto-matched Sticker: {s.name} -> {target_stickers_map[name_l]}")
                         self.engine.state.set_target_sticker_mapping(s.id, target_stickers_map[name_l])
         except Exception as e:
+            logger.error(f"Auto-matching error: {e}\n{traceback.format_exc()}")
             logger.warning(f"Auto-matching error: {e}")
 
         return {
@@ -1951,7 +2230,6 @@ class OperationPane(Container):
                     after_id = verified_id
                 elif choice == "btn_main_menu":
                     modal_prog.dismiss()
-                    self.app.switch_screen("config_selection")
                     return
                 
                 # If we are here, proceeding either via Start First or Start from ID (after_id)
@@ -2062,7 +2340,7 @@ class OperationPane(Container):
                 self.engine.is_running = False
                 await self.engine.close_connections()
                 if choice == "btn_main_menu":
-                    self.app.switch_screen("config_selection")
+                    pass
                 return
 
             modal_prog.cancel_callback = lambda: setattr(self.engine, "is_running", False)