diff --git a/src/core/discord_reader.py b/src/core/discord_reader.py index 6b28cf3..f90edfd 100644 --- a/src/core/discord_reader.py +++ b/src/core/discord_reader.py @@ -208,15 +208,28 @@ class DiscordReader: async def get_first_message(self, channel_id: int): """Returns the first (oldest) message in a channel.""" channel = await self.get_channel(channel_id) - if isinstance(channel, discord.TextChannel) or isinstance(channel, discord.Thread): + if hasattr(channel, 'history'): async for message in channel.history(limit=1, oldest_first=True): return message + elif isinstance(channel, discord.ForumChannel): + # For forums, find the oldest thread and get its starter message + threads = [] + threads.extend(channel.threads) + async for arch_thread in channel.archived_threads(limit=None): + threads.append(arch_thread) + if threads: + threads.sort(key=lambda t: t.id) + oldest_thread = threads[0] + try: + return await oldest_thread.fetch_message(oldest_thread.id) + except Exception: + pass return None async def fetch_message_history(self, channel_id: int, limit: int = None, after_id: int = None, inclusive: bool = False) -> AsyncGenerator[discord.Message, None]: """Yields messages from a given channel, optionally handling pagination.""" channel = await self.get_channel(channel_id) - if isinstance(channel, discord.TextChannel) or isinstance(channel, discord.Thread): + if hasattr(channel, 'history'): # Discord's 'after' is exclusive. To make it inclusive, we use after_id - 1 if requested. after = None if after_id: @@ -225,6 +238,32 @@ class DiscordReader: # To avoid exploding RAM, we yield items one by one async for message in channel.history(limit=limit, oldest_first=True, after=after): yield message + elif isinstance(channel, discord.ForumChannel): + logger.info(f"Fetching message history for ForumChannel {channel.name} ({channel.id}) oldest_first=True after={after_id} inclusive={inclusive}") + threads = [] + threads.extend(channel.threads) + async for arch_thread in channel.archived_threads(limit=None): + threads.append(arch_thread) + + # Sort threads chronologically (by ID) + threads.sort(key=lambda t: t.id) + + for thread in threads: + if after_id: + if not inclusive and thread.id <= after_id: + continue + if inclusive and thread.id < after_id: + continue + + try: + # In a forum, the starter message ID is the thread ID + starter = await thread.fetch_message(thread.id) + # Bind the thread so migrate_messages handles it properly + if not hasattr(starter, 'thread') or starter.thread is None: + starter.thread = thread + yield starter + except Exception as e: + logger.debug(f"Could not fetch starter message for forum thread {thread.id}: {e}") async def download_emoji(self, emoji: discord.Emoji) -> bytes: """Downloads a Discord emoji into memory.""" diff --git a/src/fluxer/migrate_message.py b/src/fluxer/migrate_message.py index 32ff0ad..605091a 100644 --- a/src/fluxer/migrate_message.py +++ b/src/fluxer/migrate_message.py @@ -187,6 +187,9 @@ async def migrate_messages( channel_id=target_channel_id, content=f"> <<< END OF THREAD >>>" ) + + if progress_callback: + await progress_callback(stats) continue else: # Use custom clean_mentions with msg mentions for accuracy @@ -371,10 +374,6 @@ async def migrate_messages( stats["last_message_content"] = content stats["last_message_author"] = msg.author.display_name - # Periodic log - if stats["messages"] % 50 == 0: - logger.info(f"Progress: Migrated {stats['messages']}/{total_to_process} messages in this channel.") - # Check for associated thread (Normal case: parent message is migrated) if hasattr(msg, 'thread') and msg.thread: thread = msg.thread diff --git a/src/stoat/migrate_message.py b/src/stoat/migrate_message.py index 3d4f7bd..8b9cf99 100644 --- a/src/stoat/migrate_message.py +++ b/src/stoat/migrate_message.py @@ -128,6 +128,7 @@ async def analyze_migration(context: MigrationContext, source_channel_id: int, a if progress_callback and stats["messages"] % 10 == 0: await progress_callback(stats) + return stats @@ -195,6 +196,9 @@ async def migrate_messages( channel_id=target_channel_id, content=f"> <<< END OF THREAD >>>" ) + + if progress_callback: + await progress_callback(stats) continue else: # Use custom clean_mentions with msg mentions for accuracy @@ -382,10 +386,6 @@ async def migrate_messages( stats["last_message_content"] = content stats["last_message_author"] = msg.author.display_name - # Periodic log - if stats["messages"] % 50 == 0: - logger.info(f"Progress: Migrated {stats['messages']} messages in this channel.") - # Check for associated thread (Normal case: parent message is migrated) if hasattr(msg, 'thread') and msg.thread: thread = msg.thread diff --git a/src/ui/modals.py b/src/ui/modals.py index b666691..44ba413 100644 --- a/src/ui/modals.py +++ b/src/ui/modals.py @@ -55,20 +55,20 @@ class ProgressScreen(Screen[None]): #prog_stats { height: auto; layout: horizontal; - border: solid cyan; - padding: 1; margin-bottom: 0; display: none; } .stat_label { width: 1fr; content-align: center middle; text-style: bold; } + #stats_rule { display: none; margin: 0; } #prog_log { height: 1fr; margin-bottom: 0; border: solid $primary; } #live_log { height: 10; margin-bottom: 0; border: solid yellow; } - #prog_item_status { margin-bottom: 1; text-style: bold; color: cyan; width: 100%; text-align: center; } + #prog_item_status { margin-bottom: 0; text-style: bold; color: cyan; width: 100%; text-align: center; } #info_container { height: auto; layout: vertical; border: solid cyan; padding: 1; margin-bottom: 0; display: none; } .info_label { text-style: bold; content-align: center middle; width: 100%; color: cyan; } + #prog_actions { height: auto; margin-top: 0; dock: bottom; margin-bottom: 0; layout: vertical; } .action_row { height: auto; layout: horizontal; } @@ -86,17 +86,18 @@ class ProgressScreen(Screen[None]): yield LoadingIndicator(id="prog_loader") yield Label("00:00", id="prog_timer") - with Horizontal(id="prog_stats"): - yield Label("Messages: 0", id="stat_messages", classes="stat_label") - yield Label("Threads: 0", id="stat_threads", classes="stat_label") - yield Label("Files: 0", id="stat_files", classes="stat_label") - - with Vertical(id="info_container"): + with Horizontal(id="prog_stats"): + yield Label("Messages: 0", id="stat_messages", classes="stat_label") + yield Label("Threads: 0", id="stat_threads", classes="stat_label") + yield Label("Files: 0", id="stat_files", classes="stat_label") + + yield Rule(id="stats_rule") yield Label("", id="info_migration_status", classes="info_label") yield Label("", id="info_new_items", classes="info_label") yield Label("", id="prog_item_status") + yield RichLog(id="prog_log", highlight=True, markup=True) yield RichLog(id="live_log", highlight=True, markup=True) @@ -221,9 +222,12 @@ class ProgressScreen(Screen[None]): def show_stats(self): try: self.query_one("#prog_stats", Horizontal).display = True + self.query_one("#stats_rule", Rule).display = True + self.query_one("#info_container", Vertical).display = True except Exception: pass + def update_stats(self, **kwargs): # kwargs can be messages, threads, files for key, val in kwargs.items(): diff --git a/src/ui/shuttle_ops.py b/src/ui/shuttle_ops.py index 9086866..ca932b2 100644 --- a/src/ui/shuttle_ops.py +++ b/src/ui/shuttle_ops.py @@ -983,7 +983,29 @@ class ShuttlePane(Container): after_id = verified_id else: logger.info("Proceeding with 'Start from First' (clean sink).") + after_id = None + # If after_id changed from the initial analysis, we must re-analyze + # to get the correct total count for the UI fraction (e.g. Messages: 8/8 instead of 8/1) + initial_after = int(last_migrated) if last_migrated else None + if after_id != initial_after: + modal.set_status("Re-analyzing channel from new starting point...") + try: + self.engine.is_running = True + stats_analysis = await migrate_mod.analyze_migration( + self.engine, + source_channel_id=source_channel.id, + after_message_id=after_id, + progress_callback=update_scan, + ) + modal.update_stats( + messages=stats_analysis['messages'], + threads=stats_analysis['threads'], + files=stats_analysis['attachments'] + ) + except Exception as e: + logger.warning(f"Failed to re-analyze for correct totals: {e}") + # If we are here, we are proceeding with migration break