bugfix: message counter during migration
This commit is contained in:
parent
e27c84b072
commit
0c97d14a2b
5 changed files with 83 additions and 19 deletions
|
|
@ -208,15 +208,28 @@ class DiscordReader:
|
||||||
async def get_first_message(self, channel_id: int):
|
async def get_first_message(self, channel_id: int):
|
||||||
"""Returns the first (oldest) message in a channel."""
|
"""Returns the first (oldest) message in a channel."""
|
||||||
channel = await self.get_channel(channel_id)
|
channel = await self.get_channel(channel_id)
|
||||||
if isinstance(channel, discord.TextChannel) or isinstance(channel, discord.Thread):
|
if hasattr(channel, 'history'):
|
||||||
async for message in channel.history(limit=1, oldest_first=True):
|
async for message in channel.history(limit=1, oldest_first=True):
|
||||||
return message
|
return message
|
||||||
|
elif isinstance(channel, discord.ForumChannel):
|
||||||
|
# For forums, find the oldest thread and get its starter message
|
||||||
|
threads = []
|
||||||
|
threads.extend(channel.threads)
|
||||||
|
async for arch_thread in channel.archived_threads(limit=None):
|
||||||
|
threads.append(arch_thread)
|
||||||
|
if threads:
|
||||||
|
threads.sort(key=lambda t: t.id)
|
||||||
|
oldest_thread = threads[0]
|
||||||
|
try:
|
||||||
|
return await oldest_thread.fetch_message(oldest_thread.id)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def fetch_message_history(self, channel_id: int, limit: int = None, after_id: int = None, inclusive: bool = False) -> AsyncGenerator[discord.Message, None]:
|
async def fetch_message_history(self, channel_id: int, limit: int = None, after_id: int = None, inclusive: bool = False) -> AsyncGenerator[discord.Message, None]:
|
||||||
"""Yields messages from a given channel, optionally handling pagination."""
|
"""Yields messages from a given channel, optionally handling pagination."""
|
||||||
channel = await self.get_channel(channel_id)
|
channel = await self.get_channel(channel_id)
|
||||||
if isinstance(channel, discord.TextChannel) or isinstance(channel, discord.Thread):
|
if hasattr(channel, 'history'):
|
||||||
# Discord's 'after' is exclusive. To make it inclusive, we use after_id - 1 if requested.
|
# Discord's 'after' is exclusive. To make it inclusive, we use after_id - 1 if requested.
|
||||||
after = None
|
after = None
|
||||||
if after_id:
|
if after_id:
|
||||||
|
|
@ -225,6 +238,32 @@ class DiscordReader:
|
||||||
# To avoid exploding RAM, we yield items one by one
|
# To avoid exploding RAM, we yield items one by one
|
||||||
async for message in channel.history(limit=limit, oldest_first=True, after=after):
|
async for message in channel.history(limit=limit, oldest_first=True, after=after):
|
||||||
yield message
|
yield message
|
||||||
|
elif isinstance(channel, discord.ForumChannel):
|
||||||
|
logger.info(f"Fetching message history for ForumChannel {channel.name} ({channel.id}) oldest_first=True after={after_id} inclusive={inclusive}")
|
||||||
|
threads = []
|
||||||
|
threads.extend(channel.threads)
|
||||||
|
async for arch_thread in channel.archived_threads(limit=None):
|
||||||
|
threads.append(arch_thread)
|
||||||
|
|
||||||
|
# Sort threads chronologically (by ID)
|
||||||
|
threads.sort(key=lambda t: t.id)
|
||||||
|
|
||||||
|
for thread in threads:
|
||||||
|
if after_id:
|
||||||
|
if not inclusive and thread.id <= after_id:
|
||||||
|
continue
|
||||||
|
if inclusive and thread.id < after_id:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
# In a forum, the starter message ID is the thread ID
|
||||||
|
starter = await thread.fetch_message(thread.id)
|
||||||
|
# Bind the thread so migrate_messages handles it properly
|
||||||
|
if not hasattr(starter, 'thread') or starter.thread is None:
|
||||||
|
starter.thread = thread
|
||||||
|
yield starter
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"Could not fetch starter message for forum thread {thread.id}: {e}")
|
||||||
|
|
||||||
async def download_emoji(self, emoji: discord.Emoji) -> bytes:
|
async def download_emoji(self, emoji: discord.Emoji) -> bytes:
|
||||||
"""Downloads a Discord emoji into memory."""
|
"""Downloads a Discord emoji into memory."""
|
||||||
|
|
|
||||||
|
|
@ -187,6 +187,9 @@ async def migrate_messages(
|
||||||
channel_id=target_channel_id,
|
channel_id=target_channel_id,
|
||||||
content=f"> <<< END OF THREAD >>>"
|
content=f"> <<< END OF THREAD >>>"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if progress_callback:
|
||||||
|
await progress_callback(stats)
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
# Use custom clean_mentions with msg mentions for accuracy
|
# Use custom clean_mentions with msg mentions for accuracy
|
||||||
|
|
@ -371,10 +374,6 @@ async def migrate_messages(
|
||||||
stats["last_message_content"] = content
|
stats["last_message_content"] = content
|
||||||
stats["last_message_author"] = msg.author.display_name
|
stats["last_message_author"] = msg.author.display_name
|
||||||
|
|
||||||
# Periodic log
|
|
||||||
if stats["messages"] % 50 == 0:
|
|
||||||
logger.info(f"Progress: Migrated {stats['messages']}/{total_to_process} messages in this channel.")
|
|
||||||
|
|
||||||
# Check for associated thread (Normal case: parent message is migrated)
|
# Check for associated thread (Normal case: parent message is migrated)
|
||||||
if hasattr(msg, 'thread') and msg.thread:
|
if hasattr(msg, 'thread') and msg.thread:
|
||||||
thread = msg.thread
|
thread = msg.thread
|
||||||
|
|
|
||||||
|
|
@ -128,6 +128,7 @@ async def analyze_migration(context: MigrationContext, source_channel_id: int, a
|
||||||
if progress_callback and stats["messages"] % 10 == 0:
|
if progress_callback and stats["messages"] % 10 == 0:
|
||||||
await progress_callback(stats)
|
await progress_callback(stats)
|
||||||
|
|
||||||
|
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -195,6 +196,9 @@ async def migrate_messages(
|
||||||
channel_id=target_channel_id,
|
channel_id=target_channel_id,
|
||||||
content=f"> <<< END OF THREAD >>>"
|
content=f"> <<< END OF THREAD >>>"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if progress_callback:
|
||||||
|
await progress_callback(stats)
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
# Use custom clean_mentions with msg mentions for accuracy
|
# Use custom clean_mentions with msg mentions for accuracy
|
||||||
|
|
@ -382,10 +386,6 @@ async def migrate_messages(
|
||||||
stats["last_message_content"] = content
|
stats["last_message_content"] = content
|
||||||
stats["last_message_author"] = msg.author.display_name
|
stats["last_message_author"] = msg.author.display_name
|
||||||
|
|
||||||
# Periodic log
|
|
||||||
if stats["messages"] % 50 == 0:
|
|
||||||
logger.info(f"Progress: Migrated {stats['messages']} messages in this channel.")
|
|
||||||
|
|
||||||
# Check for associated thread (Normal case: parent message is migrated)
|
# Check for associated thread (Normal case: parent message is migrated)
|
||||||
if hasattr(msg, 'thread') and msg.thread:
|
if hasattr(msg, 'thread') and msg.thread:
|
||||||
thread = msg.thread
|
thread = msg.thread
|
||||||
|
|
|
||||||
|
|
@ -55,20 +55,20 @@ class ProgressScreen(Screen[None]):
|
||||||
#prog_stats {
|
#prog_stats {
|
||||||
height: auto;
|
height: auto;
|
||||||
layout: horizontal;
|
layout: horizontal;
|
||||||
border: solid cyan;
|
|
||||||
padding: 1;
|
|
||||||
margin-bottom: 0;
|
margin-bottom: 0;
|
||||||
display: none;
|
display: none;
|
||||||
}
|
}
|
||||||
.stat_label { width: 1fr; content-align: center middle; text-style: bold; }
|
.stat_label { width: 1fr; content-align: center middle; text-style: bold; }
|
||||||
|
#stats_rule { display: none; margin: 0; }
|
||||||
|
|
||||||
#prog_log { height: 1fr; margin-bottom: 0; border: solid $primary; }
|
#prog_log { height: 1fr; margin-bottom: 0; border: solid $primary; }
|
||||||
#live_log { height: 10; margin-bottom: 0; border: solid yellow; }
|
#live_log { height: 10; margin-bottom: 0; border: solid yellow; }
|
||||||
|
|
||||||
#prog_item_status { margin-bottom: 1; text-style: bold; color: cyan; width: 100%; text-align: center; }
|
#prog_item_status { margin-bottom: 0; text-style: bold; color: cyan; width: 100%; text-align: center; }
|
||||||
|
|
||||||
#info_container { height: auto; layout: vertical; border: solid cyan; padding: 1; margin-bottom: 0; display: none; }
|
#info_container { height: auto; layout: vertical; border: solid cyan; padding: 1; margin-bottom: 0; display: none; }
|
||||||
.info_label { text-style: bold; content-align: center middle; width: 100%; color: cyan; }
|
.info_label { text-style: bold; content-align: center middle; width: 100%; color: cyan; }
|
||||||
|
|
||||||
|
|
||||||
#prog_actions { height: auto; margin-top: 0; dock: bottom; margin-bottom: 0; layout: vertical; }
|
#prog_actions { height: auto; margin-top: 0; dock: bottom; margin-bottom: 0; layout: vertical; }
|
||||||
.action_row { height: auto; layout: horizontal; }
|
.action_row { height: auto; layout: horizontal; }
|
||||||
|
|
@ -86,17 +86,18 @@ class ProgressScreen(Screen[None]):
|
||||||
yield LoadingIndicator(id="prog_loader")
|
yield LoadingIndicator(id="prog_loader")
|
||||||
yield Label("00:00", id="prog_timer")
|
yield Label("00:00", id="prog_timer")
|
||||||
|
|
||||||
with Horizontal(id="prog_stats"):
|
|
||||||
yield Label("Messages: 0", id="stat_messages", classes="stat_label")
|
|
||||||
yield Label("Threads: 0", id="stat_threads", classes="stat_label")
|
|
||||||
yield Label("Files: 0", id="stat_files", classes="stat_label")
|
|
||||||
|
|
||||||
|
|
||||||
with Vertical(id="info_container"):
|
with Vertical(id="info_container"):
|
||||||
|
with Horizontal(id="prog_stats"):
|
||||||
|
yield Label("Messages: 0", id="stat_messages", classes="stat_label")
|
||||||
|
yield Label("Threads: 0", id="stat_threads", classes="stat_label")
|
||||||
|
yield Label("Files: 0", id="stat_files", classes="stat_label")
|
||||||
|
|
||||||
|
yield Rule(id="stats_rule")
|
||||||
yield Label("", id="info_migration_status", classes="info_label")
|
yield Label("", id="info_migration_status", classes="info_label")
|
||||||
yield Label("", id="info_new_items", classes="info_label")
|
yield Label("", id="info_new_items", classes="info_label")
|
||||||
yield Label("", id="prog_item_status")
|
yield Label("", id="prog_item_status")
|
||||||
|
|
||||||
|
|
||||||
yield RichLog(id="prog_log", highlight=True, markup=True)
|
yield RichLog(id="prog_log", highlight=True, markup=True)
|
||||||
yield RichLog(id="live_log", highlight=True, markup=True)
|
yield RichLog(id="live_log", highlight=True, markup=True)
|
||||||
|
|
||||||
|
|
@ -221,9 +222,12 @@ class ProgressScreen(Screen[None]):
|
||||||
def show_stats(self):
|
def show_stats(self):
|
||||||
try:
|
try:
|
||||||
self.query_one("#prog_stats", Horizontal).display = True
|
self.query_one("#prog_stats", Horizontal).display = True
|
||||||
|
self.query_one("#stats_rule", Rule).display = True
|
||||||
|
self.query_one("#info_container", Vertical).display = True
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def update_stats(self, **kwargs):
|
def update_stats(self, **kwargs):
|
||||||
# kwargs can be messages, threads, files
|
# kwargs can be messages, threads, files
|
||||||
for key, val in kwargs.items():
|
for key, val in kwargs.items():
|
||||||
|
|
|
||||||
|
|
@ -983,7 +983,29 @@ class ShuttlePane(Container):
|
||||||
after_id = verified_id
|
after_id = verified_id
|
||||||
else:
|
else:
|
||||||
logger.info("Proceeding with 'Start from First' (clean sink).")
|
logger.info("Proceeding with 'Start from First' (clean sink).")
|
||||||
|
after_id = None
|
||||||
|
|
||||||
|
# If after_id changed from the initial analysis, we must re-analyze
|
||||||
|
# to get the correct total count for the UI fraction (e.g. Messages: 8/8 instead of 8/1)
|
||||||
|
initial_after = int(last_migrated) if last_migrated else None
|
||||||
|
if after_id != initial_after:
|
||||||
|
modal.set_status("Re-analyzing channel from new starting point...")
|
||||||
|
try:
|
||||||
|
self.engine.is_running = True
|
||||||
|
stats_analysis = await migrate_mod.analyze_migration(
|
||||||
|
self.engine,
|
||||||
|
source_channel_id=source_channel.id,
|
||||||
|
after_message_id=after_id,
|
||||||
|
progress_callback=update_scan,
|
||||||
|
)
|
||||||
|
modal.update_stats(
|
||||||
|
messages=stats_analysis['messages'],
|
||||||
|
threads=stats_analysis['threads'],
|
||||||
|
files=stats_analysis['attachments']
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to re-analyze for correct totals: {e}")
|
||||||
|
|
||||||
# If we are here, we are proceeding with migration
|
# If we are here, we are proceeding with migration
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue