ad3laid3/disco-reaper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve thread markers

Browse source
This commit is contained in:
rambros 2026-02-27 02:01:58 +05:30
parent 1c04a90f82
commit 0d1e0f6717
2 changed files with 85 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
src/fluxer/migrate_message.py
View file

@ -43,10 +43,7 @@ async def analyze_migration(context: MigrationContext, source_channel_id: int, a
if not context.is_running: if not context.is_running:
break break
stats["messages"] += 1 # Count thread messages and markers even if parent is skipped
stats["attachments"] += len(msg.attachments)
# Count thread messages and markers
if hasattr(msg, 'thread') and msg.thread: if hasattr(msg, 'thread') and msg.thread:
stats["threads"] += 1 stats["threads"] += 1
# Recursively count thread content # Recursively count thread content
@ -55,6 +52,13 @@ async def analyze_migration(context: MigrationContext, source_channel_id: int, a
stats["attachments"] += thread_stats["attachments"] stats["attachments"] += thread_stats["attachments"]
stats["threads"] += thread_stats["threads"] # Nested threads (rare in Discord but possible in forum channels) stats["threads"] += thread_stats["threads"] # Nested threads (rare in Discord but possible in forum channels)
# Consistent filtering with migrate_messages
if msg.type not in [discord.MessageType.default, discord.MessageType.reply, discord.MessageType.thread_starter_message]:
continue
stats["messages"] += 1
stats["attachments"] += len(msg.attachments)
if progress_callback and stats["messages"] % 10 == 0: if progress_callback and stats["messages"] % 10 == 0:
await progress_callback(stats["messages"]) await progress_callback(stats["messages"])
@ -75,11 +79,36 @@ async def migrate_messages(context: MigrationContext, source_channel_id: int, ta
if not context.is_running: if not context.is_running:
break break
# Skip system messages like "pinned a message", etc. # Skip system messages like "pinned a message", etc.
# We treat thread_starter_message (type 21) as our thread marker. # We treat thread_starter_message (type 21) as our thread marker.
if msg.type == discord.MessageType.thread_starter_message: if msg.type == discord.MessageType.thread_starter_message:
content = f"> <<< THREAD: **{msg.channel.name}** >>>" content = f"> <<< THREAD: **{msg.channel.name}** >>>"
elif msg.type not in [discord.MessageType.default, discord.MessageType.reply]: elif msg.type not in [discord.MessageType.default, discord.MessageType.reply]:
# If we are skipping the parent, we STILL need to check for a thread!
if hasattr(msg, 'thread') and msg.thread:
thread = msg.thread
logger.info(f"Detected thread '{thread.name}' on skipped message {msg.id}")
# Track thread entry
stats["threads"] += 1
# Migrate thread messages recursively
thread_stats = await migrate_messages(
context=context,
source_channel_id=thread.id,
target_channel_id=target_channel_id
)
stats["messages"] += thread_stats["messages"]
stats["attachments"] += thread_stats["attachments"]
stats["threads"] += thread_stats["threads"]
# Send End Marker
await context.fluxer_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< END OF THREAD >>>"
)
continue continue
else: else:
# Get clean content # Get clean content
@ -138,14 +167,19 @@ async def migrate_messages(context: MigrationContext, source_channel_id: int, ta
if fluxer_msg_id: if fluxer_msg_id:
context.state.set_message_mapping(str(msg.id), fluxer_msg_id) context.state.set_message_mapping(str(msg.id), fluxer_msg_id)
# Check for associated thread context.state.update_last_message_timestamp(str(source_channel_id), str(msg.created_at))
context.state.update_last_message_id(str(source_channel_id), str(msg.id))
stats["messages"] += 1
# Check for associated thread (Normal case: parent message is migrated)
if hasattr(msg, 'thread') and msg.thread: if hasattr(msg, 'thread') and msg.thread:
thread = msg.thread thread = msg.thread
logger.info(f"Detected thread '{thread.name}' on message {msg.id}") logger.info(f"Detected thread '{thread.name}' on message {msg.id}")
# Migrate thread messages # Track thread entry
# We don't pass a progress callback here to avoid confusing the UI stats["threads"] += 1
# but we do want to track count if possible.
# Migrate thread messages recursively
thread_stats = await migrate_messages( thread_stats = await migrate_messages(
context=context, context=context,
source_channel_id=thread.id, source_channel_id=thread.id,
@ -161,10 +195,6 @@ async def migrate_messages(context: MigrationContext, source_channel_id: int, ta
content=f"> <<< END OF THREAD >>>" content=f"> <<< END OF THREAD >>>"
) )
context.state.update_last_message_timestamp(str(source_channel_id), str(msg.created_at))
context.state.update_last_message_id(str(source_channel_id), str(msg.id))
stats["messages"] += 1
# Update Link Tracking (but prevent threaded messages from overwriting the parent channel pointers) # Update Link Tracking (but prevent threaded messages from overwriting the parent channel pointers)
# The 'after_message_id' param usually means it's the main function call and not a thread recursive call # The 'after_message_id' param usually means it's the main function call and not a thread recursive call
if not stats["first_message_url"]: if not stats["first_message_url"]:

52
src/stoat/migrate_message.py
View file

@ -43,10 +43,7 @@ async def analyze_migration(context: MigrationContext, source_channel_id: int, a
if not context.is_running: if not context.is_running:
break break
stats["messages"] += 1 # Count thread messages and markers even if parent is skipped
stats["attachments"] += len(msg.attachments)
# Count thread messages and markers
if hasattr(msg, 'thread') and msg.thread: if hasattr(msg, 'thread') and msg.thread:
stats["threads"] += 1 stats["threads"] += 1
thread_stats = await analyze_migration(context, msg.thread.id) thread_stats = await analyze_migration(context, msg.thread.id)
@ -54,6 +51,13 @@ async def analyze_migration(context: MigrationContext, source_channel_id: int, a
stats["attachments"] += thread_stats["attachments"] stats["attachments"] += thread_stats["attachments"]
stats["threads"] += thread_stats["threads"] stats["threads"] += thread_stats["threads"]
# Consistent filtering with migrate_messages
if msg.type not in [discord.MessageType.default, discord.MessageType.reply, discord.MessageType.thread_starter_message]:
continue
stats["messages"] += 1
stats["attachments"] += len(msg.attachments)
if progress_callback and stats["messages"] % 10 == 0: if progress_callback and stats["messages"] % 10 == 0:
await progress_callback(stats["messages"]) await progress_callback(stats["messages"])
@ -74,12 +78,39 @@ async def migrate_messages(context: MigrationContext, source_channel_id: int, ta
if not context.is_running: if not context.is_running:
break break
# Skip system messages like "pinned a message", etc. # Skip system messages like "pinned a message", etc.
# We treat thread_starter_message (type 21) as our thread marker. # We treat thread_starter_message (type 21) as our thread marker.
content = "" # Initialize content content = "" # Initialize content
if msg.type == discord.MessageType.thread_starter_message: if msg.type == discord.MessageType.thread_starter_message:
content = f"> <<< THREAD: **{msg.channel.name}** >>>" content = f"> <<< THREAD: **{msg.channel.name}** >>>"
# If it's a thread starter and we already processed the thread at the top,
# we might be double-posting. But we want it as a marker.
elif msg.type not in [discord.MessageType.default, discord.MessageType.reply]: elif msg.type not in [discord.MessageType.default, discord.MessageType.reply]:
# If we are skipping the parent, we STILL need to check for a thread!
if hasattr(msg, 'thread') and msg.thread:
thread = msg.thread
logger.info(f"Detected thread '{thread.name}' on skipped message {msg.id}")
# Track thread entry
stats["threads"] += 1
# Migrate thread messages recursively
thread_stats = await migrate_messages(
context=context,
source_channel_id=thread.id,
target_channel_id=target_channel_id
)
stats["messages"] += thread_stats["messages"]
stats["attachments"] += thread_stats["attachments"]
stats["threads"] += thread_stats["threads"]
# Send End Marker
await context.stoat_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< END OF THREAD >>>"
)
continue continue
else: else:
# Get clean content # Get clean content
@ -133,11 +164,18 @@ async def migrate_messages(context: MigrationContext, source_channel_id: int, ta
if stoat_msg_id: if stoat_msg_id:
context.state.set_message_mapping(str(msg.id), stoat_msg_id) context.state.set_message_mapping(str(msg.id), stoat_msg_id)
# Check for associated thread context.state.update_last_message_timestamp(str(source_channel_id), str(msg.created_at))
context.state.update_last_message_id(str(source_channel_id), str(msg.id))
stats["messages"] += 1
# Check for associated thread (Normal case: parent message is migrated)
if hasattr(msg, 'thread') and msg.thread: if hasattr(msg, 'thread') and msg.thread:
thread = msg.thread thread = msg.thread
logger.info(f"Detected thread '{thread.name}' on message {msg.id}") logger.info(f"Detected thread '{thread.name}' on message {msg.id}")
# Track thread entry
stats["threads"] += 1
# Migrate thread messages recursively # Migrate thread messages recursively
thread_stats = await migrate_messages( thread_stats = await migrate_messages(
context=context, context=context,
@ -154,10 +192,6 @@ async def migrate_messages(context: MigrationContext, source_channel_id: int, ta
content=f"> <<< END OF THREAD >>>" content=f"> <<< END OF THREAD >>>"
) )
context.state.update_last_message_timestamp(str(source_channel_id), str(msg.created_at))
context.state.update_last_message_id(str(source_channel_id), str(msg.id))
stats["messages"] += 1
# Update Link Tracking # Update Link Tracking
if not stats["first_message_url"]: if not stats["first_message_url"]:
stats["first_message_url"] = msg.jump_url stats["first_message_url"] = msg.jump_url