ad3laid3/disco-reaper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add logs for partial messages migration

Browse source
This commit is contained in:
rambros 2026-02-25 22:42:15 +05:30
parent 34f9c9afa4
commit 6de9a054f7
3 changed files with 222 additions and 208 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

202
src/fluxer/migrate_message.py
View file

@ -69,111 +69,115 @@ async def migrate_messages(context: MigrationContext, source_channel_id: int, ta
"first_message_url": None, "first_message_url": None,
"last_message_url": None "last_message_url": None
} }
async for msg in context.discord_reader.fetch_message_history(source_channel_id, after_id=after_message_id): try:
if not context.is_running: async for msg in context.discord_reader.fetch_message_history(source_channel_id, after_id=after_message_id):
break if not context.is_running:
break
# Process attachments # Process attachments
files = [] files = []
attachments_to_process = list(msg.attachments) attachments_to_process = list(msg.attachments)
# Check if this message is forwarded # Check if this message is forwarded
# Discord flags: forwarded (is bit 28 / 0x10000000) # Discord flags: forwarded (is bit 28 / 0x10000000)
is_forwarded = False is_forwarded = False
if hasattr(msg.flags, 'forwarded'): if hasattr(msg.flags, 'forwarded'):
is_forwarded = msg.flags.forwarded is_forwarded = msg.flags.forwarded
# If forwarded, the content and attachments might be in message_snapshots (discord.py 2.5+) # If forwarded, the content and attachments might be in message_snapshots (discord.py 2.5+)
content = msg.clean_content content = msg.clean_content
if is_forwarded: if is_forwarded:
logger.debug(f"Detected forwarded message: ID={msg.id}, Flags={msg.flags.value}") logger.debug(f"Detected forwarded message: ID={msg.id}, Flags={msg.flags.value}")
if hasattr(msg, 'message_snapshots') and msg.message_snapshots: if hasattr(msg, 'message_snapshots') and msg.message_snapshots:
# For now we handle the first snapshot # For now we handle the first snapshot
snapshot = msg.message_snapshots[0] snapshot = msg.message_snapshots[0]
if not content: if not content:
content = snapshot.content content = snapshot.content
if hasattr(msg, 'guild') and msg.guild: if hasattr(msg, 'guild') and msg.guild:
content = clean_mentions(content, msg.guild) content = clean_mentions(content, msg.guild)
# Add snapshot attachments to the list to process # Add snapshot attachments to the list to process
attachments_to_process.extend(snapshot.attachments) attachments_to_process.extend(snapshot.attachments)
logger.debug(f"Found forwarded snapshot content: {content[:50]}... and {len(snapshot.attachments)} attachments") logger.debug(f"Found forwarded snapshot content: {content[:50]}... and {len(snapshot.attachments)} attachments")
for att in attachments_to_process:
try:
att_data = await context.discord_reader.download_attachment(att)
files.append({"filename": att.filename, "data": att_data})
stats["attachments"] += 1
except Exception as e:
logger.error(f"Failed to download attachment {att.filename}: {e}")
for att in attachments_to_process:
try: try:
att_data = await context.discord_reader.download_attachment(att) # Check if this message is a reply
files.append({"filename": att.filename, "data": att_data}) reply_to_fluxer_id = None
stats["attachments"] += 1 if msg.reference and msg.reference.message_id:
reply_to_fluxer_id = context.state.get_fluxer_message_id(str(msg.reference.message_id))
fluxer_msg_id = await context.fluxer_writer.send_message(
channel_id=target_channel_id,
author_name=msg.author.display_name,
author_avatar_url=str(msg.author.display_avatar.url),
content=content,
timestamp=msg.created_at.strftime("%Y-%m-%d %H:%M:%S"),
files=files if files else None,
reply_to_message_id=reply_to_fluxer_id,
is_forwarded=is_forwarded
)
if fluxer_msg_id:
context.state.set_message_mapping(str(msg.id), fluxer_msg_id)
# Check for associated thread
if hasattr(msg, 'thread') and msg.thread:
thread = msg.thread
logger.info(f"Detected thread '{thread.name}' on message {msg.id}")
# Send Start Marker
stats["threads"] += 1
await context.fluxer_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< THREAD: **{thread.name}** >>>"
)
# Migrate thread messages
# We don't pass a progress callback here to avoid confusing the UI
# but we do want to track count if possible.
thread_stats = await migrate_messages(
context=context,
source_channel_id=thread.id,
target_channel_id=target_channel_id
)
stats["messages"] += thread_stats["messages"]
stats["attachments"] += thread_stats["attachments"]
stats["threads"] += thread_stats["threads"]
# Send End Marker
await context.fluxer_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< END OF THREAD >>>"
)
context.state.update_last_message_timestamp(str(source_channel_id), str(msg.created_at))
context.state.update_last_message_id(str(source_channel_id), str(msg.id))
stats["messages"] += 1
# Update Link Tracking (but prevent threaded messages from overwriting the parent channel pointers)
# The 'after_message_id' param usually means it's the main function call and not a thread recursive call
if not stats["first_message_url"]:
stats["first_message_url"] = msg.jump_url
stats["last_message_url"] = msg.jump_url
if progress_callback:
await progress_callback(stats["messages"])
except Exception as e: except Exception as e:
logger.error(f"Failed to download attachment {att.filename}: {e}") logger.error(f"Failed to process message {msg.id}: {e}")
import traceback
logger.error(traceback.format_exc())
try: # Delay for rate limit safety
# Check if this message is a reply await asyncio.sleep(context.config.migration.rate_limit_delay_seconds)
reply_to_fluxer_id = None except (KeyboardInterrupt, asyncio.CancelledError):
if msg.reference and msg.reference.message_id: context.is_running = False
reply_to_fluxer_id = context.state.get_fluxer_message_id(str(msg.reference.message_id)) pass
fluxer_msg_id = await context.fluxer_writer.send_message(
channel_id=target_channel_id,
author_name=msg.author.display_name,
author_avatar_url=str(msg.author.display_avatar.url),
content=content,
timestamp=msg.created_at.strftime("%Y-%m-%d %H:%M:%S"),
files=files if files else None,
reply_to_message_id=reply_to_fluxer_id,
is_forwarded=is_forwarded
)
if fluxer_msg_id:
context.state.set_message_mapping(str(msg.id), fluxer_msg_id)
# Check for associated thread
if hasattr(msg, 'thread') and msg.thread:
thread = msg.thread
logger.info(f"Detected thread '{thread.name}' on message {msg.id}")
# Send Start Marker
stats["threads"] += 1
await context.fluxer_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< THREAD: **{thread.name}** >>>"
)
# Migrate thread messages
# We don't pass a progress callback here to avoid confusing the UI
# but we do want to track count if possible.
thread_stats = await migrate_messages(
context=context,
source_channel_id=thread.id,
target_channel_id=target_channel_id
)
stats["messages"] += thread_stats["messages"]
stats["attachments"] += thread_stats["attachments"]
stats["threads"] += thread_stats["threads"]
# Send End Marker
await context.fluxer_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< END OF THREAD >>>"
)
context.state.update_last_message_timestamp(str(source_channel_id), str(msg.created_at))
context.state.update_last_message_id(str(source_channel_id), str(msg.id))
stats["messages"] += 1
# Update Link Tracking (but prevent threaded messages from overwriting the parent channel pointers)
# The 'after_message_id' param usually means it's the main function call and not a thread recursive call
if not stats["first_message_url"]:
stats["first_message_url"] = msg.jump_url
stats["last_message_url"] = msg.jump_url
if progress_callback:
await progress_callback(stats["messages"])
except Exception as e:
logger.error(f"Failed to process message {msg.id}: {e}")
import traceback
logger.error(traceback.format_exc())
# Delay for rate limit safety
await asyncio.sleep(context.config.migration.rate_limit_delay_seconds)
return stats return stats

196
src/stoat/migrate_message.py
View file

@ -68,108 +68,112 @@ async def migrate_messages(context: MigrationContext, source_channel_id: int, ta
"first_message_url": None, "first_message_url": None,
"last_message_url": None "last_message_url": None
} }
async for msg in context.discord_reader.fetch_message_history(source_channel_id, after_id=after_message_id): try:
if not context.is_running: async for msg in context.discord_reader.fetch_message_history(source_channel_id, after_id=after_message_id):
break if not context.is_running:
break
# Process attachments # Process attachments
files = [] files = []
attachments_to_process = list(msg.attachments) attachments_to_process = list(msg.attachments)
# Check if this message is forwarded # Check if this message is forwarded
is_forwarded = False is_forwarded = False
if hasattr(msg.flags, 'forwarded'): if hasattr(msg.flags, 'forwarded'):
is_forwarded = msg.flags.forwarded is_forwarded = msg.flags.forwarded
# Get clean content # Get clean content
content = msg.clean_content content = msg.clean_content
if is_forwarded: if is_forwarded:
logger.debug(f"Detected forwarded message: ID={msg.id}, Flags={msg.flags.value}") logger.debug(f"Detected forwarded message: ID={msg.id}, Flags={msg.flags.value}")
if hasattr(msg, 'message_snapshots') and msg.message_snapshots: if hasattr(msg, 'message_snapshots') and msg.message_snapshots:
snapshot = msg.message_snapshots[0] snapshot = msg.message_snapshots[0]
if not content: if not content:
content = snapshot.content content = snapshot.content
if hasattr(msg, 'guild') and msg.guild: if hasattr(msg, 'guild') and msg.guild:
content = clean_mentions(content, msg.guild) content = clean_mentions(content, msg.guild)
attachments_to_process.extend(snapshot.attachments) attachments_to_process.extend(snapshot.attachments)
logger.debug(f"Found forwarded snapshot content: {content[:50]}... and {len(snapshot.attachments)} attachments") logger.debug(f"Found forwarded snapshot content: {content[:50]}... and {len(snapshot.attachments)} attachments")
for att in attachments_to_process:
try:
att_data = await context.discord_reader.download_attachment(att)
files.append({"filename": att.filename, "data": att_data})
stats["attachments"] += 1
except Exception as e:
logger.error(f"Failed to download attachment {att.filename}: {e}")
for att in attachments_to_process:
try: try:
att_data = await context.discord_reader.download_attachment(att) # Check if this message is a reply
files.append({"filename": att.filename, "data": att_data}) reply_to_stoat_id = None
stats["attachments"] += 1 if msg.reference and msg.reference.message_id:
reply_to_stoat_id = context.state.get_target_message_id(str(msg.reference.message_id))
stoat_msg_id = await context.stoat_writer.send_message(
channel_id=target_channel_id,
author_name=msg.author.display_name,
author_avatar_url=str(msg.author.display_avatar.url),
content=content,
timestamp=msg.created_at.strftime("%Y-%m-%d %H:%M:%S"),
files=files if files else None,
reply_to_message_id=reply_to_stoat_id,
is_forwarded=is_forwarded
)
if stoat_msg_id:
context.state.set_message_mapping(str(msg.id), stoat_msg_id)
# Check for associated thread
if hasattr(msg, 'thread') and msg.thread:
thread = msg.thread
logger.info(f"Detected thread '{thread.name}' on message {msg.id}")
# Send Start Marker
stats["threads"] += 1
await context.stoat_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< THREAD: **{thread.name}** >>>"
)
# Migrate thread messages recursively
thread_stats = await migrate_messages(
context=context,
source_channel_id=thread.id,
target_channel_id=target_channel_id
)
stats["messages"] += thread_stats["messages"]
stats["attachments"] += thread_stats["attachments"]
stats["threads"] += thread_stats["threads"]
# Send End Marker
await context.stoat_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< END OF THREAD >>>"
)
context.state.update_last_message_timestamp(str(source_channel_id), str(msg.created_at))
context.state.update_last_message_id(str(source_channel_id), str(msg.id))
stats["messages"] += 1
# Update Link Tracking
if not stats["first_message_url"]:
stats["first_message_url"] = msg.jump_url
stats["last_message_url"] = msg.jump_url
if progress_callback:
await progress_callback(stats["messages"])
except Exception as e: except Exception as e:
logger.error(f"Failed to download attachment {att.filename}: {e}") # If it's a permission error, stop the entire migration
if "MissingPermission" in str(e):
raise
logger.error(f"Failed to process message {msg.id}: {e}")
import traceback
logger.error(traceback.format_exc())
try: # Delay for rate limit safety
# Check if this message is a reply await asyncio.sleep(context.config.migration.rate_limit_delay_seconds)
reply_to_stoat_id = None except (KeyboardInterrupt, asyncio.CancelledError):
if msg.reference and msg.reference.message_id: context.is_running = False
reply_to_stoat_id = context.state.get_target_message_id(str(msg.reference.message_id)) pass
stoat_msg_id = await context.stoat_writer.send_message(
channel_id=target_channel_id,
author_name=msg.author.display_name,
author_avatar_url=str(msg.author.display_avatar.url),
content=content,
timestamp=msg.created_at.strftime("%Y-%m-%d %H:%M:%S"),
files=files if files else None,
reply_to_message_id=reply_to_stoat_id,
is_forwarded=is_forwarded
)
if stoat_msg_id:
context.state.set_message_mapping(str(msg.id), stoat_msg_id)
# Check for associated thread
if hasattr(msg, 'thread') and msg.thread:
thread = msg.thread
logger.info(f"Detected thread '{thread.name}' on message {msg.id}")
# Send Start Marker
stats["threads"] += 1
await context.stoat_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< THREAD: **{thread.name}** >>>"
)
# Migrate thread messages recursively
thread_stats = await migrate_messages(
context=context,
source_channel_id=thread.id,
target_channel_id=target_channel_id
)
stats["messages"] += thread_stats["messages"]
stats["attachments"] += thread_stats["attachments"]
stats["threads"] += thread_stats["threads"]
# Send End Marker
await context.stoat_writer.send_marker(
channel_id=target_channel_id,
content=f"> <<< END OF THREAD >>>"
)
context.state.update_last_message_timestamp(str(source_channel_id), str(msg.created_at))
context.state.update_last_message_id(str(source_channel_id), str(msg.id))
stats["messages"] += 1
# Update Link Tracking
if not stats["first_message_url"]:
stats["first_message_url"] = msg.jump_url
stats["last_message_url"] = msg.jump_url
if progress_callback:
await progress_callback(stats["messages"])
except Exception as e:
# If it's a permission error, stop the entire migration
if "MissingPermission" in str(e):
raise
logger.error(f"Failed to process message {msg.id}: {e}")
import traceback
logger.error(traceback.format_exc())
# Delay for rate limit safety
await asyncio.sleep(context.config.migration.rate_limit_delay_seconds)
return stats return stats

22
src/ui/app.py
View file

@ -1335,23 +1335,29 @@ class MigrationCLI:
progress_callback=update_msg_progress progress_callback=update_msg_progress
) )
console.print(f"\n[bold green]Success! {result_stats['messages']} messages migrated to {target_channel.get('name')}.[/bold green]") if not self.engine.is_running:
console.print(f"\n[bold yellow]Migration Interrupted! {result_stats['messages']} messages migrated to {target_channel.get('name')}.[/bold yellow]")
event_title = "Message History Migration Interrupted"
else:
console.print(f"\n[bold green]Success! {result_stats['messages']} messages migrated to {target_channel.get('name')}.[/bold green]")
event_title = "Message History Migrated"
lines = [f"Successfully migrated messages from Discord #{source_channel.name} to {platform_name} #{target_channel.get('name')}:"] lines = [f"Migrated messages from Discord #{source_channel.name} to {platform_name} #{target_channel.get('name')}:"]
if result_stats.get('first_message_url') or result_stats.get('last_message_url'): if result_stats.get('first_message_url') or result_stats.get('last_message_url'):
lines.append(f"**Message Info:**") lines.append(f"**Message Info:**")
if result_stats.get('first_message_url'): if result_stats.get('first_message_url'):
lines.append(f"- First message: <{result_stats['first_message_url']}>") lines.append(f"First message: <{result_stats['first_message_url']}>")
if result_stats.get('last_message_url'): if result_stats.get('last_message_url'):
lines.append(f"- Last message: <{result_stats['last_message_url']}>") lines.append(f"Last message: <{result_stats['last_message_url']}>")
lines.append(f"**Stats:**") lines.append(f"**Stats:**")
lines.append(f"- {result_stats['messages']} messages") lines.append(f"{result_stats['messages']} messages")
lines.append(f"- {result_stats['attachments']} attachments") lines.append(f"{result_stats['attachments']} attachments")
lines.append(f"- {result_stats['threads']} threads") lines.append(f"{result_stats['threads']} threads")
audit_desc = "\n".join(lines) audit_desc = "\n".join(lines)
await log_audit_event(self.engine, "Message History Migrated", audit_desc) await log_audit_event(self.engine, event_title, audit_desc)
except Exception as e: except Exception as e:
err_str = str(e) err_str = str(e)