RamBros
GitHub
commit
a7192f7b56
8 changed files with 1047 additions and 49 deletions
|
|
@ -9,8 +9,8 @@
|
|||
|
||||
|
||||
|
||||
### Modern Terminal Interface
|
||||
The tool now features a unified, intuitive TUI (Terminal User Interface) - no more text commands
|
||||
### Video Guide - [Youtube](https://www.youtube.com/watch?v=SwIPQDxLzqA)
|
||||
|
||||
|
||||
| Features | Fluxer | Stoat |
|
||||
| :--- | :---: | :---: |
|
||||
|
|
|
|||
|
|
@ -791,6 +791,83 @@ class BackupDatabase:
|
|||
|
||||
return msg_list
|
||||
|
||||
def get_global_messages_paged(self, limit: int = 100, offset: int = 0, after_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Fetches messages across ALL channels globally, ordered by timestamp/ID ascending."""
|
||||
with self._lock:
|
||||
query = "SELECT * FROM messages"
|
||||
params = []
|
||||
|
||||
if after_id:
|
||||
query += " WHERE id > ?"
|
||||
params.append(parse_snowflake(after_id))
|
||||
|
||||
query += " ORDER BY id ASC LIMIT ? OFFSET ?"
|
||||
params.extend([limit, offset])
|
||||
|
||||
rows = self._conn.execute(query, params).fetchall()
|
||||
msg_list = [dict(r) for r in rows]
|
||||
|
||||
if msg_list:
|
||||
msg_ids = [m["id"] for m in msg_list]
|
||||
placeholders = ",".join(["?"] * len(msg_ids))
|
||||
|
||||
att_rows = self._conn.execute(f"SELECT * FROM attachments WHERE message_id IN ({placeholders})", msg_ids).fetchall()
|
||||
atts_by_msg = {}
|
||||
for ar in att_rows:
|
||||
mid = ar["message_id"]
|
||||
if mid not in atts_by_msg: atts_by_msg[mid] = []
|
||||
atts_by_msg[mid].append(dict(ar))
|
||||
|
||||
emb_rows = self._conn.execute(f"SELECT * FROM embeds WHERE message_id IN ({placeholders})", msg_ids).fetchall()
|
||||
embs_by_msg = {}
|
||||
for er in emb_rows:
|
||||
mid = er["message_id"]
|
||||
if mid not in embs_by_msg: embs_by_msg[mid] = []
|
||||
|
||||
e_dict = {
|
||||
"title": er["title"],
|
||||
"description": er["description"],
|
||||
"url": er["url"],
|
||||
"color": er["color"],
|
||||
"timestamp": er["timestamp"],
|
||||
"thumbnail": {"url": er["thumbnail_url"]} if er["thumbnail_url"] else None,
|
||||
"image": {"url": er["image_url"]} if er["image_url"] else None,
|
||||
"author": {
|
||||
"name": er["author_name"],
|
||||
"url": er["author_url"],
|
||||
"icon_url": er["author_icon_url"]
|
||||
} if er["author_name"] else None,
|
||||
"footer": {
|
||||
"text": er["footer_text"],
|
||||
"icon_url": er["footer_icon_url"]
|
||||
} if er["footer_text"] else None,
|
||||
"fields": json.loads(er["fields"]) if er["fields"] else []
|
||||
}
|
||||
embs_by_msg[mid].append(e_dict)
|
||||
|
||||
rea_rows = self._conn.execute(f"SELECT * FROM reactions WHERE message_id IN ({placeholders})", msg_ids).fetchall()
|
||||
reas_by_msg = {}
|
||||
for rr in rea_rows:
|
||||
mid = rr["message_id"]
|
||||
if mid not in reas_by_msg: reas_by_msg[mid] = []
|
||||
reas_by_msg[mid].append(dict(rr))
|
||||
|
||||
st_rows = self._conn.execute(f"SELECT * FROM message_stickers WHERE message_id IN ({placeholders})", msg_ids).fetchall()
|
||||
sts_by_msg = {}
|
||||
for sr in st_rows:
|
||||
mid = sr["message_id"]
|
||||
if mid not in sts_by_msg: sts_by_msg[mid] = []
|
||||
sts_by_msg[mid].append(dict(sr))
|
||||
|
||||
for m in msg_list:
|
||||
m_id = m["id"]
|
||||
m["attachments"] = atts_by_msg.get(m_id, [])
|
||||
m["embeds"] = embs_by_msg.get(m_id, [])
|
||||
m["reactions"] = reas_by_msg.get(m_id, [])
|
||||
m["stickers"] = sts_by_msg.get(m_id, [])
|
||||
|
||||
return msg_list
|
||||
|
||||
def delete_channel_messages(self, channel_id: Union[str, int]):
|
||||
"""Deletes all messages and related metadata for a specific channel and its threads."""
|
||||
cid = parse_snowflake(channel_id)
|
||||
|
|
|
|||
|
|
@ -1417,6 +1417,42 @@ class BackupReader:
|
|||
if len(msgs) < batch_size:
|
||||
break
|
||||
|
||||
async def fetch_global_message_history(
|
||||
self,
|
||||
limit: int = None,
|
||||
after_id: int = None
|
||||
) -> AsyncGenerator["BackupMessage", None]:
|
||||
"""Yields BackupMessages globally from SQLite across all channels, natively ordered by timestamp/ID."""
|
||||
if not self.db: return
|
||||
|
||||
offset = 0
|
||||
batch_size = 100
|
||||
count = 0
|
||||
|
||||
while True:
|
||||
actual_limit = batch_size
|
||||
if limit:
|
||||
rem = limit - count
|
||||
if rem <= 0: break
|
||||
actual_limit = min(batch_size, rem)
|
||||
|
||||
msgs = self.db.get_global_messages_paged(
|
||||
limit=actual_limit,
|
||||
offset=offset,
|
||||
after_id=str(after_id) if after_id else None
|
||||
)
|
||||
|
||||
if not msgs:
|
||||
break
|
||||
|
||||
for m in msgs:
|
||||
yield self._hydrate_message(m)
|
||||
count += 1
|
||||
|
||||
offset += len(msgs)
|
||||
if len(msgs) < batch_size:
|
||||
break
|
||||
|
||||
# ── download helpers ─────────────────────────────────────────────────
|
||||
|
||||
async def download_emoji(self, emoji: BackupEmoji) -> bytes:
|
||||
|
|
|
|||
|
|
@ -451,6 +451,49 @@ class MigrationDatabase:
|
|||
return dict(row)
|
||||
return {"last_msg_id": None, "last_msg_ts": None, "msg_count": 0, "file_count": 0}
|
||||
|
||||
|
||||
def get_global_min_last_message_id(self, all_mapped_ids: List[str]) -> Optional[int]:
|
||||
"""
|
||||
Returns the minimum last_msg_id successfully migrated across all mapped channels/threads.
|
||||
If any mapped entity has no progress record, it is treated as ID 0.
|
||||
Returns None only if NO progress has been made across ANY entity.
|
||||
"""
|
||||
if not all_mapped_ids:
|
||||
return None
|
||||
|
||||
conn = self._get_conn()
|
||||
placeholders = ",".join(["?"] * len(all_mapped_ids))
|
||||
|
||||
# 1. Get last message IDs from channel tracking
|
||||
c_rows = conn.execute(f"SELECT channel_id, last_msg_id FROM channel_tracking WHERE channel_id IN ({placeholders})", all_mapped_ids).fetchall()
|
||||
c_map = {r["channel_id"]: r["last_msg_id"] for r in c_rows}
|
||||
|
||||
# 2. Get last message IDs from thread tracking
|
||||
t_rows = conn.execute(f"SELECT thread_id, last_msg_id FROM thread_tracking WHERE thread_id IN ({placeholders})", all_mapped_ids).fetchall()
|
||||
t_map = {r["thread_id"]: r["last_msg_id"] for r in t_rows}
|
||||
|
||||
# Combine maps
|
||||
progress_map = {**c_map, **t_map}
|
||||
|
||||
# 3. Aggregate IDs
|
||||
ids = []
|
||||
has_any_progress = False
|
||||
for mid in all_mapped_ids:
|
||||
last_id = progress_map.get(mid)
|
||||
if not last_id:
|
||||
ids.append(0) # Unmigrated entity
|
||||
else:
|
||||
try:
|
||||
ids.append(int(last_id))
|
||||
has_any_progress = True
|
||||
except (ValueError, TypeError):
|
||||
ids.append(0)
|
||||
|
||||
if not has_any_progress:
|
||||
return None
|
||||
|
||||
return min(ids)
|
||||
|
||||
# Thread methods similar to channel methods
|
||||
def set_thread_message_mapping(self, channel_id: str, thread_id: str, source_id: str, target_id: str, timestamp: str = None):
|
||||
conn = self._get_conn()
|
||||
|
|
@ -496,6 +539,18 @@ class MigrationDatabase:
|
|||
return dict(row)
|
||||
return {"last_msg_id": None, "last_msg_ts": None, "msg_count": 0, "file_count": 0}
|
||||
|
||||
def get_all_channel_tracking_ids(self) -> Dict[str, str]:
|
||||
"""Returns a map of channel_id -> last_msg_id for all tracked channels."""
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute("SELECT channel_id, last_msg_id FROM channel_tracking WHERE last_msg_id IS NOT NULL").fetchall()
|
||||
return {str(row["channel_id"]): str(row["last_msg_id"]) for row in rows}
|
||||
|
||||
def get_all_thread_tracking_ids(self) -> Dict[str, str]:
|
||||
"""Returns a map of thread_id -> last_msg_id for all tracked threads."""
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute("SELECT thread_id, last_msg_id FROM thread_tracking WHERE last_msg_id IS NOT NULL").fetchall()
|
||||
return {str(row["thread_id"]): str(row["last_msg_id"]) for row in rows}
|
||||
|
||||
def clear_channel_data(self, channel_id: str):
|
||||
"""Purge all mappings and tracking data for a specific channel and its threads."""
|
||||
conn = self._get_conn()
|
||||
|
|
|
|||
|
|
@ -38,6 +38,13 @@ class MigrationState:
|
|||
if self.db:
|
||||
self.db.delete_server_mapping("channel", str(discord_id))
|
||||
|
||||
def remove_target_channel_mapping(self, discord_id: int | str):
|
||||
if self.db:
|
||||
self.db.delete_server_mapping("channel", str(discord_id))
|
||||
|
||||
def set_target_channel_id(self, discord_id: int | str, target_id: str, *args):
|
||||
"""Alias for set_channel_mapping to handle legacy calls."""
|
||||
self.set_channel_mapping(discord_id, target_id)
|
||||
|
||||
get_fluxer_channel_id = get_target_channel_id
|
||||
set_target_channel_mapping = set_channel_mapping
|
||||
|
|
@ -58,6 +65,10 @@ class MigrationState:
|
|||
if self.db:
|
||||
self.db.delete_server_mapping("category", str(discord_id))
|
||||
|
||||
def set_target_category_id(self, discord_id: int | str, target_id: str, *args):
|
||||
"""Alias for set_category_mapping to handle legacy calls."""
|
||||
self.set_category_mapping(discord_id, target_id)
|
||||
|
||||
get_fluxer_category_id = get_category_mapping
|
||||
get_target_category_id = get_category_mapping
|
||||
set_target_category_mapping = set_category_mapping
|
||||
|
|
@ -78,6 +89,10 @@ class MigrationState:
|
|||
if self.db:
|
||||
self.db.delete_server_mapping("role", str(discord_id))
|
||||
|
||||
def set_target_role_id(self, discord_id: int | str, target_id: str, *args):
|
||||
"""Alias for set_role_mapping to handle legacy calls."""
|
||||
self.set_role_mapping(discord_id, target_id)
|
||||
|
||||
get_fluxer_role_id = get_role_mapping
|
||||
get_target_role_id = get_role_mapping
|
||||
set_target_role_mapping = set_role_mapping
|
||||
|
|
@ -212,9 +227,35 @@ class MigrationState:
|
|||
|
||||
def get_last_message_id(self, target_channel_id: str) -> str | None:
|
||||
if self._ensure_db():
|
||||
return self.db.get_channel_tracking(str(target_channel_id)).get("last_msg_id")
|
||||
tracking = self.db.get_channel_tracking(str(target_channel_id))
|
||||
return tracking.get("last_msg_id") if tracking else None
|
||||
return None
|
||||
|
||||
|
||||
def get_global_min_last_message_id(self, all_mapped_ids: List[str]) -> int | None:
|
||||
"""Returns the absolute minimum last_msg_id among the given list of mapped target IDs (channels and threads)."""
|
||||
if self._ensure_db():
|
||||
return self.db.get_global_min_last_message_id(all_mapped_ids)
|
||||
return None
|
||||
|
||||
def set_waterfall_last_id(self, last_id: str | int):
|
||||
if self.db:
|
||||
self.db.set_metadata("waterfall_last_id", str(last_id))
|
||||
|
||||
def get_waterfall_last_id(self) -> int | None:
|
||||
if self.db:
|
||||
val = self.db.get_metadata("waterfall_last_id")
|
||||
return int(val) if val else None
|
||||
return None
|
||||
|
||||
def get_all_last_message_ids(self) -> Dict[str, str]:
|
||||
"""Returns a combined map of channel_id/thread_id -> last_msg_id."""
|
||||
if self._ensure_db():
|
||||
c_map = self.db.get_all_channel_tracking_ids()
|
||||
t_map = self.db.get_all_thread_tracking_ids()
|
||||
return {**c_map, **t_map}
|
||||
return {}
|
||||
|
||||
def get_thread_last_message_id(self, target_channel_id: str, thread_id: str) -> str | None:
|
||||
if self._ensure_db():
|
||||
return self.db.get_thread_tracking(str(target_channel_id), str(thread_id)).get("last_msg_id")
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import re
|
|||
import json
|
||||
import io
|
||||
from typing import Callable, Awaitable, Dict, Any, List
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from lottie.objects import Animation
|
||||
|
|
@ -27,22 +28,25 @@ def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None,
|
|||
uid = int(match.group(1))
|
||||
if anonymize_users and state:
|
||||
alias = state.get_user_alias(str(uid))
|
||||
return f"`@{alias}`"
|
||||
return f"`@{alias}`" if alias else "`@Anonymized User`"
|
||||
|
||||
# 1. Try provided guild
|
||||
member = guild.get_member(uid)
|
||||
if member:
|
||||
return f"`@{member.display_name}`"
|
||||
# 2. Try message's user_mentions
|
||||
|
||||
# 2. Try provided user_mentions
|
||||
if user_mentions:
|
||||
for u in user_mentions:
|
||||
if u.id == uid:
|
||||
return f"`@{getattr(u, 'display_name', u.name)}`"
|
||||
m = next((u for u in user_mentions if u.id == uid), None)
|
||||
if m:
|
||||
return f"`@{m.display_name}`"
|
||||
|
||||
# 3. Try global cache via guild.client
|
||||
if hasattr(guild, 'client'):
|
||||
user = guild.client.get_user(uid)
|
||||
if user:
|
||||
return f"`@{user.name}`"
|
||||
return f"`@Unknown User`"
|
||||
return "`@Unknown User`"
|
||||
|
||||
def replace_role(match):
|
||||
rid = int(match.group(1))
|
||||
|
|
@ -539,8 +543,11 @@ async def migrate_messages(
|
|||
except Exception as e:
|
||||
logger.error(f"Failed to download sticker {getattr(s, 'name', 'unknown')}: {e}")
|
||||
|
||||
# Check for existing mapping to avoid duplicates when resuming
|
||||
if context.state.get_target_message_id(target_channel_id, str(msg.id)):
|
||||
continue
|
||||
|
||||
try:
|
||||
# Check if this message is a reply
|
||||
reply_to_fluxer_id = None
|
||||
if msg.reference and msg.reference.message_id:
|
||||
reply_to_fluxer_id = context.state.get_fluxer_message_id(target_channel_id, str(msg.reference.message_id))
|
||||
|
|
@ -557,23 +564,22 @@ async def migrate_messages(
|
|||
if thread_name and stats["messages"] == 0:
|
||||
content = f"> <<< THREAD: **{thread_name}** >>>\n{content}"
|
||||
|
||||
# Get or generate alias
|
||||
# Always ensure alias is created/retrieved to populate user_alias table
|
||||
alias = context.state.get_user_alias(str(msg.author.id))
|
||||
|
||||
if context.config.anonymize_users:
|
||||
author_name = alias
|
||||
avatar_url = f"https://api.dicebear.com/9.x/fun-emoji/jpg?seed={alias}"
|
||||
anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False
|
||||
if anonymize_users:
|
||||
author_name = alias or "Anonymized User"
|
||||
author_avatar_url = None
|
||||
else:
|
||||
author_name = msg.author.display_name
|
||||
avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
|
||||
if avatar_url and not avatar_url.startswith("http"):
|
||||
avatar_url = None
|
||||
author_avatar_url = msg.author.avatar.url if hasattr(msg.author, 'avatar') and msg.author.avatar else None
|
||||
|
||||
logger.debug(f"Fluxer: Calling send_message for Discord ID {msg.id}")
|
||||
fluxer_msg_id = await context.fluxer_writer.send_message(
|
||||
channel_id=target_channel_id,
|
||||
author_name=author_name,
|
||||
author_avatar_url=avatar_url,
|
||||
author_avatar_url=author_avatar_url,
|
||||
content=content,
|
||||
timestamp=int(msg.created_at.timestamp()),
|
||||
files=files if files else None,
|
||||
|
|
@ -663,3 +669,261 @@ async def migrate_messages(
|
|||
pass
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
async def analyze_global_migration(context: MigrationContext, after_message_id: int | None = None, inclusive: bool = False, progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None) -> Dict[str, int]:
|
||||
"""
|
||||
Scans the entire server history to count messages, threads, and attachments globally.
|
||||
"""
|
||||
stats = {"messages": 0, "threads": 0, "attachments": 0}
|
||||
|
||||
# In global mode, thread messages are returned natively in timestamp order by global fetch if they're in the DB
|
||||
# However we just count them if the fetcher yields them.
|
||||
# Fetch global progress map to skip migrated messages efficiently
|
||||
progress_map = context.state.get_all_last_message_ids()
|
||||
|
||||
async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id):
|
||||
if not context.is_running:
|
||||
break
|
||||
|
||||
# Determine target channel to check for existing mapping
|
||||
if not msg.channel:
|
||||
continue
|
||||
|
||||
target_channel_id = context.state.get_target_channel_id(str(msg.channel.id))
|
||||
if not target_channel_id:
|
||||
continue
|
||||
|
||||
# Efficient skip: if message ID is <= last migrated ID for this channel/thread
|
||||
# This is the primary resume mechanism: wait until we pass the last migrated ID for this channel
|
||||
last_id = progress_map.get(str(msg.channel.id))
|
||||
if last_id and msg.id <= int(last_id):
|
||||
continue
|
||||
|
||||
if msg.type not in [
|
||||
context.discord_reader.MESSAGE_TYPE_DEFAULT,
|
||||
context.discord_reader.MESSAGE_TYPE_REPLY,
|
||||
context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
|
||||
context.discord_reader.MESSAGE_TYPE_FORWARD,
|
||||
context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
|
||||
context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
|
||||
context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
|
||||
context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
|
||||
]:
|
||||
continue
|
||||
|
||||
stats["messages"] += 1
|
||||
stats["attachments"] += len(msg.attachments)
|
||||
if hasattr(msg, 'thread') and msg.thread:
|
||||
# We don't recursively traverse here, we just count the fact there is a thread
|
||||
# The actual thread messages are also fetched by the global fetcher because they have their own timestamp/id
|
||||
stats["threads"] += 1
|
||||
|
||||
if progress_callback and stats["messages"] % 100 == 0:
|
||||
await progress_callback(stats)
|
||||
|
||||
if progress_callback:
|
||||
await progress_callback(stats)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
async def migrate_global_messages(
|
||||
context: MigrationContext,
|
||||
after_message_id: int | None = None,
|
||||
inclusive: bool = False,
|
||||
progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Migrates messages across all channels chronologically.
|
||||
"""
|
||||
stats = {
|
||||
"messages": 0,
|
||||
"threads": 0,
|
||||
"attachments": 0,
|
||||
"last_message_content": "",
|
||||
"last_message_author": "",
|
||||
"first_message_url": None,
|
||||
"last_message_url": None
|
||||
}
|
||||
|
||||
processed_threads = set()
|
||||
logger.info("Starting Global Waterfall Migration for Fluxer...")
|
||||
|
||||
# Keep track of active thread mapping natively to pass parent target IDs if needed
|
||||
thread_to_target_channel = {}
|
||||
|
||||
# Emojis and mapped users cache setup
|
||||
emoji_map = context.state.emoji_map
|
||||
db_media = context.discord_reader.db.get_all_media() if context.discord_reader.db else {}
|
||||
target_server_id = getattr(context.fluxer_writer, "server_id", None)
|
||||
|
||||
# Fetch global progress map to skip migrated messages efficiently
|
||||
progress_map = context.state.get_all_last_message_ids()
|
||||
|
||||
try:
|
||||
async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id):
|
||||
if not context.is_running:
|
||||
logger.warning("Global migration interrupted by user")
|
||||
break
|
||||
|
||||
if msg.type not in [
|
||||
context.discord_reader.MESSAGE_TYPE_DEFAULT,
|
||||
context.discord_reader.MESSAGE_TYPE_REPLY,
|
||||
context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
|
||||
context.discord_reader.MESSAGE_TYPE_FORWARD,
|
||||
context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
|
||||
context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
|
||||
context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
|
||||
context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
|
||||
]:
|
||||
continue
|
||||
|
||||
# Determine target channel
|
||||
if not msg.channel:
|
||||
continue
|
||||
|
||||
target_channel_id = context.state.get_target_channel_id(str(msg.channel.id))
|
||||
if not target_channel_id:
|
||||
continue
|
||||
|
||||
# Efficient skip: if message ID is <= last migrated ID for this channel/thread
|
||||
# This ensures we only resume a channel once we reach its last known progress point
|
||||
last_id = progress_map.get(str(target_channel_id))
|
||||
if last_id and msg.id <= int(last_id):
|
||||
continue
|
||||
|
||||
# If it's a thread message, we need to handle it based on if it's the thread starter or a reply
|
||||
parent_target_id = None
|
||||
if hasattr(msg, 'thread') and msg.thread and msg.id == msg.thread.id:
|
||||
processed_threads.add(msg.thread.id)
|
||||
stats["threads"] += 1
|
||||
elif msg.channel.type in [11, 12]: # Thread channels
|
||||
# It's a message IN a thread.
|
||||
# In Fluxer, threads might just be linear messages or threaded replies depending on schema
|
||||
# For basic migration we just send it to the parent mapped target channel.
|
||||
# The parent mapped target channel ID should already be calculated correctly by get_target_channel_id (which returns mapped thread or parent channel)
|
||||
pass
|
||||
|
||||
# Formatting
|
||||
files = []
|
||||
file_names = []
|
||||
|
||||
# Always ensure alias is created/retrieved to populate user_alias table
|
||||
alias = context.state.get_user_alias(str(msg.author.id))
|
||||
|
||||
anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False
|
||||
|
||||
if anonymize_users:
|
||||
author_name = alias or "Anonymized User"
|
||||
author_avatar_url = None
|
||||
else:
|
||||
author_name = msg.author.display_name
|
||||
author_avatar_url = msg.author.avatar.url if hasattr(msg.author, 'avatar') and msg.author.avatar else None
|
||||
|
||||
for att in msg.attachments:
|
||||
media_info = db_media.get(att.local_hash) if db_media else None
|
||||
local_path = None
|
||||
if media_info:
|
||||
local_path = Path(media_info["local_path"])
|
||||
elif hasattr(att, 'read'):
|
||||
# Fallback
|
||||
pass
|
||||
|
||||
if local_path and local_path.exists():
|
||||
files.append(local_path)
|
||||
file_names.append(att.filename)
|
||||
|
||||
content = msg.content or ""
|
||||
|
||||
# Stickers
|
||||
for sticker in msg.stickers:
|
||||
sticker_name = sticker.name
|
||||
sticker_url = sticker.url
|
||||
|
||||
# Check for uploaded media pool logic first
|
||||
s_hash = sticker.local_hash
|
||||
sticker_file = None
|
||||
s_media = db_media.get(s_hash) if db_media and s_hash else None
|
||||
if s_media:
|
||||
s_path = Path(s_media["local_path"])
|
||||
if s_path.exists():
|
||||
sticker_file = s_path
|
||||
|
||||
content += f"\n[Sticker: {sticker_name}]"
|
||||
if sticker_file:
|
||||
files.append(sticker_file)
|
||||
file_names.append(f"sticker_{sticker_name}.png")
|
||||
|
||||
content = clean_mentions(
|
||||
content=content,
|
||||
guild=context.discord_reader.guild,
|
||||
user_mentions=msg.mentions,
|
||||
role_mentions=msg.role_mentions,
|
||||
channel_mentions=msg.channel_mentions,
|
||||
emoji_map=emoji_map,
|
||||
channel_map=context.state.channel_map,
|
||||
state=context.state,
|
||||
target_server_id=target_server_id,
|
||||
channel_names=context.channel_names if hasattr(context, 'channel_names') else None,
|
||||
anonymize_users=anonymize_users
|
||||
)
|
||||
|
||||
if not content and not files:
|
||||
logger.debug(f"Message {msg.id} empty after processing, skipping.")
|
||||
continue
|
||||
|
||||
timestamp_int = int(msg.created_at.timestamp())
|
||||
|
||||
if msg.reference and msg.reference.message_id:
|
||||
# Resolve the author of the message being replied to
|
||||
source_ref_msg = await context.discord_reader.get_message(msg.channel.id, msg.reference.message_id)
|
||||
if source_ref_msg and source_ref_msg.author:
|
||||
ref_author_id = str(source_ref_msg.author.id)
|
||||
if anonymize_users:
|
||||
ref_name = context.state.get_user_alias(ref_author_id) or "Anonymized User"
|
||||
else:
|
||||
ref_name = source_ref_msg.author.display_name
|
||||
content = f"`@{ref_name}`\n{content}"
|
||||
else:
|
||||
# Fallback if author cannot be resolved (e.g. deleted/missing from backup)
|
||||
tgt_reply = context.state.get_target_message_id(target_channel_id, msg.reference.message_id)
|
||||
if tgt_reply:
|
||||
content = f"[Reply to {tgt_reply}]\n{content}"
|
||||
|
||||
try:
|
||||
fluxer_msg_id = await context.fluxer_writer.send_message(
|
||||
channel_id=target_channel_id,
|
||||
author_name=author_name,
|
||||
author_avatar_url=author_avatar_url,
|
||||
content=content,
|
||||
files=files,
|
||||
timestamp=timestamp_int,
|
||||
embeds=msg.embeds
|
||||
)
|
||||
|
||||
if fluxer_msg_id:
|
||||
context.state.set_target_message_mapping(target_channel_id, msg.id, fluxer_msg_id)
|
||||
context.state.update_last_message_id(target_channel_id, msg.id)
|
||||
context.state.set_waterfall_last_id(msg.id)
|
||||
stats["attachments"] += len(files) if files else 0
|
||||
|
||||
stats["messages"] += 1
|
||||
stats["last_message_content"] = content
|
||||
stats["last_message_author"] = author_name
|
||||
|
||||
if not stats["first_message_url"]:
|
||||
stats["first_message_url"] = msg.jump_url
|
||||
stats["last_message_url"] = msg.jump_url
|
||||
|
||||
if progress_callback:
|
||||
await progress_callback(stats)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process global message {msg.id}: {e}")
|
||||
|
||||
except (KeyboardInterrupt, asyncio.CancelledError):
|
||||
context.is_running = False
|
||||
pass
|
||||
|
||||
return stats
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import re
|
|||
import json
|
||||
import io
|
||||
from typing import Callable, Awaitable, Dict, Any, List
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from lottie.objects import Animation
|
||||
|
|
@ -27,22 +28,25 @@ def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None,
|
|||
uid = int(match.group(1))
|
||||
if anonymize_users and state:
|
||||
alias = state.get_user_alias(str(uid))
|
||||
return f"`@{alias}`"
|
||||
return f"`@{alias}`" if alias else "`@Anonymized User`"
|
||||
|
||||
# 1. Try provided guild
|
||||
member = guild.get_member(uid)
|
||||
if member:
|
||||
return f"`@{member.display_name}`"
|
||||
# 2. Try message's user_mentions
|
||||
|
||||
# 2. Try provided user_mentions
|
||||
if user_mentions:
|
||||
for u in user_mentions:
|
||||
if u.id == uid:
|
||||
return f"`@{getattr(u, 'display_name', u.name)}`"
|
||||
m = next((u for u in user_mentions if u.id == uid), None)
|
||||
if m:
|
||||
return f"`@{m.display_name}`"
|
||||
|
||||
# 3. Try global cache via guild.client
|
||||
if hasattr(guild, 'client'):
|
||||
user = guild.client.get_user(uid)
|
||||
if user:
|
||||
return f"`@{user.name}`"
|
||||
return f"`@Unknown User`"
|
||||
return "`@Unknown User`"
|
||||
|
||||
def replace_role(match):
|
||||
rid = int(match.group(1))
|
||||
|
|
@ -543,6 +547,10 @@ async def migrate_messages(
|
|||
logger.error(f"Failed to download sticker {getattr(s, 'name', 'unknown')}: {e}")
|
||||
|
||||
try:
|
||||
# Check for existing mapping to avoid duplicates when resuming
|
||||
if context.state.get_target_message_id(target_channel_id, str(msg.id)):
|
||||
continue
|
||||
|
||||
# Check if this message is a reply
|
||||
reply_to_stoat_id = None
|
||||
if msg.reference and msg.reference.message_id:
|
||||
|
|
@ -560,22 +568,24 @@ async def migrate_messages(
|
|||
if thread_name and stats["messages"] == 0:
|
||||
content = f"> <<< THREAD: **{thread_name}** >>>\n{content}"
|
||||
|
||||
# Get or generate alias
|
||||
# Always ensure alias is created/retrieved to populate user_alias table
|
||||
alias = context.state.get_user_alias(str(msg.author.id))
|
||||
|
||||
if context.config.anonymize_users:
|
||||
author_name = alias
|
||||
avatar_url = f"https://api.dicebear.com/9.x/fun-emoji/jpg?seed={alias}"
|
||||
anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False
|
||||
if anonymize_users:
|
||||
author_name = alias or "Anonymized User"
|
||||
author_avatar_url = None
|
||||
else:
|
||||
author_name = msg.author.display_name
|
||||
avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
|
||||
if avatar_url and not avatar_url.startswith("http"):
|
||||
avatar_url = None
|
||||
author_avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
|
||||
if author_avatar_url and not author_avatar_url.startswith("http"):
|
||||
author_avatar_url = None
|
||||
|
||||
logger.debug(f"Stoat: Calling send_message for Discord ID {msg.id}")
|
||||
stoat_msg_id = await context.stoat_writer.send_message(
|
||||
channel_id=target_channel_id,
|
||||
author_name=author_name,
|
||||
author_avatar_url=avatar_url,
|
||||
author_avatar_url=author_avatar_url,
|
||||
content=content,
|
||||
timestamp=int(msg.created_at.timestamp()),
|
||||
files=files if files else None,
|
||||
|
|
@ -664,3 +674,240 @@ async def migrate_messages(
|
|||
pass
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
async def analyze_global_migration(context: MigrationContext, after_message_id: int | None = None, inclusive: bool = False, progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None) -> Dict[str, int]:
|
||||
"""
|
||||
Scans the entire server history to count messages, threads, and attachments globally.
|
||||
"""
|
||||
stats = {"messages": 0, "threads": 0, "attachments": 0}
|
||||
|
||||
# Fetch global progress map to skip migrated messages efficiently
|
||||
progress_map = context.state.get_all_last_message_ids()
|
||||
|
||||
async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id):
|
||||
if not context.is_running:
|
||||
break
|
||||
|
||||
# Determine target channel to check for existing mapping
|
||||
if not msg.channel:
|
||||
continue
|
||||
|
||||
target_channel_id = context.state.get_target_channel_id(str(msg.channel.id))
|
||||
if not target_channel_id:
|
||||
continue
|
||||
|
||||
# Efficient skip: if message ID is <= last migrated ID for this channel/thread
|
||||
# This is the primary resume mechanism: wait until we pass the last migrated ID for this channel
|
||||
last_id = progress_map.get(str(msg.channel.id))
|
||||
if last_id and msg.id <= int(last_id):
|
||||
continue
|
||||
|
||||
if msg.type not in [
|
||||
context.discord_reader.MESSAGE_TYPE_DEFAULT,
|
||||
context.discord_reader.MESSAGE_TYPE_REPLY,
|
||||
context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
|
||||
context.discord_reader.MESSAGE_TYPE_FORWARD,
|
||||
context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
|
||||
context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
|
||||
context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
|
||||
context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
|
||||
]:
|
||||
continue
|
||||
|
||||
stats["messages"] += 1
|
||||
stats["attachments"] += len(msg.attachments)
|
||||
if hasattr(msg, 'thread') and msg.thread:
|
||||
stats["threads"] += 1
|
||||
|
||||
if progress_callback and stats["messages"] % 100 == 0:
|
||||
await progress_callback(stats)
|
||||
|
||||
if progress_callback:
|
||||
await progress_callback(stats)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
async def migrate_global_messages(
|
||||
context: MigrationContext,
|
||||
after_message_id: int | None = None,
|
||||
inclusive: bool = False,
|
||||
progress_callback: Callable[[Dict[str, Any]], Awaitable[None]] | None = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Migrates messages across all channels chronologically to Stoat.
|
||||
"""
|
||||
stats = {
|
||||
"messages": 0,
|
||||
"threads": 0,
|
||||
"attachments": 0,
|
||||
"last_message_content": "",
|
||||
"last_message_author": "",
|
||||
"first_message_url": None,
|
||||
"last_message_url": None
|
||||
}
|
||||
|
||||
processed_threads = set()
|
||||
logger.info("Starting Global Waterfall Migration for Stoat...")
|
||||
|
||||
emoji_map = context.state.emoji_map
|
||||
db_media = context.discord_reader.db.get_all_media() if context.discord_reader.db else {}
|
||||
# Fetch global progress map to skip migrated messages efficiently
|
||||
progress_map = context.state.get_all_last_message_ids()
|
||||
|
||||
try:
|
||||
async for msg in context.discord_reader.fetch_global_message_history(after_id=after_message_id):
|
||||
if not context.is_running:
|
||||
logger.warning("Global migration interrupted by user")
|
||||
break
|
||||
|
||||
if msg.type not in [
|
||||
context.discord_reader.MESSAGE_TYPE_DEFAULT,
|
||||
context.discord_reader.MESSAGE_TYPE_REPLY,
|
||||
context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
|
||||
context.discord_reader.MESSAGE_TYPE_FORWARD,
|
||||
context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
|
||||
context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
|
||||
context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
|
||||
context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
|
||||
]:
|
||||
continue
|
||||
|
||||
# Determine target channel
|
||||
if not msg.channel:
|
||||
continue
|
||||
|
||||
target_channel_id = context.state.get_target_channel_id(str(msg.channel.id))
|
||||
if not target_channel_id:
|
||||
continue
|
||||
|
||||
# Efficient skip: if message ID is <= last migrated ID for this channel/thread
|
||||
# This ensures we only resume a channel once we reach its last known progress point
|
||||
last_id = progress_map.get(str(target_channel_id))
|
||||
if last_id and msg.id <= int(last_id):
|
||||
continue
|
||||
|
||||
parent_target_id = None
|
||||
if hasattr(msg, 'thread') and msg.thread and msg.id == msg.thread.id:
|
||||
processed_threads.add(msg.thread.id)
|
||||
stats["threads"] += 1
|
||||
elif msg.channel.type in [11, 12]:
|
||||
pass
|
||||
|
||||
# Formatting
|
||||
files = []
|
||||
|
||||
# Always ensure alias is created/retrieved to populate user_alias table
|
||||
alias = context.state.get_user_alias(str(msg.author.id))
|
||||
|
||||
anonymize_users = context.config.anonymize_users if hasattr(context, 'config') else False
|
||||
|
||||
if anonymize_users:
|
||||
author_name = alias or "Anonymized User"
|
||||
author_avatar_url = None
|
||||
else:
|
||||
author_name = msg.author.display_name
|
||||
author_avatar_url = msg.author.avatar.url if hasattr(msg.author, 'avatar') and msg.author.avatar else None
|
||||
|
||||
for att in msg.attachments:
|
||||
media_info = db_media.get(att.local_hash) if db_media else None
|
||||
local_path = None
|
||||
if media_info:
|
||||
local_path = Path(media_info["local_path"])
|
||||
|
||||
if local_path and local_path.exists():
|
||||
try:
|
||||
with open(local_path, "rb") as f:
|
||||
files.append({"filename": att.filename, "data": f.read()})
|
||||
except Exception as fe:
|
||||
logger.error(f"Failed to read file {local_path}: {fe}")
|
||||
|
||||
content = msg.content or ""
|
||||
|
||||
for sticker in msg.stickers:
|
||||
sticker_name = sticker.name
|
||||
s_hash = sticker.local_hash
|
||||
sticker_file = None
|
||||
s_media = db_media.get(s_hash) if db_media and s_hash else None
|
||||
if s_media:
|
||||
s_path = Path(s_media["local_path"])
|
||||
if s_path.exists():
|
||||
sticker_file = s_path
|
||||
|
||||
content += f"\n[Sticker: {sticker_name}]"
|
||||
if sticker_file:
|
||||
files.append(sticker_file)
|
||||
file_names.append(f"sticker_{sticker_name}.png")
|
||||
|
||||
content = clean_mentions(
|
||||
content=content,
|
||||
guild=context.discord_reader.guild,
|
||||
user_mentions=msg.mentions,
|
||||
role_mentions=msg.role_mentions,
|
||||
channel_mentions=msg.channel_mentions,
|
||||
emoji_map=emoji_map,
|
||||
channel_map=context.state.channel_map,
|
||||
state=context.state,
|
||||
target_server_id=target_server_id,
|
||||
channel_names=context.channel_names if hasattr(context, 'channel_names') else None,
|
||||
anonymize_users=anonymize_users
|
||||
)
|
||||
|
||||
if not content and not files:
|
||||
logger.debug(f"Message {msg.id} empty after processing, skipping.")
|
||||
continue
|
||||
|
||||
timestamp_int = int(msg.created_at.timestamp())
|
||||
|
||||
if msg.reference and msg.reference.message_id:
|
||||
# Resolve the author of the message being replied to
|
||||
source_ref_msg = await context.discord_reader.get_message(msg.channel_id, msg.reference.message_id)
|
||||
if source_ref_msg and source_ref_msg.author:
|
||||
ref_author_id = str(source_ref_msg.author.id)
|
||||
if anonymize_users:
|
||||
ref_name = context.state.get_user_alias(ref_author_id) or "Anonymized User"
|
||||
else:
|
||||
ref_name = source_ref_msg.author.display_name
|
||||
content = f"`@{ref_name}`\n{content}"
|
||||
else:
|
||||
tgt_reply = context.state.get_target_message_id(target_channel_id, msg.reference.message_id)
|
||||
if tgt_reply:
|
||||
content = f"[Reply to {tgt_reply}]\n{content}"
|
||||
|
||||
try:
|
||||
stoat_msg_id = await context.stoat_writer.send_message(
|
||||
channel_id=target_channel_id,
|
||||
author_name=author_name,
|
||||
author_avatar_url=author_avatar_url,
|
||||
content=content,
|
||||
files=files,
|
||||
timestamp=timestamp_int,
|
||||
embeds=msg.embeds
|
||||
)
|
||||
|
||||
if stoat_msg_id:
|
||||
context.state.set_target_message_mapping(target_channel_id, msg.id, stoat_msg_id)
|
||||
context.state.update_last_message_id(target_channel_id, msg.id)
|
||||
context.state.set_waterfall_last_id(msg.id)
|
||||
stats["attachments"] += len(files) if files else 0
|
||||
|
||||
stats["messages"] += 1
|
||||
stats["last_message_content"] = content
|
||||
stats["last_message_author"] = author_name
|
||||
|
||||
if not stats["first_message_url"]:
|
||||
stats["first_message_url"] = msg.jump_url
|
||||
stats["last_message_url"] = msg.jump_url
|
||||
|
||||
if progress_callback:
|
||||
await progress_callback(stats)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process global message {msg.id}: {e}")
|
||||
|
||||
except (KeyboardInterrupt, asyncio.CancelledError):
|
||||
context.is_running = False
|
||||
pass
|
||||
|
||||
return stats
|
||||
|
|
|
|||
|
|
@ -160,6 +160,7 @@ class OperationPane(Container):
|
|||
yield Button("Clone Server Template", id="op_clone", disabled=True, tooltip="Clone server roles, categories, and channels to the target community")
|
||||
yield Button("Sync Server Settings", id="op_sync", disabled=True, tooltip="Sync emojis, stickers, server name, and icon to the target community")
|
||||
yield Button("Migrate Message History", id="op_messages", disabled=True, variant="primary", tooltip="Migrate message history from Discord to the target platform")
|
||||
yield Button("Waterfall Migration", id="op_waterfall", disabled=True, variant="primary", tooltip="Migrate all messages globally in chronological order to prevent broken links.\n(Available for Local Backups)")
|
||||
yield Rule(id="footer_rule")
|
||||
yield Button("Danger Zone ⚠", id="op_danger", variant="error", disabled=True, flat=True, tooltip="Dangerous operations:\ndelete channels, roles, emojis on target\n(use with caution)")
|
||||
|
||||
|
|
@ -394,7 +395,7 @@ class OperationPane(Container):
|
|||
lbl.update(f"{t_status}")
|
||||
|
||||
# Buttons
|
||||
for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_danger"):
|
||||
for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_waterfall", "#op_danger"):
|
||||
for btn in self.query(bid): btn.disabled = not self.tokens_valid
|
||||
|
||||
# ── validation ────────────────────────────────────────────────────────
|
||||
|
|
@ -415,7 +416,7 @@ class OperationPane(Container):
|
|||
|
||||
# Disable all operation buttons while validation is in progress
|
||||
if self.view_mode == "shuttle":
|
||||
for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_danger"):
|
||||
for bid in ("#op_clone", "#op_sync", "#op_messages", "#op_waterfall", "#op_danger"):
|
||||
for btn in self.query(bid): btn.disabled = True
|
||||
elif self.view_mode == "backup":
|
||||
for bid in ("#op_backup_msgs", "#op_backup_sync"):
|
||||
|
|
@ -552,7 +553,12 @@ class OperationPane(Container):
|
|||
target_ok = v.get("target_token") and v.get("target_community")
|
||||
self.tokens_valid = bool(discord_ok and target_ok)
|
||||
|
||||
|
||||
# Post validation adjustments
|
||||
if self.tokens_valid:
|
||||
is_backup = (self.config.tool_mode == "backup_transfer")
|
||||
for btn in self.query("#op_waterfall"):
|
||||
btn.disabled = not is_backup
|
||||
btn.display = is_backup
|
||||
|
||||
self._update_info_labels()
|
||||
|
||||
|
|
@ -570,6 +576,8 @@ class OperationPane(Container):
|
|||
self._open_sync_menu()
|
||||
elif bid == "op_messages":
|
||||
self.run_migrate_messages()
|
||||
elif bid == "op_waterfall":
|
||||
self.run_waterfall_migration()
|
||||
elif bid == "op_danger":
|
||||
self._open_danger_menu()
|
||||
|
||||
|
|
@ -716,7 +724,6 @@ class OperationPane(Container):
|
|||
return
|
||||
elif choice == "btn_main_menu":
|
||||
modal.dismiss()
|
||||
self.app.switch_screen("config_selection")
|
||||
return
|
||||
|
||||
force_mode = (choice == "btn_start_id")
|
||||
|
|
@ -798,7 +805,6 @@ class OperationPane(Container):
|
|||
return
|
||||
elif choice == "btn_main_menu":
|
||||
modal.dismiss()
|
||||
self.app.switch_screen("config_selection")
|
||||
return
|
||||
|
||||
force_mode = (choice == "btn_start_id")
|
||||
|
|
@ -1099,6 +1105,11 @@ class OperationPane(Container):
|
|||
source_channel = next(c for c in d_channels if c.id == src_id)
|
||||
target_channel = next(c for c in f_channels if c.get("id") == tgt_id)
|
||||
|
||||
# 2. Analyze
|
||||
modal = ProgressScreen(log_level=self.config.log_level)
|
||||
self.app.push_screen(modal)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Determine after_id status (skip for pending channels)
|
||||
if pending_create_name:
|
||||
last_migrated = None
|
||||
|
|
@ -1107,11 +1118,6 @@ class OperationPane(Container):
|
|||
last_migrated = self.engine.state.get_last_message_id(str(target_channel.get('id')))
|
||||
has_previous = bool(last_migrated)
|
||||
|
||||
# Analyze
|
||||
modal = ProgressScreen(log_level=self.config.log_level)
|
||||
self.app.push_screen(modal)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
src_server = getattr(self.engine.discord_reader, 'guild', None)
|
||||
tgt_server_info = await self.engine.writer.validate()
|
||||
tgt_server_name = tgt_server_info.get("community_name", "target community")
|
||||
|
|
@ -1243,7 +1249,6 @@ class OperationPane(Container):
|
|||
continue # Return to channel picker
|
||||
elif choice == "btn_main_menu":
|
||||
modal.dismiss()
|
||||
self.app.switch_screen("config_selection")
|
||||
self.engine.is_running = False
|
||||
await self.engine.close_connections()
|
||||
return
|
||||
|
|
@ -1294,6 +1299,14 @@ class OperationPane(Container):
|
|||
|
||||
try:
|
||||
self.engine.is_running = True
|
||||
|
||||
# Ensure state is initialized (database exists)
|
||||
if self.target_platform == "stoat":
|
||||
tid = self.config.stoat_server_id
|
||||
else:
|
||||
tid = self.config.fluxer_server_id
|
||||
self.engine.ensure_state_initialized(str(tid or ""), platform_name)
|
||||
|
||||
stats_analysis = await migrate_mod.analyze_migration(
|
||||
self.engine,
|
||||
source_channel_id=source_channel.id,
|
||||
|
|
@ -1399,6 +1412,239 @@ class OperationPane(Container):
|
|||
else:
|
||||
modal.write(f"[bold red]Error: {err}[/bold red]")
|
||||
modal.phase_report("Message Migration", "error", show_back=False)
|
||||
logger.error(f"Migration Error: {traceback.format_exc()}")
|
||||
finally:
|
||||
self.engine.is_running = False
|
||||
await self.engine.close_connections()
|
||||
|
||||
@work(exclusive=True)
|
||||
async def run_waterfall_migration(self) -> None:
|
||||
if not self.tokens_valid:
|
||||
return
|
||||
|
||||
migrate_mod = fluxer_migrate if self.target_platform == "fluxer" else stoat_migrate
|
||||
platform_name = self.target_platform.capitalize()
|
||||
|
||||
modal = ProgressScreen(log_level=self.config.log_level)
|
||||
self.app.push_screen(modal)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
try:
|
||||
modal.show_info("[bold cyan]Waterfall Migration Ready[/bold cyan]", "Checking mapping and missing channels...")
|
||||
modal.set_status("Connecting to Servers...")
|
||||
await self.engine.start_connections()
|
||||
|
||||
modal.set_status("Synchronizing entity mappings...")
|
||||
await self._perform_auto_matching()
|
||||
|
||||
# 1. Missing channels check
|
||||
if hasattr(self.engine.discord_reader, "get_all_channels"):
|
||||
full_d = await self.engine.discord_reader.get_all_channels()
|
||||
# Include TEXT (0), CATEGORY (4), and NEWS (5)
|
||||
d_channels = [c for c in full_d if c.type in [0, 4, 5]]
|
||||
else:
|
||||
full_d = await self.engine.discord_reader.get_channels()
|
||||
d_channels = [c for c in full_d if c.type in [0, 4, 5]]
|
||||
missing_channels = []
|
||||
for d in d_channels:
|
||||
if d.type == 4:
|
||||
tgt_id = self.engine.state.get_target_category_id(str(d.id))
|
||||
else:
|
||||
tgt_id = self.engine.state.get_target_channel_id(str(d.id))
|
||||
if not tgt_id:
|
||||
missing_channels.append(d)
|
||||
|
||||
if missing_channels:
|
||||
modal.write(f"\n[bold yellow]Found {len(missing_channels)} backed-up channels/categories missing from target platform:[/bold yellow]")
|
||||
for mc in missing_channels:
|
||||
prefix = "[bold cyan]📁[/bold cyan] " if mc.type == 4 else "[bold white]#[/bold white] "
|
||||
modal.write(f" {prefix}{mc.name}")
|
||||
|
||||
choice = await modal.phase_wait_confirm(
|
||||
show_continue=False,
|
||||
show_id=True,
|
||||
btn_start_label="Clone missing channels",
|
||||
btn_id_label="Skip missing channels",
|
||||
btn_start_variant="primary",
|
||||
btn_start_tooltip=f"Automatically create {len(missing_channels)} entities on target",
|
||||
btn_id_tooltip="Start migration without these channels"
|
||||
)
|
||||
|
||||
if choice == "btn_back":
|
||||
modal.dismiss()
|
||||
await self.engine.close_connections()
|
||||
return
|
||||
elif choice == "btn_main_menu":
|
||||
modal.dismiss()
|
||||
await self.engine.close_connections()
|
||||
return
|
||||
|
||||
if choice == "btn_start_first":
|
||||
modal.set_status("Cloning missing categories and channels...")
|
||||
# Sort so categories (type 4) come first
|
||||
missing_channels.sort(key=lambda x: 0 if x.type == 4 else 1)
|
||||
|
||||
for mc in missing_channels:
|
||||
try:
|
||||
parent_target_id = None
|
||||
if mc.type == 4:
|
||||
modal.write(f"Creating category '[bold cyan]{mc.name}[/bold cyan]'...")
|
||||
new_id = await self.engine.writer.create_channel(name=mc.name, type=4)
|
||||
self.engine.state.set_target_category_mapping(str(mc.id), new_id)
|
||||
modal.write(f"[green]Created Category {mc.name} ({new_id})[/green]")
|
||||
else:
|
||||
if hasattr(mc, 'category_id') and mc.category_id:
|
||||
parent_target_id = self.engine.state.get_target_category_id(str(mc.category_id))
|
||||
|
||||
modal.write(f"Creating channel '#{mc.name}'...")
|
||||
new_id = await self.engine.writer.create_channel(name=mc.name, parent_id=parent_target_id)
|
||||
self.engine.state.set_target_channel_id(str(mc.id), new_id, self.engine.target_platform)
|
||||
modal.write(f"[green]Created Channel {mc.name} ({new_id})[/green]")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create {mc.name}: {e}\n{traceback.format_exc()}")
|
||||
modal.write(f"[red]Failed to create {mc.name}: {e}[/red]")
|
||||
elif choice == "btn_id":
|
||||
# Skip missing channels: remove them from the active list
|
||||
missing_ids = {str(c.id) for c in missing_channels}
|
||||
d_channels = [c for c in d_channels if str(c.id) not in missing_ids]
|
||||
|
||||
# 2. Resumption check
|
||||
all_mapped_tgt_ids = []
|
||||
# Check regular text channels (exclude categories for resume check)
|
||||
for c in d_channels:
|
||||
if c.type == 4: continue
|
||||
did = str(c.id)
|
||||
tid = self.engine.state.get_target_channel_id(did)
|
||||
if tid: all_mapped_tgt_ids.append(tid)
|
||||
|
||||
# Also check threads (filtering to only include those belonging to active channels)
|
||||
active_channel_ids = {str(c.id) for c in d_channels}
|
||||
if hasattr(self.engine.discord_reader, "get_active_threads"):
|
||||
threads = await self.engine.discord_reader.get_active_threads()
|
||||
for t in threads:
|
||||
pid = str(getattr(t, 'parent_id', getattr(t, 'channel_id', None)))
|
||||
if pid not in active_channel_ids: continue
|
||||
tid = self.engine.state.get_target_channel_id(str(t.id))
|
||||
if tid: all_mapped_tgt_ids.append(tid)
|
||||
|
||||
# 2.5 Filter by actual content (Only for BackupReader)
|
||||
# If a channel has NO messages in the backup, it will always be at 0 progress.
|
||||
# We exclude those from the global MIN calculation to avoid pulling it to 0.
|
||||
if hasattr(self.engine.discord_reader, "get_backed_up_channel_ids"):
|
||||
backed_up_src_ids = await self.engine.discord_reader.get_backed_up_channel_ids()
|
||||
backed_up_src_ids_str = {str(sid) for sid in backed_up_src_ids}
|
||||
|
||||
filtered_tgt_ids = []
|
||||
# Find which target IDs belong to source channels that HAVE messages
|
||||
for c in d_channels: # (d_channels is already filtered for skipped)
|
||||
if str(c.id) in backed_up_src_ids_str:
|
||||
tid = self.engine.state.get_target_channel_id(str(c.id))
|
||||
if tid: filtered_tgt_ids.append(tid)
|
||||
|
||||
# Also check threads
|
||||
if hasattr(self.engine.discord_reader, "threads"):
|
||||
for t in self.engine.discord_reader.threads:
|
||||
if str(t.id) in backed_up_src_ids_str:
|
||||
tid = self.engine.state.get_target_channel_id(str(t.id))
|
||||
if tid: filtered_tgt_ids.append(tid)
|
||||
|
||||
if filtered_tgt_ids:
|
||||
all_mapped_tgt_ids = filtered_tgt_ids
|
||||
|
||||
# 2.6 Resume Point: Prioritize Global waterfall tracker, fallback to channel minimums
|
||||
min_last_id = self.engine.state.get_waterfall_last_id()
|
||||
if min_last_id is None:
|
||||
min_last_id = self.engine.state.get_global_min_last_message_id(all_mapped_tgt_ids)
|
||||
|
||||
modal.write(f"\n[bold cyan]Waterfall Migration Resume Point:[/bold cyan]")
|
||||
if min_last_id is not None:
|
||||
modal.write(f"Minimum unmigrated message ID found: [green]{min_last_id}[/green]")
|
||||
else:
|
||||
modal.write("No previous migration state found. Starting from the beginning.")
|
||||
|
||||
choice = await modal.phase_wait_confirm(
|
||||
show_continue=min_last_id is not None,
|
||||
show_id=False,
|
||||
btn_start_label="Start From Beginning",
|
||||
btn_start_tooltip="Safe, skips duplicates automatically",
|
||||
btn_start_variant="default" if min_last_id is not None else "primary",
|
||||
btn_continue_label=f"Continue from ID {min_last_id if min_last_id is not None else 0}" if min_last_id is not None else "Continue Migration",
|
||||
btn_continue_tooltip="Fastest"
|
||||
)
|
||||
|
||||
if choice == "btn_back":
|
||||
modal.dismiss()
|
||||
await self.engine.close_connections()
|
||||
return
|
||||
elif choice == "btn_main_menu":
|
||||
modal.dismiss()
|
||||
await self.engine.close_connections()
|
||||
return
|
||||
|
||||
after_id = None
|
||||
if choice == "btn_continue" and min_last_id is not None:
|
||||
after_id = int(min_last_id)
|
||||
|
||||
# Phase 3: Progress
|
||||
modal.cancel_callback = lambda: setattr(self.engine, "is_running", False)
|
||||
modal.phase_progress()
|
||||
modal.set_status("Migrating messages Globally...")
|
||||
|
||||
self.engine.is_running = True
|
||||
|
||||
# Ensure state is initialized (database exists)
|
||||
if self.target_platform == "stoat":
|
||||
tid = self.config.stoat_server_id
|
||||
else:
|
||||
tid = self.config.fluxer_server_id
|
||||
self.engine.ensure_state_initialized(str(tid or ""), platform_name)
|
||||
|
||||
modal.write("Scanning global footprint for totals ...")
|
||||
stats_analysis = await migrate_mod.analyze_global_migration(self.engine, after_message_id=after_id)
|
||||
total_messages = stats_analysis["messages"]
|
||||
|
||||
modal.write(f"[bold cyan]Global Migration Started:[/bold cyan] {total_messages} total messages to process.")
|
||||
modal.update_stats(messages=f"0/{total_messages}", threads=str(stats_analysis["threads"]), files=str(stats_analysis["attachments"]))
|
||||
|
||||
async def update_msg(current_stats):
|
||||
c_msgs = current_stats["messages"]
|
||||
c_threads = current_stats["threads"]
|
||||
c_files = current_stats["attachments"]
|
||||
|
||||
msg_stat = f"{c_msgs}/{total_messages}" if total_messages > 0 else str(c_msgs)
|
||||
modal.set_progress(c_msgs, total_messages or 100)
|
||||
modal.update_stats(messages=msg_stat, threads=str(c_threads), files=str(c_files))
|
||||
|
||||
content = current_stats.get("last_message_content", "")
|
||||
author = current_stats.get("last_message_author", "Unknown")
|
||||
if content:
|
||||
disp_content = (content[:100] + '...') if len(content) > 100 else content
|
||||
modal.write(f"[bold]{author}:[/bold] {disp_content}")
|
||||
|
||||
result = await migrate_mod.migrate_global_messages(
|
||||
self.engine,
|
||||
after_message_id=after_id,
|
||||
inclusive=False,
|
||||
progress_callback=update_msg,
|
||||
)
|
||||
|
||||
if self.engine.is_running:
|
||||
modal.write(f"[bold green]Success! {result['messages']} messages migrated globally.[/bold green]")
|
||||
modal.phase_report("Waterfall Migration", show_back=False)
|
||||
else:
|
||||
modal.write(f"[bold yellow]Interrupted! {result['messages']} messages migrated.[/bold yellow]")
|
||||
modal.phase_report("Waterfall Migration", "stopped", show_back=False)
|
||||
|
||||
lines = [f"Migrated Server Globally → {platform_name}:"]
|
||||
lines.append(f"{result['messages']} messages, {result['attachments']} attachments, {result['threads']} threads")
|
||||
await log_audit_event(self.engine, "Waterfall Migration", "\n".join(lines))
|
||||
|
||||
except Exception as e:
|
||||
err = str(e)
|
||||
modal.write(f"[bold red]Error: {err}[/bold red]")
|
||||
modal.phase_report("Waterfall Migration", "error", show_back=False)
|
||||
|
||||
logger.error(traceback.format_exc())
|
||||
finally:
|
||||
self.engine.is_running = False
|
||||
await self.engine.close_connections()
|
||||
|
|
@ -1485,7 +1731,6 @@ class OperationPane(Container):
|
|||
return
|
||||
elif choice == "btn_main_menu":
|
||||
modal.dismiss()
|
||||
self.app.switch_screen("config_selection")
|
||||
return
|
||||
|
||||
modal.cancel_callback = lambda: setattr(self.engine, "is_running", False)
|
||||
|
|
@ -1644,6 +1889,39 @@ class OperationPane(Container):
|
|||
logger.warning(f"Auto-matching: failed to fetch target data: {e}")
|
||||
return # Cannot match without target data
|
||||
|
||||
# 1.5 Cleanup deleted entities from mapping database
|
||||
# This prevents "Ghost" mappings to channels/roles that were deleted on target
|
||||
valid_chan_ids = {str(c.get("id")) for c in target_chans_raw}
|
||||
valid_cat_ids = {str(c.get("id")) for c in target_chans_raw if c.get("type") == 4}
|
||||
valid_role_ids = set(target_roles_map.values())
|
||||
valid_emoji_ids = set(target_emojis_map.values())
|
||||
|
||||
# Channels
|
||||
for src_id, tgt_id in self.engine.state.channel_map.items():
|
||||
if str(tgt_id) not in valid_chan_ids:
|
||||
logger.info(f"Auto-matching: clearing deleted channel mapping {src_id} -> {tgt_id}")
|
||||
self.engine.state.remove_target_channel_mapping(src_id)
|
||||
|
||||
# Categories
|
||||
for src_id, tgt_id in self.engine.state.category_map.items():
|
||||
if str(tgt_id) not in valid_cat_ids:
|
||||
logger.info(f"Auto-matching: clearing deleted category mapping {src_id} -> {tgt_id}")
|
||||
self.engine.state.remove_category_mapping(src_id)
|
||||
|
||||
# Roles
|
||||
for src_id, tgt_id in self.engine.state.role_map.items():
|
||||
if str(tgt_id) not in valid_role_ids:
|
||||
logger.info(f"Auto-matching: clearing deleted role mapping {src_id} -> {tgt_id}")
|
||||
self.engine.state.remove_role_mapping(src_id)
|
||||
|
||||
# Emojis
|
||||
for src_id, tgt_id in self.engine.state.emoji_map.items():
|
||||
if str(tgt_id) not in valid_emoji_ids:
|
||||
# Emojis might be URLs in some platforms, but we check if they are IDs first
|
||||
if isinstance(tgt_id, str) and tgt_id.isdigit():
|
||||
logger.info(f"Auto-matching: clearing deleted emoji mapping {src_id} -> {tgt_id}")
|
||||
self.engine.state.remove_emoji_mapping(src_id)
|
||||
|
||||
# 2. Match entities
|
||||
try:
|
||||
# Roles
|
||||
|
|
@ -1687,6 +1965,7 @@ class OperationPane(Container):
|
|||
logger.info(f"Auto-matched Sticker: {s.name} -> {target_stickers_map[name_l]}")
|
||||
self.engine.state.set_target_sticker_mapping(s.id, target_stickers_map[name_l])
|
||||
except Exception as e:
|
||||
logger.error(f"Auto-matching error: {e}\n{traceback.format_exc()}")
|
||||
logger.warning(f"Auto-matching error: {e}")
|
||||
|
||||
return {
|
||||
|
|
@ -1951,7 +2230,6 @@ class OperationPane(Container):
|
|||
after_id = verified_id
|
||||
elif choice == "btn_main_menu":
|
||||
modal_prog.dismiss()
|
||||
self.app.switch_screen("config_selection")
|
||||
return
|
||||
|
||||
# If we are here, proceeding either via Start First or Start from ID (after_id)
|
||||
|
|
@ -2062,7 +2340,7 @@ class OperationPane(Container):
|
|||
self.engine.is_running = False
|
||||
await self.engine.close_connections()
|
||||
if choice == "btn_main_menu":
|
||||
self.app.switch_screen("config_selection")
|
||||
pass
|
||||
return
|
||||
|
||||
modal_prog.cancel_callback = lambda: setattr(self.engine, "is_running", False)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue