fix embeds and gif links in backup migration

This commit is contained in:
rambros 2026-03-15 22:45:24 +05:30
parent 7bea5582e9
commit 06091423a2
7 changed files with 342 additions and 158 deletions

View file

@ -606,14 +606,6 @@ class BackupTag:
return f"BackupTag(id={self.id}, name='{self.name}')" return f"BackupTag(id={self.id}, name='{self.name}')"
class BackupMessageReference:
"""Minimal stand-in for discord.MessageReference."""
__slots__ = ("message_id", "channel_id")
def __init__(self, data: dict):
self.message_id = parse_snowflake(data["messageId"])
self.channel_id = parse_snowflake(data["channelId"])
class BackupThread: class BackupThread:
@ -769,17 +761,30 @@ class BackupMessage:
# Reference (replies/forwards) # Reference (replies/forwards)
self.reference = None self.reference = None
if data.get("message_reference"): if data.get("message_reference"):
self.reference = type("Ref", (), {"message_id": parse_snowflake(data["message_reference"]), "channel_id": self.channel_id})() self.reference = BackupMessageReference(
message_id=parse_snowflake(data["message_reference"]),
channel_id=self.channel_id
)
self.thread = None self.thread = None
self.flags = type("Flags", (), {"value": 0, "forwarded": self.type == MessageType.forward})() self.flags = BackupMessageFlags(forwarded=self.type == MessageType.forward)
# snapshots not used in latest refined structure as content is in main message
self.message_snapshots = [] self.message_snapshots = []
def __repr__(self) -> str: def __repr__(self) -> str:
return f"BackupMessage(id={self.id}, author={self.author})" return f"BackupMessage(id={self.id}, author={self.author})"
class BackupMessageReference:
__slots__ = ("message_id", "channel_id")
def __init__(self, message_id: int | None = None, channel_id: int | None = None):
self.message_id = message_id
self.channel_id = channel_id
class BackupMessageFlags:
__slots__ = ("value", "forwarded")
def __init__(self, value: int = 0, forwarded: bool = False):
self.value = value
self.forwarded = forwarded
class BackupEmbed: class BackupEmbed:
"""Minimal stand-in for discord.Embed.""" """Minimal stand-in for discord.Embed."""
@ -793,24 +798,70 @@ class BackupEmbed:
self.color = data.get("color") self.color = data.get("color")
self.timestamp = data.get("timestamp") self.timestamp = data.get("timestamp")
self.thumbnail = type("Thumbnail", (), {"url": data["thumbnail"]["url"]})() if data.get("thumbnail") and "url" in data["thumbnail"] else None self.thumbnail = BackupEmbedThumbnail(data["thumbnail"]["url"]) if data.get("thumbnail") and "url" in data["thumbnail"] else None
self.image = type("Image", (), {"url": data["image"]["url"]})() if data.get("image") and "url" in data["image"] else None self.image = BackupEmbedImage(data["image"]["url"]) if data.get("image") and "url" in data["image"] else None
author = data.get("author") author = data.get("author")
self.author = type("Author", (), { self.author = BackupEmbedAuthor(
"name": author.get("name"), name=author.get("name"),
"url": author.get("url"), url=author.get("url"),
"icon_url": author.get("icon_url") icon_url=author.get("icon_url")
})() if author else None ) if author else None
footer = data.get("footer") footer = data.get("footer")
self.footer = type("Footer", (), { self.footer = BackupEmbedFooter(
"text": footer.get("text"), text=footer.get("text"),
"icon_url": footer.get("icon_url") icon_url=footer.get("icon_url")
})() if footer else None ) if footer else None
self.fields = [BackupEmbedField(f) for f in data.get("fields", [])] self.fields = [BackupEmbedField(f) for f in data.get("fields", [])]
def to_dict(self) -> dict:
result = {
"type": "rich" # Default for bot-sent embeds
}
if self.title: result["title"] = self.title
if self.description: result["description"] = self.description
if self.url: result["url"] = self.url
if self.color: result["color"] = self.color
if self.timestamp: result["timestamp"] = self.timestamp
if self.thumbnail: result["thumbnail"] = {"url": self.thumbnail.url}
if self.image: result["image"] = {"url": self.image.url}
if self.author:
result["author"] = {
"name": self.author.name,
"url": self.author.url,
"icon_url": self.author.icon_url
}
if self.footer:
result["footer"] = {
"text": self.footer.text,
"icon_url": self.footer.icon_url
}
if self.fields:
result["fields"] = [{"name": f.name, "value": f.value, "inline": f.inline} for f in self.fields]
return result
class BackupEmbedThumbnail:
__slots__ = ("url",)
def __init__(self, url: str): self.url = url
class BackupEmbedImage:
__slots__ = ("url",)
def __init__(self, url: str): self.url = url
class BackupEmbedAuthor:
__slots__ = ("name", "url", "icon_url")
def __init__(self, name: str | None = None, url: str | None = None, icon_url: str | None = None):
self.name = name
self.url = url
self.icon_url = icon_url
class BackupEmbedFooter:
__slots__ = ("text", "icon_url")
def __init__(self, text: str | None = None, icon_url: str | None = None):
self.text = text
self.icon_url = icon_url
class BackupEmbedField: class BackupEmbedField:
"""Minimal stand-in for embed fields.""" """Minimal stand-in for embed fields."""
@ -916,7 +967,10 @@ class BackupReader:
MESSAGE_TYPE_REPLY = MessageType.reply MESSAGE_TYPE_REPLY = MessageType.reply
MESSAGE_TYPE_THREAD_STARTER = MessageType.thread_starter_message MESSAGE_TYPE_THREAD_STARTER = MessageType.thread_starter_message
MESSAGE_TYPE_FORWARD = MessageType.forward # Custom Reaper constant MESSAGE_TYPE_FORWARD = MessageType.forward # Custom Reaper constant
MESSAGE_TYPE_CHAT_INPUT_COMMAND = MessageType.chat_input_command
MESSAGE_TYPE_CONTEXT_MENU_COMMAND = MessageType.context_menu_command
MESSAGE_TYPE_POLL_RESULT = MessageType.poll_result
MESSAGE_TYPE_AUTO_MODERATION_ACTION = MessageType.auto_moderation_action
Forbidden = BackupForbidden Forbidden = BackupForbidden
CHANNEL_TYPE_TEXT = ChannelType.text CHANNEL_TYPE_TEXT = ChannelType.text

View file

@ -9,6 +9,11 @@ class DiscordReader:
MESSAGE_TYPE_DEFAULT = discord.MessageType.default MESSAGE_TYPE_DEFAULT = discord.MessageType.default
MESSAGE_TYPE_REPLY = discord.MessageType.reply MESSAGE_TYPE_REPLY = discord.MessageType.reply
MESSAGE_TYPE_THREAD_STARTER = discord.MessageType.thread_starter_message MESSAGE_TYPE_THREAD_STARTER = discord.MessageType.thread_starter_message
MESSAGE_TYPE_CHAT_INPUT_COMMAND = discord.MessageType.chat_input_command
MESSAGE_TYPE_CONTEXT_MENU_COMMAND = discord.MessageType.context_menu_command
MESSAGE_TYPE_FORWARD = getattr(discord.MessageType, 'forward', 100)
MESSAGE_TYPE_POLL_RESULT = getattr(discord.MessageType, 'poll_result', 46)
MESSAGE_TYPE_AUTO_MODERATION_ACTION = getattr(discord.MessageType, 'auto_moderation_action', 24)
# Exceptions # Exceptions
Forbidden = discord.Forbidden Forbidden = discord.Forbidden
@ -295,9 +300,28 @@ class DiscordReader:
"""Downloads a Discord emoji into memory.""" """Downloads a Discord emoji into memory."""
return await emoji.read() return await emoji.read()
@staticmethod
def get_sticker_extension(sticker) -> str:
"""Determines the correct file extension for a sticker."""
fmt = getattr(sticker, 'format', None)
if fmt:
# StickerFormatType: png=1, apng=2, lottie=3, gif=4
val = getattr(fmt, 'value', fmt)
if val == 3: return "json"
if val == 4: return "gif"
if val == 2: return "png" # APNG is often saved as PNG
# Fallback to URL parsing
url = str(getattr(sticker, 'url', ""))
if ".json" in url: return "json"
if ".gif" in url: return "gif"
if ".webp" in url: return "webp"
return "png"
async def download_sticker(self, sticker: Union[discord.GuildSticker, discord.StickerItem]) -> bytes: async def download_sticker(self, sticker: Union[discord.GuildSticker, discord.StickerItem]) -> bytes:
"""Downloads a Discord sticker into memory.""" """Downloads a Discord sticker into memory."""
logger.debug(f"Attempting to download sticker: {getattr(sticker, 'name', 'unknown')} (type: {type(sticker)})") name = getattr(sticker, 'name', 'unknown')
logger.debug(f"Attempting to download sticker: {name} (ID: {sticker.id}, type: {type(sticker)})")
# 1. Try directly reading # 1. Try directly reading
if hasattr(sticker, 'read'): if hasattr(sticker, 'read'):
@ -316,22 +340,36 @@ class DiscordReader:
except Exception as e: except Exception as e:
logger.debug(f"to_sticker fallback failed: {e}") logger.debug(f"to_sticker fallback failed: {e}")
# 3. Try downloading from URL as last resort # 3. Try download via reader's session (Robust fallback)
url = getattr(sticker, 'url', None) url = getattr(sticker, 'url', None)
if url: if url:
try: try:
# Use the internal session from discord.py if possible (it has proper headers/auth)
session = None
if self.client and hasattr(self.client, 'http') and hasattr(self.client.http, '_HTTPClient__session'):
session = self.client.http._HTTPClient__session
if session:
logger.debug(f"Attempting download for sticker '{name}' using bot's session from {url}")
async with session.get(str(url)) as resp:
if resp.status == 200:
data = await resp.read()
if data:
logger.debug(f"Successfully downloaded sticker '{name}' (size: {len(data)})")
return data
else:
# Generic fallback session
import aiohttp import aiohttp
logger.debug(f"Attempting URL download for sticker from {url}") logger.debug(f"Attempting download for sticker '{name}' using generic session from {url}")
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.get(str(url)) as resp: async with session.get(str(url)) as resp:
if resp.status == 200: if resp.status == 200:
return await resp.read() data = await resp.read()
else: if data: return data
logger.debug(f"URL download failed with status {resp.status}")
except Exception as e: except Exception as e:
logger.debug(f"URL download failed for sticker: {e}") logger.debug(f"URL download failed for sticker '{name}': {e}")
logger.warning(f"Failed to download sticker {getattr(sticker, 'name', 'unknown')} after all attempts") logger.warning(f"Failed to download sticker {name} ({sticker.id}) after all attempts")
return b"" return b""
async def download_attachment(self, attachment: discord.Attachment) -> bytes: async def download_attachment(self, attachment: discord.Attachment) -> bytes:

View file

@ -21,6 +21,7 @@ class DiscordExporter:
self.base_dir = Path(base_dir) if base_dir else Path(".") self.base_dir = Path(base_dir) if base_dir else Path(".")
self.is_running = True self.is_running = True
self.db: Optional[BackupDatabase] = None self.db: Optional[BackupDatabase] = None
self.sticker_cache: Dict[int, bytes] = {} # Deduplicate downloads in one session
async def setup(self): async def setup(self):
"""Prepares the output directory and fetches server metadata.""" """Prepares the output directory and fetches server metadata."""
@ -177,19 +178,16 @@ class DiscordExporter:
sticker_data = [] sticker_data = []
logger.info(f"Exporting {len(stickers)} stickers...") logger.info(f"Exporting {len(stickers)} stickers...")
for s in stickers: for s in stickers:
ext = "png" ext = self.reader.get_sticker_extension(s)
if s.url:
if ".json" in str(s.url): ext = "json"
elif ".gif" in str(s.url): ext = "gif"
elif ".webp" in str(s.url): ext = "webp"
filename = f"sticker_{s.id}.{ext}" filename = f"sticker_{s.id}.{ext}"
sticker_path = self.assets_path / filename sticker_path = self.assets_path / filename
try: try:
if not sticker_path.exists(): if not sticker_path.exists():
data = await self.reader.download_sticker(s) data = await self.reader.download_sticker(s)
if data:
with open(sticker_path, "wb") as f: with open(sticker_path, "wb") as f:
f.write(data) f.write(data)
mime_type = "image/png" mime_type = "image/png"
if ext == "json": mime_type = "application/json" if ext == "json": mime_type = "application/json"
elif ext == "gif": mime_type = "image/gif" elif ext == "gif": mime_type = "image/gif"
@ -197,14 +195,14 @@ class DiscordExporter:
sticker_data.append({ sticker_data.append({
"id": str(s.id), "id": str(s.id),
"name": s.name, "name": getattr(s, "name", "unknown"),
"type": "sticker", "type": "sticker",
"filename": filename, "filename": filename,
"url": str(s.url), "url": str(s.url),
"mime_type": mime_type "mime_type": mime_type
}) })
except Exception as ex: except Exception as ex:
logger.error(f"Failed to download sticker {s.name}: {ex}") logger.error(f"Failed to download sticker {getattr(s, 'name', 'unknown')}: {ex}")
# Save to database # Save to database
if self.db: if self.db:
@ -482,30 +480,29 @@ class DiscordExporter:
stickers = [] stickers = []
if msg.stickers: if msg.stickers:
for st in msg.stickers: for st in msg.stickers:
# Determine extension based on format # Deduplicate downloads for the same sticker in one session
ext = ".png" if st.id in self.sticker_cache:
if hasattr(st, "format"): st_bytes = self.sticker_cache[st.id]
try: else:
from discord import StickerFormatType st_bytes = await self.reader.download_sticker(st)
if st.format == StickerFormatType.lottie: if st_bytes:
ext = ".json" self.sticker_cache[st.id] = st_bytes
elif st.format == StickerFormatType.apng:
ext = ".png"
elif st.format == StickerFormatType.gif:
ext = ".gif"
except ImportError:
pass
if st_bytes:
ext = self.reader.get_sticker_extension(st)
st_data = await self._process_media( st_data = await self._process_media(
media_id=st.id, media_id=st.id,
url=st.url, url=st.url,
filename=f"{st.name}{ext}", filename=f"{st.name}.{ext}",
content_type=f"image/{ext[1:]}" if ext != ".json" else "application/json", content_type=f"image/{ext}" if ext != "json" else "application/json",
save_method=st.save data=st_bytes
) )
if st_data: if st_data:
st_data["name"] = st.name
st_data["format_type"] = int(st.format.value) if hasattr(st, "format") and hasattr(st.format, "value") else 1 st_data["format_type"] = int(st.format.value) if hasattr(st, "format") and hasattr(st.format, "value") else 1
stickers.append(st_data) stickers.append(st_data)
else:
logger.warning(f"Could not download message sticker {st.id} in message {msg.id}")
# 3. Embeds # 3. Embeds
embeds = [] embeds = []
@ -576,7 +573,7 @@ class DiscordExporter:
return m_data, user_data return m_data, user_data
async def _process_media(self, media_id, url, filename, size=None, content_type=None, save_method=None): async def _process_media(self, media_id, url, filename, size=None, content_type=None, save_method=None, data=None):
"""Downloads and deduplicates any media (attachment or sticker) using SHA-256 (CAS).""" """Downloads and deduplicates any media (attachment or sticker) using SHA-256 (CAS)."""
# 1. First check by URL in DB # 1. First check by URL in DB
if self.db: if self.db:
@ -594,14 +591,23 @@ class DiscordExporter:
# 2. Temporary download to calculate hash # 2. Temporary download to calculate hash
import tempfile import tempfile
import shutil import shutil
tmp_path = None
try:
with tempfile.NamedTemporaryFile(delete=False) as tmp: with tempfile.NamedTemporaryFile(delete=False) as tmp:
tmp_path = Path(tmp.name) tmp_path = Path(tmp.name)
try: if data:
if save_method: tmp.write(data)
elif save_method:
# Closing handle before save_method just in case it needs to open it's own handle
tmp.close()
await save_method(tmp_path) await save_method(tmp_path)
else: else:
return None return None
# Ensure it's closed before hashing
try: tmp.close()
except: pass
file_hash = self._calculate_sha256(tmp_path) file_hash = self._calculate_sha256(tmp_path)
actual_size = tmp_path.stat().st_size actual_size = tmp_path.stat().st_size
@ -639,7 +645,7 @@ class DiscordExporter:
} }
except Exception as e: except Exception as e:
logger.error(f"Failed to process media {filename}: {e}") logger.error(f"Failed to process media {filename}: {e}")
if tmp_path.exists(): tmp_path.unlink() if tmp_path and tmp_path.exists(): tmp_path.unlink()
return None return None
async def export_threads(self, channel_id: int, progress_callback=None, force=False, accumulated_count=0, accumulated_threads=0, accumulated_files=0, after_id: int | None = None): async def export_threads(self, channel_id: int, progress_callback=None, force=False, accumulated_count=0, accumulated_threads=0, accumulated_files=0, after_id: int | None = None):

View file

@ -58,4 +58,8 @@ def resolve_discord_links(content: str, state: MigrationState, platform: str, ta
# Fallback for unmigrated channel # Fallback for unmigrated channel
return f"[`discord-channel`](<{full_url}>)" return f"[`discord-channel`](<{full_url}>)"
return discord_link_re.sub(replace_link, content) logger.debug(f"resolve_discord_links: Processing content (len {len(content)}): {content[:100]!r}")
result = discord_link_re.sub(replace_link, content)
if result != content:
logger.debug(f"resolve_discord_links: Content resolved to (len {len(result)}): {result[:100]!r}")
return result

View file

@ -18,6 +18,8 @@ from src.core.utils import resolve_discord_links
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, emoji_map=None, channel_map=None, state=None, target_server_id=None) -> str: def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, emoji_map=None, channel_map=None, state=None, target_server_id=None) -> str:
if content is None:
return ""
if not content or not guild: if not content or not guild:
return content return content
@ -159,12 +161,18 @@ async def analyze_migration(context: MigrationContext, source_channel_id: int, a
context.discord_reader.MESSAGE_TYPE_DEFAULT, context.discord_reader.MESSAGE_TYPE_DEFAULT,
context.discord_reader.MESSAGE_TYPE_REPLY, context.discord_reader.MESSAGE_TYPE_REPLY,
context.discord_reader.MESSAGE_TYPE_THREAD_STARTER, context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
context.discord_reader.MESSAGE_TYPE_FORWARD context.discord_reader.MESSAGE_TYPE_FORWARD,
context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
]: ]:
logger.debug(f"Skipping message {msg.id} in analyze: type={msg.type} (not an allowed type)")
continue continue
stats["messages"] += 1 stats["messages"] += 1
stats["attachments"] += len(msg.attachments) stats["attachments"] += len(msg.attachments)
logger.debug(f"Analyze msg {msg.id}: type={msg.type}, content={msg.content[:50]!r}...")
if progress_callback and stats["messages"] % 10 == 0: if progress_callback and stats["messages"] % 10 == 0:
await progress_callback(stats) await progress_callback(stats)
@ -225,11 +233,16 @@ async def migrate_messages(
# Skip system messages like "pinned a message", etc. # Skip system messages like "pinned a message", etc.
logger.debug(f"Analyzing message {msg.id}: type={msg.type}, content_len={len(msg.content) if msg.content else 0}, attachments={len(msg.attachments)}, embeds={len(msg.embeds)}")
if msg.type not in [ if msg.type not in [
context.discord_reader.MESSAGE_TYPE_DEFAULT, context.discord_reader.MESSAGE_TYPE_DEFAULT,
context.discord_reader.MESSAGE_TYPE_REPLY, context.discord_reader.MESSAGE_TYPE_REPLY,
context.discord_reader.MESSAGE_TYPE_THREAD_STARTER, context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
context.discord_reader.MESSAGE_TYPE_FORWARD context.discord_reader.MESSAGE_TYPE_FORWARD,
context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
]: ]:
# If we are skipping the parent, we STILL need to check for a thread! # If we are skipping the parent, we STILL need to check for a thread!
if hasattr(msg, 'thread') and msg.thread: if hasattr(msg, 'thread') and msg.thread:
@ -263,7 +276,6 @@ async def migrate_messages(
await progress_callback(stats) await progress_callback(stats)
continue continue
else: else:
# Use custom clean_mentions with msg mentions for accuracy
# Use custom clean_mentions with msg mentions for accuracy # Use custom clean_mentions with msg mentions for accuracy
content = clean_mentions( content = clean_mentions(
msg.content, msg.content,
@ -275,6 +287,7 @@ async def migrate_messages(
state=context.state, state=context.state,
target_server_id=context.fluxer_writer.community_id target_server_id=context.fluxer_writer.community_id
) )
logger.debug(f"Message {msg.id} cleaned content length: {len(content) if content else 0}")
# Process attachments # Process attachments
files = [] files = []
@ -343,21 +356,31 @@ async def migrate_messages(
if ext == 'lottie': if ext == 'lottie':
if HAS_LOTTIE: if HAS_LOTTIE:
try: try:
logger.debug(f"Converting Lottie sticker {s.name} to WebP...") logger.debug(f"Converting Lottie sticker {s.name} (ID: {s.id}) to WebP...")
lottie_data = json.loads(sticker_data) lottie_data = json.loads(sticker_data)
animation = Animation.load(lottie_data)
gif_buf = io.BytesIO() def _convert_lottie(data):
export_gif(animation, gif_buf) anim = Animation.load(data)
gif_buf.seek(0) buf = io.BytesIO()
export_gif(anim, buf)
buf.seek(0)
return buf
gif_buf = await asyncio.to_thread(_convert_lottie, lottie_data)
# GIF → WebP via Pillow # GIF → WebP via Pillow
from PIL import Image from PIL import Image
img = Image.open(gif_buf)
webp_buf = io.BytesIO() def _convert_gif_to_webp(buf):
img = Image.open(buf)
w_buf = io.BytesIO()
if getattr(img, 'n_frames', 1) > 1: if getattr(img, 'n_frames', 1) > 1:
img.save(webp_buf, format='WEBP', save_all=True, loop=0) img.save(w_buf, format='WEBP', save_all=True, loop=0, quality=80)
else: else:
img.save(webp_buf, format='WEBP') img.save(w_buf, format='WEBP', quality=80)
sticker_data = webp_buf.getvalue() return w_buf.getvalue()
sticker_data = await asyncio.to_thread(_convert_gif_to_webp, gif_buf)
ext = 'webp' ext = 'webp'
logger.debug(f"Successfully converted Lottie sticker {s.name} to WebP") logger.debug(f"Successfully converted Lottie sticker {s.name} to WebP")
except Exception as conv_err: except Exception as conv_err:
@ -367,18 +390,21 @@ async def migrate_messages(
logger.warning(f"Lottie library not available, sending sticker {s.name} as raw JSON") logger.warning(f"Lottie library not available, sending sticker {s.name} as raw JSON")
ext = 'json' ext = 'json'
# APNG / GIF → WebP (via Pillow)
elif ext in ('apng', 'gif'): elif ext in ('apng', 'gif'):
try: try:
logger.debug(f"Converting {ext.upper()} sticker {s.name} to WebP...") logger.debug(f"Converting {ext.upper()} sticker {s.name} (ID: {s.id}) to WebP...")
from PIL import Image from PIL import Image
img = Image.open(io.BytesIO(sticker_data))
def _process_animated_sticker(data):
img = Image.open(io.BytesIO(data))
webp_buf = io.BytesIO() webp_buf = io.BytesIO()
if getattr(img, 'n_frames', 1) > 1: if getattr(img, 'n_frames', 1) > 1:
img.save(webp_buf, format='WEBP', save_all=True, loop=0) img.save(webp_buf, format='WEBP', save_all=True, loop=0, quality=80)
else: else:
img.save(webp_buf, format='WEBP') img.save(webp_buf, format='WEBP', quality=80)
sticker_data = webp_buf.getvalue() return webp_buf.getvalue()
sticker_data = await asyncio.to_thread(_process_animated_sticker, sticker_data)
ext = 'webp' ext = 'webp'
logger.debug(f"Successfully converted sticker {s.name} to WebP") logger.debug(f"Successfully converted sticker {s.name} to WebP")
except Exception as conv_err: except Exception as conv_err:
@ -386,9 +412,10 @@ async def migrate_messages(
# Keep original format as fallback # Keep original format as fallback
filename = f"sticker_{s.name}_{s.id}.{ext}" filename = f"sticker_{s.name}_{s.id}.{ext}"
sticker_size = len(sticker_data)
files.append({"filename": filename, "data": sticker_data}) files.append({"filename": filename, "data": sticker_data})
stats["attachments"] += 1 stats["attachments"] += 1
logger.debug(f"Added sticker {s.name} as attachment (extension: {ext})") logger.debug(f"Added sticker {s.name} as attachment (extension: {ext}, size: {sticker_size} bytes)")
except Exception as e: except Exception as e:
logger.error(f"Failed to download sticker {getattr(s, 'name', 'unknown')}: {e}") logger.error(f"Failed to download sticker {getattr(s, 'name', 'unknown')}: {e}")
@ -414,6 +441,7 @@ async def migrate_messages(
if avatar_url and not avatar_url.startswith("http"): if avatar_url and not avatar_url.startswith("http"):
avatar_url = None avatar_url = None
logger.debug(f"Fluxer: Calling send_message for Discord ID {msg.id}")
fluxer_msg_id = await context.fluxer_writer.send_message( fluxer_msg_id = await context.fluxer_writer.send_message(
channel_id=target_channel_id, channel_id=target_channel_id,
author_name=msg.author.display_name, author_name=msg.author.display_name,
@ -431,6 +459,8 @@ async def migrate_messages(
context.state.set_thread_message_mapping(target_channel_id, thread_id, str(msg.id), fluxer_msg_id) context.state.set_thread_message_mapping(target_channel_id, thread_id, str(msg.id), fluxer_msg_id)
else: else:
context.state.set_message_mapping(target_channel_id, str(msg.id), fluxer_msg_id) context.state.set_message_mapping(target_channel_id, str(msg.id), fluxer_msg_id)
else:
logger.warning(f"Fluxer: send_message returned None for Discord ID {msg.id} (message might have been skipped or timed out)")
if thread_id: if thread_id:
context.state.update_thread_last_message_timestamp(target_channel_id, thread_id, str(msg.created_at)) context.state.update_thread_last_message_timestamp(target_channel_id, thread_id, str(msg.created_at))
@ -481,6 +511,7 @@ async def migrate_messages(
if progress_callback: if progress_callback:
await progress_callback(stats) await progress_callback(stats)
logger.debug(f"Fluxer: Finished processing message Discord ID {msg.id}")
except Exception as e: except Exception as e:
logger.error(f"Failed to process message {msg.id}: {e}") logger.error(f"Failed to process message {msg.id}: {e}")
import traceback import traceback

View file

@ -265,16 +265,21 @@ class FluxerWriter:
Returns the ID of the sent message if available. Returns the ID of the sent message if available.
""" """
assert self.client is not None assert self.client is not None
logger.debug(f"Fluxer: send_message called for channel {channel_id}, author='{author_name}', content_len={len(content) if content else 0}, files={len(files) if files else 0}, is_forwarded={is_forwarded}")
# Ensure we are ready before sending (wait a bit if needed) # Ensure we are ready before sending (wait a bit if needed)
if not self._ready_event.is_set(): if not self._ready_event.is_set():
logger.debug(f"Fluxer: Bot not ready, waiting...")
try: try:
await asyncio.wait_for(self._ready_event.wait(), timeout=5.0) await asyncio.wait_for(self._ready_event.wait(), timeout=5.0)
except asyncio.TimeoutError: except asyncio.TimeoutError:
logger.warning(f"Fluxer: Timeout waiting for bot readiness.")
pass pass
# Use webhook for avatar/username spoofing # Use webhook for avatar/username spoofing
logger.debug(f"Fluxer: Resolving webhook for channel {channel_id}...")
webhook = await self._get_or_create_webhook(channel_id) webhook = await self._get_or_create_webhook(channel_id)
logger.debug(f"Fluxer: Webhook resolved: {webhook.id if webhook else 'None'}")
# Prepare content with subtext timestamp # Prepare content with subtext timestamp
# -# is Fluxer/Discord's subtext markdown: small, muted grey text # -# is Fluxer/Discord's subtext markdown: small, muted grey text
@ -287,6 +292,7 @@ class FluxerWriter:
display_content = f">>> {content}" display_content = f">>> {content}"
final_content = prefix + display_content if display_content else prefix final_content = prefix + display_content if display_content else prefix
logger.debug(f"Fluxer: Prepared final_content (len {len(final_content)}): {final_content!r}")
# Convert files to fluxer.File objects # Convert files to fluxer.File objects
fluxer_files = None fluxer_files = None
@ -298,24 +304,42 @@ class FluxerWriter:
if embeds: if embeds:
normalized_embeds = [] normalized_embeds = []
for e in embeds: for e in embeds:
if hasattr(e, "to_dict"): d = e.to_dict() if hasattr(e, "to_dict") else e
normalized_embeds.append(e.to_dict()) if not isinstance(d, dict):
else: continue
normalized_embeds.append(e)
# Heuristic: Skip redundant link previews to avoid "Invalid Embed" errors or duplication.
# If an embed has a URL that is already in the message content, and no complex fields, skip it.
if content and d.get("url") and str(d.get("url")) in content:
if not d.get("fields") and not d.get("description") and not d.get("title"):
logger.debug(f"Fluxer: Skipping redundant link preview embed for {d.get('url')}")
continue
normalized_embeds.append(d)
if not normalized_embeds: normalized_embeds = None
try: try:
# Current limitation: fluxer.py execute_webhook doesn't support 'message_reference' yet. # Current limitation: fluxer.py execute_webhook doesn't support 'message_reference' yet.
# So if we have a reply, we MUST use the bot's direct send method. # So if we have a reply, we MUST use the bot's direct send method.
if webhook and not reply_to_message_id: if webhook and not reply_to_message_id:
msg = await webhook.send( logger.debug(f"Fluxer: Sending message via webhook {webhook.id} for user '{author_name}'")
try:
msg = await asyncio.wait_for(
webhook.send(
content=final_content, content=final_content,
username=f"{author_name} (discord)", username=f"{author_name} (discord)",
avatar_url=author_avatar_url, avatar_url=author_avatar_url,
files=fluxer_files, files=fluxer_files,
embeds=normalized_embeds, embeds=normalized_embeds,
wait=True wait=True
),
timeout=45.0 # Increased timeout for potential large file uploads
) )
logger.debug(f"Fluxer: Webhook send complete, msg_id={msg.id if msg else 'None'}")
return str(msg.id) if msg else None return str(msg.id) if msg else None
except asyncio.TimeoutError:
logger.error(f"Fluxer: Webhook send timed out after 45s for channel {channel_id}")
return None
else: else:
# Use bot direct message (supports files and message_reference) # Use bot direct message (supports files and message_reference)
# We add the author name to the prefix since bot name won't match # We add the author name to the prefix since bot name won't match
@ -330,19 +354,28 @@ class FluxerWriter:
if reply_to_message_id: if reply_to_message_id:
message_reference = {"message_id": str(reply_to_message_id), "channel_id": str(channel_id)} message_reference = {"message_id": str(reply_to_message_id), "channel_id": str(channel_id)}
msg_data = await self.client.send_message( logger.debug(f"Fluxer: Sending message via bot for user '{author_name}'")
try:
msg_data = await asyncio.wait_for(
self.client.send_message(
channel_id=channel_id, channel_id=channel_id,
content=final_bot_content, content=final_bot_content,
files=fluxer_files, files=fluxer_files,
embeds=normalized_embeds, embeds=normalized_embeds,
message_reference=message_reference message_reference=message_reference
),
timeout=45.0
) )
logger.debug(f"Fluxer: Bot send complete, msg_id={msg_data.get('id') if msg_data else 'None'}")
return str(msg_data["id"]) if msg_data else None return str(msg_data["id"]) if msg_data else None
except asyncio.TimeoutError:
logger.error(f"Fluxer: Bot send timed out after 45s for channel {channel_id}")
return None
except Exception as e: except Exception as e:
err_msg = f"Failed to copy message: {e}" err_msg = f"Failed to copy message to Fluxer: {e}"
if hasattr(e, 'errors') and e.errors: if hasattr(e, 'errors') and e.errors:
err_msg += f" - Details: {e.errors}" err_msg += f" - Details: {e.errors}"
print(err_msg) logger.error(err_msg)
return None return None
async def send_marker(self, channel_id: str, content: str, files: list[dict] | None = None, reply_to_message_id: Optional[str] = None) -> Optional[str]: async def send_marker(self, channel_id: str, content: str, files: list[dict] | None = None, reply_to_message_id: Optional[str] = None) -> Optional[str]:

View file

@ -18,6 +18,8 @@ from src.core.utils import resolve_discord_links
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, emoji_map=None, channel_map=None, state=None, target_server_id=None) -> str: def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, emoji_map=None, channel_map=None, state=None, target_server_id=None) -> str:
if content is None:
return ""
if not content or not guild: if not content or not guild:
return content return content
@ -164,8 +166,13 @@ async def analyze_migration(context: MigrationContext, source_channel_id: int, a
context.discord_reader.MESSAGE_TYPE_DEFAULT, context.discord_reader.MESSAGE_TYPE_DEFAULT,
context.discord_reader.MESSAGE_TYPE_REPLY, context.discord_reader.MESSAGE_TYPE_REPLY,
context.discord_reader.MESSAGE_TYPE_THREAD_STARTER, context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
context.discord_reader.MESSAGE_TYPE_FORWARD context.discord_reader.MESSAGE_TYPE_FORWARD,
context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
]: ]:
logger.debug(f"Skipping message {msg.id} in analyze: type={msg.type} (not an allowed type)")
continue continue
stats["messages"] += 1 stats["messages"] += 1
@ -230,11 +237,16 @@ async def migrate_messages(
# Skip system messages like "pinned a message", etc. # Skip system messages like "pinned a message", etc.
content = "" # Initialize content content = "" # Initialize content
logger.debug(f"Analyzing message {msg.id}: type={msg.type}, content_len={len(msg.content) if msg.content else 0}, attachments={len(msg.attachments)}, embeds={len(msg.embeds)}")
if msg.type not in [ if msg.type not in [
context.discord_reader.MESSAGE_TYPE_DEFAULT, context.discord_reader.MESSAGE_TYPE_DEFAULT,
context.discord_reader.MESSAGE_TYPE_REPLY, context.discord_reader.MESSAGE_TYPE_REPLY,
context.discord_reader.MESSAGE_TYPE_THREAD_STARTER, context.discord_reader.MESSAGE_TYPE_THREAD_STARTER,
context.discord_reader.MESSAGE_TYPE_FORWARD context.discord_reader.MESSAGE_TYPE_FORWARD,
context.discord_reader.MESSAGE_TYPE_CHAT_INPUT_COMMAND,
context.discord_reader.MESSAGE_TYPE_CONTEXT_MENU_COMMAND,
context.discord_reader.MESSAGE_TYPE_POLL_RESULT,
context.discord_reader.MESSAGE_TYPE_AUTO_MODERATION_ACTION
]: ]:
# If we are skipping the parent, we STILL need to check for a thread! # If we are skipping the parent, we STILL need to check for a thread!
if hasattr(msg, 'thread') and msg.thread: if hasattr(msg, 'thread') and msg.thread:
@ -280,6 +292,7 @@ async def migrate_messages(
state=context.state, state=context.state,
target_server_id=context.stoat_writer.community_id target_server_id=context.stoat_writer.community_id
) )
logger.debug(f"Message {msg.id} cleaned content length: {len(content) if content else 0}")
# Process attachments # Process attachments
files = [] files = []
@ -345,10 +358,14 @@ async def migrate_messages(
try: try:
logger.debug(f"Converting Lottie sticker {s.name} to GIF...") logger.debug(f"Converting Lottie sticker {s.name} to GIF...")
lottie_data = json.loads(sticker_data) lottie_data = json.loads(sticker_data)
animation = Animation.load(lottie_data)
def _convert_lottie_to_gif(data):
animation = Animation.load(data)
output = io.BytesIO() output = io.BytesIO()
export_gif(animation, output) export_gif(animation, output)
sticker_data = output.getvalue() return output.getvalue()
sticker_data = await asyncio.to_thread(_convert_lottie_to_gif, lottie_data)
ext = 'gif' ext = 'gif'
logger.debug(f"Successfully converted Lottie sticker {s.name} to GIF") logger.debug(f"Successfully converted Lottie sticker {s.name} to GIF")
except Exception as conv_err: except Exception as conv_err:
@ -361,24 +378,23 @@ async def migrate_messages(
# APNG → GIF (via Pillow, with proper frame disposal) # APNG → GIF (via Pillow, with proper frame disposal)
elif ext == 'apng': elif ext == 'apng':
try: try:
logger.debug(f"Converting APNG sticker {s.name} to GIF...") logger.debug(f"Converting APNG sticker {s.name} (ID: {s.id}) to GIF for Stoat...")
from PIL import Image from PIL import Image
img = Image.open(io.BytesIO(sticker_data))
def _convert_apng_to_gif(data):
img = Image.open(io.BytesIO(data))
gif_buf = io.BytesIO() gif_buf = io.BytesIO()
if getattr(img, 'n_frames', 1) > 1: if getattr(img, 'n_frames', 1) > 1:
# Extract each frame onto a clean background to avoid overlap
frames = [] frames = []
durations = [] durations = []
for frame_idx in range(img.n_frames): # Create a RGBA canvas for disposal handling
img.seek(frame_idx) canvas = Image.new('RGBA', img.size, (0,0,0,0))
# Create fresh background and composite the frame for i in range(img.n_frames):
canvas = Image.new('RGBA', img.size, (0, 0, 0, 0)) img.seek(i)
frame = img.convert('RGBA') frame = img.convert('RGBA')
canvas.paste(frame, (0, 0), frame) canvas.paste(frame, (0, 0), frame)
# Convert to palette mode for GIF
frames.append(canvas.convert('RGBA')) frames.append(canvas.convert('RGBA'))
durations.append(img.info.get('duration', 100)) durations.append(img.info.get('duration', 100))
# Save with disposal=2 (clear frame before next)
frames[0].save( frames[0].save(
gif_buf, format='GIF', save_all=True, gif_buf, format='GIF', save_all=True,
append_images=frames[1:], loop=0, append_images=frames[1:], loop=0,
@ -386,7 +402,9 @@ async def migrate_messages(
) )
else: else:
img.save(gif_buf, format='GIF') img.save(gif_buf, format='GIF')
sticker_data = gif_buf.getvalue() return gif_buf.getvalue()
sticker_data = await asyncio.to_thread(_convert_apng_to_gif, sticker_data)
ext = 'gif' ext = 'gif'
logger.debug(f"Successfully converted APNG sticker {s.name} to GIF") logger.debug(f"Successfully converted APNG sticker {s.name} to GIF")
except Exception as conv_err: except Exception as conv_err: