improve waterfall resume operation

This commit is contained in:
rambros 2026-03-30 03:03:00 +05:30
parent ef2e945477
commit 5b315ab2bf
6 changed files with 52 additions and 19 deletions

View file

@ -97,11 +97,17 @@ class MigrationContext:
} }
# CONSISTENCY: Once target metadata is known, initialize the flat SQLite DB. # CONSISTENCY: Once target metadata is known, initialize the flat SQLite DB.
if results["target_community"] and results["target_community_name"]: if results["target_community"]:
tid = self.config.fluxer_server_id if self.target_platform == "fluxer" else self.config.stoat_server_id tid = self.config.fluxer_server_id if self.target_platform == "fluxer" else self.config.stoat_server_id
# Prefer the original discord community name for the DB file if available (e.g. from live load or backup)
db_name = results.get("discord_server_name")
if not db_name or db_name == "Not Found" or db_name == "Unknown":
db_name = results.get("target_community_name") or "Unknown"
self.ensure_state_initialized( self.ensure_state_initialized(
str(tid or ""), str(tid or ""),
results["target_community_name"] db_name
) )
return results return results
@ -120,6 +126,23 @@ class MigrationContext:
return return
import re import re
import json
# Override the target name explicitly with the original Discord source name if available.
# This fixes naming collisions and UI confusion like "Fluxer-123456.db" instead of "MyServer-123456.db"
try:
if hasattr(self.discord_reader, "guild") and getattr(self.discord_reader, "guild", None):
community_name = getattr(self.discord_reader, "guild").name
elif getattr(self, "source_mode", "live") == "backup" and hasattr(self.discord_reader, "backup_dir"):
b_dir = getattr(self.discord_reader, "backup_dir")
if b_dir and b_dir.exists():
meta_file = b_dir / "metadata.json"
if meta_file.exists():
data = json.loads(meta_file.read_text())
community_name = data.get("name", community_name)
except Exception:
pass
clean_name = re.sub(r'[^\w\s-]', '', community_name).strip() clean_name = re.sub(r'[^\w\s-]', '', community_name).strip()
clean_name = re.sub(r'[-\s]+', '_', clean_name) clean_name = re.sub(r'[-\s]+', '_', clean_name)

View file

@ -560,6 +560,15 @@ class MigrationDatabase:
conn.execute("DELETE FROM thread_tracking WHERE channel_id = ?", (str(channel_id),)) conn.execute("DELETE FROM thread_tracking WHERE channel_id = ?", (str(channel_id),))
conn.commit() conn.commit()
logger.info(f"Cleared all tracking and mapping data for channel: {channel_id}") logger.info(f"Cleared all tracking and mapping data for channel: {channel_id}")
def clear_all_migration_data(self):
"""Purge all mappings and tracking data for ALL channels and threads."""
conn = self._get_conn()
conn.execute("DELETE FROM message_mappings")
conn.execute("DELETE FROM thread_mappings")
conn.execute("DELETE FROM channel_tracking")
conn.execute("DELETE FROM thread_tracking")
conn.commit()
logger.info("Cleared ALL tracking and message mapping data globally.")
def close(self): def close(self):
if hasattr(self._local, "conn"): if hasattr(self._local, "conn"):

View file

@ -238,15 +238,11 @@ class MigrationState:
return self.db.get_global_min_last_message_id(all_mapped_ids) return self.db.get_global_min_last_message_id(all_mapped_ids)
return None return None
def set_waterfall_last_id(self, last_id: str | int):
if self.db:
self.db.set_metadata("waterfall_last_id", str(last_id))
def get_waterfall_last_id(self) -> int | None: def clear_all_migration_data(self):
if self.db: """Clears all message mapping and tracking state globally."""
val = self.db.get_metadata("waterfall_last_id") if self._ensure_db():
return int(val) if val else None self.db.clear_all_migration_data()
return None
def get_all_last_message_ids(self) -> Dict[str, str]: def get_all_last_message_ids(self) -> Dict[str, str]:
"""Returns a combined map of channel_id/thread_id -> last_msg_id.""" """Returns a combined map of channel_id/thread_id -> last_msg_id."""

View file

@ -905,7 +905,8 @@ async def migrate_global_messages(
if fluxer_msg_id: if fluxer_msg_id:
context.state.set_target_message_mapping(target_channel_id, msg.id, fluxer_msg_id) context.state.set_target_message_mapping(target_channel_id, msg.id, fluxer_msg_id)
context.state.update_last_message_id(target_channel_id, msg.id) context.state.update_last_message_id(target_channel_id, msg.id)
context.state.set_waterfall_last_id(msg.id) context.state.update_last_message_timestamp(target_channel_id, str(msg.created_at))
context.state.increment_stats(target_channel_id, messages=1, files=len(files) if files else 0)
stats["attachments"] += len(files) if files else 0 stats["attachments"] += len(files) if files else 0
stats["messages"] += 1 stats["messages"] += 1

View file

@ -896,7 +896,8 @@ async def migrate_global_messages(
if stoat_msg_id: if stoat_msg_id:
context.state.set_target_message_mapping(target_channel_id, msg.id, stoat_msg_id) context.state.set_target_message_mapping(target_channel_id, msg.id, stoat_msg_id)
context.state.update_last_message_id(target_channel_id, msg.id) context.state.update_last_message_id(target_channel_id, msg.id)
context.state.set_waterfall_last_id(msg.id) context.state.update_last_message_timestamp(target_channel_id, str(msg.created_at))
context.state.increment_stats(target_channel_id, messages=1, files=len(files) if files else 0)
stats["attachments"] += len(files) if files else 0 stats["attachments"] += len(files) if files else 0
stats["messages"] += 1 stats["messages"] += 1

View file

@ -1551,9 +1551,7 @@ class OperationPane(Container):
if filtered_tgt_ids: if filtered_tgt_ids:
all_mapped_tgt_ids = filtered_tgt_ids all_mapped_tgt_ids = filtered_tgt_ids
# 2.6 Resume Point: Prioritize Global waterfall tracker, fallback to channel minimums # 2.6 Resume Point: Calculate from global channel minimums
min_last_id = self.engine.state.get_waterfall_last_id()
if min_last_id is None:
min_last_id = self.engine.state.get_global_min_last_message_id(all_mapped_tgt_ids) min_last_id = self.engine.state.get_global_min_last_message_id(all_mapped_tgt_ids)
modal.write(f"\n[bold cyan]Waterfall Migration Resume Point:[/bold cyan]") modal.write(f"\n[bold cyan]Waterfall Migration Resume Point:[/bold cyan]")
@ -1566,7 +1564,7 @@ class OperationPane(Container):
show_continue=min_last_id is not None, show_continue=min_last_id is not None,
show_id=False, show_id=False,
btn_start_label="Start From Beginning", btn_start_label="Start From Beginning",
btn_start_tooltip="Safe, skips duplicates automatically", btn_start_tooltip="Wipes migration progress and restarts from the beginning; may create duplicates",
btn_start_variant="default" if min_last_id is not None else "primary", btn_start_variant="default" if min_last_id is not None else "primary",
btn_continue_label=f"Continue from ID {min_last_id if min_last_id is not None else 0}" if min_last_id is not None else "Continue Migration", btn_continue_label=f"Continue from ID {min_last_id if min_last_id is not None else 0}" if min_last_id is not None else "Continue Migration",
btn_continue_tooltip="Fastest" btn_continue_tooltip="Fastest"
@ -1582,7 +1580,11 @@ class OperationPane(Container):
return return
after_id = None after_id = None
if choice == "btn_continue" and min_last_id is not None: if choice == "btn_start_first":
logger.info("Proceeding with 'Start from Beginning' (global clean sink).")
self.engine.state.clear_all_migration_data()
after_id = None
elif choice == "btn_continue" and min_last_id is not None:
after_id = int(min_last_id) after_id = int(min_last_id)
# Phase 3: Progress # Phase 3: Progress
@ -1599,6 +1601,7 @@ class OperationPane(Container):
tid = self.config.fluxer_server_id tid = self.config.fluxer_server_id
self.engine.ensure_state_initialized(str(tid or ""), platform_name) self.engine.ensure_state_initialized(str(tid or ""), platform_name)
modal.show_stats()
modal.write("Scanning global footprint for totals ...") modal.write("Scanning global footprint for totals ...")
stats_analysis = await migrate_mod.analyze_global_migration(self.engine, after_message_id=after_id) stats_analysis = await migrate_mod.analyze_global_migration(self.engine, after_message_id=after_id)
total_messages = stats_analysis["messages"] total_messages = stats_analysis["messages"]