improve waterfall resume operation
This commit is contained in:
parent
ef2e945477
commit
5b315ab2bf
6 changed files with 52 additions and 19 deletions
|
|
@ -97,11 +97,17 @@ class MigrationContext:
|
||||||
}
|
}
|
||||||
|
|
||||||
# CONSISTENCY: Once target metadata is known, initialize the flat SQLite DB.
|
# CONSISTENCY: Once target metadata is known, initialize the flat SQLite DB.
|
||||||
if results["target_community"] and results["target_community_name"]:
|
if results["target_community"]:
|
||||||
tid = self.config.fluxer_server_id if self.target_platform == "fluxer" else self.config.stoat_server_id
|
tid = self.config.fluxer_server_id if self.target_platform == "fluxer" else self.config.stoat_server_id
|
||||||
|
|
||||||
|
# Prefer the original discord community name for the DB file if available (e.g. from live load or backup)
|
||||||
|
db_name = results.get("discord_server_name")
|
||||||
|
if not db_name or db_name == "Not Found" or db_name == "Unknown":
|
||||||
|
db_name = results.get("target_community_name") or "Unknown"
|
||||||
|
|
||||||
self.ensure_state_initialized(
|
self.ensure_state_initialized(
|
||||||
str(tid or ""),
|
str(tid or ""),
|
||||||
results["target_community_name"]
|
db_name
|
||||||
)
|
)
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
@ -120,6 +126,23 @@ class MigrationContext:
|
||||||
return
|
return
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
import json
|
||||||
|
|
||||||
|
# Override the target name explicitly with the original Discord source name if available.
|
||||||
|
# This fixes naming collisions and UI confusion like "Fluxer-123456.db" instead of "MyServer-123456.db"
|
||||||
|
try:
|
||||||
|
if hasattr(self.discord_reader, "guild") and getattr(self.discord_reader, "guild", None):
|
||||||
|
community_name = getattr(self.discord_reader, "guild").name
|
||||||
|
elif getattr(self, "source_mode", "live") == "backup" and hasattr(self.discord_reader, "backup_dir"):
|
||||||
|
b_dir = getattr(self.discord_reader, "backup_dir")
|
||||||
|
if b_dir and b_dir.exists():
|
||||||
|
meta_file = b_dir / "metadata.json"
|
||||||
|
if meta_file.exists():
|
||||||
|
data = json.loads(meta_file.read_text())
|
||||||
|
community_name = data.get("name", community_name)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
clean_name = re.sub(r'[^\w\s-]', '', community_name).strip()
|
clean_name = re.sub(r'[^\w\s-]', '', community_name).strip()
|
||||||
clean_name = re.sub(r'[-\s]+', '_', clean_name)
|
clean_name = re.sub(r'[-\s]+', '_', clean_name)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -560,6 +560,15 @@ class MigrationDatabase:
|
||||||
conn.execute("DELETE FROM thread_tracking WHERE channel_id = ?", (str(channel_id),))
|
conn.execute("DELETE FROM thread_tracking WHERE channel_id = ?", (str(channel_id),))
|
||||||
conn.commit()
|
conn.commit()
|
||||||
logger.info(f"Cleared all tracking and mapping data for channel: {channel_id}")
|
logger.info(f"Cleared all tracking and mapping data for channel: {channel_id}")
|
||||||
|
def clear_all_migration_data(self):
|
||||||
|
"""Purge all mappings and tracking data for ALL channels and threads."""
|
||||||
|
conn = self._get_conn()
|
||||||
|
conn.execute("DELETE FROM message_mappings")
|
||||||
|
conn.execute("DELETE FROM thread_mappings")
|
||||||
|
conn.execute("DELETE FROM channel_tracking")
|
||||||
|
conn.execute("DELETE FROM thread_tracking")
|
||||||
|
conn.commit()
|
||||||
|
logger.info("Cleared ALL tracking and message mapping data globally.")
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
if hasattr(self._local, "conn"):
|
if hasattr(self._local, "conn"):
|
||||||
|
|
|
||||||
|
|
@ -238,16 +238,12 @@ class MigrationState:
|
||||||
return self.db.get_global_min_last_message_id(all_mapped_ids)
|
return self.db.get_global_min_last_message_id(all_mapped_ids)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def set_waterfall_last_id(self, last_id: str | int):
|
|
||||||
if self.db:
|
def clear_all_migration_data(self):
|
||||||
self.db.set_metadata("waterfall_last_id", str(last_id))
|
"""Clears all message mapping and tracking state globally."""
|
||||||
|
if self._ensure_db():
|
||||||
|
self.db.clear_all_migration_data()
|
||||||
|
|
||||||
def get_waterfall_last_id(self) -> int | None:
|
|
||||||
if self.db:
|
|
||||||
val = self.db.get_metadata("waterfall_last_id")
|
|
||||||
return int(val) if val else None
|
|
||||||
return None
|
|
||||||
|
|
||||||
def get_all_last_message_ids(self) -> Dict[str, str]:
|
def get_all_last_message_ids(self) -> Dict[str, str]:
|
||||||
"""Returns a combined map of channel_id/thread_id -> last_msg_id."""
|
"""Returns a combined map of channel_id/thread_id -> last_msg_id."""
|
||||||
if self._ensure_db():
|
if self._ensure_db():
|
||||||
|
|
|
||||||
|
|
@ -905,7 +905,8 @@ async def migrate_global_messages(
|
||||||
if fluxer_msg_id:
|
if fluxer_msg_id:
|
||||||
context.state.set_target_message_mapping(target_channel_id, msg.id, fluxer_msg_id)
|
context.state.set_target_message_mapping(target_channel_id, msg.id, fluxer_msg_id)
|
||||||
context.state.update_last_message_id(target_channel_id, msg.id)
|
context.state.update_last_message_id(target_channel_id, msg.id)
|
||||||
context.state.set_waterfall_last_id(msg.id)
|
context.state.update_last_message_timestamp(target_channel_id, str(msg.created_at))
|
||||||
|
context.state.increment_stats(target_channel_id, messages=1, files=len(files) if files else 0)
|
||||||
stats["attachments"] += len(files) if files else 0
|
stats["attachments"] += len(files) if files else 0
|
||||||
|
|
||||||
stats["messages"] += 1
|
stats["messages"] += 1
|
||||||
|
|
|
||||||
|
|
@ -896,7 +896,8 @@ async def migrate_global_messages(
|
||||||
if stoat_msg_id:
|
if stoat_msg_id:
|
||||||
context.state.set_target_message_mapping(target_channel_id, msg.id, stoat_msg_id)
|
context.state.set_target_message_mapping(target_channel_id, msg.id, stoat_msg_id)
|
||||||
context.state.update_last_message_id(target_channel_id, msg.id)
|
context.state.update_last_message_id(target_channel_id, msg.id)
|
||||||
context.state.set_waterfall_last_id(msg.id)
|
context.state.update_last_message_timestamp(target_channel_id, str(msg.created_at))
|
||||||
|
context.state.increment_stats(target_channel_id, messages=1, files=len(files) if files else 0)
|
||||||
stats["attachments"] += len(files) if files else 0
|
stats["attachments"] += len(files) if files else 0
|
||||||
|
|
||||||
stats["messages"] += 1
|
stats["messages"] += 1
|
||||||
|
|
|
||||||
|
|
@ -1551,10 +1551,8 @@ class OperationPane(Container):
|
||||||
if filtered_tgt_ids:
|
if filtered_tgt_ids:
|
||||||
all_mapped_tgt_ids = filtered_tgt_ids
|
all_mapped_tgt_ids = filtered_tgt_ids
|
||||||
|
|
||||||
# 2.6 Resume Point: Prioritize Global waterfall tracker, fallback to channel minimums
|
# 2.6 Resume Point: Calculate from global channel minimums
|
||||||
min_last_id = self.engine.state.get_waterfall_last_id()
|
min_last_id = self.engine.state.get_global_min_last_message_id(all_mapped_tgt_ids)
|
||||||
if min_last_id is None:
|
|
||||||
min_last_id = self.engine.state.get_global_min_last_message_id(all_mapped_tgt_ids)
|
|
||||||
|
|
||||||
modal.write(f"\n[bold cyan]Waterfall Migration Resume Point:[/bold cyan]")
|
modal.write(f"\n[bold cyan]Waterfall Migration Resume Point:[/bold cyan]")
|
||||||
if min_last_id is not None:
|
if min_last_id is not None:
|
||||||
|
|
@ -1566,7 +1564,7 @@ class OperationPane(Container):
|
||||||
show_continue=min_last_id is not None,
|
show_continue=min_last_id is not None,
|
||||||
show_id=False,
|
show_id=False,
|
||||||
btn_start_label="Start From Beginning",
|
btn_start_label="Start From Beginning",
|
||||||
btn_start_tooltip="Safe, skips duplicates automatically",
|
btn_start_tooltip="Wipes migration progress and restarts from the beginning; may create duplicates",
|
||||||
btn_start_variant="default" if min_last_id is not None else "primary",
|
btn_start_variant="default" if min_last_id is not None else "primary",
|
||||||
btn_continue_label=f"Continue from ID {min_last_id if min_last_id is not None else 0}" if min_last_id is not None else "Continue Migration",
|
btn_continue_label=f"Continue from ID {min_last_id if min_last_id is not None else 0}" if min_last_id is not None else "Continue Migration",
|
||||||
btn_continue_tooltip="Fastest"
|
btn_continue_tooltip="Fastest"
|
||||||
|
|
@ -1582,7 +1580,11 @@ class OperationPane(Container):
|
||||||
return
|
return
|
||||||
|
|
||||||
after_id = None
|
after_id = None
|
||||||
if choice == "btn_continue" and min_last_id is not None:
|
if choice == "btn_start_first":
|
||||||
|
logger.info("Proceeding with 'Start from Beginning' (global clean sink).")
|
||||||
|
self.engine.state.clear_all_migration_data()
|
||||||
|
after_id = None
|
||||||
|
elif choice == "btn_continue" and min_last_id is not None:
|
||||||
after_id = int(min_last_id)
|
after_id = int(min_last_id)
|
||||||
|
|
||||||
# Phase 3: Progress
|
# Phase 3: Progress
|
||||||
|
|
@ -1599,6 +1601,7 @@ class OperationPane(Container):
|
||||||
tid = self.config.fluxer_server_id
|
tid = self.config.fluxer_server_id
|
||||||
self.engine.ensure_state_initialized(str(tid or ""), platform_name)
|
self.engine.ensure_state_initialized(str(tid or ""), platform_name)
|
||||||
|
|
||||||
|
modal.show_stats()
|
||||||
modal.write("Scanning global footprint for totals ...")
|
modal.write("Scanning global footprint for totals ...")
|
||||||
stats_analysis = await migrate_mod.analyze_global_migration(self.engine, after_message_id=after_id)
|
stats_analysis = await migrate_mod.analyze_global_migration(self.engine, after_message_id=after_id)
|
||||||
total_messages = stats_analysis["messages"]
|
total_messages = stats_analysis["messages"]
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue