From bc500641bfb679898850177517df4a465694509a Mon Sep 17 00:00:00 2001 From: rambros Date: Sun, 1 Mar 2026 18:09:05 +0530 Subject: [PATCH] improve channel selection & progress display --- src/disco_reaper/exporter.py | 43 +++++++++++++---- src/ui/reaper_app.py | 90 ++++++++++++++++++++++++++++++++---- 2 files changed, 116 insertions(+), 17 deletions(-) diff --git a/src/disco_reaper/exporter.py b/src/disco_reaper/exporter.py index 19b4de3..eda1ba8 100644 --- a/src/disco_reaper/exporter.py +++ b/src/disco_reaper/exporter.py @@ -55,7 +55,24 @@ class DiscordExporter: else: metadata["banner"] = None + # Add metadata fields + from datetime import datetime + metadata["last_backup"] = datetime.now().isoformat() + output_file = self.export_path / "server_profile.json" + + # Preserve ignore_channels if the file already exists + ignore_channels = [] + if output_file.exists(): + try: + with open(output_file, "r", encoding="utf-8") as f: + old_data = json.load(f) + ignore_channels = old_data.get("ignore_channels", []) + except Exception as e: + logger.warning(f"Could not read existing server_profile.json to preserve ignore_channels: {e}") + + metadata["ignore_channels"] = ignore_channels + with open(output_file, "w", encoding="utf-8") as f: json.dump(metadata, f, indent=4, ensure_ascii=False) return metadata @@ -240,10 +257,12 @@ class DiscordExporter: "nsfw": getattr(c, "nsfw", False) } - async def export_channel_messages(self, channel_id: int, progress_callback=None): - """Exports all messages from a channel, including attachments, pins, reactions.""" + async def export_channel_messages(self, channel_id: int, progress_callback=None, force=False): + """Fetches and saves message history for a channel, handling incremental sync.""" channel = await self.reader.get_channel(channel_id) - if not channel: return 0 + if not channel: + logger.error(f"Channel not found: {channel_id}") + return 0 channel_name = channel.name safe_name = channel_name.replace(" ", "-").lower() @@ -278,13 +297,21 @@ class DiscordExporter: base_filename = str(channel_id) json_file = backup_dir / f"{base_filename}.json" asset_dir = backup_dir / base_filename + + if force and asset_dir.exists(): + import shutil + try: + shutil.rmtree(asset_dir) + except Exception as e: + logger.warning(f"Failed to clear asset directory {asset_dir}: {e}") + asset_dir.mkdir(exist_ok=True) messages = [] last_id = None - # Load existing messages for incremental sync - if json_file.exists(): + # Load existing messages for incremental sync (skip if force) + if not force and json_file.exists(): try: with open(json_file, "r", encoding="utf-8") as f: old_data = json.load(f) @@ -309,7 +336,7 @@ class DiscordExporter: messages.append(msg_data) new_count += 1 if progress_callback: - await progress_callback(channel_name, count + new_count) + await progress_callback(channel_name, new_count) except discord.Forbidden: logger.error(f"403 Forbidden: Missing Access to read messages in {channel_name} ({channel_id})") if not messages: return 0 @@ -532,7 +559,7 @@ class DiscordExporter: return data - async def export_threads(self, channel_id: int): + async def export_threads(self, channel_id: int, progress_callback=None, force=False): """Exports active and archived threads for a channel.""" channel = await self.reader.get_channel(channel_id) if not hasattr(channel, "threads") and not hasattr(channel, "public_archived_threads"): @@ -562,7 +589,7 @@ class DiscordExporter: logger.info(f"Found {len(all_threads)} threads in {channel.name}. Starting backup...") for thread in all_threads: - await self.export_channel_messages(thread.id) + await self.export_channel_messages(thread.id, progress_callback=progress_callback, force=force) thread_count += 1 return thread_count diff --git a/src/ui/reaper_app.py b/src/ui/reaper_app.py index 4c5ad25..2f3bde0 100644 --- a/src/ui/reaper_app.py +++ b/src/ui/reaper_app.py @@ -3,6 +3,10 @@ import asyncio import discord import logging import time +import re +import json +from datetime import datetime +from pathlib import Path from rich.console import Console from rich.prompt import Prompt, Confirm from rich.table import Table @@ -65,6 +69,29 @@ class DiscoReaperCLI: console.print(f"[bold red]Discord validation failed: {e}[/bold red]") return False + + def get_backup_info(self): + """Checks for existing backup and returns formatted timestamp.""" + d_name = self.validation_results.get("discord_server_name") + d_id = self.config.discord_server_id + if not d_name or not d_id or d_id == "DISCORD_SERVER_ID": + return None + + safe_name = re.sub(r'[^a-zA-Z0-9_\-\.]', '_', d_name) + export_path = Path(".") / f"EXPORT-{safe_name}-{d_id}" + profile_file = export_path / "server_profile.json" + + if profile_file.exists(): + try: + with open(profile_file, "r", encoding="utf-8") as f: + data = json.load(f) + ts_str = data.get("last_backup") + if ts_str: + dt = datetime.fromisoformat(ts_str) + return dt.strftime("%d-%b-%Y %H:%M") + except Exception: + pass + return None async def run(self): await self.validate_config() @@ -81,9 +108,13 @@ class DiscoReaperCLI: b_display = f"[bold green]\"{b_name}\"[/bold green]" if b_name else "[bold red]UNKNOWN[/bold red]" console.print(f"[bold cyan]Bot name:[/bold cyan] {b_display}") + backup_ts = self.get_backup_info() + if backup_ts: + console.print(f"[bold cyan]Backup Found:[/bold cyan] [bold yellow]{backup_ts}[/bold yellow]") + console.print("\n[bold]Main Menu[/bold]") console.print("(1) Backup Server Profile") - console.print("(2) Backup Messages") + console.print("(2) Backup Channel Messages") console.print("(3) Update & Sync Backup") console.print("(4) Configuration") console.print("(Q) Exit") @@ -182,7 +213,14 @@ class DiscoReaperCLI: cat_name = cat_map.get(chan.category_id) if cat_name: display_name = f"{chan.name} [{cat_name}]" - console.print(f"({i+1}) {display_name}") + + # Check for existing backup + found_prefix = "" + backup_file = self.exporter.export_path / "message_backup" / f"{chan.id}.json" + if backup_file.exists(): + found_prefix = "[green][FOUND][/green] " + + console.print(f"({i+1}) {found_prefix}{display_name}") console.print("(A) [bold green]All Channels[/bold green]") console.print("(B) Back") @@ -209,23 +247,57 @@ class DiscoReaperCLI: console.print("[yellow]No valid channels selected.[/yellow]") return + # Check if any have [FOUND] + any_found = False + for chan in selected_channels: + if (self.exporter.export_path / "message_backup" / f"{chan.id}.json").exists(): + any_found = True + break + + force_overwrite = False + if any_found: + console.print("\n[bold yellow]Existing backup(s) found for some selected channels.[/bold yellow]") + console.print("(Y) Update & Sync Backup") + console.print("(F) [bold red]Force Overwrite Backup[/bold red]") + console.print("(B) Back") + + sync_choice = Prompt.ask("\nSelect option", choices=["Y", "F", "B"], default="Y").upper() + if sync_choice == "B": + return + force_overwrite = (sync_choice == "F") + console.print(f"\n[yellow]Starting backup for {len(selected_channels)} channels...[/yellow]") with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), - BarColumn(), - TaskProgressColumn(), + TextColumn("- [bold yellow]{task.fields[msg_count]} {task.fields[suffix]}"), console=console ) as progress: - overall_task = progress.add_task("[cyan]Exporting Channels...", total=len(selected_channels)) for chan in selected_channels: - progress.update(overall_task, description=f"[cyan]Backing up: {chan.name}") - await self.exporter.export_channel_messages(chan.id) - await self.exporter.export_threads(chan.id) - progress.advance(overall_task) + # Determine if it's a sync or fresh backup + backup_exists = (self.exporter.export_path / "message_backup" / f"{chan.id}.json").exists() + is_sync = backup_exists and not force_overwrite + + label = "Syncing Backup" if is_sync else "Backing up" + suffix = "new messages" if is_sync else "messages" + + # Create a specific task for each channel to keep it on its own line + task_id = progress.add_task(f"[cyan]{label}: {chan.name}", total=None, msg_count=0, suffix=suffix) + + async def update_msg_count(name, count, tid=task_id): + progress.update(tid, msg_count=count) + + await self.exporter.export_channel_messages(chan.id, progress_callback=update_msg_count, force=force_overwrite) + await self.exporter.export_threads(chan.id, progress_callback=update_msg_count, force=force_overwrite) + + # Mark as finished (stop spinner) + progress.stop_task(task_id) + progress.update(task_id, description=f"[green]Completed: {chan.name}") console.print("[bold green]Message backup complete![/bold green]") + # Update last_backup in server_profile.json + await self.exporter.export_metadata() except Exception as e: console.print(f"[bold red]Message backup failed: {e}[/bold red]") finally: