switch to sqlite for backups also
This commit is contained in:
rambros
parent
de28ecdbb1
commit
f402a36477
6 changed files with 1852 additions and 1010 deletions
762
src/core/backup_database.py
Normal file
762
src/core/backup_database.py
Normal file
|
|
@ -0,0 +1,762 @@
|
|||
import sqlite3
|
||||
import logging
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class BackupDatabase:
|
||||
"""Manages the SQLite database for local Discord backups."""
|
||||
|
||||
def __init__(self, db_path: Path | str):
|
||||
self.db_path = Path(db_path)
|
||||
self._lock = threading.Lock()
|
||||
self._init_db()
|
||||
|
||||
def _get_conn(self):
|
||||
conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
def _init_db(self):
|
||||
"""Initializes the database schema."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
# Guild Profile
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS guild_profile (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
description TEXT,
|
||||
icon_file TEXT,
|
||||
icon_url TEXT,
|
||||
banner_file TEXT,
|
||||
banner_url TEXT,
|
||||
owner_id TEXT,
|
||||
last_backup TEXT,
|
||||
ignore_channels TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# Roles
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS roles (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
color INTEGER,
|
||||
position INTEGER,
|
||||
permissions TEXT,
|
||||
hoist INTEGER,
|
||||
mentionable INTEGER
|
||||
)
|
||||
""")
|
||||
|
||||
# Channels
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS channels (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
type INTEGER,
|
||||
position INTEGER,
|
||||
category_id TEXT,
|
||||
topic TEXT,
|
||||
nsfw INTEGER
|
||||
)
|
||||
""")
|
||||
|
||||
# Channel Permissions
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS permissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
channel_id TEXT,
|
||||
target_id TEXT,
|
||||
target_type TEXT,
|
||||
allow INTEGER,
|
||||
deny INTEGER
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_permissions_chan ON permissions(channel_id)")
|
||||
|
||||
# Users (Author cache)
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id TEXT PRIMARY KEY,
|
||||
username TEXT,
|
||||
display_name TEXT,
|
||||
avatar_file TEXT,
|
||||
avatar_url TEXT,
|
||||
roles TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# Messages
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
channel_id TEXT,
|
||||
author_id TEXT,
|
||||
content TEXT,
|
||||
timestamp TEXT,
|
||||
type INTEGER,
|
||||
message_reference TEXT,
|
||||
is_pinned INTEGER,
|
||||
extra_data TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_messages_channel ON messages(channel_id)")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp)")
|
||||
|
||||
# Attachments
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id TEXT PRIMARY KEY,
|
||||
message_id TEXT,
|
||||
filename TEXT,
|
||||
size INTEGER,
|
||||
url TEXT,
|
||||
content_type TEXT,
|
||||
local_hash TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_attachments_msg ON attachments(message_id)")
|
||||
|
||||
# Embeds
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS embeds (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
message_id TEXT,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
url TEXT,
|
||||
color INTEGER,
|
||||
timestamp TEXT,
|
||||
thumbnail_url TEXT,
|
||||
image_url TEXT,
|
||||
author_name TEXT,
|
||||
author_url TEXT,
|
||||
author_icon_url TEXT,
|
||||
footer_text TEXT,
|
||||
footer_icon_url TEXT,
|
||||
fields TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_embeds_msg ON embeds(message_id)")
|
||||
|
||||
# Reactions
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS reactions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
message_id TEXT,
|
||||
emoji_id TEXT,
|
||||
emoji_name TEXT,
|
||||
count INTEGER
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_reactions_msg ON reactions(message_id)")
|
||||
|
||||
# Message Stickers
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS message_stickers (
|
||||
message_id TEXT,
|
||||
sticker_id TEXT,
|
||||
name TEXT,
|
||||
url TEXT,
|
||||
format_type INTEGER,
|
||||
local_hash TEXT,
|
||||
PRIMARY KEY (message_id, sticker_id)
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_message_stickers_msg ON message_stickers(message_id)")
|
||||
|
||||
# Threads
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS threads (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
type INTEGER,
|
||||
parent_id TEXT,
|
||||
message_count INTEGER,
|
||||
member_count INTEGER,
|
||||
archived INTEGER,
|
||||
archive_timestamp TEXT,
|
||||
auto_archive_duration INTEGER,
|
||||
locked INTEGER,
|
||||
applied_tags TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_threads_parent ON threads(parent_id)")
|
||||
|
||||
# Forum Tags (Definitions for a forum channel)
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS forum_tags (
|
||||
id TEXT PRIMARY KEY,
|
||||
forum_id TEXT,
|
||||
name TEXT,
|
||||
moderated INTEGER,
|
||||
emoji_id TEXT,
|
||||
emoji_name TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_forum_tags_forum ON forum_tags(forum_id)")
|
||||
|
||||
# Media Pool (CAS)
|
||||
# Maps content hashes to local storage paths
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS media_pool (
|
||||
hash TEXT PRIMARY KEY,
|
||||
local_path TEXT,
|
||||
size INTEGER,
|
||||
mime_type TEXT,
|
||||
first_seen_url TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# Server Assets (Emojis, Stickers, etc.)
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS server_assets (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
type TEXT,
|
||||
filename TEXT,
|
||||
url TEXT,
|
||||
mime_type TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def set_guild_profile(self, data: Dict[str, Any]):
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
conn.execute("""
|
||||
INSERT OR REPLACE INTO guild_profile (id, name, description, icon_file, icon_url, banner_file, banner_url, owner_id, last_backup, ignore_channels)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
str(data.get("id")), data.get("name"), data.get("description"),
|
||||
data.get("icon_file"), data.get("icon_url"),
|
||||
data.get("banner_file"), data.get("banner_url"),
|
||||
str(data.get("owner_id")),
|
||||
data.get("last_backup"), json.dumps(data.get("ignore_channels", []))
|
||||
))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def get_guild_profile(self) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute("SELECT * FROM guild_profile LIMIT 1").fetchone()
|
||||
conn.close()
|
||||
if row:
|
||||
data = dict(row)
|
||||
if data.get("ignore_channels"):
|
||||
data["ignore_channels"] = json.loads(data["ignore_channels"])
|
||||
else:
|
||||
data["ignore_channels"] = []
|
||||
return data
|
||||
return None
|
||||
|
||||
def save_roles(self, roles: List[Dict[str, Any]]):
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
# Ensure complex fields are strings if they aren't already
|
||||
formatted = []
|
||||
for r in roles:
|
||||
formatted.append({
|
||||
"id": str(r["id"]),
|
||||
"name": r["name"],
|
||||
"color": r["color"],
|
||||
"position": r["position"],
|
||||
"permissions": str(r["permissions"]),
|
||||
"hoist": 1 if r["hoist"] else 0,
|
||||
"mentionable": 1 if r["mentionable"] else 0
|
||||
})
|
||||
|
||||
conn.executemany("""
|
||||
INSERT OR REPLACE INTO roles (id, name, color, position, permissions, hoist, mentionable)
|
||||
VALUES (:id, :name, :color, :position, :permissions, :hoist, :mentionable)
|
||||
""", formatted)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def save_channels(self, channels: List[Dict[str, Any]]):
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
conn.executemany("""
|
||||
INSERT OR REPLACE INTO channels (id, name, type, position, category_id, topic, nsfw)
|
||||
VALUES (:id, :name, :type, :position, :category_id, :topic, :nsfw)
|
||||
""", channels)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def save_permissions(self, permissions: List[Dict[str, Any]]):
|
||||
"""Saves a batch of channel permission overwrites."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
conn.executemany("""
|
||||
INSERT INTO permissions (channel_id, target_id, target_type, allow, deny)
|
||||
VALUES (:channel_id, :target_id, :target_type, :allow, :deny)
|
||||
""", permissions)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def save_users(self, users: List[Dict[str, Any]]):
|
||||
"""Saves users to the author cache."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
conn.executemany("""
|
||||
INSERT OR REPLACE INTO users (id, username, display_name, avatar_file, avatar_url, roles)
|
||||
VALUES (:id, :username, :display_name, :avatar_file, :avatar_url, :roles)
|
||||
""", users)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def save_server_assets(self, assets: List[Dict[str, Any]]):
|
||||
"""Saves a batch of server assets (emojis, stickers) to the database."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
formatted = []
|
||||
for a in assets:
|
||||
formatted.append({
|
||||
"id": str(a["id"]),
|
||||
"name": a.get("name"),
|
||||
"type": a.get("type"),
|
||||
"filename": a.get("filename"),
|
||||
"url": a.get("url"),
|
||||
"mime_type": a.get("mime_type")
|
||||
})
|
||||
|
||||
conn.executemany("""
|
||||
INSERT OR REPLACE INTO server_assets (id, name, type, filename, url, mime_type)
|
||||
VALUES (:id, :name, :type, :filename, :url, :mime_type)
|
||||
""", formatted)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def save_threads(self, threads: List[Dict[str, Any]]):
|
||||
"""Saves metadata for threads to the database."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
conn.executemany("""
|
||||
INSERT OR REPLACE INTO threads (id, name, type, parent_id, message_count, member_count, archived, archive_timestamp, auto_archive_duration, locked, applied_tags)
|
||||
VALUES (:id, :name, :type, :parent_id, :message_count, :member_count, :archived, :archive_timestamp, :auto_archive_duration, :locked, :applied_tags)
|
||||
""", threads)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def save_forum_tags(self, tags: List[Dict[str, Any]]):
|
||||
"""Saves definitions for forum tags."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
conn.executemany("""
|
||||
INSERT OR REPLACE INTO forum_tags (id, forum_id, name, moderated, emoji_id, emoji_name)
|
||||
VALUES (:id, :forum_id, :name, :moderated, :emoji_id, :emoji_name)
|
||||
""", tags)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def save_messages_batch(self, messages: List[Dict[str, Any]]):
|
||||
"""Batch inserts messages and their attachments."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
# Insert messages
|
||||
conn.executemany("""
|
||||
INSERT OR REPLACE INTO messages (id, channel_id, author_id, content, timestamp, type, message_reference, is_pinned, extra_data)
|
||||
VALUES (:id, :channel_id, :author_id, :content, :timestamp, :type, :message_reference, :is_pinned, :extra_data)
|
||||
""", messages)
|
||||
|
||||
# Extract attachments, reactions, and stickers
|
||||
all_attachments = []
|
||||
all_reactions = []
|
||||
all_stickers = []
|
||||
|
||||
for msg in messages:
|
||||
# Attachments
|
||||
if "attachments" in msg:
|
||||
for att in msg["attachments"]:
|
||||
att["message_id"] = msg["id"]
|
||||
all_attachments.append(att)
|
||||
|
||||
# Reactions
|
||||
if "reactions" in msg and msg["reactions"]:
|
||||
for rea in msg["reactions"]:
|
||||
all_reactions.append({
|
||||
"message_id": msg["id"],
|
||||
"emoji_id": str(rea["emoji_id"]) if rea.get("emoji_id") else None,
|
||||
"emoji_name": rea.get("emoji_name"),
|
||||
"count": rea.get("count", 0)
|
||||
})
|
||||
|
||||
# Stickers
|
||||
if "stickers" in msg and msg["stickers"]:
|
||||
for st in msg["stickers"]:
|
||||
all_stickers.append({
|
||||
"message_id": msg["id"],
|
||||
"sticker_id": str(st["id"]),
|
||||
"name": st.get("name"),
|
||||
"url": st.get("url"),
|
||||
"format_type": st.get("format_type"),
|
||||
"local_hash": st.get("local_hash")
|
||||
})
|
||||
|
||||
if all_attachments:
|
||||
conn.executemany("""
|
||||
INSERT OR REPLACE INTO attachments (id, message_id, filename, size, url, content_type, local_hash)
|
||||
VALUES (:id, :message_id, :filename, :size, :url, :content_type, :local_hash)
|
||||
""", all_attachments)
|
||||
|
||||
# Save Embeds (Normalized with JSON Fields)
|
||||
for msg in messages:
|
||||
if "embeds" in msg and msg["embeds"]:
|
||||
for emb in msg["embeds"]:
|
||||
# Insert top-level embed
|
||||
conn.execute("""
|
||||
INSERT INTO embeds (
|
||||
message_id, title, description, url, color, timestamp,
|
||||
thumbnail_url, image_url, author_name, author_url,
|
||||
author_icon_url, footer_text, footer_icon_url, fields
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
msg["id"], emb.get("title"), emb.get("description"), emb.get("url"),
|
||||
emb.get("color"), emb.get("timestamp"),
|
||||
emb.get("thumbnail", {}).get("url") if isinstance(emb.get("thumbnail"), dict) else None,
|
||||
emb.get("image", {}).get("url") if isinstance(emb.get("image"), dict) else None,
|
||||
emb.get("author", {}).get("name") if isinstance(emb.get("author"), dict) else None,
|
||||
emb.get("author", {}).get("url") if isinstance(emb.get("author"), dict) else None,
|
||||
emb.get("author", {}).get("icon_url") if isinstance(emb.get("author"), dict) else None,
|
||||
emb.get("footer", {}).get("text") if isinstance(emb.get("footer"), dict) else None,
|
||||
emb.get("footer", {}).get("icon_url") if isinstance(emb.get("footer"), dict) else None,
|
||||
json.dumps(emb.get("fields", []))
|
||||
))
|
||||
|
||||
if all_reactions:
|
||||
conn.executemany("""
|
||||
INSERT INTO reactions (message_id, emoji_id, emoji_name, count)
|
||||
VALUES (:message_id, :emoji_id, :emoji_name, :count)
|
||||
""", all_reactions)
|
||||
|
||||
if all_stickers:
|
||||
conn.executemany("""
|
||||
INSERT OR REPLACE INTO message_stickers (message_id, sticker_id, name, url, format_type, local_hash)
|
||||
VALUES (:message_id, :sticker_id, :name, :url, :format_type, :local_hash)
|
||||
""", all_stickers)
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def get_last_message_id(self, channel_id: str) -> Optional[str]:
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute("SELECT id FROM messages WHERE channel_id = ? ORDER BY id DESC LIMIT 1", (str(channel_id),)).fetchone()
|
||||
conn.close()
|
||||
return row["id"] if row else None
|
||||
|
||||
def get_media_by_hash(self, file_hash: str) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute("SELECT * FROM media_pool WHERE hash = ?", (file_hash,)).fetchone()
|
||||
conn.close()
|
||||
return dict(row) if row else None
|
||||
|
||||
def get_media_by_url(self, url: str) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute("SELECT * FROM media_pool WHERE first_seen_url = ?", (url,)).fetchone()
|
||||
conn.close()
|
||||
return dict(row) if row else None
|
||||
|
||||
def add_media_to_pool(self, file_hash: str, local_path: str, size: int, mime_type: str, url: str):
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
conn.execute("""
|
||||
INSERT OR REPLACE INTO media_pool (hash, local_path, size, mime_type, first_seen_url)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""", (file_hash, str(local_path), size, mime_type, url))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def get_stats_by_channel(self) -> Dict[int, Dict[str, Any]]:
|
||||
"""Returns aggregate stats for all channels with backups."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
# Summary of messages per effective channel (including threads rolled up)
|
||||
msg_rows = conn.execute("""
|
||||
SELECT
|
||||
COALESCE(t.parent_id, m.channel_id) as channel_id,
|
||||
COUNT(m.id) as msg_count
|
||||
FROM messages m
|
||||
LEFT JOIN threads t ON m.channel_id = t.id
|
||||
GROUP BY channel_id
|
||||
""").fetchall()
|
||||
|
||||
# Thread counts per parent
|
||||
thread_rows = conn.execute("""
|
||||
SELECT parent_id, COUNT(*) as thread_count
|
||||
FROM threads
|
||||
GROUP BY parent_id
|
||||
""").fetchall()
|
||||
|
||||
# Summary of attachments per effective channel (including threads rolled up)
|
||||
att_rows = conn.execute("""
|
||||
SELECT
|
||||
COALESCE(t.parent_id, m.channel_id) as channel_id,
|
||||
COUNT(a.id) as att_count,
|
||||
SUM(a.size) as total_size
|
||||
FROM attachments a
|
||||
JOIN messages m ON a.message_id = m.id
|
||||
LEFT JOIN threads t ON m.channel_id = t.id
|
||||
GROUP BY channel_id
|
||||
""").fetchall()
|
||||
|
||||
conn.close()
|
||||
|
||||
stats = {}
|
||||
for r in msg_rows:
|
||||
cid_raw = r["channel_id"]
|
||||
if cid_raw is None or cid_raw == "None": continue
|
||||
cid = int(cid_raw)
|
||||
stats[cid] = {
|
||||
"message_count": r["msg_count"],
|
||||
"thread_count": 0,
|
||||
"attachment_count": 0,
|
||||
"total_size": 0
|
||||
}
|
||||
|
||||
for r in thread_rows:
|
||||
cid_raw = r["parent_id"]
|
||||
if cid_raw is None or cid_raw == "None": continue
|
||||
cid = int(cid_raw)
|
||||
if cid not in stats:
|
||||
stats[cid] = {"message_count": 0, "thread_count": 0, "attachment_count": 0, "total_size": 0}
|
||||
stats[cid]["thread_count"] = r["thread_count"]
|
||||
|
||||
for r in att_rows:
|
||||
cid_raw = r["channel_id"]
|
||||
if cid_raw is None or cid_raw == "None": continue
|
||||
cid = int(cid_raw)
|
||||
if cid not in stats:
|
||||
stats[cid] = {"message_count": 0, "thread_count": 0, "attachment_count": 0, "total_size": 0}
|
||||
stats[cid]["attachment_count"] = r["att_count"]
|
||||
stats[cid]["total_size"] = r["total_size"] or 0
|
||||
|
||||
return stats
|
||||
|
||||
def get_all_roles(self) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute("SELECT * FROM roles ORDER BY position DESC").fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
def get_all_channels(self) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute("SELECT * FROM channels ORDER BY position ASC").fetchall()
|
||||
|
||||
# Fetch all permissions
|
||||
chan_list = [dict(r) for r in rows]
|
||||
if chan_list:
|
||||
ids = [c["id"] for c in chan_list]
|
||||
placeholders = ",".join(["?"] * len(ids))
|
||||
perm_rows = conn.execute(f"SELECT * FROM permissions WHERE channel_id IN ({placeholders})", ids).fetchall()
|
||||
|
||||
perms_by_chan = {}
|
||||
for pr in perm_rows:
|
||||
cid = pr["channel_id"]
|
||||
if cid not in perms_by_chan: perms_by_chan[cid] = []
|
||||
perms_by_chan[cid].append({
|
||||
"id": pr["target_id"],
|
||||
"type": pr["target_type"],
|
||||
"allow": pr["allow"],
|
||||
"deny": pr["deny"]
|
||||
})
|
||||
|
||||
for c in chan_list:
|
||||
c["overwrites"] = perms_by_chan.get(c["id"], [])
|
||||
|
||||
# Fetch Forum Tags
|
||||
tag_rows = conn.execute(f"SELECT * FROM forum_tags WHERE forum_id IN ({placeholders})", ids).fetchall()
|
||||
tags_by_forum = {}
|
||||
for tr in tag_rows:
|
||||
fid = tr["forum_id"]
|
||||
if fid not in tags_by_forum: tags_by_forum[fid] = []
|
||||
tags_by_forum[fid].append(dict(tr))
|
||||
|
||||
for c in chan_list:
|
||||
c["available_tags"] = tags_by_forum.get(c["id"], [])
|
||||
|
||||
conn.close()
|
||||
return chan_list
|
||||
|
||||
def get_all_threads(self) -> List[Dict[str, Any]]:
|
||||
"""Returns metadata for all threads in the backup."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute("SELECT * FROM threads").fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
def get_forum_tags(self, forum_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Returns forum tag definitions."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
if forum_id:
|
||||
rows = conn.execute("SELECT * FROM forum_tags WHERE forum_id = ?", (str(forum_id),)).fetchall()
|
||||
else:
|
||||
rows = conn.execute("SELECT * FROM forum_tags").fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
def get_threads_by_parent(self, parent_id: str) -> List[Dict[str, Any]]:
|
||||
"""Returns all threads belonging to a parent channel."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute("SELECT * FROM threads WHERE parent_id = ?", (str(parent_id),)).fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
def get_thread(self, thread_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Retrieves a single thread's metadata."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute("SELECT * FROM threads WHERE id = ?", (str(thread_id),)).fetchone()
|
||||
conn.close()
|
||||
return dict(row) if row else None
|
||||
|
||||
def get_all_users(self) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute("SELECT * FROM users").fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute("SELECT * FROM users WHERE id = ?", (str(user_id),)).fetchone()
|
||||
conn.close()
|
||||
if row:
|
||||
data = dict(row)
|
||||
if data.get("roles"):
|
||||
data["roles"] = json.loads(data["roles"])
|
||||
return data
|
||||
return None
|
||||
|
||||
def get_server_assets(self, asset_type: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Returns all server assets, optionally filtered by type."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
if asset_type:
|
||||
rows = conn.execute("SELECT * FROM server_assets WHERE type = ?", (asset_type,)).fetchall()
|
||||
else:
|
||||
rows = conn.execute("SELECT * FROM server_assets").fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
def get_all_media(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Returns the entire media pool as a dictionary indexed by hash."""
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute("SELECT * FROM media_pool").fetchall()
|
||||
conn.close()
|
||||
return {r["hash"]: dict(r) for r in rows}
|
||||
|
||||
def get_messages_paged(self, channel_id: str, limit: int = 100, offset: int = 0, after_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
conn = self._get_conn()
|
||||
query = "SELECT * FROM messages WHERE channel_id = ?"
|
||||
params = [str(channel_id)]
|
||||
|
||||
if after_id:
|
||||
query += " AND id > ?"
|
||||
params.append(str(after_id))
|
||||
|
||||
query += " ORDER BY id ASC LIMIT ? OFFSET ?"
|
||||
params.extend([limit, offset])
|
||||
|
||||
rows = conn.execute(query, params).fetchall()
|
||||
msg_list = [dict(r) for r in rows]
|
||||
|
||||
if msg_list:
|
||||
# Fetch attachments for these messages
|
||||
msg_ids = [m["id"] for m in msg_list]
|
||||
placeholders = ",".join(["?"] * len(msg_ids))
|
||||
|
||||
# Attachments
|
||||
att_rows = conn.execute(f"SELECT * FROM attachments WHERE message_id IN ({placeholders})", msg_ids).fetchall()
|
||||
atts_by_msg = {}
|
||||
for ar in att_rows:
|
||||
mid = ar["message_id"]
|
||||
if mid not in atts_by_msg: atts_by_msg[mid] = []
|
||||
atts_by_msg[mid].append(dict(ar))
|
||||
|
||||
# Embeds
|
||||
emb_rows = conn.execute(f"SELECT * FROM embeds WHERE message_id IN ({placeholders})", msg_ids).fetchall()
|
||||
embs_by_msg = {}
|
||||
for er in emb_rows:
|
||||
mid = er["message_id"]
|
||||
if mid not in embs_by_msg: embs_by_msg[mid] = []
|
||||
|
||||
e_dict = {
|
||||
"title": er["title"],
|
||||
"description": er["description"],
|
||||
"url": er["url"],
|
||||
"color": er["color"],
|
||||
"timestamp": er["timestamp"],
|
||||
"thumbnail": {"url": er["thumbnail_url"]} if er["thumbnail_url"] else None,
|
||||
"image": {"url": er["image_url"]} if er["image_url"] else None,
|
||||
"author": {
|
||||
"name": er["author_name"],
|
||||
"url": er["author_url"],
|
||||
"icon_url": er["author_icon_url"]
|
||||
} if er["author_name"] else None,
|
||||
"footer": {
|
||||
"text": er["footer_text"],
|
||||
"icon_url": er["footer_icon_url"]
|
||||
} if er["footer_text"] else None,
|
||||
"fields": json.loads(er["fields"]) if er["fields"] else []
|
||||
}
|
||||
embs_by_msg[mid].append(e_dict)
|
||||
|
||||
# Reactions
|
||||
rea_rows = conn.execute(f"SELECT * FROM reactions WHERE message_id IN ({placeholders})", msg_ids).fetchall()
|
||||
reas_by_msg = {}
|
||||
for rr in rea_rows:
|
||||
mid = rr["message_id"]
|
||||
if mid not in reas_by_msg: reas_by_msg[mid] = []
|
||||
reas_by_msg[mid].append(dict(rr))
|
||||
|
||||
# Stickers (Message-specific)
|
||||
st_rows = conn.execute(f"SELECT * FROM message_stickers WHERE message_id IN ({placeholders})", msg_ids).fetchall()
|
||||
sts_by_msg = {}
|
||||
for sr in st_rows:
|
||||
mid = sr["message_id"]
|
||||
if mid not in sts_by_msg: sts_by_msg[mid] = []
|
||||
sts_by_msg[mid].append(dict(sr))
|
||||
|
||||
for m in msg_list:
|
||||
m_id = m["id"]
|
||||
m["attachments"] = atts_by_msg.get(m_id, [])
|
||||
m["embeds"] = embs_by_msg.get(m_id, [])
|
||||
m["reactions"] = reas_by_msg.get(m_id, [])
|
||||
m["stickers"] = sts_by_msg.get(m_id, [])
|
||||
|
||||
conn.close()
|
||||
return msg_list
|
||||
|
||||
def close(self):
|
||||
# We don't keep long-lived connections in this model to avoid locking issues,
|
||||
# but the method is here for parity with other DB classes.
|
||||
pass
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -279,9 +279,14 @@ class DiscordReader:
|
|||
try:
|
||||
# In a forum, the starter message ID is the thread ID
|
||||
starter = await thread.fetch_message(thread.id)
|
||||
# Bind the thread so migrate_messages handles it properly
|
||||
if not hasattr(starter, 'thread') or starter.thread is None:
|
||||
starter.thread = thread
|
||||
# Bind the thread if possible so downstream code has context
|
||||
try:
|
||||
if not hasattr(starter, 'thread') or starter.thread is None:
|
||||
starter.thread = thread
|
||||
except (AttributeError, TypeError):
|
||||
# Some versions of discord.py don't allow setting the thread property
|
||||
# or it might already be populated as a read-only property
|
||||
pass
|
||||
yield starter
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not fetch starter message for forum thread {thread.id}: {e}")
|
||||
|
|
|
|||
1000
src/core/exporter.py
1000
src/core/exporter.py
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
import asyncio
|
||||
|
|
@ -7,11 +8,15 @@ from textual.app import ComposeResult
|
|||
from textual.screen import Screen
|
||||
from textual.containers import Container, Vertical, VerticalScroll, Horizontal
|
||||
from textual.widgets import Button, Label, Rule, Tree
|
||||
from src.ui.widgets import RamDisplay
|
||||
from textual import work
|
||||
from rich.text import Text
|
||||
from rich.style import Style
|
||||
|
||||
from src.ui.widgets import RamDisplay
|
||||
from src.core.backup_reader import BackupReader, ChannelType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class BackupStatsScreen(Screen[None]):
|
||||
"""Full-screen view for displaying detailed backup statistics."""
|
||||
|
||||
|
|
@ -254,178 +259,133 @@ class BackupStatsScreen(Screen[None]):
|
|||
@work(exclusive=True)
|
||||
async def load_data(self) -> None:
|
||||
try:
|
||||
# Locate the correct backup directory passed from backup_ops
|
||||
if not self.target_dir.exists():
|
||||
raise FileNotFoundError(f"Config directory {self.target_dir} not found.")
|
||||
# Initialize BackupReader
|
||||
reader = BackupReader(self.target_dir)
|
||||
await reader.start()
|
||||
|
||||
target_dir = self.target_dir
|
||||
if not (target_dir / "server_profile" / "profile.json").exists():
|
||||
if not reader.guild:
|
||||
raise FileNotFoundError(f"No valid backup found in {self.target_dir}")
|
||||
|
||||
self.profile_path = target_dir / "server_profile"
|
||||
self.backup_path = target_dir / "message_backup"
|
||||
|
||||
# 1. Profile / Server Info
|
||||
server_name = "Unknown Server"
|
||||
server_id = "0"
|
||||
guild = reader.guild
|
||||
server_name = guild.name
|
||||
server_id = str(guild.id)
|
||||
|
||||
# Get last backup info from guild profile
|
||||
profile = reader.db.get_guild_profile()
|
||||
last_backup_str = "Never"
|
||||
profile_file = self.profile_path / "profile.json"
|
||||
if profile_file.exists():
|
||||
with open(profile_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
server_name = data.get("name", server_name)
|
||||
server_id = data.get("id", server_id)
|
||||
ts = data.get("last_backup")
|
||||
if ts:
|
||||
try:
|
||||
dt = datetime.fromisoformat(ts)
|
||||
last_backup_str = dt.strftime("%d %b, %Y - %H:%M")
|
||||
except Exception:
|
||||
last_backup_str = ts
|
||||
ts = profile.get("last_backup")
|
||||
if ts:
|
||||
try:
|
||||
dt = datetime.fromisoformat(ts)
|
||||
last_backup_str = dt.strftime("%d %b, %Y - %H:%M")
|
||||
except Exception:
|
||||
last_backup_str = ts
|
||||
|
||||
self.query_one("#bs_name", Label).update(f"{server_name}")
|
||||
self.query_one("#bs_id", Label).update(f"{server_id}")
|
||||
self.query_one("#bs_last_backup", Label).update(f"{last_backup_str}")
|
||||
|
||||
# 2. Assets / Entities
|
||||
member_count = 0
|
||||
emoji_count = 0
|
||||
sticker_count = 0
|
||||
|
||||
# Fetch members from users/user_info.json
|
||||
if self.backup_path:
|
||||
user_info_file = self.backup_path / "users" / "user_info.json"
|
||||
if user_info_file.exists():
|
||||
try:
|
||||
with open(user_info_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, list):
|
||||
member_count = len(data)
|
||||
elif isinstance(data, dict):
|
||||
member_count = len(data)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
assets_file = self.profile_path / "assets.json"
|
||||
if assets_file.exists():
|
||||
with open(assets_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
emoji_count = len(data.get("emojis", []))
|
||||
sticker_count = len(data.get("stickers", []))
|
||||
|
||||
role_count = 0
|
||||
roles_file = self.profile_path / "roles.json"
|
||||
if roles_file.exists():
|
||||
with open(roles_file, "r", encoding="utf-8") as f:
|
||||
role_count = len(json.load(f))
|
||||
# 2. Assets / Entities (using properties)
|
||||
member_count = len(reader.members)
|
||||
emoji_count = len(reader.emojis)
|
||||
sticker_count = len(reader.stickers)
|
||||
role_count = len(reader.roles)
|
||||
|
||||
self.query_one("#bs_val_members", Label).update(f"{member_count}")
|
||||
self.query_one("#bs_val_roles", Label).update(f"{role_count}")
|
||||
self.query_one("#bs_val_emojis", Label).update(f"{emoji_count}")
|
||||
self.query_one("#bs_val_stickers", Label).update(f"{sticker_count}")
|
||||
|
||||
# 3. Structure & Per-Channel Stats
|
||||
structure_file = self.profile_path / "structure.json"
|
||||
total_channels = 0
|
||||
backed_up_channels = 0
|
||||
# 3. Aggregate Stats from DB
|
||||
channel_stats = reader.db.get_stats_by_channel()
|
||||
|
||||
total_msgs = 0
|
||||
total_threads = 0
|
||||
total_files = 0
|
||||
total_size = 0
|
||||
total_msgs = sum(s["message_count"] for s in channel_stats.values())
|
||||
total_threads = sum(s["thread_count"] for s in channel_stats.values())
|
||||
total_files = sum(s["attachment_count"] for s in channel_stats.values())
|
||||
total_size = sum(s["total_size"] for s in channel_stats.values())
|
||||
|
||||
cat_nodes = [] # Collect category data
|
||||
backed_up_channel_ids = set(channel_stats.keys())
|
||||
|
||||
if structure_file.exists():
|
||||
with open(structure_file, "r", encoding="utf-8") as f:
|
||||
structure = json.load(f)
|
||||
# 4. Structure & Per-Channel Stats
|
||||
categories = reader.categories
|
||||
all_channels = reader.channels
|
||||
|
||||
for cat in structure:
|
||||
cat_name = cat.get("name", "Unknown Category").upper()
|
||||
c_msgs = 0
|
||||
c_thds = 0
|
||||
c_files = 0
|
||||
c_size = 0
|
||||
total_channels = len([c for c in all_channels if c.type != ChannelType.category])
|
||||
backed_up_channels = len([cid for cid in backed_up_channel_ids if any(c.id == cid for c in all_channels)])
|
||||
|
||||
chan_list = []
|
||||
# Helper to map channels to categories
|
||||
cat_map = {cat.id: {"cat": cat, "chans": []} for cat in categories}
|
||||
cat_map[None] = {"cat": None, "chans": []} # Uncategorized
|
||||
|
||||
for chan in cat.get("channels", []):
|
||||
total_channels += 1
|
||||
ch_id = chan.get("id")
|
||||
ch_name = f"# {chan.get('name', 'unknown')}"
|
||||
for chan in all_channels:
|
||||
if chan.type == ChannelType.category: continue
|
||||
cid = chan.category_id
|
||||
if cid not in cat_map: cid = None
|
||||
cat_map[cid]["chans"].append(chan)
|
||||
|
||||
m_count = 0
|
||||
t_count = 0
|
||||
f_count = 0
|
||||
s_bytes = 0
|
||||
is_backed_up = False
|
||||
|
||||
msg_file = self.backup_path / str(ch_id) / "messages.json"
|
||||
if msg_file.exists():
|
||||
is_backed_up = True
|
||||
backed_up_channels += 1
|
||||
try:
|
||||
with open(msg_file, "r", encoding="utf-8") as mf:
|
||||
mdata = json.load(mf)
|
||||
m_count = mdata.get("messageCount", 0)
|
||||
t_count = mdata.get("threadCount", 0)
|
||||
f_count = mdata.get("numberOfAttachments", 0)
|
||||
s_bytes = mdata.get("totalAttachmentSizeBytes", 0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
chan_list.append({
|
||||
"name": ch_name,
|
||||
"msgs": m_count if is_backed_up else "NA",
|
||||
"threads": t_count if is_backed_up else "NA",
|
||||
"files": f_count if is_backed_up else "NA",
|
||||
"size": s_bytes,
|
||||
"is_backed_up": is_backed_up
|
||||
})
|
||||
c_msgs += m_count
|
||||
c_thds += t_count
|
||||
c_files += f_count
|
||||
c_size += s_bytes
|
||||
|
||||
total_msgs += c_msgs
|
||||
total_threads += c_thds
|
||||
total_files += c_files
|
||||
total_size += c_size
|
||||
|
||||
cat_nodes.append({
|
||||
"name": cat_name,
|
||||
"channels": chan_list,
|
||||
"msgs": c_msgs,
|
||||
"threads": c_thds,
|
||||
"files": c_files,
|
||||
"size": c_size
|
||||
})
|
||||
|
||||
# 4. Global Stats
|
||||
# Global update
|
||||
self.query_one("#bs_val_msgs", Label).update(f"{total_msgs}")
|
||||
self.query_one("#bs_val_threads", Label).update(f"{total_threads}")
|
||||
self.query_one("#bs_val_files", Label).update(f"{total_files}")
|
||||
self.query_one("#bs_val_size", Label).update(f"{self._format_size(total_size)}")
|
||||
|
||||
self.query_one("#bs_val_coverage", Label).update(f"{backed_up_channels} / {total_channels}")
|
||||
|
||||
# 5. Build Tree
|
||||
for cat in cat_nodes:
|
||||
cat_lbl = self._format_tree_row(f"{cat['name']}", cat['msgs'], cat['threads'], cat['files'], self._format_size(cat['size']))
|
||||
for cat_id, info in cat_map.items():
|
||||
cat = info["cat"]
|
||||
chans = info["chans"]
|
||||
if not chans: continue
|
||||
|
||||
cat_name = cat.name.upper() if cat else "UNCATEGORIZED"
|
||||
|
||||
# Aggregate for category
|
||||
c_msgs = 0
|
||||
c_thds = 0
|
||||
c_files = 0
|
||||
c_size = 0
|
||||
|
||||
chan_nodes_data = []
|
||||
for ch in chans:
|
||||
stats = channel_stats.get(ch.id, {"message_count": 0, "thread_count": 0, "attachment_count": 0, "total_size": 0})
|
||||
is_bu = ch.id in backed_up_channel_ids
|
||||
|
||||
chan_nodes_data.append({
|
||||
"name": f"# {ch.name}",
|
||||
"msgs": stats["message_count"] if is_bu else "NA",
|
||||
"threads": stats["thread_count"] if is_bu else "NA",
|
||||
"files": stats["attachment_count"] if is_bu else "NA",
|
||||
"size": stats["total_size"] if is_bu else 0,
|
||||
"is_backed_up": is_bu
|
||||
})
|
||||
|
||||
c_msgs += stats["message_count"]
|
||||
c_thds += stats["thread_count"]
|
||||
c_files += stats["attachment_count"]
|
||||
c_size += stats["total_size"]
|
||||
|
||||
cat_lbl = self._format_tree_row(cat_name, c_msgs, c_thds, c_files, self._format_size(c_size))
|
||||
cat_lbl.stylize("bold yellow")
|
||||
node = self.stats_tree.root.add(cat_lbl, expand=True)
|
||||
|
||||
for ch in cat["channels"]:
|
||||
size_str = self._format_size(ch['size']) if ch['is_backed_up'] else "NA"
|
||||
ch_lbl = self._format_tree_row(f" {ch['name']}", ch['msgs'], ch['threads'], ch['files'], size_str)
|
||||
for ch_data in chan_nodes_data:
|
||||
size_str = self._format_size(ch_data['size']) if ch_data['is_backed_up'] else "NA"
|
||||
ch_lbl = self._format_tree_row(f" {ch_data['name']}", ch_data['msgs'], ch_data['threads'], ch_data['files'], size_str)
|
||||
|
||||
if ch['is_backed_up']:
|
||||
if ch_data['is_backed_up']:
|
||||
ch_lbl.stylize("bold white")
|
||||
else:
|
||||
ch_lbl.stylize("dim white") # Textual 'dim' looks like a dull grey
|
||||
ch_lbl.stylize("dim white")
|
||||
|
||||
node.add_leaf(ch_lbl)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
logger.exception("Failed to load backup stats")
|
||||
trace_str = traceback.format_exc()
|
||||
logger.error(f"Full Traceback: {trace_str}")
|
||||
self.query_one("#bs_name", Label).update(f"[red]Error loading data[/red]")
|
||||
self.query_one("#bs_id", Label).update(f"[red]{e}[/red]")
|
||||
finally:
|
||||
if 'reader' in locals():
|
||||
await reader.close()
|
||||
|
|
|
|||
|
|
@ -183,7 +183,6 @@ class OperationPane(Container):
|
|||
self.exporter = DiscordExporter(self.engine.discord_reader, base_dir=self._base_dir())
|
||||
|
||||
def _get_backup_info(self) -> str | None:
|
||||
import json
|
||||
if not self.config or not self.config.discord_server_id:
|
||||
return None
|
||||
|
||||
|
|
@ -191,14 +190,16 @@ class OperationPane(Container):
|
|||
if not target_dir.exists():
|
||||
return None
|
||||
|
||||
profile_file = target_dir / "server_profile" / "profile.json"
|
||||
if not profile_file.exists():
|
||||
db_file = target_dir / "backup.db"
|
||||
if not db_file.exists():
|
||||
return None
|
||||
try:
|
||||
from datetime import datetime
|
||||
with open(profile_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
ts_str = data.get("last_backup")
|
||||
from src.core.backup_database import BackupDatabase
|
||||
db = BackupDatabase(db_file)
|
||||
profile = db.get_guild_profile()
|
||||
if profile:
|
||||
ts_str = profile.get("last_backup")
|
||||
if ts_str:
|
||||
dt = datetime.fromisoformat(ts_str)
|
||||
return dt.strftime("%d-%b-%Y %H:%M")
|
||||
|
|
@ -1767,10 +1768,12 @@ class OperationPane(Container):
|
|||
|
||||
any_found = False
|
||||
backed_up_ids = set()
|
||||
for chan in eligible_channels:
|
||||
if (self.exporter.export_path / "message_backup" / str(chan.id) / "messages.json").exists():
|
||||
any_found = True
|
||||
backed_up_ids.add(chan.id)
|
||||
if self.exporter.db:
|
||||
channel_stats = self.exporter.db.get_stats_by_channel()
|
||||
for chan in eligible_channels:
|
||||
if chan.id in channel_stats:
|
||||
any_found = True
|
||||
backed_up_ids.add(chan.id)
|
||||
|
||||
self.app.pop_screen()
|
||||
|
||||
|
|
@ -1867,6 +1870,8 @@ class OperationPane(Container):
|
|||
modal_prog.write(f"[yellow]Starting backup for {total_chans} channels...[/yellow]")
|
||||
|
||||
accumulated_msgs = 0
|
||||
accumulated_threads = 0
|
||||
accumulated_files = 0
|
||||
|
||||
for i, chan in enumerate(selected_channels):
|
||||
if not self.exporter.is_running:
|
||||
|
|
@ -1874,7 +1879,7 @@ class OperationPane(Container):
|
|||
break
|
||||
await asyncio.sleep(0.01) # Yield to UI thread to keep it responsive
|
||||
|
||||
backup_exists = (self.exporter.export_path / "message_backup" / str(chan.id) / "messages.json").exists()
|
||||
backup_exists = chan.id in backed_up_ids
|
||||
is_sync = backup_exists and not force_overwrite
|
||||
|
||||
label = "Syncing Backup" if is_sync else "Backing up"
|
||||
|
|
@ -1884,20 +1889,17 @@ class OperationPane(Container):
|
|||
logger.info(f"{label} for channel: #{chan.name} ({chan.id})")
|
||||
|
||||
_msg_log_counter = 0
|
||||
async def update_msg_count(name, count, author_name=None, message_preview=None):
|
||||
async def update_msg_count(name, count, author_name=None, message_preview=None, thread_count=0, file_count=0):
|
||||
nonlocal _msg_log_counter
|
||||
modal_prog.update_stats(messages=str(count))
|
||||
modal_prog.update_stats(messages=str(count), threads=str(thread_count), files=str(file_count))
|
||||
_msg_log_counter += 1
|
||||
if author_name and message_preview and _msg_log_counter % 10 == 0:
|
||||
modal_prog.write(f"[bold]{author_name}:[/bold] {message_preview}")
|
||||
|
||||
accumulated_msgs = await self.exporter.export_channel_messages(
|
||||
accumulated_msgs, accumulated_threads, accumulated_files = await self.exporter.export_channel_messages(
|
||||
chan.id, progress_callback=update_msg_count, force=force_overwrite,
|
||||
accumulated_count=accumulated_msgs, after_id=after_id
|
||||
)
|
||||
accumulated_msgs = await self.exporter.export_threads(
|
||||
chan.id, progress_callback=update_msg_count, force=force_overwrite,
|
||||
accumulated_count=accumulated_msgs
|
||||
accumulated_count=accumulated_msgs, accumulated_threads=accumulated_threads, accumulated_files=accumulated_files,
|
||||
after_id=after_id
|
||||
)
|
||||
|
||||
modal_prog.write(f"[green]Completed: {chan.name}[/green]")
|
||||
|
|
@ -1981,9 +1983,14 @@ class OperationPane(Container):
|
|||
]
|
||||
]
|
||||
|
||||
# Get channels that have messages in the database
|
||||
backed_up_channel_ids = set()
|
||||
if self.exporter.db:
|
||||
backed_up_channel_ids = set(self.exporter.db.get_stats_by_channel().keys())
|
||||
|
||||
selected_channels = [
|
||||
c for c in eligible_channels
|
||||
if (self.exporter.export_path / "message_backup" / str(c.id) / "messages.json").exists()
|
||||
if c.id in backed_up_channel_ids
|
||||
]
|
||||
|
||||
if not selected_channels:
|
||||
|
|
@ -1999,6 +2006,8 @@ class OperationPane(Container):
|
|||
modal_prog.cancel_callback = lambda: setattr(self.exporter, "is_running", False)
|
||||
|
||||
accumulated_msgs = 0
|
||||
accumulated_threads = 0
|
||||
accumulated_files = 0
|
||||
|
||||
for i, chan in enumerate(selected_channels):
|
||||
if not self.exporter.is_running:
|
||||
|
|
@ -2012,20 +2021,16 @@ class OperationPane(Container):
|
|||
logger.info(f"Syncing backup for channel: #{chan.name} ({chan.id})")
|
||||
|
||||
_msg_log_counter = 0
|
||||
async def update_msg_count(name, count, author_name=None, message_preview=None):
|
||||
async def update_msg_count(name, count, author_name=None, message_preview=None, thread_count=0, file_count=0):
|
||||
nonlocal _msg_log_counter
|
||||
modal_prog.update_stats(messages=str(count))
|
||||
modal_prog.update_stats(messages=str(count), threads=str(thread_count), files=str(file_count))
|
||||
_msg_log_counter += 1
|
||||
if author_name and message_preview and _msg_log_counter % 10 == 0:
|
||||
modal_prog.write(f"[bold]{author_name}:[/bold] {message_preview}")
|
||||
|
||||
accumulated_msgs = await self.exporter.export_channel_messages(
|
||||
accumulated_msgs, accumulated_threads, accumulated_files = await self.exporter.export_channel_messages(
|
||||
chan.id, progress_callback=update_msg_count, force=False,
|
||||
accumulated_count=accumulated_msgs
|
||||
)
|
||||
accumulated_msgs = await self.exporter.export_threads(
|
||||
chan.id, progress_callback=update_msg_count, force=False,
|
||||
accumulated_count=accumulated_msgs
|
||||
accumulated_count=accumulated_msgs, accumulated_threads=accumulated_threads, accumulated_files=accumulated_files
|
||||
)
|
||||
modal_prog.write(f"[green]Synced: {chan.name}[/green]")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue