add ability to anonymize users
This commit is contained in:
parent
3a86487fc2
commit
518438cb09
6 changed files with 54 additions and 22 deletions
|
|
@ -11,6 +11,7 @@ class AppConfig(BaseModel):
|
||||||
target_bot_token: Optional[str] = Field(default=None)
|
target_bot_token: Optional[str] = Field(default=None)
|
||||||
target_server_id: Optional[str] = Field(default=None)
|
target_server_id: Optional[str] = Field(default=None)
|
||||||
target_api_url: Optional[str] = Field(default=None)
|
target_api_url: Optional[str] = Field(default=None)
|
||||||
|
anonymize_users: bool = Field(default=False)
|
||||||
log_level: str = Field(default="DEBUG")
|
log_level: str = Field(default="DEBUG")
|
||||||
|
|
||||||
# ── backward‑compat shims (read‑only) ────────────────────────────────
|
# ── backward‑compat shims (read‑only) ────────────────────────────────
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ from src.core.utils import resolve_discord_links
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, channel_mentions=None, emoji_map=None, channel_map=None, state=None, target_server_id=None, channel_names=None) -> str:
|
def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, channel_mentions=None, emoji_map=None, channel_map=None, state=None, target_server_id=None, channel_names=None, anonymize_users=False) -> str:
|
||||||
if content is None:
|
if content is None:
|
||||||
return ""
|
return ""
|
||||||
if not content or not guild:
|
if not content or not guild:
|
||||||
|
|
@ -25,6 +25,9 @@ def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None,
|
||||||
|
|
||||||
def replace_user(match):
|
def replace_user(match):
|
||||||
uid = int(match.group(1))
|
uid = int(match.group(1))
|
||||||
|
if anonymize_users and state:
|
||||||
|
alias = state.get_user_alias(str(uid))
|
||||||
|
return f"`@{alias}`"
|
||||||
# 1. Try provided guild
|
# 1. Try provided guild
|
||||||
member = guild.get_member(uid)
|
member = guild.get_member(uid)
|
||||||
if member:
|
if member:
|
||||||
|
|
@ -395,7 +398,8 @@ async def migrate_messages(
|
||||||
context.state.channel_map,
|
context.state.channel_map,
|
||||||
state=context.state,
|
state=context.state,
|
||||||
target_server_id=context.fluxer_writer.community_id,
|
target_server_id=context.fluxer_writer.community_id,
|
||||||
channel_names=context.channel_names if hasattr(context, 'channel_names') else None
|
channel_names=context.channel_names if hasattr(context, 'channel_names') else None,
|
||||||
|
anonymize_users=context.config.anonymize_users
|
||||||
)
|
)
|
||||||
logger.debug(f"Message {msg.id} cleaned content length: {len(content) if content else 0}")
|
logger.debug(f"Message {msg.id} cleaned content length: {len(content) if content else 0}")
|
||||||
|
|
||||||
|
|
@ -429,7 +433,8 @@ async def migrate_messages(
|
||||||
context.state.channel_map,
|
context.state.channel_map,
|
||||||
state=context.state,
|
state=context.state,
|
||||||
target_server_id=context.fluxer_writer.community_id,
|
target_server_id=context.fluxer_writer.community_id,
|
||||||
channel_names=context.channel_names if hasattr(context, 'channel_names') else None
|
channel_names=context.channel_names if hasattr(context, 'channel_names') else None,
|
||||||
|
anonymize_users=context.config.anonymize_users
|
||||||
)
|
)
|
||||||
# Add snapshot attachments to the list to process
|
# Add snapshot attachments to the list to process
|
||||||
attachments_to_process.extend(snapshot.attachments)
|
attachments_to_process.extend(snapshot.attachments)
|
||||||
|
|
@ -549,17 +554,22 @@ async def migrate_messages(
|
||||||
if thread_name and stats["messages"] == 0:
|
if thread_name and stats["messages"] == 0:
|
||||||
content = f"> <<< THREAD: **{thread_name}** >>>\n{content}"
|
content = f"> <<< THREAD: **{thread_name}** >>>\n{content}"
|
||||||
|
|
||||||
avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
|
# Get or generate alias
|
||||||
if avatar_url and not avatar_url.startswith("http"):
|
alias = context.state.get_user_alias(str(msg.author.id))
|
||||||
avatar_url = None
|
|
||||||
|
|
||||||
# Trigger alias generation/storage in DB without replacing the display name yet
|
if context.config.anonymize_users:
|
||||||
context.state.get_user_alias(str(msg.author.id))
|
author_name = alias
|
||||||
|
avatar_url = f"https://api.dicebear.com/9.x/fun-emoji/jpg?seed={alias}"
|
||||||
|
else:
|
||||||
|
author_name = msg.author.display_name
|
||||||
|
avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
|
||||||
|
if avatar_url and not avatar_url.startswith("http"):
|
||||||
|
avatar_url = None
|
||||||
|
|
||||||
logger.debug(f"Fluxer: Calling send_message for Discord ID {msg.id}")
|
logger.debug(f"Fluxer: Calling send_message for Discord ID {msg.id}")
|
||||||
fluxer_msg_id = await context.fluxer_writer.send_message(
|
fluxer_msg_id = await context.fluxer_writer.send_message(
|
||||||
channel_id=target_channel_id,
|
channel_id=target_channel_id,
|
||||||
author_name=msg.author.display_name,
|
author_name=author_name,
|
||||||
author_avatar_url=avatar_url,
|
author_avatar_url=avatar_url,
|
||||||
content=content,
|
content=content,
|
||||||
timestamp=int(msg.created_at.timestamp()),
|
timestamp=int(msg.created_at.timestamp()),
|
||||||
|
|
|
||||||
|
|
@ -285,7 +285,7 @@ class FluxerWriter:
|
||||||
# -# is Fluxer/Discord's subtext markdown: small, muted grey text
|
# -# is Fluxer/Discord's subtext markdown: small, muted grey text
|
||||||
prefix = f"-# <t:{timestamp}:D>\n"
|
prefix = f"-# <t:{timestamp}:D>\n"
|
||||||
if is_forwarded:
|
if is_forwarded:
|
||||||
prefix += "-# -->*forwarded*\n"
|
prefix += "-# ⤷*forwarded*\n"
|
||||||
|
|
||||||
display_content = content
|
display_content = content
|
||||||
if is_forwarded and content:
|
if is_forwarded and content:
|
||||||
|
|
@ -345,7 +345,7 @@ class FluxerWriter:
|
||||||
# We add the author name to the prefix since bot name won't match
|
# We add the author name to the prefix since bot name won't match
|
||||||
bot_prefix = f"-# <t:{timestamp}:D>\n"
|
bot_prefix = f"-# <t:{timestamp}:D>\n"
|
||||||
if is_forwarded:
|
if is_forwarded:
|
||||||
bot_prefix += "-# -->*forwarded*\n"
|
bot_prefix += "-# ⤷*forwarded*\n"
|
||||||
bot_prefix += f"-# · {author_name}\n"
|
bot_prefix += f"-# · {author_name}\n"
|
||||||
|
|
||||||
final_bot_content = bot_prefix + display_content if display_content else bot_prefix
|
final_bot_content = bot_prefix + display_content if display_content else bot_prefix
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ from src.core.utils import resolve_discord_links
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, channel_mentions=None, emoji_map=None, channel_map=None, state=None, target_server_id=None, channel_names=None) -> str:
|
def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None, channel_mentions=None, emoji_map=None, channel_map=None, state=None, target_server_id=None, channel_names=None, anonymize_users=False) -> str:
|
||||||
if content is None:
|
if content is None:
|
||||||
return ""
|
return ""
|
||||||
if not content or not guild:
|
if not content or not guild:
|
||||||
|
|
@ -25,6 +25,9 @@ def clean_mentions(content: str, guild, user_mentions=None, role_mentions=None,
|
||||||
|
|
||||||
def replace_user(match):
|
def replace_user(match):
|
||||||
uid = int(match.group(1))
|
uid = int(match.group(1))
|
||||||
|
if anonymize_users and state:
|
||||||
|
alias = state.get_user_alias(str(uid))
|
||||||
|
return f"`@{alias}`"
|
||||||
# 1. Try provided guild
|
# 1. Try provided guild
|
||||||
member = guild.get_member(uid)
|
member = guild.get_member(uid)
|
||||||
if member:
|
if member:
|
||||||
|
|
@ -398,7 +401,8 @@ async def migrate_messages(
|
||||||
context.state.channel_map,
|
context.state.channel_map,
|
||||||
state=context.state,
|
state=context.state,
|
||||||
target_server_id=context.stoat_writer.community_id,
|
target_server_id=context.stoat_writer.community_id,
|
||||||
channel_names=context.channel_names if hasattr(context, 'channel_names') else None
|
channel_names=context.channel_names if hasattr(context, 'channel_names') else None,
|
||||||
|
anonymize_users=context.config.anonymize_users
|
||||||
)
|
)
|
||||||
logger.debug(f"Message {msg.id} cleaned content length: {len(content) if content else 0}")
|
logger.debug(f"Message {msg.id} cleaned content length: {len(content) if content else 0}")
|
||||||
|
|
||||||
|
|
@ -428,7 +432,8 @@ async def migrate_messages(
|
||||||
context.state.channel_map,
|
context.state.channel_map,
|
||||||
state=context.state,
|
state=context.state,
|
||||||
target_server_id=context.stoat_writer.community_id,
|
target_server_id=context.stoat_writer.community_id,
|
||||||
channel_names=context.channel_names if hasattr(context, 'channel_names') else None
|
channel_names=context.channel_names if hasattr(context, 'channel_names') else None,
|
||||||
|
anonymize_users=context.config.anonymize_users
|
||||||
)
|
)
|
||||||
attachments_to_process.extend(snapshot.attachments)
|
attachments_to_process.extend(snapshot.attachments)
|
||||||
logger.debug(f"Found forwarded snapshot content: {content[:50]}... and {len(snapshot.attachments)} attachments")
|
logger.debug(f"Found forwarded snapshot content: {content[:50]}... and {len(snapshot.attachments)} attachments")
|
||||||
|
|
@ -546,16 +551,21 @@ async def migrate_messages(
|
||||||
if thread_name and stats["messages"] == 0:
|
if thread_name and stats["messages"] == 0:
|
||||||
content = f"> <<< THREAD: **{thread_name}** >>>\n{content}"
|
content = f"> <<< THREAD: **{thread_name}** >>>\n{content}"
|
||||||
|
|
||||||
avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
|
# Get or generate alias
|
||||||
if avatar_url and not avatar_url.startswith("http"):
|
alias = context.state.get_user_alias(str(msg.author.id))
|
||||||
avatar_url = None
|
|
||||||
|
|
||||||
# Trigger alias generation/storage in DB without replacing the display name yet
|
if context.config.anonymize_users:
|
||||||
context.state.get_user_alias(str(msg.author.id))
|
author_name = alias
|
||||||
|
avatar_url = f"https://api.dicebear.com/9.x/fun-emoji/jpg?seed={alias}"
|
||||||
|
else:
|
||||||
|
author_name = msg.author.display_name
|
||||||
|
avatar_url = str(msg.author.display_avatar.url) if msg.author.display_avatar.url else None
|
||||||
|
if avatar_url and not avatar_url.startswith("http"):
|
||||||
|
avatar_url = None
|
||||||
|
|
||||||
stoat_msg_id = await context.stoat_writer.send_message(
|
stoat_msg_id = await context.stoat_writer.send_message(
|
||||||
channel_id=target_channel_id,
|
channel_id=target_channel_id,
|
||||||
author_name=msg.author.display_name,
|
author_name=author_name,
|
||||||
author_avatar_url=avatar_url,
|
author_avatar_url=avatar_url,
|
||||||
content=content,
|
content=content,
|
||||||
timestamp=int(msg.created_at.timestamp()),
|
timestamp=int(msg.created_at.timestamp()),
|
||||||
|
|
|
||||||
|
|
@ -316,7 +316,7 @@ class StoatWriter:
|
||||||
# Build content with timestamp prefix
|
# Build content with timestamp prefix
|
||||||
prefix = f"###### <t:{timestamp}:D>\n"
|
prefix = f"###### <t:{timestamp}:D>\n"
|
||||||
if is_forwarded:
|
if is_forwarded:
|
||||||
prefix += "##### -->*forwarded*\n"
|
prefix += "##### ⤷*forwarded*\n"
|
||||||
|
|
||||||
display_content = content
|
display_content = content
|
||||||
if is_forwarded and content:
|
if is_forwarded and content:
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ from textual.app import App, ComposeResult
|
||||||
from textual.containers import Container, Horizontal, Vertical, VerticalScroll, Center
|
from textual.containers import Container, Horizontal, Vertical, VerticalScroll, Center
|
||||||
from textual.widgets import (
|
from textual.widgets import (
|
||||||
Header, Footer, Button, Label, Input, ListItem,
|
Header, Footer, Button, Label, Input, ListItem,
|
||||||
ListView, Rule, RadioButton, RadioSet, Select, Static, Markdown
|
ListView, Rule, RadioButton, RadioSet, Select, Static, Markdown, Switch
|
||||||
)
|
)
|
||||||
from textual.screen import Screen, ModalScreen
|
from textual.screen import Screen, ModalScreen
|
||||||
|
|
||||||
|
|
@ -252,6 +252,8 @@ class ConfigScreen(Screen):
|
||||||
.fetch_row Input { width: 1fr; }
|
.fetch_row Input { width: 1fr; }
|
||||||
.fetch_row Button { width: auto; margin-left: 1; }
|
.fetch_row Button { width: auto; margin-left: 1; }
|
||||||
#inp_discord_server { margin-bottom: 1; }
|
#inp_discord_server { margin-bottom: 1; }
|
||||||
|
.switch_row { height: auto; align: left middle; margin-top: 1; margin-bottom: 1; }
|
||||||
|
#lbl_anonymize { margin-right: 2; margin-top: 0; }
|
||||||
"""
|
"""
|
||||||
|
|
||||||
BINDINGS = [("escape", "go_back", "Back")]
|
BINDINGS = [("escape", "go_back", "Back")]
|
||||||
|
|
@ -349,6 +351,13 @@ class ConfigScreen(Screen):
|
||||||
tooltip="Enter the custom API url\nfor self hosted instances"
|
tooltip="Enter the custom API url\nfor self hosted instances"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
yield Horizontal(
|
||||||
|
Label("Anonymize Users:", id="lbl_anonymize"),
|
||||||
|
Switch(value=self.config.anonymize_users, id="inp_anonymize_users", tooltip="Anonymize user Names and Avatars during migration"),
|
||||||
|
id="anonymize_row",
|
||||||
|
classes="switch_row"
|
||||||
|
)
|
||||||
|
|
||||||
yield Rule(id="footer_rule")
|
yield Rule(id="footer_rule")
|
||||||
with Horizontal(id="cfg_actions"):
|
with Horizontal(id="cfg_actions"):
|
||||||
yield Button("Save Configuration", variant="success", id="btn_save", tooltip="Save all changes to config.yaml")
|
yield Button("Save Configuration", variant="success", id="btn_save", tooltip="Save all changes to config.yaml")
|
||||||
|
|
@ -514,6 +523,8 @@ class ConfigScreen(Screen):
|
||||||
|
|
||||||
target_api = self.query_one("#inp_target_api", Input).value.strip()
|
target_api = self.query_one("#inp_target_api", Input).value.strip()
|
||||||
self.config.target_api_url = target_api or None
|
self.config.target_api_url = target_api or None
|
||||||
|
|
||||||
|
self.config.anonymize_users = self.query_one("#inp_anonymize_users", Switch).value
|
||||||
else:
|
else:
|
||||||
self.config.target_platform = "none"
|
self.config.target_platform = "none"
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue