#!/usr/bin/env python3
"""Scramble node text in a Tana workspace snapshot JSON file, keeping its graph.

Usage: python3 -I scramble_tana.py <in.json> <out.json>
       python3 -I scramble_tana.py --verify <in.json> <out.json> [sensitive-term ...]

- Scrub node text, cached names, code, AI payload text and editor account labels.
- Each word maps to a same-length pseudo-word via HMAC with a random per-run key
  (never stored). Word lengths, digit counts, case and repetitions remain visible.
- Markup tags, inline node refs and inline dates are kept; hrefs, URLs and emails
  are replaced with example.com placeholders.
- Keep built-in names, expressions, calendar dates, settings and view operators.
- Editor emails become editor<n>@example.com.

Links in node text, including attachment links, become placeholders.
Graph and JSON protocol metadata remain and need review before sharing.
Free-text substring/regex searches and untyped numeric literals can change meaning.
Unsupported regex escapes stop the run and need manual scrubbing.
"""
import argparse
import collections
import hashlib
import hmac
import html
import json
import re
import secrets
import sys
from zoneinfo import available_timezones

KEY = secrets.token_bytes(32)

LOWER = 'abcdefghijklmnopqrstuvwxyz'
TWO_DIGITS = tuple(f'{value:02}' for value in range(100))
OTHER_LETTERS = ''.join(chr(i) for i in range(0xc0, 0x530) if chr(i).isalpha() and chr(i).islower() and len(chr(i).upper()) == 1)
OTHER_NUMBERS = '¼½¾' + ''.join(chr(i) for i in range(0x2150, 0x218a) if chr(i).isnumeric())
CJK_POOL = (
    '的一是不了人我在有他这中大来上国个到说们为子和你地出道也时年得就那要下以生会自着去之过家学对可她里后小么心多天而能好都然没日于起还发成事只作当想看文无开手十用主行方又如前所本见经头面公同三已老从动两长知民样现分将外但身些与高意进把法此实回二理美点月明其种声全工己话儿者向情部正名定女问力机给等几很业最间新什打便位因重被走电四第门相次东政海口使教西再平真听世气信北少关并内加化由却代军产入先山五太水万市眼体别处总才场师书比住员九笑性通目华报立马命张活难神数件安表原车白应路期叫死常提感金何更反合放做系计或司利受光王果亲界及今京务制解各任至清物台象记边共风战干接它许八特觉望直服毛林题建南度统色字请交爱让认算论百吃义科怎元社术结六功指思非流每青管夫连远资队跟带花快条院变联言权往展该领传近留红治决周保达办运武半候七必城父强步完革深区'
)

URL_RE = re.compile(r'''(?:https?|ftp)://[^\s<>\"']+|www\.(?=[^\W_])[^\s<>\"']+''', re.IGNORECASE)
EMAIL_RE = re.compile(r'[\w.+-]+@[\w-]+(?:\.[\w-]+)+')
TAG_RE = re.compile(r'''<!--.*?-->|<(?:(?:"[^"]*"|'[^']*')|[^'">])*>''', re.DOTALL)
ATTR_RE = re.compile(r'''([\w:-]+)(\s*=\s*)(?:(["'])(.*?)\3|([^\s>]+))''', re.DOTALL)
TOKEN_RE = re.compile(r'[^\W\d_]+|\d+', re.UNICODE)
SYSTEM_ID_RE = re.compile(r'^SYS_')
INTERPOLATION_RE = re.compile(r'\$\{([^}]*)\}')
CALENDAR_PREFIX_RE = re.compile(r'^(?:\d{4}(?:-(?:W\d{2}|\d{2})(?:-\d{2})?)?|Week \d{1,2})(?=$|\s)')
RELATIVE_RE = re.compile(r'^(PARENT|GRANDPARENT|INSTANCE(?: PARENT)?)(?:\.([^+\-]*))?([+\-]\d+(?:\s*(?:days?|weeks?|months?|years?|[dDwWmMyY]))?)?$')
SEARCH_KEYWORDS = {
    'AND', 'OR', 'NOT', 'PARENTS DESCENDANTS', 'GRANDPARENTS DESCENDANTS',
    'PARENTS DESCENDANTS WITH REFS', 'GRANDPARENTS DESCENDANTS WITH REFS',
    'PARENTS DESCENDANTS WITH NON SEARCH REFS', 'GRANDPARENTS DESCENDANTS WITH NON SEARCH REFS',
    'HAS ATTRIBUTE', 'HAS FIELD', 'HAS TAG', 'IS PUBLISHED', 'HAS MEDIA', 'HAS AUDIO',
    'HAS VIDEO', 'HAS IMAGE', 'IS SEARCH NODE', 'IS CHAT', 'IN LIBRARY', 'ON DAY NODE',
    'IS CALENDAR NODE', 'ANY ACCESSIBLE WORKSPACE', 'DONE', 'NOT DONE', 'TODO',
    'IS TEMPLATE', 'IS TAG', 'IS FIELD', 'FROM CALENDAR', 'FROM CAPTURE', 'FROM MOBILE',
    'IS COMMAND', 'IS CODE', 'IS ENTITY'
}
SEARCH_DATE_RE = re.compile(
    r'^(?:FOR RELATIVE DATE (?:TODAY|YESTERDAY|TOMORROW|NOW|(?:THIS|LAST|NEXT)(?: \d+)? (?:DAY|WEEK|MONTH|YEAR)S?)'
    r'|(?:CREATED|DONE|EDITED) LAST \d+ DAYS?|OVERDUE(?: \d+ DAYS?)?)$',
    re.IGNORECASE
)
WEEKDAYS = {'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'}
SEARCH_ROOT_ATTRIBUTES = {'SYS_A15', 'SYS_A18', 'SYS_MP005'}
EXPRESSION_NAMES = {'name', 'cdate', 'ctime', 'mdate', 'mtime', 'start', 'end'}
SYSTEM_EXPRESSION_NAMES = {
    'description', 'createdAt', 'lastModifiedAt', 'lastModifiedBy', 'lastEditedAt',
    'lastEditedBy', 'modifiedBy', 'editedBy', 'owner', 'dateFromDayNode',
    'dateFromCalendarNode', 'doneTime', 'nodeId', 'nodeURL', 'tags', 'context',
    'source', 'chat', 'sourceOrContent', 'content', 'currentDate', 'currentDateTime',
    'userInput', 'actionItemTags'
}
STRUCTURAL_ATTRIBUTES = {
    'data-inlineref-node', 'data-inlineref-date', 'class', 'style', 'lang', 'dir',
    'role', 'contenteditable', 'target', 'rel', 'width', 'height', 'href', 'src'
}
HTML_TAGS = {
    'a', 'b', 'i', 'u', 'em', 'strong', 'span', 'mark', 'del', 'strike', 's',
    'code', 'pre', 'div', 'p', 'br', 'hr', 'blockquote', 'img', 'video', 'audio',
    'source', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'ul', 'ol', 'li',
    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'sub', 'sup'
}
SETTINGS_SECTIONS = {
    'Plugins', 'Notifications seen', 'Journaling', 'Progress stats', 'Preferences',
    'Private keyboard shortcuts', 'Search', 'Google Calendar settings',
    'Mobile app settings', 'Move targets', 'Date/Time settings', 'API Keys', 'Publishing'
}
SETTINGS_KEYS = {
    'Preferences': set('''Enable dark mode
Use system theme
Show sidebar
Warn when editing referenced nodes
Show release notes
Notify about release notes
Notify about Tana Labs and live streams
Notify about tutorials and videos
Notify about Tana Currents
Show URL embeds
Show reference counter
Hide sidebar
Show Inbox in sidebar
Show Daily Notes in sidebar
Show AI Chat in sidebar
Show Recent in sidebar
Show Supertags in sidebar
Show image generation warning
Show credits in topbar
Show keyboard shortcuts in topbar
Show help in topbar
Collapse references section by default
Collapse hidden fields
Save audio from transcriptions
Show reminder content in system notifications
Default content width
Default recording tag
Default transcription language
DateTime preferences'''.splitlines()),
    'Journaling': set('''day
week
month
year
Default journal'''.splitlines()),
    'Search': set('''Exclude'''.splitlines()),
    'Mobile app settings': set('''Share to Tana tag'''.splitlines()),
    'Google Calendar settings': set('''Event supertag
Attendee supertag
Date field
Attendee email field
Attendees field
Enabled Calendars
Event URL field
Event meet link URL
Event details field
Event Status field
Event Destination
Contacts Destination
Google Calendar Events
Google Calendar Contacts
Event Classification Config'''.splitlines()),
    'User guidance': set('''Supertag configuration
Field configuration
Today node
Search node
References and inline references
Navigating in Tana
Calendars view
Custom commands
AI-enhanced fields
Audio-enabled fields
Audio-enabled tags
Prompt Workbench
Capture Inbox
Right-click on nodes
AI Chat
Quick start guides
Quick start guide for supertags
Quick start guide for meetings
Quick start guide for voice memos
Show account upgrade notice
Voice Memo Notice
Invite Meeting Agent
Calendar classification
Save audio during transcription'''.splitlines()),
    'Promotions': set('''Invite a friend'''.splitlines()),
    'Notifications seen': set('''Remaining invites
Pending workspaces'''.splitlines())
}
TRANSCRIPTION_LANGUAGES = {
    'auto', 'English', 'Afrikaans', 'Albanian', 'Amharic', 'Arabic', 'Armenian', 'Assamese',
    'Azerbaijani', 'Bashkir', 'Belarusian', 'Bengali', 'Bosnian', 'Breton', 'Bulgarian',
    'Cantonese', 'Catalan', 'Chinese', 'Croatian', 'Czech', 'Danish', 'Dutch', 'Estonian',
    'Faroese', 'Finnish', 'French', 'Galician', 'Georgian', 'German', 'Greek', 'Gujarati',
    'Haitian creole', 'Hausa', 'Hawaiian', 'Hebrew', 'Hindi', 'Hungarian', 'Icelandic',
    'Indonesian', 'Italian', 'Japanese', 'Javanese', 'Kannada', 'Kazakh', 'Khmer', 'Korean', 'Lao',
    'Latin', 'Latvian', 'Lingala', 'Lithuanian', 'Luxembourgish', 'Macedonian', 'Malagasy',
    'Malay', 'Malayalam', 'Maltese', 'Maori', 'Marathi', 'Mongolian', 'Myanmar', 'Nepali',
    'Norwegian', 'Occitan', 'Pashto', 'Persian', 'Polish', 'Portuguese', 'Punjabi', 'Romanian',
    'Russian', 'Sanskrit', 'Serbian', 'Shona', 'Sindhi', 'Sinhala', 'Slovak', 'Slovenian',
    'Somali', 'Spanish', 'Sundanese', 'Swahili', 'Swedish', 'Tagalog', 'Tajik', 'Tamil', 'Tatar',
    'Telugu', 'Thai', 'Tibetan', 'Turkish', 'Turkmen', 'Ukrainian', 'Urdu', 'Uzbek', 'Vietnamese',
    'Welsh', 'Yiddish', 'Yoruba'
}
EVENT_CLASSES = {'meeting', 'block', 'full day event', 'task', 'investor meeting', 'user interview', 'hiring interview', 'out of office', 'two person meeting', 'one-on-one', 'standup'}
FORMULA_FUNCTIONS = {
    'top', 'first', 'pickRandom', 'bottom', 'last', 'sum', 'subtract', 'count', 'max',
    'min', 'average', 'mean', 'round', '+', '-', 'multiply', '*', 'divide', '/',
    'findInstance', 'findFieldValues', 'findInstances', 'lookup', 'lookupField',
    'filter', 'childrenOf', 'nodesBelow', 'SELF', 'CREATED', 'CURRENT_USER',
    'CURRENT_USER_REF', 'CURRENT_DATE_REF', 'CURRENT_DATETIME_REF', 'concat',
    'strlen', 'blockFilter', 'uniq', 'backlink', 'ownerOf', 'formatDate', 'emojiCount'
}
FORMULA_NO_PARAMS = {'SELF', 'CREATED', 'CURRENT_USER', 'CURRENT_USER_REF', 'CURRENT_DATE_REF', 'CURRENT_DATETIME_REF'}
SORT_DIRECTIONS = {'ASC', 'DESC', 'NUM_ASC', 'NUM_DESC'}
CALCULATIONS = {'SUM', 'AVG', 'MEDIAN', 'MIN', 'MAX', 'COUNT_ALL', 'COUNT_VALUES', 'COUNT_UNIQUE'}
TIMEZONES = available_timezones()
CONTENT_FIELDS = ('name', 'description', '_lastComputedName', '_textContent', '_jsonData')
JSON_METADATA_KEYS = {
    '$schema', '$ref', 'allowedTools', 'format', 'function', 'id', 'mediaType', 'model',
    'required', 'role', 'status', 'timestamp', 'thoughtSignature', 'tool', 'toolCallId',
    'toolName', 'tools', 'type'
}
JSON_PROTOCOL_VALUES = {
    'role': {'user', 'assistant', 'system', 'tool', 'greeting'},
    'status': {'pending', 'approved', 'rejected'},
    'type': {
        'user_message', 'assistant_message', 'system_message', 'greeting_message',
        'tool_response', 'tool_call', 'text', 'image', 'file', 'reasoning',
        'tool-call', 'tool-result', 'tool_use', 'function', 'json', 'error',
        'object', 'array', 'string', 'number', 'integer', 'boolean', 'null'
    }
}
_JSON_CONTENT_MARKER = object()

_cache = {}
_placeholders = {'url': 0, 'email': 0}
_redactions = {}
_view_accounts = {}
_reverse_cache = {
    token.casefold(): None
    for name in SEARCH_KEYWORDS | EXPRESSION_NAMES | SYSTEM_EXPRESSION_NAMES | WEEKDAYS | SETTINGS_SECTIONS | FORMULA_FUNCTIONS | SORT_DIRECTIONS | CALCULATIONS | {'Default', 'Calendar', 'Months', 'Week'}
    for token in TOKEN_RE.findall(name)
}
_short_maps = {}
_preserved_fields = set(EXPRESSION_NAMES)


def _stream(token):
    return hmac.new(KEY, token.encode('utf-8'), hashlib.sha256).digest()


def scramble_token(tok):
    hit = _cache.get(tok)
    if hit is not None:
        return hit
    canonical = tok.casefold()
    for alphabet in (LOWER, '0123456789', TWO_DIGITS):
        if alphabet not in _short_maps:
            shuffled = sorted(alphabet, key=lambda c: _stream('single:' + c))
            for i, ch in enumerate(alphabet):
                if shuffled[i] == ch:
                    other = (i + 1) % len(alphabet)
                    shuffled[i], shuffled[other] = shuffled[other], shuffled[i]
            _short_maps[alphabet] = dict(zip(alphabet, shuffled))
            _reverse_cache.update({output: source for source, output in _short_maps[alphabet].items()})
    if len(tok) <= 2 and tok.isascii() and tok.isdigit():
        alphabet = '0123456789' if len(tok) == 1 else TWO_DIGITS
        res = _short_maps[alphabet][tok]
        _cache[tok] = res
        return res
    if len(tok) == 1 and tok.isascii() and tok.isalpha():
        alphabet = LOWER
        res = _short_maps[alphabet][canonical]
        res = res.upper() if tok.isupper() else res
        _cache[tok] = res
        return res
    for attempt in range(2048):
        h = _stream(canonical + (':' + str(attempt) if attempt else ''))
        out = []
        for i, ch in enumerate(tok):
            b = int.from_bytes(h[2 * i % 32:2 * i % 32 + 2], 'big') ^ (i * 131 & 0xFF)
            if ch.isascii() and ch.isdigit():
                out.append(str(b % 10))
            elif '⺀' <= ch <= '鿿' or '豈' <= ch <= '\ufaff':
                out.append(CJK_POOL[(b + i) % len(CJK_POOL)])
            elif ch.isnumeric():
                out.append(OTHER_NUMBERS[b % len(OTHER_NUMBERS)])
            elif ch.isalpha() and not ch.isascii():
                c = OTHER_LETTERS[b % len(OTHER_LETTERS)]
                out.append(c.upper() if ch.isupper() else c)
            elif ch.isalpha():
                c = LOWER[b % len(LOWER)]
                out.append(c.upper() if ch.isupper() else c)
            else:
                out.append(ch)
        res = ''.join(out)
        output_key = res.casefold()
        if output_key.endswith(('www', 'http', 'https', 'ftp')) or output_key == canonical or (output_key in _reverse_cache and _reverse_cache[output_key] != canonical):
            continue
        _reverse_cache[output_key] = canonical
        _cache[tok] = res
        return res
    raise ValueError('Could not create a distinct scrambled word of length ' + str(len(tok)))


def _placeholder(kind):
    _placeholders[kind] += 1
    n = _placeholders[kind]
    return f'https://example.com/u{n}' if kind == 'url' else f'user{n}@example.com'


def redact(kind, value):
    pattern = r'https://example\.com/u\d+' if kind == 'url' else r'(?:user|editor)\d+@example\.com'
    if re.fullmatch(pattern, value):
        return value
    key = (kind, value.casefold() if kind == 'email' else value)
    if key not in _redactions:
        _redactions[key] = _placeholder(kind)
    return _redactions[key]


def scramble_interpolation(expression):
    field_path, separator, formatting = expression.partition('|')
    steps = field_path.split('.')
    for i, step in enumerate(steps):
        if step in _preserved_fields or (
            step.startswith('sys:') and step[4:] in SYSTEM_EXPRESSION_NAMES
        ):
            continue
        steps[i] = scramble_plain(step)
    if not re.fullmatch(r'\d*(?:\.\.\.|…)?n?\??', formatting):
        formatting = TOKEN_RE.sub(lambda m: scramble_token(m.group()), formatting)
    return '${' + '.'.join(steps) + separator + formatting + '}'


def scramble_field_path(path):
    return scramble_interpolation(path)[2:-1]


def scramble_plain(text):
    text = URL_RE.sub(lambda m: redact('url', m.group()), text)
    text = EMAIL_RE.sub(lambda m: redact('email', m.group()), text)
    parts = re.split(r'(\$\{[^}]*\}|https://example\.com/u\d+|(?:user|editor)\d+@example\.com)', text)
    for i in range(0, len(parts), 2):
        parts[i] = TOKEN_RE.sub(lambda m: scramble_token(m.group()), parts[i])
    for i in range(1, len(parts), 2):
        match = INTERPOLATION_RE.fullmatch(parts[i])
        if match:
            parts[i] = scramble_interpolation(match[1])
    return ''.join(parts)


def scramble_tag(tag):
    if tag.startswith('<!--'):
        return '<!--' + scramble_plain(html.unescape(tag[4:-3])) + '-->'
    tag_name = re.match(r'^</?([a-zA-Z][a-zA-Z0-9]*)(?=[\s/>])', tag)
    if not tag_name or tag_name[1].lower() not in HTML_TAGS:
        return html.escape(scramble_plain(html.unescape(tag)), quote=False)
    def attribute(match):
        name, equals, quote, quoted_value, bare_value = match.groups()
        quote = quote or ''
        value = quoted_value if quoted_value is not None else bare_value
        decoded = html.unescape(value)
        attribute_name = name.casefold()
        if attribute_name in {'href', 'src'}:
            value = html.escape(redact('url', decoded), quote=True)
        elif attribute_name == 'data-inlineref-user':
            value = html.escape(_view_accounts.get(decoded) or scramble_plain(decoded), quote=True)
        elif attribute_name in STRUCTURAL_ATTRIBUTES:
            redacted = EMAIL_RE.sub(lambda m: redact('email', m.group()),
                                    URL_RE.sub(lambda m: redact('url', m.group()), decoded))
            if redacted != decoded:
                value = html.escape(redacted, quote=True)
        else:
            value = html.escape(scramble_plain(decoded), quote=True)
        if not quote and re.search(r'''[\s"'`=<>]''', value):
            quote = '"'
        return name + equals + quote + value + quote
    return URL_RE.sub(lambda m: redact('url', html.unescape(m.group())), ATTR_RE.sub(attribute, tag))


def scramble_text(s):
    out = []
    last = 0
    for m in TAG_RE.finditer(s):
        out.append(html.escape(scramble_plain(html.unescape(s[last:m.start()])), quote=False))
        out.append(scramble_tag(m.group()))
        last = m.end()
    out.append(html.escape(scramble_plain(html.unescape(s[last:])), quote=False))
    return ''.join(out)


def search_clause_ids(docs):
    nodes = {node['id']: node for node in docs}
    pending = [node['id'] for node in docs if (node.get('children') or [None])[0] in SEARCH_ROOT_ATTRIBUTES]
    seen = set()
    while pending:
        node_id = pending.pop()
        if node_id in seen:
            continue
        seen.add(node_id)
        node = nodes.get(node_id)
        if node and node.get('props', {}).get('_docType') == 'tuple':
            pending.extend(node.get('children', [])[1:])
    return seen


def calendar_container_ids(docs):
    nodes = {node['id']: node for node in docs}
    containers = set()
    for node in docs:
        props = node.get('props') or {}
        parent = nodes.get(props.get('_ownerId'), {}).get('props') or {}
        if props.get('name') == 'Calendar' and (
            props.get('_docType') == 'journal' or parent.get('_docType') == 'home'
        ):
            containers.add(node['id'])
        if props.get('name') == 'Months' and parent.get('_docType') == 'journalPart' and re.fullmatch(r'\d{4}', parent.get('name', '')):
            containers.add(node['id'])
    children = {}
    for node in docs:
        owner = (node.get('props') or {}).get('_ownerId')
        children.setdefault(owner, []).append(node)
    pending = [(node_id, 'root') for node_id in containers if nodes[node_id]['props'].get('name') == 'Calendar']
    visited = set()
    while pending:
        parent_id, kind = pending.pop()
        if parent_id in visited:
            continue
        visited.add(parent_id)
        for child in children.get(parent_id, []):
            name = (child.get('props') or {}).get('name', '')
            child_kind = None
            if kind == 'root' and re.fullmatch(r'\d{4}', name):
                child_kind = 'year'
            elif kind == 'year' and name == 'Months':
                child_kind = 'months'
            elif kind == 'year' and re.fullmatch(r'Week \d{1,2}', name):
                child_kind = 'week'
            elif kind == 'months' and re.fullmatch(r'\d{4}-\d{2}', name):
                child_kind = 'month'
            elif kind == 'week' and re.fullmatch(r'\d{4}-\d{2}-\d{2} - (?:' + '|'.join(WEEKDAYS) + ')', name):
                child_kind = 'day'
            if child_kind:
                containers.add(child['id'])
                pending.append((child['id'], child_kind))
    return containers


def reserved_name_ids(docs):
    reserved = calendar_container_ids(docs)
    nodes = {node['id']: node for node in docs}
    for node in docs:
        children = node.get('children') or []
        if (node.get('props') or {}).get('_docType') != 'tuple' or not children:
            continue
        if children[0] == 'SYS_A98':
            reserved.update(child_id for child_id in children[1:] if (nodes.get(child_id, {}).get('props') or {}).get('name') in TIMEZONES)
        if len(children) > 2 and children[0] in {'SYS_A20', 'SYS_A72'}:
            valid_names = SORT_DIRECTIONS if children[0] == 'SYS_A20' else CALCULATIONS
            if (nodes.get(children[2], {}).get('props') or {}).get('name') in valid_names:
                reserved.add(children[2])
        if children[0] != 'SYS_A69':
            continue
        for settings_id in children[1:]:
            settings = nodes.get(settings_id, {})
            if (settings.get('props') or {}).get('name') == 'Settings':
                reserved.add(settings_id)
            for child_id in settings.get('children', []):
                child = nodes.get(child_id, {})
                name = (child.get('props') or {}).get('name')
                if name not in SETTINGS_SECTIONS:
                    continue
                reserved.add(child_id)
                for nested_id in child.get('children', []):
                    nested = nodes.get(nested_id, {})
                    nested_name = (nested.get('props') or {}).get('name')
                    if name == 'Progress stats' and nested_name in {'User guidance', 'Promotions'}:
                        reserved.add(nested_id)
                    if name == 'Date/Time settings' and nested_name == 'Display timezone':
                        reserved.add(nested_id)
                        reserved.update(value_id for value_id in nested.get('children', []) if (nodes.get(value_id, {}).get('props') or {}).get('name') in TIMEZONES)
                pending_sections = [(child_id, name)]
                while pending_sections:
                    section_id, section_name = pending_sections.pop()
                    for tuple_id in nodes.get(section_id, {}).get('children', []):
                        setting = nodes.get(tuple_id, {})
                        setting_name = (setting.get('props') or {}).get('name')
                        if section_name == 'Progress stats' and setting_name in {'User guidance', 'Promotions'}:
                            pending_sections.append((tuple_id, setting_name))
                        values = setting.get('children') or []
                        if (setting.get('props') or {}).get('_docType') != 'tuple' or not values:
                            continue
                        key = (nodes.get(values[0], {}).get('props') or {}).get('name')
                        if key not in SETTINGS_KEYS.get(section_name, ()):
                            continue
                        reserved.add(values[0])
                        enums = ()
                        if key == 'Default content width':
                            enums = {'auto', 'medium', 'full'}
                            for options_id in nodes.get(values[0], {}).get('children', []):
                                options = nodes.get(options_id, {}).get('children') or []
                                if options and options[0] == 'SYS_T03':
                                    reserved.update(value_id for value_id in options[1:] if (nodes.get(value_id, {}).get('props') or {}).get('name') in enums)
                        elif key == 'Default transcription language':
                            enums = TRANSCRIPTION_LANGUAGES
                        elif key == 'Event Destination':
                            enums = {'day'}
                        elif key == 'Event Classification Config':
                            enums = EVENT_CLASSES
                        reserved.update(value_id for value_id in values[1:] if (nodes.get(value_id, {}).get('props') or {}).get('name') in enums)
                        if key == 'Event Classification Config':
                            for config_id in values[1:]:
                                reserved.update(value_id for value_id in nodes.get(config_id, {}).get('children', []) if (nodes.get(value_id, {}).get('props') or {}).get('name') in enums)
    formula_attributes = {
        node['id'] for node in docs
        if any((nodes.get(child_id, {}).get('children') or [])[:2] in [['SYS_T06', 'SYS_D07'], ['SYS_A26', 'SYS_V03']] for child_id in node.get('children', []))
    }
    pending = [(node['children'][1], None) for node in docs if (node.get('props') or {}).get('_docType') == 'tuple' and len(node.get('children', [])) > 1 and node['children'][0] in formula_attributes]
    visited = set()
    while pending:
        node_id, parent_id = pending.pop()
        node = nodes.get(node_id, {})
        props = node.get('props') or {}
        name = props.get('name', '')
        if parent_id and props.get('_ownerId') != parent_id and name not in FORMULA_NO_PARAMS:
            continue
        if node_id in visited:
            continue
        visited.add(node_id)
        children = node.get('children', [])
        if props.get('_docType') == 'tuple':
            pending.extend((child_id, node_id) for child_id in children[1:])
        elif name in FORMULA_FUNCTIONS and (children or name in FORMULA_NO_PARAMS):
            reserved.add(node_id)
            pending.extend((child_id, node_id) for child_id in children)
    return reserved


def prepare_fields(docs, reserved_names):
    public_fields = {(node.get('props') or {}).get('name', '') for node in docs
                     if (node.get('props') or {}).get('_docType') == 'attrDef' and (node['id'].startswith('SYS_') or node['id'] in reserved_names)}
    _preserved_fields.update(public_fields)
    for name in public_fields:
        for token in TOKEN_RE.findall(name):
            _reverse_cache.setdefault(token.casefold(), None)
    return public_fields


def scramble_name(node, clauses, reserved_names=()):
    props = node.get('props') or {}
    name = props.get('name', '')
    if node['id'] in reserved_names:
        return name
    if props.get('_docType') == 'attrDef' and (
        name in _preserved_fields or (name.startswith('sys:') and name[4:] in SYSTEM_EXPRESSION_NAMES)
    ):
        return name
    if props.get('_docType') == 'viewDef':
        if name == 'Default':
            return name
        prefix, separator, account = name.partition(':')
        if separator and prefix in {'user', 'view'}:
            return prefix + ':' + (_view_accounts.get(account) or scramble_plain(account))
    if props.get('_docType') == 'journalPart':
        if re.fullmatch(r'\d{4}-\d{2}-\d{2} - (?:' + '|'.join(WEEKDAYS) + ')', name):
            return name
        match = CALENDAR_PREFIX_RE.match(name)
        if match:
            return match[0] + scramble_text(name[match.end():])
    if node['id'] in clauses:
        if name.replace('_', ' ').upper() in SEARCH_KEYWORDS or SEARCH_DATE_RE.fullmatch(name):
            return name
        match = RELATIVE_RE.fullmatch(name)
        if match:
            field = '.' + scramble_field_path(match[2]) if match[2] is not None else ''
            return match[1] + field + (match[3] or '')
        editor_query = re.fullmatch(r'((?:MODIFIED|EDITED) BY )(\S+)( (?:LAST \d+ DAYS?|ANYTIME))', name, re.IGNORECASE)
        if editor_query:
            return editor_query[1] + scramble_plain(editor_query[2]) + editor_query[3]
        if name.upper().startswith('SIBLING NAMED '):
            return name[:14] + scramble_text(name[14:])
        regex = re.fullmatch(r'/(.*)/([dgimsuvy]*)', name, re.DOTALL)
        if regex:
            if re.search(r'\\[xupPkKc]|\\u\{', regex[1]):
                raise ValueError('Encoded, Unicode-property and named-reference regex escapes need manual scrubbing')
            protected = re.compile(r'\\[dDsSwWbBfntvr0]|\\[1-9]\d*|\\[^\w]|\{\d+(?:,\d*)?\}|\[\^?(?:[a-zA-Z0-9]-[a-zA-Z0-9]|\\[dDsSwW]|_)+\]')
            parts = protected.split(regex[1])
            syntax = protected.findall(regex[1])
            body = ''.join(TOKEN_RE.sub(lambda m: scramble_token(m.group()), part) + (syntax[i] if i < len(syntax) else '') for i, part in enumerate(parts))
            return '/' + body + '/' + regex[2]
    return scramble_text(name)


def require_timezone_data():
    if not TIMEZONES:
        raise SystemExit('Timezone data is missing. Run python3 -m pip install tzdata (Windows: py -m pip install tzdata), then try again.')


def workspace_data(data):
    return data.get('storeData', data) if isinstance(data, dict) else data


def read_workspace(path):
    with open(path, encoding='utf-8') as f:
        data = json.load(f)
    workspace = workspace_data(data)
    if not isinstance(workspace, dict) or not isinstance(workspace.get('docs'), list) or 'docMap' in workspace:
        raise SystemExit('This helper requires workspace snapshot JSON with a docs array. TIF exports are not supported. Ask Tana support for a snapshot JSON file.')
    return data


def map_json_text(value, transform, key=None, parent=None, parent_key=None):
    if isinstance(value, str):
        structural_name = key == 'name' and isinstance(parent, dict) and (
            'inputSchema' in parent or
            (parent_key == 'function' and ('parameters' in parent or 'arguments' in parent)) or
            (isinstance(parent.get('type'), str) and parent['type'] in {'function', 'tool_call', 'tool_use'})
        )
        metadata = key in JSON_METADATA_KEYS and (key not in JSON_PROTOCOL_VALUES or value in JSON_PROTOCOL_VALUES[key])
        if (metadata or structural_name) and not (URL_RE.search(value) or EMAIL_RE.search(value)):
            return value
        return transform(value)
    if isinstance(value, list):
        return [map_json_text(item, transform, key, parent, parent_key) for item in value]
    if isinstance(value, dict):
        return {child_key: map_json_text(item, transform, child_key, value, key) for child_key, item in value.items()}
    return value


def scramble_json_text(text):
    try:
        parsed = json.loads(text)
    except ValueError:
        return scramble_text(text)
    return json.dumps(map_json_text(parsed, scramble_text), ensure_ascii=False, separators=(',', ':'))


def node_texts(node):
    props = node.get('props') or {}
    for field in CONTENT_FIELDS:
        value = props.get(field)
        if not isinstance(value, str) or not value:
            continue
        if field == '_jsonData':
            texts = []
            try:
                map_json_text(json.loads(value), texts.append)
            except ValueError:
                texts.append(value)
            for text in texts:
                yield field, text
        else:
            yield field, value


def main(src, dst):
    require_timezone_data()
    data = read_workspace(src)
    workspace = workspace_data(data)

    for i, ed in enumerate(workspace.get('editors', []), 1):
        if isinstance(ed, list) and ed and isinstance(ed[0], str):
            original = ed[0]
            ed[0] = f'editor{i}@example.com'
            _redactions[('email', original.casefold())] = ed[0]
            _view_accounts[original] = ed[0]
            _view_accounts[original.replace('.', '_')] = ed[0].replace('.', '_')

    clauses = search_clause_ids(workspace['docs'])
    reserved_names = reserved_name_ids(workspace['docs'])
    prepare_fields(workspace['docs'], reserved_names)
    kept = scrambled = 0
    for node in workspace['docs']:
        props = node.get('props') or {}
        system = bool(SYSTEM_ID_RE.match(node['id']))
        original_name = props.get('name')
        for field in CONTENT_FIELDS:
            v = props.get(field)
            if not isinstance(v, str) or not v:
                continue
            if system:
                kept += 1
                continue
            if field == 'name':
                props[field] = scramble_name(node, clauses, reserved_names)
            elif field == '_lastComputedName' and v == original_name:
                props[field] = props['name']
            elif field == '_jsonData':
                props[field] = scramble_json_text(v)
            elif field == '_textContent':
                props[field] = scramble_plain(v)
            else:
                props[field] = scramble_text(v)
            scrambled += 1

    with open(dst, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, separators=(',', ':'))
    print(f"scrambled {scrambled} strings, kept {kept} system strings, "
          f"{len(_cache)} distinct words, {_placeholders['url']} urls, {_placeholders['email']} emails")


FAKE_EMAIL_RE = re.compile(r'(?:user|editor)\d+@example\.com')
REGEX_RE = re.compile(r'/(.*)/([dgimsuvy]*)', re.DOTALL)
EDITOR_QUERY_RE = re.compile(r'((?:MODIFIED|EDITED) BY )(\S+)( (?:LAST \d+ DAYS?|ANYTIME))', re.IGNORECASE)
RESERVED_WORDS = {
    token.casefold()
    for name in SEARCH_KEYWORDS | EXPRESSION_NAMES | SYSTEM_EXPRESSION_NAMES | WEEKDAYS | SETTINGS_SECTIONS | FORMULA_FUNCTIONS | SORT_DIRECTIONS | CALCULATIONS | TIMEZONES | {
        'Default', 'Calendar', 'Months', 'Week', 'Settings', 'Display timezone', 'User guidance', 'Promotions', 'auto', 'medium', 'full', 'PARENT', 'GRANDPARENT', 'INSTANCE',
        'SIBLING NAMED', 'FOR RELATIVE DATE TODAY YESTERDAY TOMORROW NOW THIS LAST NEXT DAY WEEK MONTH YEAR DAYS WEEKS MONTHS YEARS CREATED EDITED MODIFIED BY OVERDUE ANYTIME'
    }
    for token in TOKEN_RE.findall(name)
}


def structure(data):
    result = dict(data)
    if 'storeData' in data:
        result['storeData'] = structure(data['storeData'])
        return result
    result['docs'] = []
    for node in data['docs']:
        stripped = dict(node)
        props = node.get('props')
        if isinstance(props, dict) and not node['id'].startswith('SYS_'):
            stripped['props'] = {k: v for k, v in props.items() if k not in CONTENT_FIELDS or not isinstance(v, str) or not v}
            if isinstance(props.get('_jsonData'), str) and props['_jsonData']:
                try:
                    stripped['props']['_jsonData'] = map_json_text(json.loads(props['_jsonData']), lambda _: _JSON_CONTENT_MARKER)
                except ValueError:
                    pass
        result['docs'].append(stripped)
    if 'editors' in data:
        result['editors'] = [[None, *row[1:]] if isinstance(row, list) and row and isinstance(row[0], str) else row for row in data['editors']]
    return result


def inline_references(text):
    return [(name, html.unescape(quoted if quote else bare)) for tag in TAG_RE.findall(text)
            for name, _, quote, quoted, bare in ATTR_RE.findall(tag)
            if name in {'data-inlineref-node', 'data-inlineref-date'}]


def field_signature(path):
    steps = path.split('.')
    reserved = [(i, step) for i, step in enumerate(steps) if step in _preserved_fields or (step.startswith('sys:') and step[4:] in SYSTEM_EXPRESSION_NAMES)]
    return len(steps), reserved


def expression_signature(text):
    result = []
    text = URL_RE.sub('', html.unescape(text))
    for expression in INTERPOLATION_RE.findall(text):
        path, separator, formatting = expression.partition('|')
        result.append((field_signature(path), separator, formatting if re.fullmatch(r'\d*(?:\.\.\.|…)?n?\??', formatting) else None))
    return result


def name_signature(node, name, clauses, reserved_names):
    props = node.get('props') or {}
    doc_type = props.get('_docType')
    if node['id'] in reserved_names or (doc_type == 'attrDef' and (name in _preserved_fields or (name.startswith('sys:') and name[4:] in SYSTEM_EXPRESSION_NAMES))):
        return 'exact', name
    if doc_type == 'viewDef':
        if name == 'Default':
            return 'exact', name
        prefix, separator, _ = name.partition(':')
        if separator and prefix in {'user', 'view'}:
            return 'view', prefix
    if doc_type == 'journalPart':
        if re.fullmatch(r'\d{4}-\d{2}-\d{2} - (?:' + '|'.join(WEEKDAYS) + ')', name):
            return 'exact', name
        match = CALENDAR_PREFIX_RE.match(name)
        if match:
            return 'calendar', match[0]
    if node['id'] in clauses:
        if name.replace('_', ' ').upper() in SEARCH_KEYWORDS or SEARCH_DATE_RE.fullmatch(name):
            return 'exact', name
        relative = RELATIVE_RE.fullmatch(name)
        if relative:
            return 'relative', relative[1], field_signature(relative[2]) if relative[2] is not None else None, relative[3]
        editor = EDITOR_QUERY_RE.fullmatch(name)
        if editor:
            return 'editor', editor[1], editor[3]
        if name.upper().startswith('SIBLING NAMED '):
            return 'sibling', name[:14]
        regex = REGEX_RE.fullmatch(name)
        if regex:
            return 'regex', regex[2]
    return None


def human_text(text):
    attributes = [html.unescape(quoted if quote else bare) for tag in TAG_RE.findall(text)
                  for name, _, quote, quoted, bare in ATTR_RE.findall(tag)
                  if name not in STRUCTURAL_ATTRIBUTES and name != 'data-inlineref-user']
    return html.unescape(TAG_RE.sub(' ', text)) + ' ' + ' '.join(attributes)


def fixed_name_text(signature, text):
    if signature is None:
        return False
    if signature[0] == 'exact':
        return True
    if signature[0] == 'calendar':
        return signature[1] == text
    if signature[0] == 'relative':
        return signature[2] is None or signature[2][0] == len(signature[2][1])
    return False


def vocabulary(data):
    data = workspace_data(data)
    result = collections.Counter()
    clauses = search_clause_ids(data['docs'])
    reserved = reserved_name_ids(data['docs'])
    for node in data['docs']:
        if node['id'].startswith('SYS_'):
            continue
        for field, value in node_texts(node):
            named_field = field == 'name' or (field == '_lastComputedName' and value == (node.get('props') or {}).get('name'))
            if named_field and fixed_name_text(name_signature(node, value, clauses, reserved), value):
                continue
            text = value if field == '_textContent' else human_text(value)
            text = re.sub(r'https://example\.com/u\d+|(?:user|editor)\d+@example[._]com', ' ', text)
            result.update(word.casefold() for word in re.findall(r'[^\W\d_]{5,}', text) if word.casefold() not in RESERVED_WORDS)
    return result


def verify(source, output, sensitive_terms=()):
    ok = True

    def check(name, condition, detail=''):
        nonlocal ok
        ok = ok and bool(condition)
        print(('PASS ' if condition else 'FAIL ') + name + (' ' + detail if detail else ''))

    same_structure = structure(source) == structure(output)
    full_output = output
    source, output = workspace_data(source), workspace_data(output)
    check('structure, built-in text and editor metadata unchanged', same_structure, f"{len(output['docs'])} nodes")
    if [node['id'] for node in source['docs']] != [node['id'] for node in output['docs']]:
        return False
    clauses = search_clause_ids(source['docs'])
    reserved_names = reserved_name_ids(source['docs'])
    public_fields = prepare_fields(source['docs'], reserved_names)
    RESERVED_WORDS.update(token.casefold() for name in public_fields for token in TOKEN_RE.findall(name))
    RESERVED_WORDS.update(token.casefold() for name in TRANSCRIPTION_LANGUAGES | EVENT_CLASSES for token in TOKEN_RE.findall(name))
    refs_ok = syntax_ok = accounts_ok = True
    unchanged = 0
    account_map = {}
    for i, row in enumerate(source.get('editors', []), 1):
        if isinstance(row, list) and row and isinstance(row[0], str):
            target = f'editor{i}@example.com'
            account_map[row[0]] = target
            account_map[row[0].replace('.', '_')] = target.replace('.', '_')
            accounts_ok = accounts_ok and output.get('editors', [])[i - 1][0] == target
    for original, scrambled in zip(source['docs'], output['docs']):
        original_texts, scrambled_texts = list(node_texts(original)), list(node_texts(scrambled))
        syntax_ok = syntax_ok and len(original_texts) == len(scrambled_texts)
        for (field, before), (after_field, after) in zip(original_texts, scrambled_texts):
            syntax_ok = syntax_ok and field == after_field
            if field != '_textContent':
                refs_ok = refs_ok and inline_references(before) == inline_references(after)
                syntax_ok = syntax_ok and expression_signature(before) == expression_signature(after)
            cached_name = field == '_lastComputedName' and before == (original.get('props') or {}).get('name')
            if cached_name:
                syntax_ok = syntax_ok and after == (scrambled.get('props') or {}).get('name')
            signature = name_signature(original, before, clauses, reserved_names) if field == 'name' or cached_name else None
            if field == 'name':
                syntax_ok = syntax_ok and signature == name_signature(original, after, clauses, reserved_names)
                if (original.get('props') or {}).get('_docType') == 'viewDef':
                    prefix, separator, account = before.partition(':')
                    if separator and prefix in {'user', 'view'} and account in account_map:
                        accounts_ok = accounts_ok and after == prefix + ':' + account_map[account]
            safe_unchanged = fixed_name_text(signature, before) or scramble_text(before) == before
            if not original['id'].startswith('SYS_') and before == after and not safe_unchanged and len(re.findall(r'[^\W\d_]', human_text(before))) >= 3:
                unchanged += 1
    check('inline node references and dates unchanged', refs_ok)
    check('reserved names and expression syntax unchanged', syntax_ok)
    check('view accounts match anonymized editors', accounts_ok)
    check('human text changed', unchanged == 0, f'{unchanged} unchanged strings')
    original_words, output_words = vocabulary(source), vocabulary(output)
    overlap = original_words.keys() & output_words.keys()
    check('original words removed', not overlap or len(overlap) < 0.001 * len(output_words), f'{len(overlap)} coincident words of {len(output_words)} output words')
    fields = {}
    field_names_ok = True
    for original, scrambled in zip(source['docs'], output['docs']):
        if (original.get('props') or {}).get('_docType') == 'attrDef':
            before = original['props'].get('name', '').casefold()
            after = scrambled['props'].get('name', '').casefold()
            field_names_ok = field_names_ok and fields.get(after, before) == before
            fields[after] = before
    check('distinct field names remain distinct', field_names_ok)
    output_texts = [text for node in output['docs'] if not node['id'].startswith('SYS_') for _, text in node_texts(node)]
    user_text = '\n'.join(html.unescape(text) for text in output_texts)
    all_output = html.unescape(json.dumps(full_output, ensure_ascii=False)) + '\n' + user_text
    emails = [email for email in EMAIL_RE.findall(all_output) if not FAKE_EMAIL_RE.fullmatch(email)]
    check('real emails absent', not emails, f'{len(emails)} found')
    urls = [url for url in URL_RE.findall(user_text) if not re.fullmatch(r'https://example\.com/u\d+', url)]
    check('real URLs absent from user text', not urls, f'{len(urls)} found')
    link_targets = [html.unescape(quoted if quote else bare)
                    for text in output_texts
                    for tag in TAG_RE.findall(text)
                    if not tag.startswith('<!--')
                    for name, _, quote, quoted, bare in ATTR_RE.findall(tag)
                    if name.casefold() in {'href', 'src'}]
    private_targets = [target for target in link_targets if not re.fullmatch(r'https://example\.com/u\d+', target)]
    check('markup link targets replaced with placeholders', not private_targets, f'{len(private_targets)} found')
    original_editors = [row[0] for row in source.get('editors', []) if isinstance(row, list) and row and isinstance(row[0], str) and not FAKE_EMAIL_RE.fullmatch(row[0])]
    check('original editor accounts absent', not any(email.casefold() in all_output.casefold() for email in original_editors))
    hits = [term for term in sensitive_terms if term.casefold() in all_output.casefold()]
    check('supplied sensitive terms absent, including encoded text', not hits, f'{len(hits)} terms found')
    return ok


def cli(argv=None):
    parser = argparse.ArgumentParser(description='Scramble a Tana workspace snapshot or verify its scrambled copy.')
    parser.add_argument('--verify', action='store_true', help='Check a scrambled copy instead of creating one')
    parser.add_argument('source', help='Original workspace snapshot JSON file')
    parser.add_argument('output', help='Scrambled workspace JSON file')
    parser.add_argument('sensitive_terms', nargs='*', help='Words or phrases to check with --verify')
    args = parser.parse_args(argv)
    if args.sensitive_terms and not args.verify:
        parser.error('Sensitive terms are only supported with --verify')
    require_timezone_data()
    if args.verify:
        source = read_workspace(args.source)
        output = read_workspace(args.output)
        ok = verify(source, output, args.sensitive_terms)
        print('OVERALL ' + ('PASS' if ok else 'FAIL'))
        return 0 if ok else 1
    main(args.source, args.output)
    return 0


if __name__ == '__main__':
    sys.exit(cli())
