diff --git a/brain/persona-brain-db/import_batch.py b/brain/persona-brain-db/import_batch.py new file mode 100644 index 0000000..95998b8 --- /dev/null +++ b/brain/persona-brain-db/import_batch.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Batch import multiple Notion exports into brain.db (append mode)""" +import sqlite3, os, re + +DB = "/Users/bingshuolingdianyuanhe/WorkBuddy/2026-06-04-13-44-41/persona-brain-db/brain.db" + +EXPORTS = [ + ("人格记忆总索引", "/tmp/batch-import/人格记忆总索引/export"), + ("光湖世界入口导航", "/tmp/batch-import/光湖世界入口导航/export"), + ("曜冥纪元", "/tmp/batch-import/曜冥纪元/export"), +] + +def parse_md(filepath): + fn = os.path.basename(filepath).replace('.md', '') + uuid = None + m = re.search(r'([a-f0-9]{32})$', fn) + if m: + uuid = m.group(1) + # Strip UUID suffix from title + if uuid and fn.endswith(uuid): + title = fn[:fn.rindex(uuid)].strip().rstrip(' ·').strip() + else: + title = fn + with open(filepath, 'r', encoding='utf-8', errors='replace') as f: + raw = f.read()[:12000] + return {"title": title, "uuid": uuid, "content": raw} + +def process_export(name, content_dir): + conn = sqlite3.connect(DB) + c = conn.cursor() + + # Build title → uuid + dir → uuid mappings + title_to_uuid = {} + dir_to_uuid = {} + + for walk_root, dirs, files in os.walk(content_dir): + for f in files: + if not f.endswith('.md'): continue + fpath = os.path.join(walk_root, f) + page = parse_md(fpath) + if not page['uuid'] or not page['title']: continue + title_to_uuid[page['title'].lower().strip()] = page['uuid'] + + for walk_root, dirs, files in os.walk(content_dir): + for d in dirs: + dpath = os.path.join(walk_root, d) + dname = d.lower().strip() + if dname in title_to_uuid: + dir_to_uuid[dpath] = title_to_uuid[dname] + else: + for title, uuid in title_to_uuid.items(): + if len(dname) > 4 and len(title) > 4: + if title.startswith(dname) or dname.startswith(title): + dir_to_uuid[dpath] = uuid + break + + # Import pages + new_count = 0 + skip_count = 0 + pages = [] + + for walk_root, dirs, files in sorted(os.walk(content_dir)): + for f in sorted(files): + if not f.endswith('.md'): continue + fpath = os.path.join(walk_root, f) + page = parse_md(fpath) + if not page['uuid']: continue + rel = os.path.relpath(fpath, content_dir) + parent_id = dir_to_uuid.get(walk_root) + pages.append({**page, "parent_id": parent_id, "path": rel}) + + c.execute("BEGIN TRANSACTION") + for i, p in enumerate(pages): + c.execute("SELECT 1 FROM documents WHERE document_id=?", (p['uuid'],)) + if c.fetchone(): + skip_count += 1 + continue + c.execute(""" + INSERT INTO documents (document_id, title, content, parent_id, path, content_type) + VALUES (?, ?, ?, ?, ?, 'notion') + """, (p['uuid'], p['title'], p['content'], p['parent_id'], p['path'])) + new_count += 1 + if new_count % 500 == 0: + conn.commit() + c.execute("BEGIN TRANSACTION") + print(f" {new_count} new...") + conn.commit() + + c.execute("SELECT count(*) FROM documents") + total = c.fetchone()[0] + conn.close() + print(f" {name}: {new_count} new + {skip_count} skipped (总{total}页)") + return new_count, skip_count + +def main(): + total_new = 0 + total_skip = 0 + for name, path in EXPORTS: + if not os.path.isdir(path): + print(f" SKIP {name}: {path} not found") + continue + new, skip = process_export(name, path) + total_new += new + total_skip += skip + + conn = sqlite3.connect(DB) + c = conn.cursor() + c.execute("SELECT count(*) FROM documents") + final = c.fetchone()[0] + c.execute("SELECT count(*) FROM documents WHERE parent_id IS NULL") + roots = c.fetchone()[0] + conn.close() + print(f"\n总计: {total_new}新增 + {total_skip}跳过 => 数据库共{final}页({roots}入口)") + import os as _os + print(f"数据库大小: {_os.path.getsize(DB)/1024/1024:.1f}MB") + +if __name__ == '__main__': + main() diff --git a/brain/persona-brain-db/import_notion.py b/brain/persona-brain-db/import_notion.py index e319de6..7703ace 100644 --- a/brain/persona-brain-db/import_notion.py +++ b/brain/persona-brain-db/import_notion.py @@ -1,87 +1,103 @@ #!/usr/bin/env python3 -"""Notion Markdown Export → SQLite brain.db importer""" -import sqlite3, os, re, json, uuid +"""Notion Full Export → SQLite brain.db importer""" +import sqlite3, os, re DB = "/Users/bingshuolingdianyuanhe/WorkBuddy/2026-06-04-13-44-41/persona-brain-db/brain.db" -CONTENT = "/tmp/fifth-domain-extract/content/私人与共享" +CONTENT = "/tmp/full-export/export" def parse_md(filepath): - """Parse a Notion-exported .md file""" fn = os.path.basename(filepath).replace('.md', '') uuid = None title = fn m = re.search(r'([a-f0-9]{32})$', fn) if m: uuid = m.group(1) - title = fn[:fn.rindex(uuid)].strip().rstrip(' ·').strip() - with open(filepath, 'r', encoding='utf-8') as f: - raw = f.read()[:8000] - return {"title": title, "uuid": uuid, "content": raw, "path": filepath} + title = fn[:fn.rindex(uuid)].strip().rstrip(' ·').strip() + with open(filepath, 'r', encoding='utf-8', errors='replace') as f: + raw = f.read()[:12000] + return {"title": title, "uuid": uuid, "content": raw} def main(): conn = sqlite3.connect(DB) c = conn.cursor() - - # Clear existing imported data c.execute("DELETE FROM documents") - c.execute("DELETE FROM persona_memory") conn.commit() - pages = [] - dir_to_uuid = {} # directory path -> uuid of the Notion page it represents - - # First pass: build dir_to_uuid map + # 第一遍: 建立目录→UUID映射 + # Notion export: page .md filename = "Title UUID.md", child directory = "Title/" + # Directory name may NOT include UUID, only the title + dir_to_uuid = {} + # Build a mapping from title-only to uuid + title_to_uuid = {} for walk_root, dirs, files in os.walk(CONTENT): for f in files: if not f.endswith('.md'): continue fpath = os.path.join(walk_root, f) page = parse_md(fpath) - if page['uuid']: - # This file IS the Notion page for walk_root dir - dir_to_uuid[walk_root] = page['uuid'] + if not page['uuid'] or not page['title']: continue + # Map this page's uuid to any subdirectory whose basename matches the title + # Also check the full filename (without .md) as fallback + title_to_uuid[page['title'].lower().strip()] = page['uuid'] - # Second pass: build pages with parent relationships + # Second pass on dirs: match dir names to page titles for walk_root, dirs, files in os.walk(CONTENT): + for d in dirs: + dpath = os.path.join(walk_root, d) + dname = d.lower().strip() + if dname in title_to_uuid: + dir_to_uuid[dpath] = title_to_uuid[dname] + else: + # Try prefix match: directory name might be truncated + for title, uuid in title_to_uuid.items(): + if title.startswith(dname) or dname.startswith(title): + if len(dname) > 3 and len(title) > 3: + dir_to_uuid[dpath] = uuid + break + + # 第二遍: 导入所有页面 + pages = [] + for walk_root, dirs, files in sorted(os.walk(CONTENT)): for f in sorted(files): if not f.endswith('.md'): continue fpath = os.path.join(walk_root, f) page = parse_md(fpath) if not page['uuid']: continue - rel = os.path.relpath(fpath, CONTENT) depth = max(0, rel.count('/')) + parent_id = dir_to_uuid.get(walk_root) + pages.append({**page, "depth": depth, "parent_id": parent_id, "path": rel}) - # Find parent: the directory that contains this file - # is the parent dir; its Notion page UUID is in dir_to_uuid - parent_id = dir_to_uuid.get(walk_root) if depth > 0 else None - - pages.append({ - "depth": depth, - "parent_id": parent_id, - "path": rel, - **page - }) - - for p in pages: + # 批量插入 + c.execute("BEGIN TRANSACTION") + for i, p in enumerate(pages): c.execute(""" INSERT OR REPLACE INTO documents (document_id, title, content, parent_id, path, content_type) VALUES (?, ?, ?, ?, ?, 'notion') - """, (p['uuid'], p['title'], p['content'][:8000], p['parent_id'], p['path'])) - + """, (p['uuid'], p['title'], p['content'], p['parent_id'], p['path'])) + if (i+1) % 1000 == 0: + conn.commit() + c.execute("BEGIN TRANSACTION") + print(f" {i+1}/{len(pages)}...") conn.commit() + # 统计 c.execute("SELECT count(*) FROM documents") total = c.fetchone()[0] c.execute("SELECT count(*) FROM documents WHERE parent_id IS NULL") roots = c.fetchone()[0] - print(f"Imported {total} pages ({roots} root pages)") + c.execute("SELECT count(DISTINCT parent_id) FROM documents WHERE parent_id IS NOT NULL") + parents = c.fetchone()[0] - c.execute("SELECT document_id, title FROM documents WHERE parent_id IS NULL") - for row in c.fetchall(): + print(f"\n导入完成: {total}页 ({roots}根页面, {parents}父节点)") + + c.execute("SELECT document_id, title FROM documents WHERE parent_id IS NULL ORDER BY title") + for row in c.fetchall()[:30]: print(f" [{row[0][:8]}..] {row[1][:70]}") + c.execute("SELECT count(*) FROM sqlite_master WHERE type='table'") + tables = c.fetchone()[0] conn.close() - print("Done.") + print(f"Done. {tables} tables total.") if __name__ == '__main__': main()