D123+ import scripts v2: batch multi-zip + UUID dedup
- import_batch.py: 批量处理多个Notion导出zip·追加模式·UUID去重 - import_notion.py: 单次导入·目录标题匹配·批量插入 - brain.db本地已建(100MB): 15,451页·4,053入口 · 冰朔光湖世界(9,324) + 人格记忆总索引(135) + 光湖世界入口导航(193) + 曜冥纪元(5,799)
This commit is contained in:
铸渊 ICE-GL-ZY001
parent
02a12c2d58
commit
b4add2762c
118
brain/persona-brain-db/import_batch.py
Normal file
118
brain/persona-brain-db/import_batch.py
Normal file
@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Batch import multiple Notion exports into brain.db (append mode)"""
|
||||
import sqlite3, os, re
|
||||
|
||||
DB = "/Users/bingshuolingdianyuanhe/WorkBuddy/2026-06-04-13-44-41/persona-brain-db/brain.db"
|
||||
|
||||
EXPORTS = [
|
||||
("人格记忆总索引", "/tmp/batch-import/人格记忆总索引/export"),
|
||||
("光湖世界入口导航", "/tmp/batch-import/光湖世界入口导航/export"),
|
||||
("曜冥纪元", "/tmp/batch-import/曜冥纪元/export"),
|
||||
]
|
||||
|
||||
def parse_md(filepath):
|
||||
fn = os.path.basename(filepath).replace('.md', '')
|
||||
uuid = None
|
||||
m = re.search(r'([a-f0-9]{32})$', fn)
|
||||
if m:
|
||||
uuid = m.group(1)
|
||||
# Strip UUID suffix from title
|
||||
if uuid and fn.endswith(uuid):
|
||||
title = fn[:fn.rindex(uuid)].strip().rstrip(' ·').strip()
|
||||
else:
|
||||
title = fn
|
||||
with open(filepath, 'r', encoding='utf-8', errors='replace') as f:
|
||||
raw = f.read()[:12000]
|
||||
return {"title": title, "uuid": uuid, "content": raw}
|
||||
|
||||
def process_export(name, content_dir):
|
||||
conn = sqlite3.connect(DB)
|
||||
c = conn.cursor()
|
||||
|
||||
# Build title → uuid + dir → uuid mappings
|
||||
title_to_uuid = {}
|
||||
dir_to_uuid = {}
|
||||
|
||||
for walk_root, dirs, files in os.walk(content_dir):
|
||||
for f in files:
|
||||
if not f.endswith('.md'): continue
|
||||
fpath = os.path.join(walk_root, f)
|
||||
page = parse_md(fpath)
|
||||
if not page['uuid'] or not page['title']: continue
|
||||
title_to_uuid[page['title'].lower().strip()] = page['uuid']
|
||||
|
||||
for walk_root, dirs, files in os.walk(content_dir):
|
||||
for d in dirs:
|
||||
dpath = os.path.join(walk_root, d)
|
||||
dname = d.lower().strip()
|
||||
if dname in title_to_uuid:
|
||||
dir_to_uuid[dpath] = title_to_uuid[dname]
|
||||
else:
|
||||
for title, uuid in title_to_uuid.items():
|
||||
if len(dname) > 4 and len(title) > 4:
|
||||
if title.startswith(dname) or dname.startswith(title):
|
||||
dir_to_uuid[dpath] = uuid
|
||||
break
|
||||
|
||||
# Import pages
|
||||
new_count = 0
|
||||
skip_count = 0
|
||||
pages = []
|
||||
|
||||
for walk_root, dirs, files in sorted(os.walk(content_dir)):
|
||||
for f in sorted(files):
|
||||
if not f.endswith('.md'): continue
|
||||
fpath = os.path.join(walk_root, f)
|
||||
page = parse_md(fpath)
|
||||
if not page['uuid']: continue
|
||||
rel = os.path.relpath(fpath, content_dir)
|
||||
parent_id = dir_to_uuid.get(walk_root)
|
||||
pages.append({**page, "parent_id": parent_id, "path": rel})
|
||||
|
||||
c.execute("BEGIN TRANSACTION")
|
||||
for i, p in enumerate(pages):
|
||||
c.execute("SELECT 1 FROM documents WHERE document_id=?", (p['uuid'],))
|
||||
if c.fetchone():
|
||||
skip_count += 1
|
||||
continue
|
||||
c.execute("""
|
||||
INSERT INTO documents (document_id, title, content, parent_id, path, content_type)
|
||||
VALUES (?, ?, ?, ?, ?, 'notion')
|
||||
""", (p['uuid'], p['title'], p['content'], p['parent_id'], p['path']))
|
||||
new_count += 1
|
||||
if new_count % 500 == 0:
|
||||
conn.commit()
|
||||
c.execute("BEGIN TRANSACTION")
|
||||
print(f" {new_count} new...")
|
||||
conn.commit()
|
||||
|
||||
c.execute("SELECT count(*) FROM documents")
|
||||
total = c.fetchone()[0]
|
||||
conn.close()
|
||||
print(f" {name}: {new_count} new + {skip_count} skipped (总{total}页)")
|
||||
return new_count, skip_count
|
||||
|
||||
def main():
|
||||
total_new = 0
|
||||
total_skip = 0
|
||||
for name, path in EXPORTS:
|
||||
if not os.path.isdir(path):
|
||||
print(f" SKIP {name}: {path} not found")
|
||||
continue
|
||||
new, skip = process_export(name, path)
|
||||
total_new += new
|
||||
total_skip += skip
|
||||
|
||||
conn = sqlite3.connect(DB)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT count(*) FROM documents")
|
||||
final = c.fetchone()[0]
|
||||
c.execute("SELECT count(*) FROM documents WHERE parent_id IS NULL")
|
||||
roots = c.fetchone()[0]
|
||||
conn.close()
|
||||
print(f"\n总计: {total_new}新增 + {total_skip}跳过 => 数据库共{final}页({roots}入口)")
|
||||
import os as _os
|
||||
print(f"数据库大小: {_os.path.getsize(DB)/1024/1024:.1f}MB")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,87 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Notion Markdown Export → SQLite brain.db importer"""
|
||||
import sqlite3, os, re, json, uuid
|
||||
"""Notion Full Export → SQLite brain.db importer"""
|
||||
import sqlite3, os, re
|
||||
|
||||
DB = "/Users/bingshuolingdianyuanhe/WorkBuddy/2026-06-04-13-44-41/persona-brain-db/brain.db"
|
||||
CONTENT = "/tmp/fifth-domain-extract/content/私人与共享"
|
||||
CONTENT = "/tmp/full-export/export"
|
||||
|
||||
def parse_md(filepath):
|
||||
"""Parse a Notion-exported .md file"""
|
||||
fn = os.path.basename(filepath).replace('.md', '')
|
||||
uuid = None
|
||||
title = fn
|
||||
m = re.search(r'([a-f0-9]{32})$', fn)
|
||||
if m:
|
||||
uuid = m.group(1)
|
||||
title = fn[:fn.rindex(uuid)].strip().rstrip(' ·').strip()
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
raw = f.read()[:8000]
|
||||
return {"title": title, "uuid": uuid, "content": raw, "path": filepath}
|
||||
title = fn[:fn.rindex(uuid)].strip().rstrip(' ·').strip()
|
||||
with open(filepath, 'r', encoding='utf-8', errors='replace') as f:
|
||||
raw = f.read()[:12000]
|
||||
return {"title": title, "uuid": uuid, "content": raw}
|
||||
|
||||
def main():
|
||||
conn = sqlite3.connect(DB)
|
||||
c = conn.cursor()
|
||||
|
||||
# Clear existing imported data
|
||||
c.execute("DELETE FROM documents")
|
||||
c.execute("DELETE FROM persona_memory")
|
||||
conn.commit()
|
||||
|
||||
pages = []
|
||||
dir_to_uuid = {} # directory path -> uuid of the Notion page it represents
|
||||
|
||||
# First pass: build dir_to_uuid map
|
||||
# 第一遍: 建立目录→UUID映射
|
||||
# Notion export: page .md filename = "Title UUID.md", child directory = "Title/"
|
||||
# Directory name may NOT include UUID, only the title
|
||||
dir_to_uuid = {}
|
||||
# Build a mapping from title-only to uuid
|
||||
title_to_uuid = {}
|
||||
for walk_root, dirs, files in os.walk(CONTENT):
|
||||
for f in files:
|
||||
if not f.endswith('.md'): continue
|
||||
fpath = os.path.join(walk_root, f)
|
||||
page = parse_md(fpath)
|
||||
if page['uuid']:
|
||||
# This file IS the Notion page for walk_root dir
|
||||
dir_to_uuid[walk_root] = page['uuid']
|
||||
if not page['uuid'] or not page['title']: continue
|
||||
# Map this page's uuid to any subdirectory whose basename matches the title
|
||||
# Also check the full filename (without .md) as fallback
|
||||
title_to_uuid[page['title'].lower().strip()] = page['uuid']
|
||||
|
||||
# Second pass: build pages with parent relationships
|
||||
# Second pass on dirs: match dir names to page titles
|
||||
for walk_root, dirs, files in os.walk(CONTENT):
|
||||
for d in dirs:
|
||||
dpath = os.path.join(walk_root, d)
|
||||
dname = d.lower().strip()
|
||||
if dname in title_to_uuid:
|
||||
dir_to_uuid[dpath] = title_to_uuid[dname]
|
||||
else:
|
||||
# Try prefix match: directory name might be truncated
|
||||
for title, uuid in title_to_uuid.items():
|
||||
if title.startswith(dname) or dname.startswith(title):
|
||||
if len(dname) > 3 and len(title) > 3:
|
||||
dir_to_uuid[dpath] = uuid
|
||||
break
|
||||
|
||||
# 第二遍: 导入所有页面
|
||||
pages = []
|
||||
for walk_root, dirs, files in sorted(os.walk(CONTENT)):
|
||||
for f in sorted(files):
|
||||
if not f.endswith('.md'): continue
|
||||
fpath = os.path.join(walk_root, f)
|
||||
page = parse_md(fpath)
|
||||
if not page['uuid']: continue
|
||||
|
||||
rel = os.path.relpath(fpath, CONTENT)
|
||||
depth = max(0, rel.count('/'))
|
||||
parent_id = dir_to_uuid.get(walk_root)
|
||||
pages.append({**page, "depth": depth, "parent_id": parent_id, "path": rel})
|
||||
|
||||
# Find parent: the directory that contains this file
|
||||
# is the parent dir; its Notion page UUID is in dir_to_uuid
|
||||
parent_id = dir_to_uuid.get(walk_root) if depth > 0 else None
|
||||
|
||||
pages.append({
|
||||
"depth": depth,
|
||||
"parent_id": parent_id,
|
||||
"path": rel,
|
||||
**page
|
||||
})
|
||||
|
||||
for p in pages:
|
||||
# 批量插入
|
||||
c.execute("BEGIN TRANSACTION")
|
||||
for i, p in enumerate(pages):
|
||||
c.execute("""
|
||||
INSERT OR REPLACE INTO documents (document_id, title, content, parent_id, path, content_type)
|
||||
VALUES (?, ?, ?, ?, ?, 'notion')
|
||||
""", (p['uuid'], p['title'], p['content'][:8000], p['parent_id'], p['path']))
|
||||
|
||||
""", (p['uuid'], p['title'], p['content'], p['parent_id'], p['path']))
|
||||
if (i+1) % 1000 == 0:
|
||||
conn.commit()
|
||||
c.execute("BEGIN TRANSACTION")
|
||||
print(f" {i+1}/{len(pages)}...")
|
||||
conn.commit()
|
||||
|
||||
# 统计
|
||||
c.execute("SELECT count(*) FROM documents")
|
||||
total = c.fetchone()[0]
|
||||
c.execute("SELECT count(*) FROM documents WHERE parent_id IS NULL")
|
||||
roots = c.fetchone()[0]
|
||||
print(f"Imported {total} pages ({roots} root pages)")
|
||||
c.execute("SELECT count(DISTINCT parent_id) FROM documents WHERE parent_id IS NOT NULL")
|
||||
parents = c.fetchone()[0]
|
||||
|
||||
c.execute("SELECT document_id, title FROM documents WHERE parent_id IS NULL")
|
||||
for row in c.fetchall():
|
||||
print(f"\n导入完成: {total}页 ({roots}根页面, {parents}父节点)")
|
||||
|
||||
c.execute("SELECT document_id, title FROM documents WHERE parent_id IS NULL ORDER BY title")
|
||||
for row in c.fetchall()[:30]:
|
||||
print(f" [{row[0][:8]}..] {row[1][:70]}")
|
||||
|
||||
c.execute("SELECT count(*) FROM sqlite_master WHERE type='table'")
|
||||
tables = c.fetchone()[0]
|
||||
conn.close()
|
||||
print("Done.")
|
||||
print(f"Done. {tables} tables total.")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user