添加 Notion zip 上传导入功能(adm-zip + multer)
This commit is contained in:
parent
86a7f6e27c
commit
ecbb8f57d8
85
persona-brain-db/api/routes/upload.js
Normal file
85
persona-brain-db/api/routes/upload.js
Normal file
@ -0,0 +1,85 @@
|
||||
const { Router } = require('express');
|
||||
const multer = require('multer');
|
||||
const AdmZip = require('adm-zip');
|
||||
const crypto = require('crypto');
|
||||
const os = require('os');
|
||||
const fs = require('fs');
|
||||
|
||||
const router = Router();
|
||||
const upload = multer({ dest: os.tmpdir(), limits: { fileSize: 50 * 1024 * 1024 } });
|
||||
|
||||
function parseFilename(filename) {
|
||||
const m = filename.match(/^(.+?)\s+([a-f0-9]{32})\.md$/);
|
||||
if (m) return { title: m[1], id: m[2] };
|
||||
return { title: filename.replace('.md', ''), id: null };
|
||||
}
|
||||
|
||||
router.post('/import', upload.single('file'), async (req, res) => {
|
||||
if (!req.file) return res.status(400).json({ error: 'No file' });
|
||||
const zipPath = req.file.path;
|
||||
const docs = [];
|
||||
const pathToDocId = {};
|
||||
|
||||
try {
|
||||
let zip = new AdmZip(zipPath);
|
||||
let entries = zip.getEntries();
|
||||
|
||||
const mdEntries = entries.filter(e => !e.isDirectory && e.entryName.endsWith('.md'));
|
||||
const zipEntries = entries.filter(e => !e.isDirectory && e.entryName.endsWith('.zip'));
|
||||
|
||||
// 双层打包
|
||||
if (mdEntries.length === 0 && zipEntries.length > 0) {
|
||||
for (const ze of zipEntries) {
|
||||
const innerBuf = ze.getData();
|
||||
const innerZip = new AdmZip(innerBuf);
|
||||
entries = innerZip.getEntries();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const allMd = entries.filter(e => !e.isDirectory && e.entryName.toLowerCase().endsWith('.md'));
|
||||
|
||||
for (const entry of allMd) {
|
||||
const filename = entry.entryName;
|
||||
let content;
|
||||
try { content = entry.getData().toString('utf8'); } catch (e) { continue; }
|
||||
if (!content || content.length < 10) continue;
|
||||
|
||||
const parts = filename.replace(/\\/g, '/').split('/').filter(p => p);
|
||||
if (parts.length === 0) continue;
|
||||
|
||||
const fname = parts[parts.length - 1];
|
||||
const { title, id } = parseFilename(fname);
|
||||
const docParts = [...parts.slice(0, -1), title];
|
||||
const docPath = '/' + docParts.join('/');
|
||||
const docId = id ? 'DOC-NTN-' + id.slice(0, 12) : 'DOC-' + Date.now() + '-' + crypto.randomBytes(4).toString('hex');
|
||||
|
||||
docs.push({ document_id: docId, persona_id: 'SY-001', parent_id: null, path: docPath, title, content, content_type: 'markdown', source: 'upload_import' });
|
||||
}
|
||||
|
||||
for (const doc of docs) {
|
||||
const parentParts = doc.path.split('/').slice(0, -1);
|
||||
if (parentParts.length > 0) {
|
||||
const parentPath = parentParts.join('/');
|
||||
doc.parent_id = pathToDocId[parentPath] || null;
|
||||
}
|
||||
pathToDocId[doc.path] = doc.document_id;
|
||||
}
|
||||
|
||||
const stmt = req.db.prepare('INSERT OR REPLACE INTO documents (document_id, persona_id, parent_id, path, title, content, content_type, source) VALUES (?,?,?,?,?,?,?,?)');
|
||||
let inserted = 0, skipped = 0;
|
||||
for (const doc of docs) {
|
||||
try {
|
||||
stmt.run(doc.document_id, doc.persona_id, doc.parent_id, doc.path, doc.title, doc.content.slice(0,500000), doc.content_type, doc.source);
|
||||
inserted++;
|
||||
} catch (e) { skipped++; }
|
||||
}
|
||||
fs.unlinkSync(zipPath);
|
||||
res.json({ ok: true, inserted, skipped, total: docs.length });
|
||||
} catch (err) {
|
||||
try { fs.unlinkSync(zipPath); } catch {}
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
Loading…
x
Reference in New Issue
Block a user