guanghulab/server/novel-db/app/services/upload-engine.js

/**
 * ═══════════════════════════════════════════════════════════
 * 上传引擎 · Upload Engine Service
 * ═══════════════════════════════════════════════════════════
 *
 * 智库节点 Phase 4 · ZY-SVR-006
 * 守护: 铸渊 · ICE-GL-ZY001
 * 版权: 国作登字-2026-A-00037559
 *
 * 功能:
 *   - 接收用户上传的小说文件
 *   - 自动检测文件格式 (TXT/EPUB/HTML/MD/DOCX)
 *   - 转换为统一 TXT 格式存储
 *   - 自动触发分章引擎
 *   - 支持多种编码自动检测
 *
 * 支持格式:
 *   - .txt  (纯文本 · 自动检测编码)
 *   - .epub (电子书 · 提取正文)
 *   - .html (网页 · 剥离标签)
 *   - .md   (Markdown · 转纯文本)
 *   - .docx (Word文档 · 提取正文，需额外解析)
 *
 * ═══════════════════════════════════════════════════════════
 */

'use strict';

const fs     = require('fs');
const path   = require('path');
const crypto = require('crypto');

const BOOKS_DIR   = process.env.BOOKS_DIR || path.join(__dirname, '..', 'data', 'books');
const UPLOAD_DIR  = path.join(__dirname, '..', 'data', 'uploads');
const UPLOAD_FILE = path.join(__dirname, '..', 'data', 'uploads-index.json');

// 支持的格式及 MIME 类型
const SUPPORTED_FORMATS = {
  '.txt':  { mime: ['text/plain'], name: '纯文本' },
  '.epub': { mime: ['application/epub+zip'], name: '电子书' },
  '.html': { mime: ['text/html'], name: '网页' },
  '.htm':  { mime: ['text/html'], name: '网页' },
  '.md':   { mime: ['text/markdown', 'text/plain'], name: 'Markdown' },
  '.docx': { mime: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'], name: 'Word文档' }
};

const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB

let uploadsIndex = [];

// ─── 初始化 ───
function init() {
  if (!fs.existsSync(UPLOAD_DIR)) fs.mkdirSync(UPLOAD_DIR, { recursive: true });
  if (!fs.existsSync(BOOKS_DIR)) fs.mkdirSync(BOOKS_DIR, { recursive: true });
  loadIndex();
}

function loadIndex() {
  try {
    if (fs.existsSync(UPLOAD_FILE)) {
      uploadsIndex = JSON.parse(fs.readFileSync(UPLOAD_FILE, 'utf8'));
    }
  } catch {
    uploadsIndex = [];
  }
}

function saveIndex() {
  try {
    const dir = path.dirname(UPLOAD_FILE);
    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
    fs.writeFileSync(UPLOAD_FILE, JSON.stringify(uploadsIndex, null, 2), 'utf8');
  } catch (err) {
    console.error('[UploadEngine] 保存索引失败:', err.message);
  }
}

/**
 * 检测文件格式
 */
function detectFormat(filename, mimeType) {
  const ext = path.extname(filename).toLowerCase();

  if (SUPPORTED_FORMATS[ext]) {
    return {
      ext,
      format: SUPPORTED_FORMATS[ext].name,
      supported: true
    };
  }

  // 尝试从 MIME 类型推断
  for (const [formatExt, cfg] of Object.entries(SUPPORTED_FORMATS)) {
    if (cfg.mime.includes(mimeType)) {
      return { ext: formatExt, format: cfg.name, supported: true };
    }
  }

  return { ext, format: '未知格式', supported: false };
}

/**
 * 将 HTML 内容转为纯文本（安全剥离所有标签）
 */
function htmlToText(html) {
  let text = html;
  // 移除 script 和 style 标签及内容（使用 split/join 避免 ReDoS）
  text = removeTagWithContent(text, 'script');
  text = removeTagWithContent(text, 'style');
  // 段落和换行
  text = text
    .replace(/<\/p>/gi, '\n\n')
    .replace(/<br\s*\/?>/gi, '\n')
    .replace(/<\/div>/gi, '\n')
    .replace(/<\/h[1-6]>/gi, '\n\n')
    .replace(/<h[1-6][^>]*>/gi, '\n');
  // 移除所有 HTML 标签
  text = text.replace(/<[^>]+>/g, '');
  // 解码 HTML 实体
  text = text
    .replace(/&nbsp;/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&#x27;/g, "'");
  // 清理多余空行
  text = text.replace(/\n{3,}/g, '\n\n').trim();
  return text;
}

/**
 * 安全移除指定标签及其内容（避免正则 ReDoS）
 */
function removeTagWithContent(html, tagName) {
  const openTag = '<' + tagName;
  const closeTag = '</' + tagName + '>';
  let result = html;
  let maxIterations = 100;
  while (maxIterations-- > 0) {
    const openIdx = result.toLowerCase().indexOf(openTag);
    if (openIdx === -1) break;
    const closeIdx = result.toLowerCase().indexOf(closeTag, openIdx);
    if (closeIdx === -1) {
      // No close tag, remove from open tag to end of string
      result = result.slice(0, openIdx);
      break;
    }
    result = result.slice(0, openIdx) + result.slice(closeIdx + closeTag.length);
  }
  return result;
}

/**
 * 将 Markdown 转为纯文本
 */
function markdownToText(md) {
  return md
    // 标题
    .replace(/^#{1,6}\s+/gm, '')
    // 加粗/斜体
    .replace(/\*\*(.+?)\*\*/g, '$1')
    .replace(/\*(.+?)\*/g, '$1')
    .replace(/__(.+?)__/g, '$1')
    .replace(/_(.+?)_/g, '$1')
    // 链接
    .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
    // 图片
    .replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
    // 代码块
    .replace(/```[\s\S]*?```/g, '')
    .replace(/`([^`]+)`/g, '$1')
    // 引用
    .replace(/^>\s+/gm, '')
    // 分隔线
    .replace(/^[-*_]{3,}$/gm, '')
    // 清理
    .replace(/\n{3,}/g, '\n\n')
    .trim();
}

/**
 * 提取 EPUB 文本内容（简易版）
 *
 * EPUB 本质上是 ZIP 文件，内部包含 XHTML 文件
 * 注意: 此简易实现通过字符扫描提取文本，不解压 ZIP
 * 对于复杂 EPUB，建议转为 TXT 后重新上传
 */
function extractEpubText(buffer) {
  // Convert buffer to string, ignoring non-UTF8 bytes
  const content = buffer.toString('utf8');

  // Extract text between <body> and </body> tags using indexOf (safe from ReDoS)
  const textParts = [];
  let searchStart = 0;
  const maxParts = 200;

  while (textParts.length < maxParts) {
    const bodyOpen = content.toLowerCase().indexOf('<body', searchStart);
    if (bodyOpen === -1) break;
    const bodyTagEnd = content.indexOf('>', bodyOpen);
    if (bodyTagEnd === -1) break;
    const bodyClose = content.toLowerCase().indexOf('</body>', bodyTagEnd);
    if (bodyClose === -1) break;

    const bodyContent = content.slice(bodyTagEnd + 1, bodyClose);
    const text = htmlToText(bodyContent);
    if (text.length > 20) {
      textParts.push(text);
    }
    searchStart = bodyClose + 7;
  }

  if (textParts.length > 0) {
    return textParts.join('\n\n');
  }

  // 降级: 提取可见纯文本行
  const lines = content.split('\n').filter(line => {
    const trimmed = line.trim();
    return trimmed.length > 10
      && !trimmed.startsWith('<')
      && !trimmed.startsWith('PK')
      && !/[\x00-\x08]/.test(trimmed);
  });

  return lines.join('\n') || '（EPUB解析受限，建议转为TXT后重新上传）';
}

/**
 * 转换上传文件为纯文本
 */
function convertToText(buffer, format) {
  let text;

  switch (format.ext) {
    case '.txt':
      text = buffer.toString('utf8');
      break;

    case '.html':
    case '.htm':
      text = htmlToText(buffer.toString('utf8'));
      break;

    case '.md':
      text = markdownToText(buffer.toString('utf8'));
      break;

    case '.epub':
      text = extractEpubText(buffer);
      break;

    case '.docx':
      // DOCX 是 ZIP 文件包含 XML
      // 简易提取：类似 EPUB
      text = extractEpubText(buffer);
      if (text.length < 50) {
        text = '（DOCX格式建议先用Word另存为TXT后重新上传）';
      }
      break;

    default:
      // 尝试作为纯文本处理
      text = buffer.toString('utf8');
      break;
  }

  // 统一换行符
  text = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');

  return text;
}

/**
 * 处理上传文件
 *
 * @param {object} file  { originalname, mimetype, buffer, size }
 * @param {object} meta  { title, author, uploader }
 * @returns {object} 上传结果
 */
function processUpload(file, meta) {
  if (!file || !file.buffer) {
    throw new Error('没有收到文件数据');
  }

  if (file.size > MAX_FILE_SIZE) {
    throw new Error(`文件过大 (${(file.size / 1024 / 1024).toFixed(1)}MB)，最大支持 50MB`);
  }

  // 检测格式
  const format = detectFormat(file.originalname, file.mimetype);

  // 转换为文本
  const text = convertToText(file.buffer, format);
  const wordCount = text.replace(/\s/g, '').length;

  if (wordCount < 10) {
    throw new Error('文件内容为空或无法解析，请确认文件格式正确');
  }

  // 生成安全的文件名
  const baseName = (meta.title || path.basename(file.originalname, path.extname(file.originalname)))
    .replace(/[<>:"/\\|?*\x00-\x1F]/g, '_')
    .slice(0, 100);
  const author = (meta.author || '未知')
    .replace(/[<>:"/\\|?*\x00-\x1F]/g, '_')
    .slice(0, 50);
  const uploadId = `UPL-${Date.now()}-${crypto.randomBytes(3).toString('hex')}`;
  const targetFilename = `${baseName}_${author}_upload.txt`;
  const targetPath = path.join(BOOKS_DIR, targetFilename);

  // 保存为 TXT
  fs.writeFileSync(targetPath, text, 'utf8');

  // 记录索引
  const record = {
    upload_id:       uploadId,
    original_name:   file.originalname,
    original_format: format.format,
    original_ext:    format.ext,
    original_size:   file.size,
    converted_file:  targetFilename,
    title:           meta.title || baseName,
    author:          meta.author || '未知',
    uploader:        meta.uploader || 'unknown',
    word_count:      wordCount,
    chapter_count:   0,
    created_at:      new Date().toISOString()
  };

  uploadsIndex.push(record);
  saveIndex();

  return record;
}

/**
 * 获取上传列表
 */
function listUploads(options) {
  let result = [...uploadsIndex];
  if (options && options.uploader) {
    result = result.filter(u => u.uploader === options.uploader);
  }
  result.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
  if (options && options.limit) {
    result = result.slice(0, options.limit);
  }
  return result;
}

/**
 * 获取支持的格式列表
 */
function getSupportedFormats() {
  return Object.entries(SUPPORTED_FORMATS).map(([ext, cfg]) => ({
    ext,
    name: cfg.name,
    mime: cfg.mime
  }));
}

/**
 * 统计
 */
function getStats() {
  return {
    total_uploads: uploadsIndex.length,
    total_words:   uploadsIndex.reduce((s, u) => s + (u.word_count || 0), 0),
    formats:       uploadsIndex.reduce((acc, u) => {
      const f = u.original_format || '未知';
      acc[f] = (acc[f] || 0) + 1;
      return acc;
    }, {})
  };
}

init();

module.exports = {
  processUpload,
  listUploads,
  detectFormat,
  getSupportedFormats,
  getStats
};