const fs = require('fs'); const path = require('path'); const https = require('https'); const http = require('http'); const { DatabaseSync } = require('node:sqlite'); const XML_FILE = path.join(__dirname, '../pengzhan.WordPress.2026-09-15.xml'); const CONTENT_DIR = path.join(__dirname, '../sample-content'); const POSTS_DIR = path.join(CONTENT_DIR, 'posts'); const STATIC_DIR = path.join(CONTENT_DIR, 'static'); const DOCS_DIR = path.join(STATIC_DIR, 'documents'); const DB_PATH = path.join(CONTENT_DIR, 'data/site.db'); const PUBLIC_DIR = path.join(__dirname, '../public'); // Ensure destination directories exist [POSTS_DIR, STATIC_DIR, DOCS_DIR, path.dirname(DB_PATH)].forEach(dir => { if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); }); // Helper to decode HTML entities function decodeHtmlEntities(str) { if (!str) return ''; return str .replace(/&/g, '&') .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') .replace(/'/g, "'") .replace(/’/g, "'") .replace(/‘/g, "'") .replace(/“/g, '"') .replace(/”/g, '"') .replace(/–/g, 'β') .replace(/—/g, 'β') .replace(/…/g, 'β¦') .replace(/ /g, ' '); } // Download file helper with follow redirects function downloadFile(url, destPath) { return new Promise((resolve, reject) => { if (fs.existsSync(destPath) && fs.statSync(destPath).size > 0) { return resolve(destPath); } const destDir = path.dirname(destPath); if (!fs.existsSync(destDir)) fs.mkdirSync(destDir, { recursive: true }); function get(currentUrl, redirectCount = 0) { if (redirectCount > 5) return reject(new Error('Too many redirects')); const client = currentUrl.startsWith('https') ? https : http; const req = client.get(currentUrl, { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' }, timeout: 15000 }, res => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { const nextUrl = res.headers.location.startsWith('http') ? res.headers.location : new URL(res.headers.location, currentUrl).href; res.resume(); return get(nextUrl, redirectCount + 1); } if (res.statusCode !== 200) { res.resume(); return reject(new Error(`HTTP ${res.statusCode} for ${currentUrl}`)); } const fileStream = fs.createWriteStream(destPath); res.pipe(fileStream); fileStream.on('finish', () => { fileStream.close(); resolve(destPath); }); fileStream.on('error', err => { fs.unlink(destPath, () => {}); reject(err); }); }); req.on('error', err => reject(err)); req.on('timeout', () => { req.destroy(); reject(new Error(`Timeout downloading ${currentUrl}`)); }); } get(url); }); } // Convert WordPress Gutenberg HTML to clean Markdown function htmlToMarkdown(html) { if (!html) return ''; let md = html; // Remove Gutenberg comments md = md.replace(//g, ''); // Handle
...
md = md.replace(/]*>]*class="[^"]*language-([a-zA-Z0-9_-]+)[^"]*"[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (m, lang, code) => {
return `\n\n\`\`\`${lang}\n${decodeHtmlEntities(code).trim()}\n\`\`\`\n\n`;
});
md = md.replace(/]*>]*>([\s\S]*?)<\/code><\/pre>/gi, (m, code) => {
const raw = decodeHtmlEntities(code).trim();
let lang = '';
if (raw.startsWith('#') || raw.startsWith('$ ') || raw.includes('sudo ') || raw.includes('kubectl ') || raw.includes('./')) {
lang = 'bash';
} else if (raw.includes('def ') || raw.includes('import ') || raw.includes('print(')) {
lang = 'python';
} else if (raw.includes('func ') || raw.includes('package ')) {
lang = 'go';
}
return `\n\n\`\`\`${lang}\n${raw}\n\`\`\`\n\n`;
});
md = md.replace(/]*>([\s\S]*?)<\/pre>/gi, (m, code) => {
return `\n\n\`\`\`\n${decodeHtmlEntities(code).trim()}\n\`\`\`\n\n`;
});
// Images inside figures: 
...
md = md.replace(/]*>\s*
]*src="([^"]+)"[^>]*alt="([^"]*)"[^>]*\/?>\s*(?:]*>([\s\S]*?)<\/figcaption>)?\s*<\/figure>/gi, (m, src, alt, caption) => {
const title = caption ? ` "${decodeHtmlEntities(caption).trim()}"` : '';
return `\n\n\n\n`;
});
// Plain
tags
md = md.replace(/
]*src="([^"]+)"[^>]*alt="([^"]*)"[^>]*\/?>/gi, (m, src, alt) => {
return ``;
});
// Headings
md = md.replace(/]*>([\s\S]*?)<\/h1>/gi, (m, c) => `\n\n# ${decodeHtmlEntities(c).trim()}\n\n`);
md = md.replace(/]*>([\s\S]*?)<\/h2>/gi, (m, c) => `\n\n## ${decodeHtmlEntities(c).trim()}\n\n`);
md = md.replace(/]*>([\s\S]*?)<\/h3>/gi, (m, c) => `\n\n### ${decodeHtmlEntities(c).trim()}\n\n`);
md = md.replace(/]*>([\s\S]*?)<\/h4>/gi, (m, c) => `\n\n#### ${decodeHtmlEntities(c).trim()}\n\n`);
md = md.replace(/]*>([\s\S]*?)<\/h5>/gi, (m, c) => `\n\n##### ${decodeHtmlEntities(c).trim()}\n\n`);
md = md.replace(/]*>([\s\S]*?)<\/h6>/gi, (m, c) => `\n\n###### ${decodeHtmlEntities(c).trim()}\n\n`);
// Blockquotes
md = md.replace(/]*>([\s\S]*?)<\/blockquote>/gi, (m, c) => {
const lines = decodeHtmlEntities(c).trim().split('\n');
return '\n\n' + lines.map(l => `> ${l.trim()}`).join('\n') + '\n\n';
});
// Lists
md = md.replace(/]*>([\s\S]*?)<\/li>/gi, (m, c) => `\n- ${decodeHtmlEntities(c).trim()}`);
md = md.replace(/<\/?ul[^>]*>/gi, '\n');
md = md.replace(/<\/?ol[^>]*>/gi, '\n');
// Paragraphs
md = md.replace(/