const fs = require('fs'); const path = require('path'); const https = require('https'); const http = require('http'); const { DatabaseSync } = require('node:sqlite'); const XML_FILE = path.join(__dirname, '../pengzhan.WordPress.2026-09-15.xml'); const CONTENT_DIR = path.join(__dirname, '../sample-content'); const POSTS_DIR = path.join(CONTENT_DIR, 'posts'); const STATIC_DIR = path.join(CONTENT_DIR, 'static'); const DOCS_DIR = path.join(STATIC_DIR, 'documents'); const DB_PATH = path.join(CONTENT_DIR, 'data/site.db'); const PUBLIC_DIR = path.join(__dirname, '../public'); // Ensure destination directories exist [POSTS_DIR, STATIC_DIR, DOCS_DIR, path.dirname(DB_PATH)].forEach(dir => { if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); }); // Helper to decode HTML entities function decodeHtmlEntities(str) { if (!str) return ''; return str .replace(/&/g, '&') .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') .replace(/'/g, "'") .replace(/’/g, "'") .replace(/‘/g, "'") .replace(/“/g, '"') .replace(/”/g, '"') .replace(/–/g, '–') .replace(/—/g, 'β€”') .replace(/…/g, '…') .replace(/ /g, ' '); } // Download file helper with follow redirects function downloadFile(url, destPath) { return new Promise((resolve, reject) => { if (fs.existsSync(destPath) && fs.statSync(destPath).size > 0) { return resolve(destPath); } const destDir = path.dirname(destPath); if (!fs.existsSync(destDir)) fs.mkdirSync(destDir, { recursive: true }); function get(currentUrl, redirectCount = 0) { if (redirectCount > 5) return reject(new Error('Too many redirects')); const client = currentUrl.startsWith('https') ? https : http; const req = client.get(currentUrl, { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' }, timeout: 15000 }, res => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { const nextUrl = res.headers.location.startsWith('http') ? res.headers.location : new URL(res.headers.location, currentUrl).href; res.resume(); return get(nextUrl, redirectCount + 1); } if (res.statusCode !== 200) { res.resume(); return reject(new Error(`HTTP ${res.statusCode} for ${currentUrl}`)); } const fileStream = fs.createWriteStream(destPath); res.pipe(fileStream); fileStream.on('finish', () => { fileStream.close(); resolve(destPath); }); fileStream.on('error', err => { fs.unlink(destPath, () => {}); reject(err); }); }); req.on('error', err => reject(err)); req.on('timeout', () => { req.destroy(); reject(new Error(`Timeout downloading ${currentUrl}`)); }); } get(url); }); } // Convert WordPress Gutenberg HTML to clean Markdown function htmlToMarkdown(html) { if (!html) return ''; let md = html; // Remove Gutenberg comments md = md.replace(//g, ''); // Handle
...
md = md.replace(/]*>]*class="[^"]*language-([a-zA-Z0-9_-]+)[^"]*"[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (m, lang, code) => { return `\n\n\`\`\`${lang}\n${decodeHtmlEntities(code).trim()}\n\`\`\`\n\n`; }); md = md.replace(/]*>]*>([\s\S]*?)<\/code><\/pre>/gi, (m, code) => { const raw = decodeHtmlEntities(code).trim(); let lang = ''; if (raw.startsWith('#') || raw.startsWith('$ ') || raw.includes('sudo ') || raw.includes('kubectl ') || raw.includes('./')) { lang = 'bash'; } else if (raw.includes('def ') || raw.includes('import ') || raw.includes('print(')) { lang = 'python'; } else if (raw.includes('func ') || raw.includes('package ')) { lang = 'go'; } return `\n\n\`\`\`${lang}\n${raw}\n\`\`\`\n\n`; }); md = md.replace(/]*>([\s\S]*?)<\/pre>/gi, (m, code) => { return `\n\n\`\`\`\n${decodeHtmlEntities(code).trim()}\n\`\`\`\n\n`; }); // Images inside figures:
...
...
md = md.replace(/]*>\s*]*src="([^"]+)"[^>]*alt="([^"]*)"[^>]*\/?>\s*(?:]*>([\s\S]*?)<\/figcaption>)?\s*<\/figure>/gi, (m, src, alt, caption) => { const title = caption ? ` "${decodeHtmlEntities(caption).trim()}"` : ''; return `\n\n![${decodeHtmlEntities(alt || caption || '').trim()}](${src}${title})\n\n`; }); // Plain tags md = md.replace(/]*src="([^"]+)"[^>]*alt="([^"]*)"[^>]*\/?>/gi, (m, src, alt) => { return `![${decodeHtmlEntities(alt || '').trim()}](${src})`; }); // Headings md = md.replace(/]*>([\s\S]*?)<\/h1>/gi, (m, c) => `\n\n# ${decodeHtmlEntities(c).trim()}\n\n`); md = md.replace(/]*>([\s\S]*?)<\/h2>/gi, (m, c) => `\n\n## ${decodeHtmlEntities(c).trim()}\n\n`); md = md.replace(/]*>([\s\S]*?)<\/h3>/gi, (m, c) => `\n\n### ${decodeHtmlEntities(c).trim()}\n\n`); md = md.replace(/]*>([\s\S]*?)<\/h4>/gi, (m, c) => `\n\n#### ${decodeHtmlEntities(c).trim()}\n\n`); md = md.replace(/]*>([\s\S]*?)<\/h5>/gi, (m, c) => `\n\n##### ${decodeHtmlEntities(c).trim()}\n\n`); md = md.replace(/]*>([\s\S]*?)<\/h6>/gi, (m, c) => `\n\n###### ${decodeHtmlEntities(c).trim()}\n\n`); // Blockquotes md = md.replace(/]*>([\s\S]*?)<\/blockquote>/gi, (m, c) => { const lines = decodeHtmlEntities(c).trim().split('\n'); return '\n\n' + lines.map(l => `> ${l.trim()}`).join('\n') + '\n\n'; }); // Lists md = md.replace(/]*>([\s\S]*?)<\/li>/gi, (m, c) => `\n- ${decodeHtmlEntities(c).trim()}`); md = md.replace(/<\/?ul[^>]*>/gi, '\n'); md = md.replace(/<\/?ol[^>]*>/gi, '\n'); // Paragraphs md = md.replace(/]*>([\s\S]*?)<\/p>/gi, (m, c) => `\n\n${decodeHtmlEntities(c).trim()}\n\n`); // Inline formatting md = md.replace(/<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi, (m, tag, c) => `**${decodeHtmlEntities(c).trim()}**`); md = md.replace(/<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi, (m, tag, c) => `*${decodeHtmlEntities(c).trim()}*`); md = md.replace(/]*>([\s\S]*?)<\/code>/gi, (m, c) => `\`${decodeHtmlEntities(c).trim()}\``); md = md.replace(/]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi, (m, href, c) => `[${decodeHtmlEntities(c).trim()}](${href})`); // Horizontal rules md = md.replace(/]*\/?>/gi, '\n\n---\n\n'); // Stripped tags md = md.replace(//gi, '\n'); md = md.replace(/<[^>]+>/g, ''); // Clean excessive newlines md = md.replace(/\n{3,}/g, '\n\n').trim(); return md; } async function runMigration() { console.log('===================================================='); console.log(' Starting WordPress Migration for pengzhan.dev'); console.log(` XML Source: ${XML_FILE}`); console.log('===================================================='); if (!fs.existsSync(XML_FILE)) { console.error(`❌ Export file not found at ${XML_FILE}`); process.exit(1); } const xml = fs.readFileSync(XML_FILE, 'utf8'); const items = xml.split(''); console.log(`Found ${items.length - 1} total items in XML export.`); // 1. Map all Attachments const attachments = new Map(); // id -> { id, title, url, filename, postName } const attachmentsByUrl = new Map(); for (let i = 1; i < items.length; i++) { const item = items[i]; const typeMatch = item.match(/(?:)?<\/wp:post_type>/); if (typeMatch && typeMatch[1] === 'attachment') { const idMatch = item.match(/(\d+)<\/wp:post_id>/); const titleMatch = item.match(/(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/title>/s); const urlMatch = item.match(/<wp:attachment_url>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/wp:attachment_url>/s); const postNameMatch = item.match(/<wp:post_name>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/wp:post_name>/s); if (idMatch && urlMatch) { const id = idMatch[1]; const url = urlMatch[1].trim(); const filename = path.basename(new URL(url).pathname); const attachment = { id, title: titleMatch ? titleMatch[1].trim() : filename, url, filename, postName: postNameMatch ? postNameMatch[1].trim() : '' }; attachments.set(id, attachment); attachmentsByUrl.set(url, attachment); } } } console.log(`\n[Phase 1] Cataloged ${attachments.size} media attachments.`); // Download key attachments (PDFs, favicons, site logo) console.log('\n[Phase 2] Downloading static documents, publication PDFs, and site icons...'); for (const [id, att] of attachments.entries()) { const { url, filename } = att; // Academic publications / PDFs -> sample-content/static/documents/ if (filename.endsWith('.pdf')) { const docPath = path.join(DOCS_DIR, filename); process.stdout.write(` πŸ“„ Downloading PDF: ${filename} ... `); try { await downloadFile(url, docPath); console.log('βœ“'); } catch (err) { console.log(`⚠️ Failed (${err.message})`); } } // Site icons / favicons -> public/ and sample-content/static/ if (filename.includes('android-chrome') || filename.includes('logo') || filename.endsWith('.ico')) { const pubPath = path.join(PUBLIC_DIR, filename); const staticPath = path.join(STATIC_DIR, filename); process.stdout.write(` 🎨 Downloading icon/logo: ${filename} ... `); try { await downloadFile(url, pubPath); fs.copyFileSync(pubPath, staticPath); console.log('βœ“'); } catch (err) { console.log(`⚠️ Failed (${err.message})`); } } // Fonts -> sample-content/static/fonts/ if (filename.endsWith('.otf') || filename.endsWith('.ttf') || filename.endsWith('.woff2')) { const fontsDir = path.join(STATIC_DIR, 'fonts'); if (!fs.existsSync(fontsDir)) fs.mkdirSync(fontsDir, { recursive: true }); const destPath = path.join(fontsDir, filename); process.stdout.write(` πŸ”€ Downloading font: ${filename} ... `); try { await downloadFile(url, destPath); console.log('βœ“'); } catch (err) { console.log(`⚠️ Failed (${err.message})`); } } } // Create standard favicon aliases in public/ const baseIcon = [ path.join(PUBLIC_DIR, 'cropped-android-chrome-192x192-1.png'), path.join(PUBLIC_DIR, 'android-chrome-192x192-1.png'), path.join(PUBLIC_DIR, 'cropped-android-chrome-192x192-1-32x32.png') ].find(f => fs.existsSync(f)); if (baseIcon) { [ path.join(PUBLIC_DIR, 'favicon-32x32.png'), path.join(PUBLIC_DIR, 'favicon.ico'), path.join(PUBLIC_DIR, 'android-chrome-192x192.png'), path.join(PUBLIC_DIR, 'apple-touch-icon.png'), path.join(STATIC_DIR, 'favicon.ico'), path.join(STATIC_DIR, 'favicon-32x32.png'), path.join(STATIC_DIR, 'android-chrome-192x192.png'), path.join(STATIC_DIR, 'apple-touch-icon.png') ].forEach(dst => { try { fs.copyFileSync(baseIcon, dst); } catch (_) {} }); } // 2. Parse & Migrate Published Posts console.log('\n[Phase 3] Migrating published blog posts to Markdown...'); const postsMigrated = []; for (let i = 1; i < items.length; i++) { const item = items[i]; const typeMatch = item.match(/<wp:post_type>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/wp:post_type>/); const statusMatch = item.match(/<wp:status>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/wp:status>/); const type = typeMatch ? typeMatch[1] : ''; const status = statusMatch ? statusMatch[1] : ''; if (type !== 'post' || status !== 'publish') continue; const titleMatch = item.match(/<title>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/title>/s); const title = titleMatch ? decodeHtmlEntities(titleMatch[1].trim()) : 'Untitled'; const slugMatch = item.match(/<wp:post_name>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/wp:post_name>/s); let slug = slugMatch ? slugMatch[1].trim() : ''; if (!slug) { slug = title.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, ''); } const dateMatch = item.match(/<wp:post_date>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/wp:post_date>/s); const dateStr = dateMatch ? dateMatch[1].trim() : new Date().toISOString(); const date = new Date(dateStr); const categories = [...item.matchAll(/<category domain=\"category\" nicename=\"(.*?)\">(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/category>/g)].map(m => decodeHtmlEntities(m[2].trim())); const tags = [...item.matchAll(/<category domain=\"post_tag\" nicename=\"(.*?)\">(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/category>/g)].map(m => decodeHtmlEntities(m[2].trim())); const category = categories[0] || (tags.includes('Kubernetes') || tags.includes('Debug') ? 'Tech' : 'Tech'); const contentMatch = item.match(/<content:encoded>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/content:encoded>/); const rawHtml = contentMatch ? contentMatch[1] : ''; // Convert body HTML to Markdown let mdBody = htmlToMarkdown(rawHtml); // Find thumbnail / cover image const thumbMatch = item.match(/<wp:meta_key>(?:<!\[CDATA\[)?_thumbnail_id(?:\]\]>)?<\/wp:meta_key>\s*<wp:meta_value>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/wp:meta_value>/); let coverUrl = null; let localCoverRel = null; if (thumbMatch && thumbMatch[1] && attachments.has(thumbMatch[1])) { const att = attachments.get(thumbMatch[1]); coverUrl = att.url; } // Support bilingual posts (e.g. ai-can-change-coding-for-kids) let isBilingualEn = (slug === 'ai-can-change-coding-for-kids-en'); let targetSlug = isBilingualEn ? 'ai-can-change-coding-for-kids' : slug; let mdFilename = isBilingualEn ? 'en.md' : 'index.md'; let lang = isBilingualEn ? 'en' : (slug === 'ai-can-change-coding-for-kids' ? 'zh' : 'en'); // Prepare post folder: sample-content/posts/<targetSlug>/ const postDir = path.join(POSTS_DIR, targetSlug); const postImagesDir = path.join(postDir, 'images'); if (!fs.existsSync(postImagesDir)) fs.mkdirSync(postImagesDir, { recursive: true }); // Download cover image to post's images folder if (coverUrl) { const coverExt = path.extname(new URL(coverUrl).pathname) || '.webp'; const coverFilename = isBilingualEn ? `cover-en${coverExt}` : `cover${coverExt}`; const localCoverPath = path.join(postImagesDir, coverFilename); try { await downloadFile(coverUrl, localCoverPath); localCoverRel = `/posts-assets/${targetSlug}/images/${coverFilename}`; } catch (err) { localCoverRel = coverUrl; } } // Extract inline images and video in post body and download them const inlineImageMatches = [...mdBody.matchAll(/!\[(.*?)\]\((https?:\/\/pengzhan\.dev\/wp-content\/uploads\/[^)]+)\)/g)]; for (const imgMatch of inlineImageMatches) { const remoteImgUrl = imgMatch[2].split(' ')[0].replace(/['"]/g, ''); const imgFilename = path.basename(new URL(remoteImgUrl).pathname); const localImgPath = path.join(postImagesDir, imgFilename); try { await downloadFile(remoteImgUrl, localImgPath); mdBody = mdBody.replace(remoteImgUrl, `/posts-assets/${targetSlug}/images/${imgFilename}`); } catch (_) {} } // Extract video if present const videoMatches = [...rawHtml.matchAll(/<video[^>]*src=\"(https?:\/\/pengzhan\.dev\/wp-content\/uploads\/[^"]+)\"/g)]; for (const vMatch of videoMatches) { const remoteVidUrl = vMatch[1]; const vidFilename = path.basename(new URL(remoteVidUrl).pathname); const localVidPath = path.join(postImagesDir, vidFilename); try { await downloadFile(remoteVidUrl, localVidPath); const vidHtml = `\n\n<video controls loop muted playsinline style="max-width: 100%; border-radius: 8px; margin: 1.5rem 0;" src="/posts-assets/${targetSlug}/images/${vidFilename}"></video>\n\n`; mdBody = mdBody.replace("Here's a demo:", "Here's a demo:" + vidHtml); } catch (_) {} } // Clean up any manual cross-link prefixes in bilingual posts mdBody = mdBody.replace(/^\[(English Version|δΈ­ζ–‡εŽŸζ–‡)\]\(https?:\/\/[^)]+\)\s*\n+/i, ''); // Generate excerpt from first paragraph const firstPara = (mdBody.split('\n\n').find(p => p.trim() && !p.startsWith('#') && !p.startsWith('!') && !p.startsWith('<video')) || '') .replace(/[#*`_\[\]]/g, '') .slice(0, 180) .trim(); const excerpt = firstPara ? `${firstPara}...` : ''; // Write YAML Frontmatter + Markdown const frontmatter = [ '---', `title: ${JSON.stringify(title)}`, `date: ${date.toISOString().split('T')[0]}`, `author: "Pengzhan Hao"`, `category: ${JSON.stringify(category)}`, `tags: ${JSON.stringify(tags)}`, localCoverRel ? `cover: ${JSON.stringify(localCoverRel)}` : null, excerpt ? `excerpt: ${JSON.stringify(excerpt)}` : null, (lang !== 'en' || isBilingualEn) ? `lang: ${JSON.stringify(lang)}` : null, '---', '' ].filter(Boolean).join('\n'); const fullContent = `${frontmatter}\n${mdBody}\n`; fs.writeFileSync(path.join(postDir, mdFilename), fullContent, 'utf8'); postsMigrated.push({ slug: targetSlug, title, lang, date: date.toISOString().split('T')[0] }); console.log(` βœ“ Migrated: [${date.toISOString().split('T')[0]}] ${title} (/${targetSlug}${isBilingualEn ? '/en' : ''})`); } // 3. Migrate Comments to SQLite site.db console.log('\n[Phase 4] Migrating historical comments into SQLite site.db...'); const db = new DatabaseSync(DB_PATH); // Ensure comments table exists db.exec(` CREATE TABLE IF NOT EXISTS comments ( id INTEGER PRIMARY KEY AUTOINCREMENT, post_slug TEXT NOT NULL, author_name TEXT NOT NULL, author_email TEXT, author_website TEXT, body TEXT NOT NULL, created_at TEXT NOT NULL, approved INTEGER DEFAULT 0, ip TEXT ); CREATE INDEX IF NOT EXISTS idx_comments_slug ON comments(post_slug, approved); CREATE INDEX IF NOT EXISTS idx_comments_email ON comments(author_email); `); const insertCommentStmt = db.prepare(` INSERT INTO comments (id, post_slug, author_name, author_email, author_website, body, created_at, approved, ip) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) `); const fallbackCommentStmt = db.prepare(` INSERT INTO comments (post_slug, author_name, author_email, author_website, body, created_at, approved, ip) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `); const parsedComments = []; for (let i = 1; i < items.length; i++) { const item = items[i]; const postNameMatch = item.match(/<wp:post_name>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/wp:post_name>/s); const postSlug = postNameMatch ? postNameMatch[1].trim() : ''; if (!postSlug) continue; const commentMatches = item.match(/<wp:comment>[\s\S]*?<\/wp:comment>/g) || []; for (const cXml of commentMatches) { const idMatch = cXml.match(/<wp:comment_id>(\d+)<\/wp:comment_id>/); const authorMatch = cXml.match(/<wp:comment_author>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/wp:comment_author>/); const emailMatch = cXml.match(/<wp:comment_author_email>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/wp:comment_author_email>/); const urlMatch = cXml.match(/<wp:comment_author_url>([\s\S]*?)<\/wp:comment_author_url>/); const ipMatch = cXml.match(/<wp:comment_author_IP>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/wp:comment_author_IP>/); const dateMatch = cXml.match(/<wp:comment_date>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/wp:comment_date>/); const contentMatch = cXml.match(/<wp:comment_content>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/wp:comment_content>/); const approvedMatch = cXml.match(/<wp:comment_approved>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/wp:comment_approved>/); const approvedRaw = approvedMatch ? approvedMatch[1].trim() : '0'; let approved = 0; if (approvedRaw === '1') approved = 1; else if (approvedRaw === 'spam' || approvedRaw === '-1') approved = -1; else approved = 0; const body = decodeHtmlEntities(contentMatch ? contentMatch[1].trim() : ''); if (!body) continue; parsedComments.push({ id: idMatch ? Number(idMatch[1]) : null, post_slug: postSlug, author_name: decodeHtmlEntities(authorMatch ? authorMatch[1].trim() : 'Anonymous'), author_email: emailMatch ? emailMatch[1].trim() : null, author_website: urlMatch ? urlMatch[1].trim() : null, body, created_at: dateMatch ? dateMatch[1].trim() : new Date().toISOString(), approved, ip: ipMatch ? ipMatch[1].trim() : null }); } } // Clear existing comments before full sync db.exec('DELETE FROM comments'); db.exec('BEGIN'); let commentCount = 0; for (const c of parsedComments) { try { insertCommentStmt.run( c.id, c.post_slug, c.author_name, c.author_email, c.author_website, c.body, c.created_at, c.approved, c.ip ); commentCount++; } catch (_) { try { fallbackCommentStmt.run( c.post_slug, c.author_name, c.author_email, c.author_website, c.body, c.created_at, c.approved, c.ip ); commentCount++; } catch (_) {} } } db.exec('COMMIT'); db.close(); console.log(` βœ“ Inserted ${commentCount} historical comments into SQLite site.db.`); console.log('\n===================================================='); console.log(` MIGRATION COMPLETE!`); console.log(` - ${postsMigrated.length} Blog posts migrated to Markdown`); console.log(` - ${attachments.size} Attachments cataloged & localized`); console.log(` - ${commentCount} Comments synced to SQLite`); console.log('====================================================\n'); } runMigration().catch(err => { console.error('Migration failed with error:', err); process.exit(1); });