Add reviews and support tutorial management
This commit is contained in:
Codex
@@ -0,0 +1,378 @@
|
||||
import crypto from 'node:crypto'
|
||||
import { execFile } from 'node:child_process'
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import 'dotenv/config'
|
||||
|
||||
process.env.PAYLOAD_DB_PUSH = 'false'
|
||||
|
||||
import { getPayload } from 'payload'
|
||||
import sharp from 'sharp'
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
|
||||
const CACHE_DIR = path.join(ROOT, '.tmp', 'review-covers')
|
||||
const SCREENSHOT_DIR = path.join(ROOT, '.tmp', 'review-page-shots')
|
||||
const LOCALE = process.argv.includes('--zh') ? 'zh' : 'en'
|
||||
const LIMIT_ARG = process.argv.find((arg) => arg.startsWith('--limit='))
|
||||
const LIMIT = LIMIT_ARG ? Number(LIMIT_ARG.split('=')[1]) : 500
|
||||
const IDS_ARG = process.argv.find((arg) => arg.startsWith('--ids='))
|
||||
const IDS = IDS_ARG ? new Set(IDS_ARG.split('=')[1].split(',').map((id) => Number(id.trim())).filter(Boolean)) : null
|
||||
const SCREENSHOT_MISSING = process.argv.includes('--screenshots')
|
||||
const MISSING_ONLY = process.argv.includes('--missing-only')
|
||||
const execFileAsync = promisify(execFile)
|
||||
const CHROME_PATH = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
|
||||
|
||||
const SOURCE_BY_HOST = new Map([
|
||||
['6moons.com', '6moons'],
|
||||
['alpha-audio.net', 'Alpha Audio'],
|
||||
['areadvd.de', 'Area DVD'],
|
||||
['audiophile-heaven.com', 'Audiophile Heaven'],
|
||||
['audiopt.com', 'Audio & Cinema em Casa'],
|
||||
['audiosciencereview.com', 'Audio Science Review'],
|
||||
['av-online.hu', 'AV Online'],
|
||||
['avforums.com', 'AVForums'],
|
||||
['darko.audio', 'Darko.Audio'],
|
||||
['fairaudio.de', 'fairaudio'],
|
||||
['forbes.com', 'Forbes'],
|
||||
['fwd.nl', 'FWD'],
|
||||
['head-fi.org', 'Head-Fi'],
|
||||
['headfonia.com', 'Headfonia'],
|
||||
['headfonics.com', 'Headfonics'],
|
||||
['hfc.com.pl', 'Hi-Fi Class'],
|
||||
['hifi-ifas.de', 'HIFI IFAs'],
|
||||
['hifi-voice.com', 'Hi-Fi Voice'],
|
||||
['hifi.nl', 'HiFi.nl'],
|
||||
['hificlube.net', 'HiFiClube'],
|
||||
['hifinews.com', 'Hi-Fi News'],
|
||||
['hifipig.com', 'HiFi Pig'],
|
||||
['hifistatement.net', 'HIFISTATEMENT'],
|
||||
['lite-magazin.de', 'lite-magazin'],
|
||||
['lowbeats.de', 'LowBeats'],
|
||||
['lp-magazin.de', 'LP Magazin'],
|
||||
['on-mag.fr', 'ON-mag'],
|
||||
['pursuitperfectsystem.com', 'Pursuit Perfect System'],
|
||||
['sempre-audio.at', 'Sempre Audio'],
|
||||
['silviutudor.ro', 'Silviu Tudor'],
|
||||
['sound-advice.online', 'Sound Advice'],
|
||||
['soundnews.net', 'SoundNews'],
|
||||
['stereonet.com', 'StereoNET'],
|
||||
['stevehuffphoto.com', 'Steve Huff Photo'],
|
||||
['stereophile.com', 'Stereophile'],
|
||||
['theabsolutesound.com', 'The Absolute Sound'],
|
||||
])
|
||||
|
||||
function normalizeHost(value) {
|
||||
try {
|
||||
return new URL(value).hostname.replace(/^www\./i, '').toLowerCase()
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
function inferSourceName(url, html = '') {
|
||||
const host = normalizeHost(url)
|
||||
if (SOURCE_BY_HOST.has(host)) return SOURCE_BY_HOST.get(host)
|
||||
|
||||
const siteName = extractMeta(html, ['og:site_name', 'application-name'])
|
||||
if (siteName) return cleanText(siteName)
|
||||
|
||||
const parts = host.split('.').filter(Boolean)
|
||||
const core = parts.length > 1 ? parts.at(-2) : parts[0]
|
||||
return core ? core.replace(/[-_]+/g, ' ').replace(/\b\w/g, (char) => char.toUpperCase()) : ''
|
||||
}
|
||||
|
||||
function cleanText(value) {
|
||||
return decodeHtml(value)
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/\s*(?:[|]|-|\u2013)\s*$/u, '')
|
||||
.trim()
|
||||
}
|
||||
function decodeHtml(value) {
|
||||
return value
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'|'/g, "'")
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
}
|
||||
|
||||
function extractMeta(html, names) {
|
||||
for (const name of names) {
|
||||
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
const patterns = [
|
||||
new RegExp(`<meta[^>]+(?:property|name)=["']${escaped}["'][^>]+content=["']([^"']+)["'][^>]*>`, 'i'),
|
||||
new RegExp(`<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["']${escaped}["'][^>]*>`, 'i'),
|
||||
]
|
||||
for (const pattern of patterns) {
|
||||
const match = html.match(pattern)
|
||||
if (match?.[1]) return cleanText(match[1])
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function extractImageCandidates(html, pageUrl) {
|
||||
const values = [
|
||||
extractMeta(html, ['og:image:secure_url', 'og:image', 'twitter:image', 'twitter:image:src']),
|
||||
]
|
||||
|
||||
const linkMatch = html.match(/<link[^>]+rel=["'][^"']*image_src[^"']*["'][^>]+href=["']([^"']+)["']/i)
|
||||
if (linkMatch?.[1]) values.push(cleanText(linkMatch[1]))
|
||||
|
||||
const imgPattern = /<img[^>]+(?:src|data-src|data-lazy-src)=["']([^"']+)["'][^>]*>/gi
|
||||
let match
|
||||
while ((match = imgPattern.exec(html))) {
|
||||
values.push(cleanText(match[1]))
|
||||
}
|
||||
|
||||
const seen = new Set()
|
||||
return values
|
||||
.filter(Boolean)
|
||||
.map((value) => {
|
||||
try {
|
||||
return new URL(value, pageUrl).toString()
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
})
|
||||
.filter((value) => {
|
||||
if (!value || seen.has(value)) return false
|
||||
seen.add(value)
|
||||
return !/\.(svg|ico)(?:[?#]|$)/i.test(value)
|
||||
})
|
||||
}
|
||||
|
||||
async function fetchText(url) {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
accept: 'text/html,application/xhtml+xml',
|
||||
'user-agent':
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36',
|
||||
},
|
||||
redirect: 'follow',
|
||||
signal: AbortSignal.timeout(15000),
|
||||
})
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`)
|
||||
return response.text()
|
||||
}
|
||||
|
||||
function extensionFromContentType(contentType, url) {
|
||||
if (/png/i.test(contentType)) return '.png'
|
||||
if (/webp/i.test(contentType)) return '.webp'
|
||||
if (/gif/i.test(contentType)) return '.gif'
|
||||
if (/jpe?g/i.test(contentType)) return '.jpg'
|
||||
const ext = path.extname(new URL(url).pathname).toLowerCase()
|
||||
return ext && ext.length <= 5 ? ext : '.jpg'
|
||||
}
|
||||
|
||||
async function downloadImage(url, reviewId) {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
|
||||
referer: new URL(url).origin,
|
||||
'user-agent':
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36',
|
||||
},
|
||||
redirect: 'follow',
|
||||
signal: AbortSignal.timeout(15000),
|
||||
})
|
||||
if (!response.ok) throw new Error(`image HTTP ${response.status}`)
|
||||
|
||||
const contentType = response.headers.get('content-type') || ''
|
||||
if (!/image\//i.test(contentType)) throw new Error(`not an image: ${contentType}`)
|
||||
|
||||
const bytes = Buffer.from(await response.arrayBuffer())
|
||||
if (bytes.length < 5000) throw new Error('image too small')
|
||||
|
||||
const meta = await sharp(bytes, { animated: false }).metadata()
|
||||
if ((meta.width ?? 0) < 240 || (meta.height ?? 0) < 160) {
|
||||
throw new Error(`image dimensions too small: ${meta.width}x${meta.height}`)
|
||||
}
|
||||
|
||||
await fs.mkdir(CACHE_DIR, { recursive: true })
|
||||
const hash = crypto.createHash('sha1').update(url).digest('hex').slice(0, 10)
|
||||
const filename = `review-${LOCALE}-${reviewId}-${hash}${extensionFromContentType(contentType, url)}`
|
||||
const filePath = path.join(CACHE_DIR, filename)
|
||||
await fs.writeFile(filePath, bytes)
|
||||
return { filePath, filename, height: meta.height, width: meta.width }
|
||||
}
|
||||
|
||||
async function findMediaByFilename(payload, filename) {
|
||||
const existing = await payload.find({
|
||||
collection: 'media',
|
||||
limit: 1,
|
||||
where: {
|
||||
filename: {
|
||||
equals: filename,
|
||||
},
|
||||
},
|
||||
})
|
||||
return existing.docs[0] ?? null
|
||||
}
|
||||
|
||||
async function ensureMedia(payload, review, imageUrl, sourceName) {
|
||||
const downloaded = await downloadImage(imageUrl, review.id)
|
||||
const existing = await findMediaByFilename(payload, downloaded.filename)
|
||||
if (existing) return existing.id
|
||||
|
||||
const created = await payload.create({
|
||||
collection: 'media',
|
||||
data: {
|
||||
alt: `${review.title} cover`,
|
||||
credit: sourceName || undefined,
|
||||
},
|
||||
filePath: downloaded.filePath,
|
||||
})
|
||||
return created.id
|
||||
}
|
||||
|
||||
async function capturePageScreenshot(url, reviewId) {
|
||||
await fs.mkdir(SCREENSHOT_DIR, { recursive: true })
|
||||
const filePath = path.join(SCREENSHOT_DIR, `review-${LOCALE}-${reviewId}-page.png`)
|
||||
await execFileAsync(
|
||||
CHROME_PATH,
|
||||
[
|
||||
'--headless=new',
|
||||
'--disable-gpu',
|
||||
'--no-sandbox',
|
||||
'--hide-scrollbars',
|
||||
'--window-size=1280,720',
|
||||
`--screenshot=${filePath}`,
|
||||
url,
|
||||
],
|
||||
{ timeout: 25000 },
|
||||
)
|
||||
|
||||
const meta = await sharp(filePath).metadata()
|
||||
if ((meta.width ?? 0) < 800 || (meta.height ?? 0) < 400) {
|
||||
throw new Error(`screenshot dimensions too small: ${meta.width}x${meta.height}`)
|
||||
}
|
||||
return filePath
|
||||
}
|
||||
|
||||
async function ensureScreenshotMedia(payload, review, sourceName) {
|
||||
const filename = `review-${LOCALE}-${review.id}-page.png`
|
||||
const existing = await findMediaByFilename(payload, filename)
|
||||
if (existing) return existing.id
|
||||
|
||||
const filePath = await capturePageScreenshot(review.reviewUrl, review.id)
|
||||
const created = await payload.create({
|
||||
collection: 'media',
|
||||
data: {
|
||||
alt: `${review.title} source page`,
|
||||
credit: sourceName || undefined,
|
||||
},
|
||||
filePath,
|
||||
})
|
||||
return created.id
|
||||
}
|
||||
|
||||
async function processReview(payload, review) {
|
||||
let html = ''
|
||||
try {
|
||||
html = await fetchText(review.reviewUrl)
|
||||
} catch (error) {
|
||||
const sourceName = inferSourceName(review.reviewUrl)
|
||||
let thumbnail = null
|
||||
let reason = error.message
|
||||
if (SCREENSHOT_MISSING) {
|
||||
try {
|
||||
thumbnail = await ensureScreenshotMedia(payload, review, sourceName)
|
||||
reason = ''
|
||||
} catch (screenshotError) {
|
||||
reason = `${error.message}; screenshot ${screenshotError.message}`
|
||||
}
|
||||
}
|
||||
await payload.update({
|
||||
collection: 'productReviews',
|
||||
id: review.id,
|
||||
locale: LOCALE,
|
||||
data: { sourceName, ...(thumbnail ? { thumbnail } : {}) },
|
||||
})
|
||||
return { id: review.id, sourceName, status: thumbnail ? 'screenshot' : 'source-only', reason }
|
||||
}
|
||||
|
||||
const sourceName = inferSourceName(review.reviewUrl, html)
|
||||
const candidates = extractImageCandidates(html, review.reviewUrl)
|
||||
|
||||
let thumbnail = null
|
||||
let imageUrl = ''
|
||||
let lastError = ''
|
||||
|
||||
for (const candidate of candidates.slice(0, 8)) {
|
||||
try {
|
||||
thumbnail = await ensureMedia(payload, review, candidate, sourceName)
|
||||
imageUrl = candidate
|
||||
break
|
||||
} catch (error) {
|
||||
lastError = error.message
|
||||
}
|
||||
}
|
||||
|
||||
if (!thumbnail && SCREENSHOT_MISSING) {
|
||||
try {
|
||||
thumbnail = await ensureScreenshotMedia(payload, review, sourceName)
|
||||
} catch (error) {
|
||||
lastError = `screenshot ${error.message}`
|
||||
}
|
||||
}
|
||||
|
||||
await payload.update({
|
||||
collection: 'productReviews',
|
||||
id: review.id,
|
||||
locale: LOCALE,
|
||||
data: {
|
||||
sourceName,
|
||||
...(thumbnail ? { thumbnail } : {}),
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
id: review.id,
|
||||
imageUrl,
|
||||
sourceName,
|
||||
status: thumbnail ? (imageUrl ? 'updated' : 'screenshot') : 'source-only',
|
||||
reason: thumbnail ? '' : lastError || 'no image candidates',
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { default: config } = await import('@payload-config')
|
||||
const payload = await getPayload({ config })
|
||||
const reviews = await payload.find({
|
||||
collection: 'productReviews',
|
||||
depth: 0,
|
||||
limit: LIMIT,
|
||||
locale: LOCALE,
|
||||
sort: 'id',
|
||||
where: {
|
||||
docLocale: {
|
||||
equals: LOCALE,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const results = []
|
||||
const docs = (MISSING_ONLY ? reviews.docs.filter((review) => !review.thumbnail) : reviews.docs).filter((review) => !IDS || IDS.has(review.id))
|
||||
for (const review of docs) {
|
||||
const result = await processReview(payload, review)
|
||||
results.push(result)
|
||||
console.log(`${result.status.padEnd(12)} #${result.id} ${result.sourceName}${result.reason ? ` (${result.reason})` : ''}`)
|
||||
}
|
||||
|
||||
const summary = results.reduce(
|
||||
(acc, item) => {
|
||||
acc[item.status] = (acc[item.status] || 0) + 1
|
||||
return acc
|
||||
},
|
||||
{},
|
||||
)
|
||||
console.log(JSON.stringify(summary, null, 2))
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -0,0 +1,104 @@
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import 'dotenv/config'
|
||||
|
||||
process.env.PAYLOAD_DB_PUSH = 'false'
|
||||
|
||||
import { getPayload } from 'payload'
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
|
||||
const CACHE_DIR = path.join(ROOT, '.tmp', 'support-tutorial-thumbnails')
|
||||
|
||||
const THUMBNAILS = [
|
||||
{
|
||||
match: 'M-A12Hsh6_E',
|
||||
url: 'https://www.eversolo.com/Uploads/Tutorial/8915f1ae1dcb86ae1071fbbea329c638.jpg',
|
||||
},
|
||||
{
|
||||
match: 'o4HZkExoCMM',
|
||||
url: 'https://www.eversolo.com/Uploads/Tutorial/ec8c53e2453a97dd75ac585af416b976.jpg',
|
||||
},
|
||||
{
|
||||
match: 'rRnoWb54vdk',
|
||||
url: 'https://www.eversolo.com/Uploads/Tutorial/1089ccef2f0cead17c5190442a27b2a6.jpg',
|
||||
},
|
||||
]
|
||||
|
||||
function filenameFor(url) {
|
||||
return `support-tutorial-${path.basename(new URL(url).pathname)}`
|
||||
}
|
||||
|
||||
async function download(url) {
|
||||
await fs.mkdir(CACHE_DIR, { recursive: true })
|
||||
const filename = filenameFor(url)
|
||||
const filePath = path.join(CACHE_DIR, filename)
|
||||
try {
|
||||
await fs.access(filePath)
|
||||
return { filename, filePath }
|
||||
} catch {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) throw new Error(`Failed to download ${url}: ${response.status}`)
|
||||
await fs.writeFile(filePath, Buffer.from(await response.arrayBuffer()))
|
||||
return { filename, filePath }
|
||||
}
|
||||
}
|
||||
|
||||
async function findMedia(payload, filename) {
|
||||
const existing = await payload.find({
|
||||
collection: 'media',
|
||||
limit: 1,
|
||||
where: { filename: { equals: filename } },
|
||||
})
|
||||
return existing.docs[0] ?? null
|
||||
}
|
||||
|
||||
async function ensureMedia(payload, item) {
|
||||
const { filename, filePath } = await download(item.url)
|
||||
const existing = await findMedia(payload, filename)
|
||||
if (existing) return existing.id
|
||||
|
||||
const created = await payload.create({
|
||||
collection: 'media',
|
||||
data: {
|
||||
alt: 'Eversolo video tutorial thumbnail',
|
||||
credit: 'Eversolo',
|
||||
},
|
||||
filePath,
|
||||
})
|
||||
return created.id
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { default: config } = await import('@payload-config')
|
||||
const payload = await getPayload({ config })
|
||||
|
||||
for (const item of THUMBNAILS) {
|
||||
const mediaId = await ensureMedia(payload, item)
|
||||
const videos = await payload.find({
|
||||
collection: 'supportTutorialVideos',
|
||||
limit: 20,
|
||||
where: {
|
||||
videoUrl: {
|
||||
contains: item.match,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
for (const video of videos.docs) {
|
||||
await payload.update({
|
||||
collection: 'supportTutorialVideos',
|
||||
id: video.id,
|
||||
data: {
|
||||
thumbnail: mediaId,
|
||||
},
|
||||
})
|
||||
console.log(`updated #${video.id} -> media #${mediaId}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().then(() => process.exit(0)).catch((error) => {
|
||||
console.error(error)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -0,0 +1,255 @@
|
||||
import 'dotenv/config'
|
||||
import { execFileSync } from 'node:child_process'
|
||||
|
||||
const DATABASE_URI = process.env.DATABASE_URI
|
||||
|
||||
if (!DATABASE_URI) {
|
||||
throw new Error('DATABASE_URI is required')
|
||||
}
|
||||
|
||||
function psql(args, input) {
|
||||
return execFileSync('psql', [DATABASE_URI, ...args], {
|
||||
encoding: 'utf8',
|
||||
input,
|
||||
maxBuffer: 1024 * 1024 * 24,
|
||||
})
|
||||
}
|
||||
|
||||
function sqlString(value) {
|
||||
if (value === null || value === undefined) return 'NULL'
|
||||
return `'${String(value).replace(/'/g, "''")}'`
|
||||
}
|
||||
|
||||
function sqlJson(value) {
|
||||
return `${sqlString(JSON.stringify(value))}::jsonb`
|
||||
}
|
||||
|
||||
function createTextNode(text) {
|
||||
return {
|
||||
detail: 0,
|
||||
format: 0,
|
||||
mode: 'normal',
|
||||
style: '',
|
||||
text,
|
||||
type: 'text',
|
||||
version: 1,
|
||||
}
|
||||
}
|
||||
|
||||
function createParagraphNode(text) {
|
||||
return {
|
||||
children: [createTextNode(text)],
|
||||
direction: null,
|
||||
format: '',
|
||||
indent: 0,
|
||||
textFormat: 0,
|
||||
textStyle: '',
|
||||
type: 'paragraph',
|
||||
version: 1,
|
||||
}
|
||||
}
|
||||
|
||||
function createRichTextFromParagraphs(paragraphs) {
|
||||
const children = paragraphs.map((paragraph) => paragraph.trim()).filter(Boolean).map(createParagraphNode)
|
||||
if (children.length === 0) return null
|
||||
|
||||
return {
|
||||
root: {
|
||||
children,
|
||||
direction: null,
|
||||
format: '',
|
||||
indent: 0,
|
||||
type: 'root',
|
||||
version: 1,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function decodeHtml(value) {
|
||||
return value
|
||||
.replace(/ |\u00a0/g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCharCode(Number.parseInt(code, 16)))
|
||||
}
|
||||
|
||||
function normalizeText(value) {
|
||||
return decodeHtml(value)
|
||||
.replace(/\r/g, '\n')
|
||||
.replace(/[ \t\f\v]+/g, ' ')
|
||||
.replace(/\n[ \t]+/g, '\n')
|
||||
.replace(/[ \t]+\n/g, '\n')
|
||||
.trim()
|
||||
}
|
||||
|
||||
function stripHtmlToBlocks(html) {
|
||||
const prepared = html
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, ' ')
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
|
||||
.replace(/<\/(p|div|section|article|h[1-6]|li|tr|table|ul|ol)>/gi, '\n')
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
|
||||
const seen = new Set()
|
||||
return normalizeText(prepared)
|
||||
.split(/\n+/)
|
||||
.map((line) => normalizeText(line))
|
||||
.filter((line) => line.length > 12)
|
||||
.filter((line) => !/^(copyright|首页|产品|support|about|news|dealer|app)$/i.test(line))
|
||||
.filter((line) => {
|
||||
const key = line.toLowerCase()
|
||||
if (seen.has(key)) return false
|
||||
seen.add(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
function extractTableSpecs(html) {
|
||||
const specs = []
|
||||
const rowMatches = html.matchAll(/<tr[\s\S]*?<\/tr>/gi)
|
||||
|
||||
for (const rowMatch of rowMatches) {
|
||||
const cells = [...rowMatch[0].matchAll(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi)]
|
||||
.map((match) => normalizeText(match[1].replace(/<[^>]+>/g, ' ')))
|
||||
.filter(Boolean)
|
||||
|
||||
if (cells.length >= 2) {
|
||||
const label = cells[0]
|
||||
const value = cells.slice(1).join(' / ')
|
||||
if (label.length <= 80 && value.length <= 240) specs.push({ label, value })
|
||||
}
|
||||
}
|
||||
|
||||
return specs.slice(0, 40)
|
||||
}
|
||||
|
||||
function extractInlineSpecs(blocks) {
|
||||
const specs = []
|
||||
|
||||
for (const block of blocks) {
|
||||
const match = block.match(/^(.{2,50}?)[：:]\s*(.{2,220})$/)
|
||||
if (!match) continue
|
||||
|
||||
const label = match[1].trim()
|
||||
const value = match[2].trim()
|
||||
if (/^(http|https|www\.)/i.test(value)) continue
|
||||
specs.push({ label, value })
|
||||
}
|
||||
|
||||
return specs.slice(0, 32)
|
||||
}
|
||||
|
||||
function extractFeatureBlocks(blocks, locale) {
|
||||
const candidates = blocks
|
||||
.filter((block) => block.length >= 16 && block.length <= 140)
|
||||
.filter((block) => !/^[\d\s.,:：/-]+$/.test(block))
|
||||
.filter((block) => !/(copyright|privacy|cookie|download|manual|firmware)/i.test(block))
|
||||
|
||||
const features = []
|
||||
for (const candidate of candidates) {
|
||||
if (features.some((item) => item.title === candidate)) continue
|
||||
features.push({
|
||||
title: candidate,
|
||||
description: createRichTextFromParagraphs([
|
||||
locale === 'zh'
|
||||
? '由旧站产品详情内容整理而来，可在后台继续精修为更准确的卖点说明。'
|
||||
: 'Migrated from the legacy product detail content and ready for editorial refinement.',
|
||||
]),
|
||||
})
|
||||
if (features.length >= 4) break
|
||||
}
|
||||
|
||||
return features
|
||||
}
|
||||
|
||||
function pickOverviewParagraphs(blocks) {
|
||||
return blocks
|
||||
.filter((block) => block.length >= 24)
|
||||
.filter((block) => !/(copyright|privacy|cookie|download|manual|firmware)/i.test(block))
|
||||
.slice(0, 12)
|
||||
}
|
||||
|
||||
function buildMigrationSql(doc) {
|
||||
const legacyHtml = String(doc.legacyContentHtml || '')
|
||||
const blocks = stripHtmlToBlocks(legacyHtml)
|
||||
const statements = []
|
||||
|
||||
if (doc.template === 'default') {
|
||||
const overview = createRichTextFromParagraphs(pickOverviewParagraphs(blocks))
|
||||
if (overview) {
|
||||
statements.push(
|
||||
`update products_locales set overview = ${sqlJson(overview)} where _parent_id = ${doc.id} and _locale = ${sqlString(doc.locale)}::public._locales;`,
|
||||
)
|
||||
}
|
||||
|
||||
if (Number(doc.keyFeatureCount || 0) === 0) {
|
||||
extractFeatureBlocks(blocks, doc.locale).forEach((feature, index) => {
|
||||
const featureId = `legacy-${doc.id}-${doc.locale}-feature-${index + 1}`
|
||||
statements.push(
|
||||
`insert into products_key_features (id, _order, _parent_id, _locale, title, description) values (${sqlString(featureId)}, ${index + 1}, ${doc.id}, ${sqlString(doc.locale)}::public._locales, ${sqlString(feature.title)}, ${sqlJson(feature.description)}) on conflict (id) do update set title = excluded.title, description = excluded.description;`,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
if (Number(doc.specificationCount || 0) === 0) {
|
||||
const specs = extractTableSpecs(legacyHtml)
|
||||
const finalSpecs = specs.length > 0 ? specs : extractInlineSpecs(blocks)
|
||||
|
||||
if (finalSpecs.length > 0) {
|
||||
const specId = `legacy-${doc.id}-${doc.locale}-specs`
|
||||
statements.push(
|
||||
`insert into products_specifications (id, _order, _parent_id, order, _locale, group_name) values (${sqlString(specId)}, 1, ${doc.id}, 0, ${sqlString(doc.locale)}::public._locales, ${sqlString(doc.locale === 'zh' ? '规格参数' : 'Specifications')}) on conflict (id) do update set group_name = excluded.group_name;`,
|
||||
)
|
||||
|
||||
finalSpecs.forEach((spec, index) => {
|
||||
const itemId = `${specId}-item-${index + 1}`
|
||||
statements.push(
|
||||
`insert into products_specifications_items (id, _order, _parent_id, _locale, label, value, note) values (${sqlString(itemId)}, ${index + 1}, ${sqlString(specId)}, ${sqlString(doc.locale)}::public._locales, ${sqlString(spec.label)}, ${sqlString(spec.value)}, NULL) on conflict (id) do update set label = excluded.label, value = excluded.value, note = excluded.note;`,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return statements
|
||||
}
|
||||
|
||||
const raw = psql([
|
||||
'-tA',
|
||||
'-c',
|
||||
`
|
||||
select coalesce(jsonb_agg(jsonb_build_object(
|
||||
'id', p.id,
|
||||
'model', p.model,
|
||||
'template', p.template,
|
||||
'locale', l._locale,
|
||||
'legacyContentHtml', l.legacy_content_html,
|
||||
'keyFeatureCount', (select count(*) from products_key_features k where k._parent_id = p.id),
|
||||
'specificationCount', (select count(*) from products_specifications s where s._parent_id = p.id)
|
||||
) order by p.id, l._locale), '[]'::jsonb)::text
|
||||
from products p
|
||||
join products_locales l on l._parent_id = p.id and l._locale = p.doc_locale::text::public._locales
|
||||
where coalesce(l.legacy_content_html, '') <> '';
|
||||
`,
|
||||
]).trim()
|
||||
|
||||
const docs = JSON.parse(raw || '[]')
|
||||
const statements = ['begin;']
|
||||
let migratedDefaultCount = 0
|
||||
|
||||
for (const doc of docs) {
|
||||
const docStatements = buildMigrationSql(doc)
|
||||
if (doc.template === 'default' && docStatements.length > 0) migratedDefaultCount += 1
|
||||
statements.push(...docStatements)
|
||||
}
|
||||
|
||||
statements.push("update products_locales set legacy_content_html = '' where coalesce(legacy_content_html, '') <> '';")
|
||||
statements.push('commit;')
|
||||
|
||||
psql(['-v', 'ON_ERROR_STOP=1'], statements.join('\n'))
|
||||
|
||||
console.log(`Migrated ${migratedDefaultCount} editor-driven product records and cleared legacy HTML on ${docs.length} localized product records.`)
|
||||
@@ -14,6 +14,10 @@ const routes = [
|
||||
'/zh/about',
|
||||
'/en/contact',
|
||||
'/zh/contact',
|
||||
'/en/dealers',
|
||||
'/zh/dealers',
|
||||
'/en/dealers?tab=stores',
|
||||
'/zh/dealers?tab=stores',
|
||||
'/en/support/contact',
|
||||
'/zh/support/contact',
|
||||
'/favicon.ico',
|
||||
|
||||
Reference in New Issue
Block a user