/** * 从 phonerecommend.top 同步处理器(SoC)档案到本项目的 chips 表 / 导入 JSON。 * * 示例: * pnpm sync:processors * pnpm sync:processors -- --write * pnpm sync:processors -- --write --limit 10 */ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; import { chipSchema } from '../src/lib/schema'; import { importEntities } from '../src/lib/repo'; import { getDb } from '../src/lib/db'; const BASE = 'https://www.phonerecommend.top'; const HEADERS = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', Accept: 'application/json, text/plain, */*', Referer: `${BASE}/processors`, Origin: BASE, }; /** 源站中文品牌 → 本项目规范名 */ const BRAND_MAP = { 高通: 'Qualcomm', 联发科: 'MediaTek', 苹果: 'Apple', 小米: 'Xiaomi', 海思: 'HiSilicon', 三星: 'Samsung', 谷歌: 'Google', 紫光展锐: 'UNISOC', }; const BRAND_WORDS = [ '高通', '联发科', '苹果', '小米', '海思', '三星', '谷歌', '紫光展锐', 'qualcomm', 'mediatek', 'apple', 'xiaomi', 'hisilicon', 'samsung', 'google', 'unisoc', ]; function clean(value) { if (typeof value === 'string') return value.trim(); if (typeof value === 'number') return String(value); if (value == null) return ''; return ''; } function text(value) { if (Array.isArray(value)) return value.map(text).filter(Boolean).join(' / '); if (value && typeof value === 'object') { return Object.entries(value) .map(([key, item]) => `${key}: ${text(item)}`) .filter(Boolean) .join(';'); } return clean(value); } function normalizeProcess(value) { return clean(value).replace(/nanometers?/gi, 'nm').replace(/\s+/g, ' ').trim(); } function numberOrUndefined(value) { return typeof value === 'number' && Number.isFinite(value) ? value : undefined; } function unique(values) { return [...new Set(values.map(clean).filter(Boolean))]; } function stripBrand(value, prefixes) { for (const prefix of prefixes) { if (prefix && value.toLowerCase().startsWith(prefix.toLowerCase())) { return value.slice(prefix.length).trim(); } } return value; } function splitFeatures(value) { return unique(String(value ?? '').split(/[,,、;;]/)); } function normalizeChipName(value) { let name = String(value ?? '') .toLowerCase() .replace(/[®™©]/g, '') .replace(/\s+/g, '') .replace(/移动平台|处理器|平台/g, '') .replace(/[()()]/g, ''); for (const word of BRAND_WORDS) name = name.split(word).join(''); return name; } const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); async function requestJson(path, attempt = 1) { try { await sleep(80); const response = await fetch(`${BASE}${path}`, { headers: HEADERS, signal: AbortSignal.timeout(20000), }); if (!response.ok) { const error = new Error(`HTTP ${response.status}`); error.status = response.status; throw error; } const json = await response.json(); if (!json.success) throw new Error(json.message ?? '接口返回失败'); return json; } catch (error) { if (attempt >= 5) throw error; // 源站对高频请求会返回 429 / 403,退避后重试 const limited = error?.status === 429 || error?.status === 403; await sleep((limited ? 1500 : 400) * attempt); return requestJson(path, attempt + 1); } } async function fetchAllProcessors(limit) { const first = await requestJson('/api/processors?limit=100&page=1'); const pages = Math.max(1, Number(first.pagination?.pages ?? 1)); const items = [...(first.data ?? [])]; for (let page = 2; page <= pages; page += 1) { const json = await requestJson(`/api/processors?limit=100&page=${page}`); items.push(...(json.data ?? [])); } console.log(`源站共 ${first.pagination?.total ?? items.length} 颗处理器,已获取 ${items.length} 颗`); return limit > 0 ? items.slice(0, limit) : items; } async function fetchDetails(items, concurrency) { const results = new Array(items.length); let cursor = 0; let done = 0; async function worker() { while (cursor < items.length) { const index = cursor; cursor += 1; const item = items[index]; try { const json = await requestJson(`/api/processors/${encodeURIComponent(item.key)}`); results[index] = json.data ?? item; } catch (error) { console.warn(` 详情失败 ${item.key}:${error instanceof Error ? error.message : error}`); results[index] = item; } done += 1; if (done % 50 === 0) console.log(` 详情进度 ${done}/${items.length}`); } } await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker())); return results; } function mapProcessor(remote) { const brandSource = clean(remote.brand); const brand = BRAND_MAP[brandSource] ?? brandSource ?? 'Unknown'; const nameZh = clean(remote.name_zh); const nameEn = clean(remote.name_en); const rawName = nameZh || nameEn || clean(remote.key); const model = stripBrand(rawName, [brandSource, brand]) || rawName; const cpu = remote.cpu ?? {}; const gpu = remote.gpu; const npu = remote.npu; const benchmark = remote.benchmark ?? {}; const nanoreview = remote.nanoreview ?? {}; const nrSpecs = nanoreview.specs ?? {}; const specZh = remote.spec_zh ?? {}; const specGroupsSource = Object.keys(specZh).length > 0 ? specZh : (remote.spec_groups ?? {}); const specGroups = {}; for (const [groupName, fields] of Object.entries(specGroupsSource)) { if (!fields || typeof fields !== 'object' || Array.isArray(fields)) continue; const rows = {}; for (const [key, value] of Object.entries(fields)) { const content = text(value); if (content) rows[key] = content; } if (Object.keys(rows).length > 0) specGroups[groupName] = rows; } const cache = cpu.cache ?? {}; const cpuClusters = (Array.isArray(cpu.clusters) ? cpu.clusters : []).map((cluster, index) => { const type = clean(cluster.type); return { name: type && !/^cluster\s*\d+$/i.test(type) ? type : `簇 ${index + 1}`, cores: [ { microArch: clean(cluster.arch) || undefined, count: numberOrUndefined(cluster.cores), clockMax: clean(cluster.freq) || undefined, l2: clean(cluster.l2) || undefined, }, ], }; }); const cpuCores = numberOrUndefined(cpu.cores) ?? cpuClusters.reduce( (sum, cluster) => sum + cluster.cores.reduce((n, core) => n + (core.count ?? 0), 0), 0, ); const gpuText = clean(gpu) || clean(specZh['图形']?.['名称']); const gpuApis = splitFeatures(specZh['图形']?.['图形接口']); const gpuFeatures = unique([specZh['图形']?.['特性']]); const camera = remote.camera ?? {}; const display = remote.display ?? {}; const connectivity = remote.connectivity ?? {}; const memory = remote.memory ?? {}; const benchmarks = []; const geekbenchVersion = clean(benchmark.geekbench_version) || 'Geekbench'; if (numberOrUndefined(benchmark.geekbench_single) !== undefined) { benchmarks.push({ name: `${geekbenchVersion} 单核`, score: String(benchmark.geekbench_single), }); } if (numberOrUndefined(benchmark.geekbench_multi) !== undefined) { benchmarks.push({ name: `${geekbenchVersion} 多核`, score: String(benchmark.geekbench_multi), }); } if (numberOrUndefined(nanoreview.gb6_single) !== undefined) { benchmarks.push({ name: 'Geekbench 6 单核', score: String(nanoreview.gb6_single) }); } if (numberOrUndefined(nanoreview.gb6_multi) !== undefined) { benchmarks.push({ name: 'Geekbench 6 多核', score: String(nanoreview.gb6_multi) }); } if (numberOrUndefined(nanoreview.antutu11_total) !== undefined) { benchmarks.push({ name: '安兔兔 v11', score: String(nanoreview.antutu11_total), unit: '分' }); } if (numberOrUndefined(nanoreview['3dmark_snl']) !== undefined) { benchmarks.push({ name: '3DMark Steel Nomad Light', score: String(nanoreview['3dmark_snl']), unit: '分', }); } if (numberOrUndefined(benchmark.cpu_index) !== undefined) { benchmarks.push({ name: 'CPU 性能指数', score: String(benchmark.cpu_index), unit: '骁龙 865=100', }); } const slowMotion = clean(specZh['相机']?.['慢动作录制']); const cameraBlock = { isp: clean(camera.isp) || clean(specZh['相机']?.['ISP 名称']) || undefined, maxSensor: clean(camera.max_sensor) || clean(specZh['相机']?.['单摄最高像素']) || undefined, videoCapture: clean(camera.video_capture) || clean(specZh['相机']?.['视频录制']) || undefined, slowMotion: slowMotion || undefined, features: unique([specZh['相机']?.['特性']]), }; const detail = { overview: { nameEn: nameEn || undefined, nameZh: nameZh || undefined, series: clean(remote.series) || undefined, tier: clean(remote.performance_tier) || undefined, tierScore: numberOrUndefined(remote.tier_score), releaseYear: numberOrUndefined(remote.release_year), phoneCount: numberOrUndefined(remote.phone_count), source: clean(remote.source) || undefined, }, cpuClusters, cpuBlock: { config: clean(cpu.config) || undefined, arch: clean(cpu.arch) || undefined, archType: clean(cpu.arch_type) || undefined, maxFreq: clean(cpu.max_freq) || undefined, cores: cpuCores || undefined, cacheL3: clean(cache.l3) || undefined, systemCache: clean(cache.system_level) || undefined, note: clean(cache.note) || undefined, }, gpuBlock: { name: gpuText || undefined, apis: gpuApis, features: gpuFeatures, }, npuBlock: { name: clean(npu) || undefined, features: [], }, systemCache: clean(cache.system_level) || undefined, ispBlock: cameraBlock.isp, modemBlock: clean(remote.modem) || undefined, memorySupport: { type: clean(memory.type) || clean(specZh['内存']?.['类型']) || undefined, maxClock: clean(specZh['内存']?.['频率']) || undefined, bandwidth: clean(nrSpecs['Max bandwidth']) || undefined, }, storageSupport: clean(remote.storage) || clean(specZh['存储']?.['UFS']) || undefined, displaySupport: clean(display.max_resolution) || clean(display.max_refresh) || undefined, displayBlock: { maxResolution: clean(display.max_resolution) || undefined, maxRefresh: clean(display.max_refresh) || undefined, externalMax: clean(specZh['屏幕']?.['外接最高分辨率']) || undefined, hdr: clean(specZh['屏幕']?.['HDR']) || undefined, colorDepth: clean(specZh['屏幕']?.['色深']) || undefined, colorGamut: clean(specZh['屏幕']?.['色域']) || undefined, }, cameraBlock, connectivityBlock: { wifi: clean(connectivity.wifi) || undefined, wifiStandard: clean(connectivity.wifi_standard) || undefined, bluetooth: clean(connectivity.bluetooth) || undefined, peakSpeed: clean(connectivity.peak_speed) || undefined, }, benchmarkInfo: { version: clean(benchmark.geekbench_version) || undefined, cpuIndex: numberOrUndefined(benchmark.cpu_index), cpuIndexNote: clean(benchmark.cpu_index_note) || undefined, note: clean(benchmark.note) || undefined, antutu: nanoreview.antutu11 && typeof nanoreview.antutu11 === 'object' ? { cpu: numberOrUndefined(nanoreview.antutu11.cpu), gpu: numberOrUndefined(nanoreview.antutu11.gpu), memory: numberOrUndefined(nanoreview.antutu11.memory), ux: numberOrUndefined(nanoreview.antutu11.ux), } : undefined, }, benchmarks, traits: unique(remote.traits ?? []), caveats: unique(remote.caveats ?? []), sourceUrls: unique(remote.source_urls ?? []), specGroups, extra: {}, }; const slug = /^[a-z0-9]+(-[a-z0-9]+)*$/.test(clean(remote.key)) ? clean(remote.key) : clean(remote.key) .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, ''); return { slug, type: 'soc', brand, model, codename: clean(remote.codename) || clean(specZh['型号']?.['型号代号']) || undefined, aliases: unique([nameEn, nameZh, clean(remote.codename), ...(remote.aliases ?? [])]), process: normalizeProcess( clean(remote.process_node) || clean(specZh['制程']?.['制程']) || clean(nrSpecs['Process']), ) || undefined, arch: clean(nrSpecs['Instruction set']) || clean(cpu.arch) || undefined, detail, meta: [], }; } function linkDevices(chips) { const nameMap = new Map(); for (const chip of chips) { const names = [ chip.model, chip.codename, chip.slug, ...(chip.aliases ?? []), chip.detail.overview?.nameEn, chip.detail.overview?.nameZh, ]; for (const name of names) { const normalized = normalizeChipName(name); if (normalized) nameMap.set(normalized, chip.slug); } } const db = getDb(); const rows = db.prepare('SELECT slug, chip_refs_json, specs_json FROM devices').all(); const now = new Date().toISOString(); let linked = 0; db.exec('BEGIN'); try { for (const row of rows) { let specs = {}; try { specs = JSON.parse(row.specs_json ?? '{}'); } catch { continue; } const chipset = specs.platform?.chipset; if (!chipset) continue; const chipSlug = nameMap.get(normalizeChipName(chipset)); if (!chipSlug) continue; let refs = {}; try { refs = JSON.parse(row.chip_refs_json ?? '{}'); } catch { refs = {}; } if (refs.soc === chipSlug) continue; refs.soc = chipSlug; db.prepare('UPDATE devices SET chip_refs_json = ?, updated_at = ? WHERE slug = ?').run( JSON.stringify(refs), now, row.slug, ); linked += 1; } db.exec('COMMIT'); } catch (error) { db.exec('ROLLBACK'); throw error; } return linked; } function parseArgs(argv) { const options = { write: false, limit: 0, out: '', concurrency: 2 }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === '--write') options.write = true; else if (arg === '--limit') options.limit = Number(argv[(index += 1)] ?? 0); else if (arg === '--out') options.out = argv[(index += 1)] ?? ''; else if (arg === '--concurrency') options.concurrency = Number(argv[(index += 1)] ?? 6); else throw new Error(`未知参数:${arg}`); } return options; } async function main() { const options = parseArgs(process.argv.slice(2)); console.log(`拉取 ${BASE}/api/processors ...`); const items = await fetchAllProcessors(options.limit); console.log('拉取处理器详情 ...'); const details = await fetchDetails(items, options.concurrency); const chips = []; const issues = []; const seen = new Set(); for (const remote of details) { try { const chip = mapProcessor(remote); if (seen.has(chip.slug)) { console.warn(` slug 重复,跳过:${chip.slug}(${chip.brand} ${chip.model})`); continue; } const parsed = chipSchema.safeParse(chip); if (!parsed.success) { issues.push({ name: `${chip.brand} ${chip.model}`, issues: parsed.error.issues .slice(0, 8) .map((issue) => `${issue.path.join('.')}: ${issue.message}`), }); continue; } seen.add(chip.slug); chips.push(parsed.data); } catch (error) { issues.push({ name: remote?.key ?? '?', issues: [String(error)] }); } } if (issues.length > 0) { console.error(`\n有 ${issues.length} 颗处理器映射失败:`); for (const issue of issues.slice(0, 20)) { console.error(`- ${issue.name}`); for (const line of issue.issues) console.error(` ${line}`); } throw new Error('映射失败,已中止,未写入任何数据'); } console.log(`映射成功 ${chips.length} 颗`); const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); const outPath = options.out || `data/import/processors-${stamp}.json`; const payload = { formatVersion: 1, generator: 'LFSpecs / sync-processors', exportedAt: new Date().toISOString(), devices: [], chips, }; await mkdir(dirname(outPath), { recursive: true }); await writeFile(outPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); console.log(`JSON 已写入 ${outPath}(${chips.length} 颗)`); if (!options.write) { console.log('未指定 --write,本次只产出 JSON。'); return; } console.log('写入数据库 ...'); const counts = importEntities([], chips); console.log(`数据库写入完成:devices ${counts.devices} / chips ${counts.chips}`); const linked = linkDevices(chips); console.log(`已关联 ${linked} 款设备的 SoC 引用`); } main().catch((error) => { console.error(error); process.exit(1); });