Files
LFSpecs/scripts/sync-processors.mjs
2026-11-06 15:07:02 +08:00

533 lines
16 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* 从 phonerecommend.top 同步处理器(SoC)档案到本项目的 chips 表 / 导入 JSON。
*
* 示例:
* pnpm sync:processors
* pnpm sync:processors -- --write
* pnpm sync:processors -- --write --limit 10
*/
import { mkdir, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import { chipSchema } from '../src/lib/schema';
import { importEntities } from '../src/lib/repo';
import { getDb } from '../src/lib/db';
const BASE = 'https://www.phonerecommend.top';
const HEADERS = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
Accept: 'application/json, text/plain, */*',
Referer: `${BASE}/processors`,
Origin: BASE,
};
/** 源站中文品牌 → 本项目规范名 */
const BRAND_MAP = {
高通: 'Qualcomm',
联发科: 'MediaTek',
苹果: 'Apple',
小米: 'Xiaomi',
海思: 'HiSilicon',
三星: 'Samsung',
谷歌: 'Google',
紫光展锐: 'UNISOC',
};
const BRAND_WORDS = [
'高通',
'联发科',
'苹果',
'小米',
'海思',
'三星',
'谷歌',
'紫光展锐',
'qualcomm',
'mediatek',
'apple',
'xiaomi',
'hisilicon',
'samsung',
'google',
'unisoc',
];
function clean(value) {
if (typeof value === 'string') return value.trim();
if (typeof value === 'number') return String(value);
if (value == null) return '';
return '';
}
function text(value) {
if (Array.isArray(value)) return value.map(text).filter(Boolean).join(' / ');
if (value && typeof value === 'object') {
return Object.entries(value)
.map(([key, item]) => `${key}: ${text(item)}`)
.filter(Boolean)
.join(';');
}
return clean(value);
}
function normalizeProcess(value) {
return clean(value).replace(/nanometers?/gi, 'nm').replace(/\s+/g, ' ').trim();
}
function numberOrUndefined(value) {
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
}
function unique(values) {
return [...new Set(values.map(clean).filter(Boolean))];
}
function stripBrand(value, prefixes) {
for (const prefix of prefixes) {
if (prefix && value.toLowerCase().startsWith(prefix.toLowerCase())) {
return value.slice(prefix.length).trim();
}
}
return value;
}
function splitFeatures(value) {
return unique(String(value ?? '').split(/[,,、;;]/));
}
function normalizeChipName(value) {
let name = String(value ?? '')
.toLowerCase()
.replace(/[®™©]/g, '')
.replace(/\s+/g, '')
.replace(/移动平台|处理器|平台/g, '')
.replace(/[()()]/g, '');
for (const word of BRAND_WORDS) name = name.split(word).join('');
return name;
}
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function requestJson(path, attempt = 1) {
try {
await sleep(80);
const response = await fetch(`${BASE}${path}`, {
headers: HEADERS,
signal: AbortSignal.timeout(20000),
});
if (!response.ok) {
const error = new Error(`HTTP ${response.status}`);
error.status = response.status;
throw error;
}
const json = await response.json();
if (!json.success) throw new Error(json.message ?? '接口返回失败');
return json;
} catch (error) {
if (attempt >= 5) throw error;
// 源站对高频请求会返回 429 / 403,退避后重试
const limited = error?.status === 429 || error?.status === 403;
await sleep((limited ? 1500 : 400) * attempt);
return requestJson(path, attempt + 1);
}
}
async function fetchAllProcessors(limit) {
const first = await requestJson('/api/processors?limit=100&page=1');
const pages = Math.max(1, Number(first.pagination?.pages ?? 1));
const items = [...(first.data ?? [])];
for (let page = 2; page <= pages; page += 1) {
const json = await requestJson(`/api/processors?limit=100&page=${page}`);
items.push(...(json.data ?? []));
}
console.log(`源站共 ${first.pagination?.total ?? items.length} 颗处理器,已获取 ${items.length} 颗`);
return limit > 0 ? items.slice(0, limit) : items;
}
async function fetchDetails(items, concurrency) {
const results = new Array(items.length);
let cursor = 0;
let done = 0;
async function worker() {
while (cursor < items.length) {
const index = cursor;
cursor += 1;
const item = items[index];
try {
const json = await requestJson(`/api/processors/${encodeURIComponent(item.key)}`);
results[index] = json.data ?? item;
} catch (error) {
console.warn(` 详情失败 ${item.key}:${error instanceof Error ? error.message : error}`);
results[index] = item;
}
done += 1;
if (done % 50 === 0) console.log(` 详情进度 ${done}/${items.length}`);
}
}
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
return results;
}
function mapProcessor(remote) {
const brandSource = clean(remote.brand);
const brand = BRAND_MAP[brandSource] ?? brandSource ?? 'Unknown';
const nameZh = clean(remote.name_zh);
const nameEn = clean(remote.name_en);
const rawName = nameZh || nameEn || clean(remote.key);
const model = stripBrand(rawName, [brandSource, brand]) || rawName;
const cpu = remote.cpu ?? {};
const gpu = remote.gpu;
const npu = remote.npu;
const benchmark = remote.benchmark ?? {};
const nanoreview = remote.nanoreview ?? {};
const nrSpecs = nanoreview.specs ?? {};
const specZh = remote.spec_zh ?? {};
const specGroupsSource =
Object.keys(specZh).length > 0 ? specZh : (remote.spec_groups ?? {});
const specGroups = {};
for (const [groupName, fields] of Object.entries(specGroupsSource)) {
if (!fields || typeof fields !== 'object' || Array.isArray(fields)) continue;
const rows = {};
for (const [key, value] of Object.entries(fields)) {
const content = text(value);
if (content) rows[key] = content;
}
if (Object.keys(rows).length > 0) specGroups[groupName] = rows;
}
const cache = cpu.cache ?? {};
const cpuClusters = (Array.isArray(cpu.clusters) ? cpu.clusters : []).map((cluster, index) => {
const type = clean(cluster.type);
return {
name: type && !/^cluster\s*\d+$/i.test(type) ? type : `簇 ${index + 1}`,
cores: [
{
microArch: clean(cluster.arch) || undefined,
count: numberOrUndefined(cluster.cores),
clockMax: clean(cluster.freq) || undefined,
l2: clean(cluster.l2) || undefined,
},
],
};
});
const cpuCores =
numberOrUndefined(cpu.cores) ??
cpuClusters.reduce(
(sum, cluster) => sum + cluster.cores.reduce((n, core) => n + (core.count ?? 0), 0),
0,
);
const gpuText = clean(gpu) || clean(specZh['图形']?.['名称']);
const gpuApis = splitFeatures(specZh['图形']?.['图形接口']);
const gpuFeatures = unique([specZh['图形']?.['特性']]);
const camera = remote.camera ?? {};
const display = remote.display ?? {};
const connectivity = remote.connectivity ?? {};
const memory = remote.memory ?? {};
const benchmarks = [];
const geekbenchVersion = clean(benchmark.geekbench_version) || 'Geekbench';
if (numberOrUndefined(benchmark.geekbench_single) !== undefined) {
benchmarks.push({
name: `${geekbenchVersion} 单核`,
score: String(benchmark.geekbench_single),
});
}
if (numberOrUndefined(benchmark.geekbench_multi) !== undefined) {
benchmarks.push({
name: `${geekbenchVersion} 多核`,
score: String(benchmark.geekbench_multi),
});
}
if (numberOrUndefined(nanoreview.gb6_single) !== undefined) {
benchmarks.push({ name: 'Geekbench 6 单核', score: String(nanoreview.gb6_single) });
}
if (numberOrUndefined(nanoreview.gb6_multi) !== undefined) {
benchmarks.push({ name: 'Geekbench 6 多核', score: String(nanoreview.gb6_multi) });
}
if (numberOrUndefined(nanoreview.antutu11_total) !== undefined) {
benchmarks.push({ name: '安兔兔 v11', score: String(nanoreview.antutu11_total), unit: '分' });
}
if (numberOrUndefined(nanoreview['3dmark_snl']) !== undefined) {
benchmarks.push({
name: '3DMark Steel Nomad Light',
score: String(nanoreview['3dmark_snl']),
unit: '分',
});
}
if (numberOrUndefined(benchmark.cpu_index) !== undefined) {
benchmarks.push({
name: 'CPU 性能指数',
score: String(benchmark.cpu_index),
unit: '骁龙 865=100',
});
}
const slowMotion = clean(specZh['相机']?.['慢动作录制']);
const cameraBlock = {
isp: clean(camera.isp) || clean(specZh['相机']?.['ISP 名称']) || undefined,
maxSensor: clean(camera.max_sensor) || clean(specZh['相机']?.['单摄最高像素']) || undefined,
videoCapture: clean(camera.video_capture) || clean(specZh['相机']?.['视频录制']) || undefined,
slowMotion: slowMotion || undefined,
features: unique([specZh['相机']?.['特性']]),
};
const detail = {
overview: {
nameEn: nameEn || undefined,
nameZh: nameZh || undefined,
series: clean(remote.series) || undefined,
tier: clean(remote.performance_tier) || undefined,
tierScore: numberOrUndefined(remote.tier_score),
releaseYear: numberOrUndefined(remote.release_year),
phoneCount: numberOrUndefined(remote.phone_count),
source: clean(remote.source) || undefined,
},
cpuClusters,
cpuBlock: {
config: clean(cpu.config) || undefined,
arch: clean(cpu.arch) || undefined,
archType: clean(cpu.arch_type) || undefined,
maxFreq: clean(cpu.max_freq) || undefined,
cores: cpuCores || undefined,
cacheL3: clean(cache.l3) || undefined,
systemCache: clean(cache.system_level) || undefined,
note: clean(cache.note) || undefined,
},
gpuBlock: {
name: gpuText || undefined,
apis: gpuApis,
features: gpuFeatures,
},
npuBlock: {
name: clean(npu) || undefined,
features: [],
},
systemCache: clean(cache.system_level) || undefined,
ispBlock: cameraBlock.isp,
modemBlock: clean(remote.modem) || undefined,
memorySupport: {
type: clean(memory.type) || clean(specZh['内存']?.['类型']) || undefined,
maxClock: clean(specZh['内存']?.['频率']) || undefined,
bandwidth: clean(nrSpecs['Max bandwidth']) || undefined,
},
storageSupport:
clean(remote.storage) || clean(specZh['存储']?.['UFS']) || undefined,
displaySupport:
clean(display.max_resolution) || clean(display.max_refresh) || undefined,
displayBlock: {
maxResolution: clean(display.max_resolution) || undefined,
maxRefresh: clean(display.max_refresh) || undefined,
externalMax: clean(specZh['屏幕']?.['外接最高分辨率']) || undefined,
hdr: clean(specZh['屏幕']?.['HDR']) || undefined,
colorDepth: clean(specZh['屏幕']?.['色深']) || undefined,
colorGamut: clean(specZh['屏幕']?.['色域']) || undefined,
},
cameraBlock,
connectivityBlock: {
wifi: clean(connectivity.wifi) || undefined,
wifiStandard: clean(connectivity.wifi_standard) || undefined,
bluetooth: clean(connectivity.bluetooth) || undefined,
peakSpeed: clean(connectivity.peak_speed) || undefined,
},
benchmarkInfo: {
version: clean(benchmark.geekbench_version) || undefined,
cpuIndex: numberOrUndefined(benchmark.cpu_index),
cpuIndexNote: clean(benchmark.cpu_index_note) || undefined,
note: clean(benchmark.note) || undefined,
antutu:
nanoreview.antutu11 && typeof nanoreview.antutu11 === 'object'
? {
cpu: numberOrUndefined(nanoreview.antutu11.cpu),
gpu: numberOrUndefined(nanoreview.antutu11.gpu),
memory: numberOrUndefined(nanoreview.antutu11.memory),
ux: numberOrUndefined(nanoreview.antutu11.ux),
}
: undefined,
},
benchmarks,
traits: unique(remote.traits ?? []),
caveats: unique(remote.caveats ?? []),
sourceUrls: unique(remote.source_urls ?? []),
specGroups,
extra: {},
};
const slug = /^[a-z0-9]+(-[a-z0-9]+)*$/.test(clean(remote.key))
? clean(remote.key)
: clean(remote.key)
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '');
return {
slug,
type: 'soc',
brand,
model,
codename:
clean(remote.codename) ||
clean(specZh['型号']?.['型号代号']) ||
undefined,
aliases: unique([nameEn, nameZh, clean(remote.codename), ...(remote.aliases ?? [])]),
process:
normalizeProcess(
clean(remote.process_node) ||
clean(specZh['制程']?.['制程']) ||
clean(nrSpecs['Process']),
) || undefined,
arch: clean(nrSpecs['Instruction set']) || clean(cpu.arch) || undefined,
detail,
meta: [],
};
}
function linkDevices(chips) {
const nameMap = new Map();
for (const chip of chips) {
const names = [
chip.model,
chip.codename,
chip.slug,
...(chip.aliases ?? []),
chip.detail.overview?.nameEn,
chip.detail.overview?.nameZh,
];
for (const name of names) {
const normalized = normalizeChipName(name);
if (normalized) nameMap.set(normalized, chip.slug);
}
}
const db = getDb();
const rows = db.prepare('SELECT slug, chip_refs_json, specs_json FROM devices').all();
const now = new Date().toISOString();
let linked = 0;
db.exec('BEGIN');
try {
for (const row of rows) {
let specs = {};
try {
specs = JSON.parse(row.specs_json ?? '{}');
} catch {
continue;
}
const chipset = specs.platform?.chipset;
if (!chipset) continue;
const chipSlug = nameMap.get(normalizeChipName(chipset));
if (!chipSlug) continue;
let refs = {};
try {
refs = JSON.parse(row.chip_refs_json ?? '{}');
} catch {
refs = {};
}
if (refs.soc === chipSlug) continue;
refs.soc = chipSlug;
db.prepare('UPDATE devices SET chip_refs_json = ?, updated_at = ? WHERE slug = ?').run(
JSON.stringify(refs),
now,
row.slug,
);
linked += 1;
}
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
return linked;
}
function parseArgs(argv) {
const options = { write: false, limit: 0, out: '', concurrency: 2 };
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--write') options.write = true;
else if (arg === '--limit') options.limit = Number(argv[(index += 1)] ?? 0);
else if (arg === '--out') options.out = argv[(index += 1)] ?? '';
else if (arg === '--concurrency') options.concurrency = Number(argv[(index += 1)] ?? 6);
else throw new Error(`未知参数:${arg}`);
}
return options;
}
async function main() {
const options = parseArgs(process.argv.slice(2));
console.log(`拉取 ${BASE}/api/processors ...`);
const items = await fetchAllProcessors(options.limit);
console.log('拉取处理器详情 ...');
const details = await fetchDetails(items, options.concurrency);
const chips = [];
const issues = [];
const seen = new Set();
for (const remote of details) {
try {
const chip = mapProcessor(remote);
if (seen.has(chip.slug)) {
console.warn(` slug 重复,跳过:${chip.slug}(${chip.brand} ${chip.model})`);
continue;
}
const parsed = chipSchema.safeParse(chip);
if (!parsed.success) {
issues.push({
name: `${chip.brand} ${chip.model}`,
issues: parsed.error.issues
.slice(0, 8)
.map((issue) => `${issue.path.join('.')}: ${issue.message}`),
});
continue;
}
seen.add(chip.slug);
chips.push(parsed.data);
} catch (error) {
issues.push({ name: remote?.key ?? '?', issues: [String(error)] });
}
}
if (issues.length > 0) {
console.error(`\n有 ${issues.length} 颗处理器映射失败:`);
for (const issue of issues.slice(0, 20)) {
console.error(`- ${issue.name}`);
for (const line of issue.issues) console.error(` ${line}`);
}
throw new Error('映射失败,已中止,未写入任何数据');
}
console.log(`映射成功 ${chips.length} 颗`);
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
const outPath = options.out || `data/import/processors-${stamp}.json`;
const payload = {
formatVersion: 1,
generator: 'LFSpecs / sync-processors',
exportedAt: new Date().toISOString(),
devices: [],
chips,
};
await mkdir(dirname(outPath), { recursive: true });
await writeFile(outPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
console.log(`JSON 已写入 ${outPath}(${chips.length} 颗)`);
if (!options.write) {
console.log('未指定 --write,本次只产出 JSON。');
return;
}
console.log('写入数据库 ...');
const counts = importEntities([], chips);
console.log(`数据库写入完成:devices ${counts.devices} / chips ${counts.chips}`);
const linked = linkDevices(chips);
console.log(`已关联 ${linked} 款设备的 SoC 引用`);
}
main().catch((error) => {
console.error(error);
process.exit(1);
});