533 lines
16 KiB
JavaScript
533 lines
16 KiB
JavaScript
/**
|
||
* 从 phonerecommend.top 同步处理器(SoC)档案到本项目的 chips 表 / 导入 JSON。
|
||
*
|
||
* 示例:
|
||
* pnpm sync:processors
|
||
* pnpm sync:processors -- --write
|
||
* pnpm sync:processors -- --write --limit 10
|
||
*/
|
||
import { mkdir, writeFile } from 'node:fs/promises';
|
||
import { dirname } from 'node:path';
|
||
import { chipSchema } from '../src/lib/schema';
|
||
import { importEntities } from '../src/lib/repo';
|
||
import { getDb } from '../src/lib/db';
|
||
|
||
const BASE = 'https://www.phonerecommend.top';
|
||
const HEADERS = {
|
||
'User-Agent':
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||
Accept: 'application/json, text/plain, */*',
|
||
Referer: `${BASE}/processors`,
|
||
Origin: BASE,
|
||
};
|
||
|
||
/** 源站中文品牌 → 本项目规范名 */
|
||
const BRAND_MAP = {
|
||
高通: 'Qualcomm',
|
||
联发科: 'MediaTek',
|
||
苹果: 'Apple',
|
||
小米: 'Xiaomi',
|
||
海思: 'HiSilicon',
|
||
三星: 'Samsung',
|
||
谷歌: 'Google',
|
||
紫光展锐: 'UNISOC',
|
||
};
|
||
|
||
const BRAND_WORDS = [
|
||
'高通',
|
||
'联发科',
|
||
'苹果',
|
||
'小米',
|
||
'海思',
|
||
'三星',
|
||
'谷歌',
|
||
'紫光展锐',
|
||
'qualcomm',
|
||
'mediatek',
|
||
'apple',
|
||
'xiaomi',
|
||
'hisilicon',
|
||
'samsung',
|
||
'google',
|
||
'unisoc',
|
||
];
|
||
|
||
function clean(value) {
|
||
if (typeof value === 'string') return value.trim();
|
||
if (typeof value === 'number') return String(value);
|
||
if (value == null) return '';
|
||
return '';
|
||
}
|
||
|
||
function text(value) {
|
||
if (Array.isArray(value)) return value.map(text).filter(Boolean).join(' / ');
|
||
if (value && typeof value === 'object') {
|
||
return Object.entries(value)
|
||
.map(([key, item]) => `${key}: ${text(item)}`)
|
||
.filter(Boolean)
|
||
.join(';');
|
||
}
|
||
return clean(value);
|
||
}
|
||
|
||
function normalizeProcess(value) {
|
||
return clean(value).replace(/nanometers?/gi, 'nm').replace(/\s+/g, ' ').trim();
|
||
}
|
||
|
||
function numberOrUndefined(value) {
|
||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
||
}
|
||
|
||
function unique(values) {
|
||
return [...new Set(values.map(clean).filter(Boolean))];
|
||
}
|
||
|
||
function stripBrand(value, prefixes) {
|
||
for (const prefix of prefixes) {
|
||
if (prefix && value.toLowerCase().startsWith(prefix.toLowerCase())) {
|
||
return value.slice(prefix.length).trim();
|
||
}
|
||
}
|
||
return value;
|
||
}
|
||
|
||
function splitFeatures(value) {
|
||
return unique(String(value ?? '').split(/[,,、;;]/));
|
||
}
|
||
|
||
function normalizeChipName(value) {
|
||
let name = String(value ?? '')
|
||
.toLowerCase()
|
||
.replace(/[®™©]/g, '')
|
||
.replace(/\s+/g, '')
|
||
.replace(/移动平台|处理器|平台/g, '')
|
||
.replace(/[()()]/g, '');
|
||
for (const word of BRAND_WORDS) name = name.split(word).join('');
|
||
return name;
|
||
}
|
||
|
||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||
|
||
async function requestJson(path, attempt = 1) {
|
||
try {
|
||
await sleep(80);
|
||
const response = await fetch(`${BASE}${path}`, {
|
||
headers: HEADERS,
|
||
signal: AbortSignal.timeout(20000),
|
||
});
|
||
if (!response.ok) {
|
||
const error = new Error(`HTTP ${response.status}`);
|
||
error.status = response.status;
|
||
throw error;
|
||
}
|
||
const json = await response.json();
|
||
if (!json.success) throw new Error(json.message ?? '接口返回失败');
|
||
return json;
|
||
} catch (error) {
|
||
if (attempt >= 5) throw error;
|
||
// 源站对高频请求会返回 429 / 403,退避后重试
|
||
const limited = error?.status === 429 || error?.status === 403;
|
||
await sleep((limited ? 1500 : 400) * attempt);
|
||
return requestJson(path, attempt + 1);
|
||
}
|
||
}
|
||
|
||
async function fetchAllProcessors(limit) {
|
||
const first = await requestJson('/api/processors?limit=100&page=1');
|
||
const pages = Math.max(1, Number(first.pagination?.pages ?? 1));
|
||
const items = [...(first.data ?? [])];
|
||
for (let page = 2; page <= pages; page += 1) {
|
||
const json = await requestJson(`/api/processors?limit=100&page=${page}`);
|
||
items.push(...(json.data ?? []));
|
||
}
|
||
console.log(`源站共 ${first.pagination?.total ?? items.length} 颗处理器,已获取 ${items.length} 颗`);
|
||
return limit > 0 ? items.slice(0, limit) : items;
|
||
}
|
||
|
||
async function fetchDetails(items, concurrency) {
|
||
const results = new Array(items.length);
|
||
let cursor = 0;
|
||
let done = 0;
|
||
|
||
async function worker() {
|
||
while (cursor < items.length) {
|
||
const index = cursor;
|
||
cursor += 1;
|
||
const item = items[index];
|
||
try {
|
||
const json = await requestJson(`/api/processors/${encodeURIComponent(item.key)}`);
|
||
results[index] = json.data ?? item;
|
||
} catch (error) {
|
||
console.warn(` 详情失败 ${item.key}:${error instanceof Error ? error.message : error}`);
|
||
results[index] = item;
|
||
}
|
||
done += 1;
|
||
if (done % 50 === 0) console.log(` 详情进度 ${done}/${items.length}`);
|
||
}
|
||
}
|
||
|
||
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
|
||
return results;
|
||
}
|
||
|
||
function mapProcessor(remote) {
|
||
const brandSource = clean(remote.brand);
|
||
const brand = BRAND_MAP[brandSource] ?? brandSource ?? 'Unknown';
|
||
const nameZh = clean(remote.name_zh);
|
||
const nameEn = clean(remote.name_en);
|
||
const rawName = nameZh || nameEn || clean(remote.key);
|
||
const model = stripBrand(rawName, [brandSource, brand]) || rawName;
|
||
|
||
const cpu = remote.cpu ?? {};
|
||
const gpu = remote.gpu;
|
||
const npu = remote.npu;
|
||
const benchmark = remote.benchmark ?? {};
|
||
const nanoreview = remote.nanoreview ?? {};
|
||
const nrSpecs = nanoreview.specs ?? {};
|
||
const specZh = remote.spec_zh ?? {};
|
||
const specGroupsSource =
|
||
Object.keys(specZh).length > 0 ? specZh : (remote.spec_groups ?? {});
|
||
|
||
const specGroups = {};
|
||
for (const [groupName, fields] of Object.entries(specGroupsSource)) {
|
||
if (!fields || typeof fields !== 'object' || Array.isArray(fields)) continue;
|
||
const rows = {};
|
||
for (const [key, value] of Object.entries(fields)) {
|
||
const content = text(value);
|
||
if (content) rows[key] = content;
|
||
}
|
||
if (Object.keys(rows).length > 0) specGroups[groupName] = rows;
|
||
}
|
||
|
||
const cache = cpu.cache ?? {};
|
||
const cpuClusters = (Array.isArray(cpu.clusters) ? cpu.clusters : []).map((cluster, index) => {
|
||
const type = clean(cluster.type);
|
||
return {
|
||
name: type && !/^cluster\s*\d+$/i.test(type) ? type : `簇 ${index + 1}`,
|
||
cores: [
|
||
{
|
||
microArch: clean(cluster.arch) || undefined,
|
||
count: numberOrUndefined(cluster.cores),
|
||
clockMax: clean(cluster.freq) || undefined,
|
||
l2: clean(cluster.l2) || undefined,
|
||
},
|
||
],
|
||
};
|
||
});
|
||
|
||
const cpuCores =
|
||
numberOrUndefined(cpu.cores) ??
|
||
cpuClusters.reduce(
|
||
(sum, cluster) => sum + cluster.cores.reduce((n, core) => n + (core.count ?? 0), 0),
|
||
0,
|
||
);
|
||
|
||
const gpuText = clean(gpu) || clean(specZh['图形']?.['名称']);
|
||
const gpuApis = splitFeatures(specZh['图形']?.['图形接口']);
|
||
const gpuFeatures = unique([specZh['图形']?.['特性']]);
|
||
const camera = remote.camera ?? {};
|
||
const display = remote.display ?? {};
|
||
const connectivity = remote.connectivity ?? {};
|
||
const memory = remote.memory ?? {};
|
||
|
||
const benchmarks = [];
|
||
const geekbenchVersion = clean(benchmark.geekbench_version) || 'Geekbench';
|
||
if (numberOrUndefined(benchmark.geekbench_single) !== undefined) {
|
||
benchmarks.push({
|
||
name: `${geekbenchVersion} 单核`,
|
||
score: String(benchmark.geekbench_single),
|
||
});
|
||
}
|
||
if (numberOrUndefined(benchmark.geekbench_multi) !== undefined) {
|
||
benchmarks.push({
|
||
name: `${geekbenchVersion} 多核`,
|
||
score: String(benchmark.geekbench_multi),
|
||
});
|
||
}
|
||
if (numberOrUndefined(nanoreview.gb6_single) !== undefined) {
|
||
benchmarks.push({ name: 'Geekbench 6 单核', score: String(nanoreview.gb6_single) });
|
||
}
|
||
if (numberOrUndefined(nanoreview.gb6_multi) !== undefined) {
|
||
benchmarks.push({ name: 'Geekbench 6 多核', score: String(nanoreview.gb6_multi) });
|
||
}
|
||
if (numberOrUndefined(nanoreview.antutu11_total) !== undefined) {
|
||
benchmarks.push({ name: '安兔兔 v11', score: String(nanoreview.antutu11_total), unit: '分' });
|
||
}
|
||
if (numberOrUndefined(nanoreview['3dmark_snl']) !== undefined) {
|
||
benchmarks.push({
|
||
name: '3DMark Steel Nomad Light',
|
||
score: String(nanoreview['3dmark_snl']),
|
||
unit: '分',
|
||
});
|
||
}
|
||
if (numberOrUndefined(benchmark.cpu_index) !== undefined) {
|
||
benchmarks.push({
|
||
name: 'CPU 性能指数',
|
||
score: String(benchmark.cpu_index),
|
||
unit: '骁龙 865=100',
|
||
});
|
||
}
|
||
|
||
const slowMotion = clean(specZh['相机']?.['慢动作录制']);
|
||
const cameraBlock = {
|
||
isp: clean(camera.isp) || clean(specZh['相机']?.['ISP 名称']) || undefined,
|
||
maxSensor: clean(camera.max_sensor) || clean(specZh['相机']?.['单摄最高像素']) || undefined,
|
||
videoCapture: clean(camera.video_capture) || clean(specZh['相机']?.['视频录制']) || undefined,
|
||
slowMotion: slowMotion || undefined,
|
||
features: unique([specZh['相机']?.['特性']]),
|
||
};
|
||
|
||
const detail = {
|
||
overview: {
|
||
nameEn: nameEn || undefined,
|
||
nameZh: nameZh || undefined,
|
||
series: clean(remote.series) || undefined,
|
||
tier: clean(remote.performance_tier) || undefined,
|
||
tierScore: numberOrUndefined(remote.tier_score),
|
||
releaseYear: numberOrUndefined(remote.release_year),
|
||
phoneCount: numberOrUndefined(remote.phone_count),
|
||
source: clean(remote.source) || undefined,
|
||
},
|
||
cpuClusters,
|
||
cpuBlock: {
|
||
config: clean(cpu.config) || undefined,
|
||
arch: clean(cpu.arch) || undefined,
|
||
archType: clean(cpu.arch_type) || undefined,
|
||
maxFreq: clean(cpu.max_freq) || undefined,
|
||
cores: cpuCores || undefined,
|
||
cacheL3: clean(cache.l3) || undefined,
|
||
systemCache: clean(cache.system_level) || undefined,
|
||
note: clean(cache.note) || undefined,
|
||
},
|
||
gpuBlock: {
|
||
name: gpuText || undefined,
|
||
apis: gpuApis,
|
||
features: gpuFeatures,
|
||
},
|
||
npuBlock: {
|
||
name: clean(npu) || undefined,
|
||
features: [],
|
||
},
|
||
systemCache: clean(cache.system_level) || undefined,
|
||
ispBlock: cameraBlock.isp,
|
||
modemBlock: clean(remote.modem) || undefined,
|
||
memorySupport: {
|
||
type: clean(memory.type) || clean(specZh['内存']?.['类型']) || undefined,
|
||
maxClock: clean(specZh['内存']?.['频率']) || undefined,
|
||
bandwidth: clean(nrSpecs['Max bandwidth']) || undefined,
|
||
},
|
||
storageSupport:
|
||
clean(remote.storage) || clean(specZh['存储']?.['UFS']) || undefined,
|
||
displaySupport:
|
||
clean(display.max_resolution) || clean(display.max_refresh) || undefined,
|
||
displayBlock: {
|
||
maxResolution: clean(display.max_resolution) || undefined,
|
||
maxRefresh: clean(display.max_refresh) || undefined,
|
||
externalMax: clean(specZh['屏幕']?.['外接最高分辨率']) || undefined,
|
||
hdr: clean(specZh['屏幕']?.['HDR']) || undefined,
|
||
colorDepth: clean(specZh['屏幕']?.['色深']) || undefined,
|
||
colorGamut: clean(specZh['屏幕']?.['色域']) || undefined,
|
||
},
|
||
cameraBlock,
|
||
connectivityBlock: {
|
||
wifi: clean(connectivity.wifi) || undefined,
|
||
wifiStandard: clean(connectivity.wifi_standard) || undefined,
|
||
bluetooth: clean(connectivity.bluetooth) || undefined,
|
||
peakSpeed: clean(connectivity.peak_speed) || undefined,
|
||
},
|
||
benchmarkInfo: {
|
||
version: clean(benchmark.geekbench_version) || undefined,
|
||
cpuIndex: numberOrUndefined(benchmark.cpu_index),
|
||
cpuIndexNote: clean(benchmark.cpu_index_note) || undefined,
|
||
note: clean(benchmark.note) || undefined,
|
||
antutu:
|
||
nanoreview.antutu11 && typeof nanoreview.antutu11 === 'object'
|
||
? {
|
||
cpu: numberOrUndefined(nanoreview.antutu11.cpu),
|
||
gpu: numberOrUndefined(nanoreview.antutu11.gpu),
|
||
memory: numberOrUndefined(nanoreview.antutu11.memory),
|
||
ux: numberOrUndefined(nanoreview.antutu11.ux),
|
||
}
|
||
: undefined,
|
||
},
|
||
benchmarks,
|
||
traits: unique(remote.traits ?? []),
|
||
caveats: unique(remote.caveats ?? []),
|
||
sourceUrls: unique(remote.source_urls ?? []),
|
||
specGroups,
|
||
extra: {},
|
||
};
|
||
|
||
const slug = /^[a-z0-9]+(-[a-z0-9]+)*$/.test(clean(remote.key))
|
||
? clean(remote.key)
|
||
: clean(remote.key)
|
||
.toLowerCase()
|
||
.replace(/[^a-z0-9]+/g, '-')
|
||
.replace(/^-+|-+$/g, '');
|
||
|
||
return {
|
||
slug,
|
||
type: 'soc',
|
||
brand,
|
||
model,
|
||
codename:
|
||
clean(remote.codename) ||
|
||
clean(specZh['型号']?.['型号代号']) ||
|
||
undefined,
|
||
aliases: unique([nameEn, nameZh, clean(remote.codename), ...(remote.aliases ?? [])]),
|
||
process:
|
||
normalizeProcess(
|
||
clean(remote.process_node) ||
|
||
clean(specZh['制程']?.['制程']) ||
|
||
clean(nrSpecs['Process']),
|
||
) || undefined,
|
||
arch: clean(nrSpecs['Instruction set']) || clean(cpu.arch) || undefined,
|
||
detail,
|
||
meta: [],
|
||
};
|
||
}
|
||
|
||
function linkDevices(chips) {
|
||
const nameMap = new Map();
|
||
for (const chip of chips) {
|
||
const names = [
|
||
chip.model,
|
||
chip.codename,
|
||
chip.slug,
|
||
...(chip.aliases ?? []),
|
||
chip.detail.overview?.nameEn,
|
||
chip.detail.overview?.nameZh,
|
||
];
|
||
for (const name of names) {
|
||
const normalized = normalizeChipName(name);
|
||
if (normalized) nameMap.set(normalized, chip.slug);
|
||
}
|
||
}
|
||
|
||
const db = getDb();
|
||
const rows = db.prepare('SELECT slug, chip_refs_json, specs_json FROM devices').all();
|
||
const now = new Date().toISOString();
|
||
let linked = 0;
|
||
db.exec('BEGIN');
|
||
try {
|
||
for (const row of rows) {
|
||
let specs = {};
|
||
try {
|
||
specs = JSON.parse(row.specs_json ?? '{}');
|
||
} catch {
|
||
continue;
|
||
}
|
||
const chipset = specs.platform?.chipset;
|
||
if (!chipset) continue;
|
||
const chipSlug = nameMap.get(normalizeChipName(chipset));
|
||
if (!chipSlug) continue;
|
||
let refs = {};
|
||
try {
|
||
refs = JSON.parse(row.chip_refs_json ?? '{}');
|
||
} catch {
|
||
refs = {};
|
||
}
|
||
if (refs.soc === chipSlug) continue;
|
||
refs.soc = chipSlug;
|
||
db.prepare('UPDATE devices SET chip_refs_json = ?, updated_at = ? WHERE slug = ?').run(
|
||
JSON.stringify(refs),
|
||
now,
|
||
row.slug,
|
||
);
|
||
linked += 1;
|
||
}
|
||
db.exec('COMMIT');
|
||
} catch (error) {
|
||
db.exec('ROLLBACK');
|
||
throw error;
|
||
}
|
||
return linked;
|
||
}
|
||
|
||
function parseArgs(argv) {
|
||
const options = { write: false, limit: 0, out: '', concurrency: 2 };
|
||
for (let index = 0; index < argv.length; index += 1) {
|
||
const arg = argv[index];
|
||
if (arg === '--write') options.write = true;
|
||
else if (arg === '--limit') options.limit = Number(argv[(index += 1)] ?? 0);
|
||
else if (arg === '--out') options.out = argv[(index += 1)] ?? '';
|
||
else if (arg === '--concurrency') options.concurrency = Number(argv[(index += 1)] ?? 6);
|
||
else throw new Error(`未知参数:${arg}`);
|
||
}
|
||
return options;
|
||
}
|
||
|
||
async function main() {
|
||
const options = parseArgs(process.argv.slice(2));
|
||
console.log(`拉取 ${BASE}/api/processors ...`);
|
||
const items = await fetchAllProcessors(options.limit);
|
||
console.log('拉取处理器详情 ...');
|
||
const details = await fetchDetails(items, options.concurrency);
|
||
|
||
const chips = [];
|
||
const issues = [];
|
||
const seen = new Set();
|
||
for (const remote of details) {
|
||
try {
|
||
const chip = mapProcessor(remote);
|
||
if (seen.has(chip.slug)) {
|
||
console.warn(` slug 重复,跳过:${chip.slug}(${chip.brand} ${chip.model})`);
|
||
continue;
|
||
}
|
||
const parsed = chipSchema.safeParse(chip);
|
||
if (!parsed.success) {
|
||
issues.push({
|
||
name: `${chip.brand} ${chip.model}`,
|
||
issues: parsed.error.issues
|
||
.slice(0, 8)
|
||
.map((issue) => `${issue.path.join('.')}: ${issue.message}`),
|
||
});
|
||
continue;
|
||
}
|
||
seen.add(chip.slug);
|
||
chips.push(parsed.data);
|
||
} catch (error) {
|
||
issues.push({ name: remote?.key ?? '?', issues: [String(error)] });
|
||
}
|
||
}
|
||
|
||
if (issues.length > 0) {
|
||
console.error(`\n有 ${issues.length} 颗处理器映射失败:`);
|
||
for (const issue of issues.slice(0, 20)) {
|
||
console.error(`- ${issue.name}`);
|
||
for (const line of issue.issues) console.error(` ${line}`);
|
||
}
|
||
throw new Error('映射失败,已中止,未写入任何数据');
|
||
}
|
||
console.log(`映射成功 ${chips.length} 颗`);
|
||
|
||
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
||
const outPath = options.out || `data/import/processors-${stamp}.json`;
|
||
const payload = {
|
||
formatVersion: 1,
|
||
generator: 'LFSpecs / sync-processors',
|
||
exportedAt: new Date().toISOString(),
|
||
devices: [],
|
||
chips,
|
||
};
|
||
await mkdir(dirname(outPath), { recursive: true });
|
||
await writeFile(outPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
|
||
console.log(`JSON 已写入 ${outPath}(${chips.length} 颗)`);
|
||
|
||
if (!options.write) {
|
||
console.log('未指定 --write,本次只产出 JSON。');
|
||
return;
|
||
}
|
||
|
||
console.log('写入数据库 ...');
|
||
const counts = importEntities([], chips);
|
||
console.log(`数据库写入完成:devices ${counts.devices} / chips ${counts.chips}`);
|
||
const linked = linkDevices(chips);
|
||
console.log(`已关联 ${linked} 款设备的 SoC 引用`);
|
||
}
|
||
|
||
main().catch((error) => {
|
||
console.error(error);
|
||
process.exit(1);
|
||
});
|