This commit is contained in:
zhongluofeng committed 2026-11-06 15:07:02 +08:00
1 parent 787544eec1
commit 4ff9146910
81 files changed
+18417 -545

No files matched your search

+532
View File
@@ -0,0 +1,532 @@
/**
* 从 phonerecommend.top 同步处理器(SoC)档案到本项目的 chips 表 / 导入 JSON。
*
* 示例:
* pnpm sync:processors
* pnpm sync:processors -- --write
* pnpm sync:processors -- --write --limit 10
*/
import { mkdir, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import { chipSchema } from '../src/lib/schema';
import { importEntities } from '../src/lib/repo';
import { getDb } from '../src/lib/db';
const BASE = 'https://www.phonerecommend.top';
const HEADERS = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
Accept: 'application/json, text/plain, */*',
Referer: `${BASE}/processors`,
Origin: BASE,
};
/** 源站中文品牌 → 本项目规范名 */
const BRAND_MAP = {
高通: 'Qualcomm',
联发科: 'MediaTek',
苹果: 'Apple',
小米: 'Xiaomi',
海思: 'HiSilicon',
三星: 'Samsung',
谷歌: 'Google',
紫光展锐: 'UNISOC',
};
const BRAND_WORDS = [
'高通',
'联发科',
'苹果',
'小米',
'海思',
'三星',
'谷歌',
'紫光展锐',
'qualcomm',
'mediatek',
'apple',
'xiaomi',
'hisilicon',
'samsung',
'google',
'unisoc',
];
function clean(value) {
if (typeof value === 'string') return value.trim();
if (typeof value === 'number') return String(value);
if (value == null) return '';
return '';
}
function text(value) {
if (Array.isArray(value)) return value.map(text).filter(Boolean).join(' / ');
if (value && typeof value === 'object') {
return Object.entries(value)
.map(([key, item]) => `${key}: ${text(item)}`)
.filter(Boolean)
.join(';');
}
return clean(value);
}
function normalizeProcess(value) {
return clean(value).replace(/nanometers?/gi, 'nm').replace(/\s+/g, ' ').trim();
}
function numberOrUndefined(value) {
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
}
function unique(values) {
return [...new Set(values.map(clean).filter(Boolean))];
}
function stripBrand(value, prefixes) {
for (const prefix of prefixes) {
if (prefix && value.toLowerCase().startsWith(prefix.toLowerCase())) {
return value.slice(prefix.length).trim();
}
}
return value;
}
function splitFeatures(value) {
return unique(String(value ?? '').split(/[,,、;;]/));
}
function normalizeChipName(value) {
let name = String(value ?? '')
.toLowerCase()
.replace(/[®™©]/g, '')
.replace(/\s+/g, '')
.replace(/移动平台|处理器|平台/g, '')
.replace(/[()()]/g, '');
for (const word of BRAND_WORDS) name = name.split(word).join('');
return name;
}
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function requestJson(path, attempt = 1) {
try {
await sleep(80);
const response = await fetch(`${BASE}${path}`, {
headers: HEADERS,
signal: AbortSignal.timeout(20000),
});
if (!response.ok) {
const error = new Error(`HTTP ${response.status}`);
error.status = response.status;
throw error;
}
const json = await response.json();
if (!json.success) throw new Error(json.message ?? '接口返回失败');
return json;
} catch (error) {
if (attempt >= 5) throw error;
// 源站对高频请求会返回 429 / 403,退避后重试
const limited = error?.status === 429 || error?.status === 403;
await sleep((limited ? 1500 : 400) * attempt);
return requestJson(path, attempt + 1);
}
}
async function fetchAllProcessors(limit) {
const first = await requestJson('/api/processors?limit=100&page=1');
const pages = Math.max(1, Number(first.pagination?.pages ?? 1));
const items = [...(first.data ?? [])];
for (let page = 2; page <= pages; page += 1) {
const json = await requestJson(`/api/processors?limit=100&page=${page}`);
items.push(...(json.data ?? []));
}
console.log(`源站共 ${first.pagination?.total ?? items.length} 颗处理器,已获取 ${items.length} 颗`);
return limit > 0 ? items.slice(0, limit) : items;
}
async function fetchDetails(items, concurrency) {
const results = new Array(items.length);
let cursor = 0;
let done = 0;
async function worker() {
while (cursor < items.length) {
const index = cursor;
cursor += 1;
const item = items[index];
try {
const json = await requestJson(`/api/processors/${encodeURIComponent(item.key)}`);
results[index] = json.data ?? item;
} catch (error) {
console.warn(` 详情失败 ${item.key}:${error instanceof Error ? error.message : error}`);
results[index] = item;
}
done += 1;
if (done % 50 === 0) console.log(` 详情进度 ${done}/${items.length}`);
}
}
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
return results;
}
function mapProcessor(remote) {
const brandSource = clean(remote.brand);
const brand = BRAND_MAP[brandSource] ?? brandSource ?? 'Unknown';
const nameZh = clean(remote.name_zh);
const nameEn = clean(remote.name_en);
const rawName = nameZh || nameEn || clean(remote.key);
const model = stripBrand(rawName, [brandSource, brand]) || rawName;
const cpu = remote.cpu ?? {};
const gpu = remote.gpu;
const npu = remote.npu;
const benchmark = remote.benchmark ?? {};
const nanoreview = remote.nanoreview ?? {};
const nrSpecs = nanoreview.specs ?? {};
const specZh = remote.spec_zh ?? {};
const specGroupsSource =
Object.keys(specZh).length > 0 ? specZh : (remote.spec_groups ?? {});
const specGroups = {};
for (const [groupName, fields] of Object.entries(specGroupsSource)) {
if (!fields || typeof fields !== 'object' || Array.isArray(fields)) continue;
const rows = {};
for (const [key, value] of Object.entries(fields)) {
const content = text(value);
if (content) rows[key] = content;
}
if (Object.keys(rows).length > 0) specGroups[groupName] = rows;
}
const cache = cpu.cache ?? {};
const cpuClusters = (Array.isArray(cpu.clusters) ? cpu.clusters : []).map((cluster, index) => {
const type = clean(cluster.type);
return {
name: type && !/^cluster\s*\d+$/i.test(type) ? type : `簇 ${index + 1}`,
cores: [
{
microArch: clean(cluster.arch) || undefined,
count: numberOrUndefined(cluster.cores),
clockMax: clean(cluster.freq) || undefined,
l2: clean(cluster.l2) || undefined,
},
],
};
});
const cpuCores =
numberOrUndefined(cpu.cores) ??
cpuClusters.reduce(
(sum, cluster) => sum + cluster.cores.reduce((n, core) => n + (core.count ?? 0), 0),
0,
);
const gpuText = clean(gpu) || clean(specZh['图形']?.['名称']);
const gpuApis = splitFeatures(specZh['图形']?.['图形接口']);
const gpuFeatures = unique([specZh['图形']?.['特性']]);
const camera = remote.camera ?? {};
const display = remote.display ?? {};
const connectivity = remote.connectivity ?? {};
const memory = remote.memory ?? {};
const benchmarks = [];
const geekbenchVersion = clean(benchmark.geekbench_version) || 'Geekbench';
if (numberOrUndefined(benchmark.geekbench_single) !== undefined) {
benchmarks.push({
name: `${geekbenchVersion} 单核`,
score: String(benchmark.geekbench_single),
});
}
if (numberOrUndefined(benchmark.geekbench_multi) !== undefined) {
benchmarks.push({
name: `${geekbenchVersion} 多核`,
score: String(benchmark.geekbench_multi),
});
}
if (numberOrUndefined(nanoreview.gb6_single) !== undefined) {
benchmarks.push({ name: 'Geekbench 6 单核', score: String(nanoreview.gb6_single) });
}
if (numberOrUndefined(nanoreview.gb6_multi) !== undefined) {
benchmarks.push({ name: 'Geekbench 6 多核', score: String(nanoreview.gb6_multi) });
}
if (numberOrUndefined(nanoreview.antutu11_total) !== undefined) {
benchmarks.push({ name: '安兔兔 v11', score: String(nanoreview.antutu11_total), unit: '分' });
}
if (numberOrUndefined(nanoreview['3dmark_snl']) !== undefined) {
benchmarks.push({
name: '3DMark Steel Nomad Light',
score: String(nanoreview['3dmark_snl']),
unit: '分',
});
}
if (numberOrUndefined(benchmark.cpu_index) !== undefined) {
benchmarks.push({
name: 'CPU 性能指数',
score: String(benchmark.cpu_index),
unit: '骁龙 865=100',
});
}
const slowMotion = clean(specZh['相机']?.['慢动作录制']);
const cameraBlock = {
isp: clean(camera.isp) || clean(specZh['相机']?.['ISP 名称']) || undefined,
maxSensor: clean(camera.max_sensor) || clean(specZh['相机']?.['单摄最高像素']) || undefined,
videoCapture: clean(camera.video_capture) || clean(specZh['相机']?.['视频录制']) || undefined,
slowMotion: slowMotion || undefined,
features: unique([specZh['相机']?.['特性']]),
};
const detail = {
overview: {
nameEn: nameEn || undefined,
nameZh: nameZh || undefined,
series: clean(remote.series) || undefined,
tier: clean(remote.performance_tier) || undefined,
tierScore: numberOrUndefined(remote.tier_score),
releaseYear: numberOrUndefined(remote.release_year),
phoneCount: numberOrUndefined(remote.phone_count),
source: clean(remote.source) || undefined,
},
cpuClusters,
cpuBlock: {
config: clean(cpu.config) || undefined,
arch: clean(cpu.arch) || undefined,
archType: clean(cpu.arch_type) || undefined,
maxFreq: clean(cpu.max_freq) || undefined,
cores: cpuCores || undefined,
cacheL3: clean(cache.l3) || undefined,
systemCache: clean(cache.system_level) || undefined,
note: clean(cache.note) || undefined,
},
gpuBlock: {
name: gpuText || undefined,
apis: gpuApis,
features: gpuFeatures,
},
npuBlock: {
name: clean(npu) || undefined,
features: [],
},
systemCache: clean(cache.system_level) || undefined,
ispBlock: cameraBlock.isp,
modemBlock: clean(remote.modem) || undefined,
memorySupport: {
type: clean(memory.type) || clean(specZh['内存']?.['类型']) || undefined,
maxClock: clean(specZh['内存']?.['频率']) || undefined,
bandwidth: clean(nrSpecs['Max bandwidth']) || undefined,
},
storageSupport:
clean(remote.storage) || clean(specZh['存储']?.['UFS']) || undefined,
displaySupport:
clean(display.max_resolution) || clean(display.max_refresh) || undefined,
displayBlock: {
maxResolution: clean(display.max_resolution) || undefined,
maxRefresh: clean(display.max_refresh) || undefined,
externalMax: clean(specZh['屏幕']?.['外接最高分辨率']) || undefined,
hdr: clean(specZh['屏幕']?.['HDR']) || undefined,
colorDepth: clean(specZh['屏幕']?.['色深']) || undefined,
colorGamut: clean(specZh['屏幕']?.['色域']) || undefined,
},
cameraBlock,
connectivityBlock: {
wifi: clean(connectivity.wifi) || undefined,
wifiStandard: clean(connectivity.wifi_standard) || undefined,
bluetooth: clean(connectivity.bluetooth) || undefined,
peakSpeed: clean(connectivity.peak_speed) || undefined,
},
benchmarkInfo: {
version: clean(benchmark.geekbench_version) || undefined,
cpuIndex: numberOrUndefined(benchmark.cpu_index),
cpuIndexNote: clean(benchmark.cpu_index_note) || undefined,
note: clean(benchmark.note) || undefined,
antutu:
nanoreview.antutu11 && typeof nanoreview.antutu11 === 'object'
? {
cpu: numberOrUndefined(nanoreview.antutu11.cpu),
gpu: numberOrUndefined(nanoreview.antutu11.gpu),
memory: numberOrUndefined(nanoreview.antutu11.memory),
ux: numberOrUndefined(nanoreview.antutu11.ux),
}
: undefined,
},
benchmarks,
traits: unique(remote.traits ?? []),
caveats: unique(remote.caveats ?? []),
sourceUrls: unique(remote.source_urls ?? []),
specGroups,
extra: {},
};
const slug = /^[a-z0-9]+(-[a-z0-9]+)*$/.test(clean(remote.key))
? clean(remote.key)
: clean(remote.key)
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '');
return {
slug,
type: 'soc',
brand,
model,
codename:
clean(remote.codename) ||
clean(specZh['型号']?.['型号代号']) ||
undefined,
aliases: unique([nameEn, nameZh, clean(remote.codename), ...(remote.aliases ?? [])]),
process:
normalizeProcess(
clean(remote.process_node) ||
clean(specZh['制程']?.['制程']) ||
clean(nrSpecs['Process']),
) || undefined,
arch: clean(nrSpecs['Instruction set']) || clean(cpu.arch) || undefined,
detail,
meta: [],
};
}
function linkDevices(chips) {
const nameMap = new Map();
for (const chip of chips) {
const names = [
chip.model,
chip.codename,
chip.slug,
...(chip.aliases ?? []),
chip.detail.overview?.nameEn,
chip.detail.overview?.nameZh,
];
for (const name of names) {
const normalized = normalizeChipName(name);
if (normalized) nameMap.set(normalized, chip.slug);
}
}
const db = getDb();
const rows = db.prepare('SELECT slug, chip_refs_json, specs_json FROM devices').all();
const now = new Date().toISOString();
let linked = 0;
db.exec('BEGIN');
try {
for (const row of rows) {
let specs = {};
try {
specs = JSON.parse(row.specs_json ?? '{}');
} catch {
continue;
}
const chipset = specs.platform?.chipset;
if (!chipset) continue;
const chipSlug = nameMap.get(normalizeChipName(chipset));
if (!chipSlug) continue;
let refs = {};
try {
refs = JSON.parse(row.chip_refs_json ?? '{}');
} catch {
refs = {};
}
if (refs.soc === chipSlug) continue;
refs.soc = chipSlug;
db.prepare('UPDATE devices SET chip_refs_json = ?, updated_at = ? WHERE slug = ?').run(
JSON.stringify(refs),
now,
row.slug,
);
linked += 1;
}
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
return linked;
}
function parseArgs(argv) {
const options = { write: false, limit: 0, out: '', concurrency: 2 };
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--write') options.write = true;
else if (arg === '--limit') options.limit = Number(argv[(index += 1)] ?? 0);
else if (arg === '--out') options.out = argv[(index += 1)] ?? '';
else if (arg === '--concurrency') options.concurrency = Number(argv[(index += 1)] ?? 6);
else throw new Error(`未知参数:${arg}`);
}
return options;
}
async function main() {
const options = parseArgs(process.argv.slice(2));
console.log(`拉取 ${BASE}/api/processors ...`);
const items = await fetchAllProcessors(options.limit);
console.log('拉取处理器详情 ...');
const details = await fetchDetails(items, options.concurrency);
const chips = [];
const issues = [];
const seen = new Set();
for (const remote of details) {
try {
const chip = mapProcessor(remote);
if (seen.has(chip.slug)) {
console.warn(` slug 重复,跳过:${chip.slug}(${chip.brand} ${chip.model})`);
continue;
}
const parsed = chipSchema.safeParse(chip);
if (!parsed.success) {
issues.push({
name: `${chip.brand} ${chip.model}`,
issues: parsed.error.issues
.slice(0, 8)
.map((issue) => `${issue.path.join('.')}: ${issue.message}`),
});
continue;
}
seen.add(chip.slug);
chips.push(parsed.data);
} catch (error) {
issues.push({ name: remote?.key ?? '?', issues: [String(error)] });
}
}
if (issues.length > 0) {
console.error(`\n有 ${issues.length} 颗处理器映射失败:`);
for (const issue of issues.slice(0, 20)) {
console.error(`- ${issue.name}`);
for (const line of issue.issues) console.error(` ${line}`);
}
throw new Error('映射失败,已中止,未写入任何数据');
}
console.log(`映射成功 ${chips.length} 颗`);
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
const outPath = options.out || `data/import/processors-${stamp}.json`;
const payload = {
formatVersion: 1,
generator: 'LFSpecs / sync-processors',
exportedAt: new Date().toISOString(),
devices: [],
chips,
};
await mkdir(dirname(outPath), { recursive: true });
await writeFile(outPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
console.log(`JSON 已写入 ${outPath}(${chips.length} 颗)`);
if (!options.write) {
console.log('未指定 --write,本次只产出 JSON。');
return;
}
console.log('写入数据库 ...');
const counts = importEntities([], chips);
console.log(`数据库写入完成:devices ${counts.devices} / chips ${counts.chips}`);
const linked = linkDevices(chips);
console.log(`已关联 ${linked} 款设备的 SoC 引用`);
}
main().catch((error) => {
console.error(error);
process.exit(1);
});