调整
This commit is contained in:
1 parent
787544eec1
commit
4ff9146910
81 files changed
+18417
-545
No files matched your search
@@ -0,0 +1,532 @@
|
||||
/**
|
||||
* 从 phonerecommend.top 同步处理器(SoC)档案到本项目的 chips 表 / 导入 JSON。
|
||||
*
|
||||
* 示例:
|
||||
* pnpm sync:processors
|
||||
* pnpm sync:processors -- --write
|
||||
* pnpm sync:processors -- --write --limit 10
|
||||
*/
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { chipSchema } from '../src/lib/schema';
|
||||
import { importEntities } from '../src/lib/repo';
|
||||
import { getDb } from '../src/lib/db';
|
||||
|
||||
const BASE = 'https://www.phonerecommend.top';
|
||||
const HEADERS = {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
Referer: `${BASE}/processors`,
|
||||
Origin: BASE,
|
||||
};
|
||||
|
||||
/** 源站中文品牌 → 本项目规范名 */
|
||||
const BRAND_MAP = {
|
||||
高通: 'Qualcomm',
|
||||
联发科: 'MediaTek',
|
||||
苹果: 'Apple',
|
||||
小米: 'Xiaomi',
|
||||
海思: 'HiSilicon',
|
||||
三星: 'Samsung',
|
||||
谷歌: 'Google',
|
||||
紫光展锐: 'UNISOC',
|
||||
};
|
||||
|
||||
const BRAND_WORDS = [
|
||||
'高通',
|
||||
'联发科',
|
||||
'苹果',
|
||||
'小米',
|
||||
'海思',
|
||||
'三星',
|
||||
'谷歌',
|
||||
'紫光展锐',
|
||||
'qualcomm',
|
||||
'mediatek',
|
||||
'apple',
|
||||
'xiaomi',
|
||||
'hisilicon',
|
||||
'samsung',
|
||||
'google',
|
||||
'unisoc',
|
||||
];
|
||||
|
||||
function clean(value) {
|
||||
if (typeof value === 'string') return value.trim();
|
||||
if (typeof value === 'number') return String(value);
|
||||
if (value == null) return '';
|
||||
return '';
|
||||
}
|
||||
|
||||
function text(value) {
|
||||
if (Array.isArray(value)) return value.map(text).filter(Boolean).join(' / ');
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.entries(value)
|
||||
.map(([key, item]) => `${key}: ${text(item)}`)
|
||||
.filter(Boolean)
|
||||
.join(';');
|
||||
}
|
||||
return clean(value);
|
||||
}
|
||||
|
||||
function normalizeProcess(value) {
|
||||
return clean(value).replace(/nanometers?/gi, 'nm').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function numberOrUndefined(value) {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function unique(values) {
|
||||
return [...new Set(values.map(clean).filter(Boolean))];
|
||||
}
|
||||
|
||||
function stripBrand(value, prefixes) {
|
||||
for (const prefix of prefixes) {
|
||||
if (prefix && value.toLowerCase().startsWith(prefix.toLowerCase())) {
|
||||
return value.slice(prefix.length).trim();
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function splitFeatures(value) {
|
||||
return unique(String(value ?? '').split(/[,,、;;]/));
|
||||
}
|
||||
|
||||
function normalizeChipName(value) {
|
||||
let name = String(value ?? '')
|
||||
.toLowerCase()
|
||||
.replace(/[®™©]/g, '')
|
||||
.replace(/\s+/g, '')
|
||||
.replace(/移动平台|处理器|平台/g, '')
|
||||
.replace(/[()()]/g, '');
|
||||
for (const word of BRAND_WORDS) name = name.split(word).join('');
|
||||
return name;
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
async function requestJson(path, attempt = 1) {
|
||||
try {
|
||||
await sleep(80);
|
||||
const response = await fetch(`${BASE}${path}`, {
|
||||
headers: HEADERS,
|
||||
signal: AbortSignal.timeout(20000),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = new Error(`HTTP ${response.status}`);
|
||||
error.status = response.status;
|
||||
throw error;
|
||||
}
|
||||
const json = await response.json();
|
||||
if (!json.success) throw new Error(json.message ?? '接口返回失败');
|
||||
return json;
|
||||
} catch (error) {
|
||||
if (attempt >= 5) throw error;
|
||||
// 源站对高频请求会返回 429 / 403,退避后重试
|
||||
const limited = error?.status === 429 || error?.status === 403;
|
||||
await sleep((limited ? 1500 : 400) * attempt);
|
||||
return requestJson(path, attempt + 1);
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchAllProcessors(limit) {
|
||||
const first = await requestJson('/api/processors?limit=100&page=1');
|
||||
const pages = Math.max(1, Number(first.pagination?.pages ?? 1));
|
||||
const items = [...(first.data ?? [])];
|
||||
for (let page = 2; page <= pages; page += 1) {
|
||||
const json = await requestJson(`/api/processors?limit=100&page=${page}`);
|
||||
items.push(...(json.data ?? []));
|
||||
}
|
||||
console.log(`源站共 ${first.pagination?.total ?? items.length} 颗处理器,已获取 ${items.length} 颗`);
|
||||
return limit > 0 ? items.slice(0, limit) : items;
|
||||
}
|
||||
|
||||
async function fetchDetails(items, concurrency) {
|
||||
const results = new Array(items.length);
|
||||
let cursor = 0;
|
||||
let done = 0;
|
||||
|
||||
async function worker() {
|
||||
while (cursor < items.length) {
|
||||
const index = cursor;
|
||||
cursor += 1;
|
||||
const item = items[index];
|
||||
try {
|
||||
const json = await requestJson(`/api/processors/${encodeURIComponent(item.key)}`);
|
||||
results[index] = json.data ?? item;
|
||||
} catch (error) {
|
||||
console.warn(` 详情失败 ${item.key}:${error instanceof Error ? error.message : error}`);
|
||||
results[index] = item;
|
||||
}
|
||||
done += 1;
|
||||
if (done % 50 === 0) console.log(` 详情进度 ${done}/${items.length}`);
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
|
||||
return results;
|
||||
}
|
||||
|
||||
function mapProcessor(remote) {
|
||||
const brandSource = clean(remote.brand);
|
||||
const brand = BRAND_MAP[brandSource] ?? brandSource ?? 'Unknown';
|
||||
const nameZh = clean(remote.name_zh);
|
||||
const nameEn = clean(remote.name_en);
|
||||
const rawName = nameZh || nameEn || clean(remote.key);
|
||||
const model = stripBrand(rawName, [brandSource, brand]) || rawName;
|
||||
|
||||
const cpu = remote.cpu ?? {};
|
||||
const gpu = remote.gpu;
|
||||
const npu = remote.npu;
|
||||
const benchmark = remote.benchmark ?? {};
|
||||
const nanoreview = remote.nanoreview ?? {};
|
||||
const nrSpecs = nanoreview.specs ?? {};
|
||||
const specZh = remote.spec_zh ?? {};
|
||||
const specGroupsSource =
|
||||
Object.keys(specZh).length > 0 ? specZh : (remote.spec_groups ?? {});
|
||||
|
||||
const specGroups = {};
|
||||
for (const [groupName, fields] of Object.entries(specGroupsSource)) {
|
||||
if (!fields || typeof fields !== 'object' || Array.isArray(fields)) continue;
|
||||
const rows = {};
|
||||
for (const [key, value] of Object.entries(fields)) {
|
||||
const content = text(value);
|
||||
if (content) rows[key] = content;
|
||||
}
|
||||
if (Object.keys(rows).length > 0) specGroups[groupName] = rows;
|
||||
}
|
||||
|
||||
const cache = cpu.cache ?? {};
|
||||
const cpuClusters = (Array.isArray(cpu.clusters) ? cpu.clusters : []).map((cluster, index) => {
|
||||
const type = clean(cluster.type);
|
||||
return {
|
||||
name: type && !/^cluster\s*\d+$/i.test(type) ? type : `簇 ${index + 1}`,
|
||||
cores: [
|
||||
{
|
||||
microArch: clean(cluster.arch) || undefined,
|
||||
count: numberOrUndefined(cluster.cores),
|
||||
clockMax: clean(cluster.freq) || undefined,
|
||||
l2: clean(cluster.l2) || undefined,
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
const cpuCores =
|
||||
numberOrUndefined(cpu.cores) ??
|
||||
cpuClusters.reduce(
|
||||
(sum, cluster) => sum + cluster.cores.reduce((n, core) => n + (core.count ?? 0), 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const gpuText = clean(gpu) || clean(specZh['图形']?.['名称']);
|
||||
const gpuApis = splitFeatures(specZh['图形']?.['图形接口']);
|
||||
const gpuFeatures = unique([specZh['图形']?.['特性']]);
|
||||
const camera = remote.camera ?? {};
|
||||
const display = remote.display ?? {};
|
||||
const connectivity = remote.connectivity ?? {};
|
||||
const memory = remote.memory ?? {};
|
||||
|
||||
const benchmarks = [];
|
||||
const geekbenchVersion = clean(benchmark.geekbench_version) || 'Geekbench';
|
||||
if (numberOrUndefined(benchmark.geekbench_single) !== undefined) {
|
||||
benchmarks.push({
|
||||
name: `${geekbenchVersion} 单核`,
|
||||
score: String(benchmark.geekbench_single),
|
||||
});
|
||||
}
|
||||
if (numberOrUndefined(benchmark.geekbench_multi) !== undefined) {
|
||||
benchmarks.push({
|
||||
name: `${geekbenchVersion} 多核`,
|
||||
score: String(benchmark.geekbench_multi),
|
||||
});
|
||||
}
|
||||
if (numberOrUndefined(nanoreview.gb6_single) !== undefined) {
|
||||
benchmarks.push({ name: 'Geekbench 6 单核', score: String(nanoreview.gb6_single) });
|
||||
}
|
||||
if (numberOrUndefined(nanoreview.gb6_multi) !== undefined) {
|
||||
benchmarks.push({ name: 'Geekbench 6 多核', score: String(nanoreview.gb6_multi) });
|
||||
}
|
||||
if (numberOrUndefined(nanoreview.antutu11_total) !== undefined) {
|
||||
benchmarks.push({ name: '安兔兔 v11', score: String(nanoreview.antutu11_total), unit: '分' });
|
||||
}
|
||||
if (numberOrUndefined(nanoreview['3dmark_snl']) !== undefined) {
|
||||
benchmarks.push({
|
||||
name: '3DMark Steel Nomad Light',
|
||||
score: String(nanoreview['3dmark_snl']),
|
||||
unit: '分',
|
||||
});
|
||||
}
|
||||
if (numberOrUndefined(benchmark.cpu_index) !== undefined) {
|
||||
benchmarks.push({
|
||||
name: 'CPU 性能指数',
|
||||
score: String(benchmark.cpu_index),
|
||||
unit: '骁龙 865=100',
|
||||
});
|
||||
}
|
||||
|
||||
const slowMotion = clean(specZh['相机']?.['慢动作录制']);
|
||||
const cameraBlock = {
|
||||
isp: clean(camera.isp) || clean(specZh['相机']?.['ISP 名称']) || undefined,
|
||||
maxSensor: clean(camera.max_sensor) || clean(specZh['相机']?.['单摄最高像素']) || undefined,
|
||||
videoCapture: clean(camera.video_capture) || clean(specZh['相机']?.['视频录制']) || undefined,
|
||||
slowMotion: slowMotion || undefined,
|
||||
features: unique([specZh['相机']?.['特性']]),
|
||||
};
|
||||
|
||||
const detail = {
|
||||
overview: {
|
||||
nameEn: nameEn || undefined,
|
||||
nameZh: nameZh || undefined,
|
||||
series: clean(remote.series) || undefined,
|
||||
tier: clean(remote.performance_tier) || undefined,
|
||||
tierScore: numberOrUndefined(remote.tier_score),
|
||||
releaseYear: numberOrUndefined(remote.release_year),
|
||||
phoneCount: numberOrUndefined(remote.phone_count),
|
||||
source: clean(remote.source) || undefined,
|
||||
},
|
||||
cpuClusters,
|
||||
cpuBlock: {
|
||||
config: clean(cpu.config) || undefined,
|
||||
arch: clean(cpu.arch) || undefined,
|
||||
archType: clean(cpu.arch_type) || undefined,
|
||||
maxFreq: clean(cpu.max_freq) || undefined,
|
||||
cores: cpuCores || undefined,
|
||||
cacheL3: clean(cache.l3) || undefined,
|
||||
systemCache: clean(cache.system_level) || undefined,
|
||||
note: clean(cache.note) || undefined,
|
||||
},
|
||||
gpuBlock: {
|
||||
name: gpuText || undefined,
|
||||
apis: gpuApis,
|
||||
features: gpuFeatures,
|
||||
},
|
||||
npuBlock: {
|
||||
name: clean(npu) || undefined,
|
||||
features: [],
|
||||
},
|
||||
systemCache: clean(cache.system_level) || undefined,
|
||||
ispBlock: cameraBlock.isp,
|
||||
modemBlock: clean(remote.modem) || undefined,
|
||||
memorySupport: {
|
||||
type: clean(memory.type) || clean(specZh['内存']?.['类型']) || undefined,
|
||||
maxClock: clean(specZh['内存']?.['频率']) || undefined,
|
||||
bandwidth: clean(nrSpecs['Max bandwidth']) || undefined,
|
||||
},
|
||||
storageSupport:
|
||||
clean(remote.storage) || clean(specZh['存储']?.['UFS']) || undefined,
|
||||
displaySupport:
|
||||
clean(display.max_resolution) || clean(display.max_refresh) || undefined,
|
||||
displayBlock: {
|
||||
maxResolution: clean(display.max_resolution) || undefined,
|
||||
maxRefresh: clean(display.max_refresh) || undefined,
|
||||
externalMax: clean(specZh['屏幕']?.['外接最高分辨率']) || undefined,
|
||||
hdr: clean(specZh['屏幕']?.['HDR']) || undefined,
|
||||
colorDepth: clean(specZh['屏幕']?.['色深']) || undefined,
|
||||
colorGamut: clean(specZh['屏幕']?.['色域']) || undefined,
|
||||
},
|
||||
cameraBlock,
|
||||
connectivityBlock: {
|
||||
wifi: clean(connectivity.wifi) || undefined,
|
||||
wifiStandard: clean(connectivity.wifi_standard) || undefined,
|
||||
bluetooth: clean(connectivity.bluetooth) || undefined,
|
||||
peakSpeed: clean(connectivity.peak_speed) || undefined,
|
||||
},
|
||||
benchmarkInfo: {
|
||||
version: clean(benchmark.geekbench_version) || undefined,
|
||||
cpuIndex: numberOrUndefined(benchmark.cpu_index),
|
||||
cpuIndexNote: clean(benchmark.cpu_index_note) || undefined,
|
||||
note: clean(benchmark.note) || undefined,
|
||||
antutu:
|
||||
nanoreview.antutu11 && typeof nanoreview.antutu11 === 'object'
|
||||
? {
|
||||
cpu: numberOrUndefined(nanoreview.antutu11.cpu),
|
||||
gpu: numberOrUndefined(nanoreview.antutu11.gpu),
|
||||
memory: numberOrUndefined(nanoreview.antutu11.memory),
|
||||
ux: numberOrUndefined(nanoreview.antutu11.ux),
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
benchmarks,
|
||||
traits: unique(remote.traits ?? []),
|
||||
caveats: unique(remote.caveats ?? []),
|
||||
sourceUrls: unique(remote.source_urls ?? []),
|
||||
specGroups,
|
||||
extra: {},
|
||||
};
|
||||
|
||||
const slug = /^[a-z0-9]+(-[a-z0-9]+)*$/.test(clean(remote.key))
|
||||
? clean(remote.key)
|
||||
: clean(remote.key)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
|
||||
return {
|
||||
slug,
|
||||
type: 'soc',
|
||||
brand,
|
||||
model,
|
||||
codename:
|
||||
clean(remote.codename) ||
|
||||
clean(specZh['型号']?.['型号代号']) ||
|
||||
undefined,
|
||||
aliases: unique([nameEn, nameZh, clean(remote.codename), ...(remote.aliases ?? [])]),
|
||||
process:
|
||||
normalizeProcess(
|
||||
clean(remote.process_node) ||
|
||||
clean(specZh['制程']?.['制程']) ||
|
||||
clean(nrSpecs['Process']),
|
||||
) || undefined,
|
||||
arch: clean(nrSpecs['Instruction set']) || clean(cpu.arch) || undefined,
|
||||
detail,
|
||||
meta: [],
|
||||
};
|
||||
}
|
||||
|
||||
function linkDevices(chips) {
|
||||
const nameMap = new Map();
|
||||
for (const chip of chips) {
|
||||
const names = [
|
||||
chip.model,
|
||||
chip.codename,
|
||||
chip.slug,
|
||||
...(chip.aliases ?? []),
|
||||
chip.detail.overview?.nameEn,
|
||||
chip.detail.overview?.nameZh,
|
||||
];
|
||||
for (const name of names) {
|
||||
const normalized = normalizeChipName(name);
|
||||
if (normalized) nameMap.set(normalized, chip.slug);
|
||||
}
|
||||
}
|
||||
|
||||
const db = getDb();
|
||||
const rows = db.prepare('SELECT slug, chip_refs_json, specs_json FROM devices').all();
|
||||
const now = new Date().toISOString();
|
||||
let linked = 0;
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
for (const row of rows) {
|
||||
let specs = {};
|
||||
try {
|
||||
specs = JSON.parse(row.specs_json ?? '{}');
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const chipset = specs.platform?.chipset;
|
||||
if (!chipset) continue;
|
||||
const chipSlug = nameMap.get(normalizeChipName(chipset));
|
||||
if (!chipSlug) continue;
|
||||
let refs = {};
|
||||
try {
|
||||
refs = JSON.parse(row.chip_refs_json ?? '{}');
|
||||
} catch {
|
||||
refs = {};
|
||||
}
|
||||
if (refs.soc === chipSlug) continue;
|
||||
refs.soc = chipSlug;
|
||||
db.prepare('UPDATE devices SET chip_refs_json = ?, updated_at = ? WHERE slug = ?').run(
|
||||
JSON.stringify(refs),
|
||||
now,
|
||||
row.slug,
|
||||
);
|
||||
linked += 1;
|
||||
}
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return linked;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = { write: false, limit: 0, out: '', concurrency: 2 };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--write') options.write = true;
|
||||
else if (arg === '--limit') options.limit = Number(argv[(index += 1)] ?? 0);
|
||||
else if (arg === '--out') options.out = argv[(index += 1)] ?? '';
|
||||
else if (arg === '--concurrency') options.concurrency = Number(argv[(index += 1)] ?? 6);
|
||||
else throw new Error(`未知参数:${arg}`);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
console.log(`拉取 ${BASE}/api/processors ...`);
|
||||
const items = await fetchAllProcessors(options.limit);
|
||||
console.log('拉取处理器详情 ...');
|
||||
const details = await fetchDetails(items, options.concurrency);
|
||||
|
||||
const chips = [];
|
||||
const issues = [];
|
||||
const seen = new Set();
|
||||
for (const remote of details) {
|
||||
try {
|
||||
const chip = mapProcessor(remote);
|
||||
if (seen.has(chip.slug)) {
|
||||
console.warn(` slug 重复,跳过:${chip.slug}(${chip.brand} ${chip.model})`);
|
||||
continue;
|
||||
}
|
||||
const parsed = chipSchema.safeParse(chip);
|
||||
if (!parsed.success) {
|
||||
issues.push({
|
||||
name: `${chip.brand} ${chip.model}`,
|
||||
issues: parsed.error.issues
|
||||
.slice(0, 8)
|
||||
.map((issue) => `${issue.path.join('.')}: ${issue.message}`),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
seen.add(chip.slug);
|
||||
chips.push(parsed.data);
|
||||
} catch (error) {
|
||||
issues.push({ name: remote?.key ?? '?', issues: [String(error)] });
|
||||
}
|
||||
}
|
||||
|
||||
if (issues.length > 0) {
|
||||
console.error(`\n有 ${issues.length} 颗处理器映射失败:`);
|
||||
for (const issue of issues.slice(0, 20)) {
|
||||
console.error(`- ${issue.name}`);
|
||||
for (const line of issue.issues) console.error(` ${line}`);
|
||||
}
|
||||
throw new Error('映射失败,已中止,未写入任何数据');
|
||||
}
|
||||
console.log(`映射成功 ${chips.length} 颗`);
|
||||
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
||||
const outPath = options.out || `data/import/processors-${stamp}.json`;
|
||||
const payload = {
|
||||
formatVersion: 1,
|
||||
generator: 'LFSpecs / sync-processors',
|
||||
exportedAt: new Date().toISOString(),
|
||||
devices: [],
|
||||
chips,
|
||||
};
|
||||
await mkdir(dirname(outPath), { recursive: true });
|
||||
await writeFile(outPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
|
||||
console.log(`JSON 已写入 ${outPath}(${chips.length} 颗)`);
|
||||
|
||||
if (!options.write) {
|
||||
console.log('未指定 --write,本次只产出 JSON。');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('写入数据库 ...');
|
||||
const counts = importEntities([], chips);
|
||||
console.log(`数据库写入完成:devices ${counts.devices} / chips ${counts.chips}`);
|
||||
const linked = linkDevices(chips);
|
||||
console.log(`已关联 ${linked} 款设备的 SoC 引用`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in new issue
Block a user