调整
This commit is contained in:
1 parent
787544eec1
commit
4ff9146910
81 files changed
+18417
-545
No files matched your search
@@ -0,0 +1,278 @@
|
||||
import type { PhoneDevice } from '../schema';
|
||||
import type { Confidence, FieldMeta } from '../schema/meta';
|
||||
import { getValueAtPath, hasValueAtPath } from '../spec-fields';
|
||||
import type { AiLookupField } from './types';
|
||||
|
||||
/** low 置信度且未核实、但表单里确实有值的字段:保存前必须处理 */
|
||||
export function unverifiedLowFields(entity: { meta: FieldMeta[] }): FieldMeta[] {
|
||||
return entity.meta.filter(
|
||||
(item) => item.confidence === 'low' && !item.verified && hasValueAtPath(entity, item.path),
|
||||
);
|
||||
}
|
||||
|
||||
const CONFIDENCE_RANK: Record<Confidence, number> = { high: 3, medium: 2, low: 1 };
|
||||
|
||||
const PLACEHOLDER_PATTERN =
|
||||
/^(?:-+|—|–|未知|不详|暂无|暂无数据|无数据|待补充|待定|n\/a|na|null|undefined|其他|无)$/i;
|
||||
|
||||
/** 去掉空格、全角符号、单位写法和营销修饰后再比较,避免「6.83 英寸」和「6.83英寸」被当成冲突 */
|
||||
export function normalizeAiValue(value: unknown): string {
|
||||
if (value === undefined || value === null) return '';
|
||||
if (typeof value === 'boolean') return value ? 'true' : 'false';
|
||||
if (typeof value === 'number') return String(value);
|
||||
|
||||
let text = String(value)
|
||||
.normalize('NFKC')
|
||||
.toLowerCase()
|
||||
.replace(/ |\u00a0/g, ' ')
|
||||
.replace(/<[^>]+>/g, '')
|
||||
.replace(/([^)]*)/g, '')
|
||||
.replace(/\([^)]*\)/g, '')
|
||||
.replace(/[×✕]/g, 'x')
|
||||
.replace(/(\d+(?:\.\d+)?)亿像素/g, (_, number) => `${Number(number) * 100}mp`)
|
||||
.replace(/(\d+(?:\.\d+)?)万像素/g, (_, number) => `${Number(number) / 100}mp`)
|
||||
.replace(/毫米/g, 'mm')
|
||||
.replace(/厘米/g, 'cm')
|
||||
.replace(/微米|μm|um/g, 'um')
|
||||
.replace(/英寸|吋/g, 'in')
|
||||
.replace(/尼特|nits?/g, 'nit')
|
||||
.replace(/毫安时?/g, 'mah')
|
||||
.replace(/克/g, 'g')
|
||||
.replace(/瓦/g, 'w')
|
||||
.replace(/线性马达/g, '马达')
|
||||
.replace(/双扬声器/g, '2扬声器')
|
||||
.replace(/单扬声器/g, '1扬声器')
|
||||
.replace(/usb\s*type-?c/g, 'usbc')
|
||||
.replace(/type-?c/g, 'usbc')
|
||||
.replace(/usb-?c/g, 'usbc')
|
||||
.replace(/wi-?fi/g, 'wifi')
|
||||
.replace(/^支持(?:最高|最大)?/, '')
|
||||
.replace(/^(?:配备|采用|搭载|内置)/, '')
|
||||
.replace(/约/g, '')
|
||||
.replace(/["'“”‘’]/g, '')
|
||||
.replace(/[\s,,、;;·•]/g, '')
|
||||
.replace(/[。.!!??]+$/g, '');
|
||||
|
||||
text = text.trim();
|
||||
return PLACEHOLDER_PATTERN.test(text) ? '' : text;
|
||||
}
|
||||
|
||||
function normalizeList(value: unknown): string[] {
|
||||
const items = Array.isArray(value) ? value : [value];
|
||||
return items
|
||||
.flatMap((item) => String(item ?? '').split(/[,,、;;||]/))
|
||||
.map((item) => normalizeAiValue(item))
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
}
|
||||
|
||||
function numericTokens(value: unknown): string[] {
|
||||
return String(value ?? '').match(/\d+(?:\.\d+)?/g) ?? [];
|
||||
}
|
||||
|
||||
function comparableNumbers(value: unknown): string {
|
||||
return numericTokens(value).join('|');
|
||||
}
|
||||
|
||||
function stripUnitWords(value: unknown): string {
|
||||
return normalizeAiValue(value).replace(
|
||||
/gb|tb|mhz|ghz|hz|mah|nit|ppi|gbps|mbps|mm|cm|nm|um|mp|in|w|g|fps/g,
|
||||
'',
|
||||
);
|
||||
}
|
||||
|
||||
function toBooleanLike(value: unknown): boolean | undefined {
|
||||
if (typeof value === 'boolean') return value;
|
||||
const text = normalizeAiValue(value);
|
||||
if (/^(?:支持|是|有|true|yes)$/.test(text)) return true;
|
||||
if (/^(?:不支持|否|没有|false|no)$/.test(text)) return false;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** 当前值与 AI 值是否表达同一件事(顺序、空格、单位写法差异都视为一致) */
|
||||
export function aiValuesEquivalent(current: unknown, incoming: unknown): boolean {
|
||||
if (Array.isArray(current) || Array.isArray(incoming)) {
|
||||
const left = normalizeList(current);
|
||||
const right = normalizeList(incoming);
|
||||
return left.length > 0 && left.length === right.length && left.every((item, index) => item === right[index]);
|
||||
}
|
||||
|
||||
const leftBoolean = toBooleanLike(current);
|
||||
const rightBoolean = toBooleanLike(incoming);
|
||||
if (leftBoolean !== undefined && rightBoolean !== undefined) return leftBoolean === rightBoolean;
|
||||
|
||||
const left = normalizeAiValue(current);
|
||||
const right = normalizeAiValue(incoming);
|
||||
if (!left || !right) return false;
|
||||
if (left === right) return true;
|
||||
|
||||
// 「120」和「120Hz」「234.5」和「234.5g」这类只差单位省略的写法
|
||||
const leftNumbers = comparableNumbers(current);
|
||||
const rightNumbers = comparableNumbers(incoming);
|
||||
if (leftNumbers && leftNumbers === rightNumbers) {
|
||||
return stripUnitWords(current) === stripUnitWords(incoming);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 当前值已经完整包含 AI 值:属于「信息更少」的重复项,直接忽略,避免列表噪音 */
|
||||
function isRedundantReply(current: unknown, incoming: unknown): boolean {
|
||||
if (Array.isArray(current) || Array.isArray(incoming)) {
|
||||
const currentList = normalizeList(current);
|
||||
const incomingList = normalizeList(incoming);
|
||||
return (
|
||||
incomingList.length > 0 &&
|
||||
incomingList.length < currentList.length &&
|
||||
incomingList.every((item) => currentList.includes(item))
|
||||
);
|
||||
}
|
||||
if (typeof current !== 'string' || typeof incoming !== 'string') return false;
|
||||
|
||||
const loose = (value: string) =>
|
||||
value.normalize('NFKC').replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
const whole = loose(current);
|
||||
const part = loose(incoming);
|
||||
// 末尾是数字时容易把 UFS4 当成 UFS4.1 的子串,交给信息量规则处理
|
||||
if (!part || /\d$/.test(part)) return false;
|
||||
|
||||
const index = whole.indexOf(part);
|
||||
if (index < 0) return false;
|
||||
const isBoundary = (char: string | undefined) =>
|
||||
!char || /[\s,,、;;/·|()()<>]/.test(char);
|
||||
return isBoundary(whole[index - 1]) && isBoundary(whole[index + part.length]);
|
||||
}
|
||||
|
||||
function isBlankValue(value: unknown): boolean {
|
||||
if (value === undefined || value === null) return true;
|
||||
if (typeof value === 'string') return normalizeAiValue(value) === '';
|
||||
if (Array.isArray(value)) return value.every((item) => isBlankValue(item));
|
||||
if (typeof value === 'object') return Object.keys(value).length === 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 粗略的信息量评分:数字、单位越多,信息越具体;营销词会被扣分 */
|
||||
function informationScore(value: unknown): number {
|
||||
if (isBlankValue(value)) return 0;
|
||||
if (Array.isArray(value)) {
|
||||
return value.reduce((total, item) => total + Math.max(1, informationScore(item)), 0) + value.length * 2;
|
||||
}
|
||||
if (typeof value === 'boolean') return 1;
|
||||
|
||||
const text = String(value).trim();
|
||||
const numbers = (text.match(/\d+(?:\.\d+)?/g) ?? []).length;
|
||||
const units = (
|
||||
text.match(/(?:mm|cm|g|克|英寸|寸|hz|nit|nits|mah|w|gb|tb|mp|ppi|gbps|nm|ghz|mhz|fps|核|台|个)/gi) ?? []
|
||||
).length;
|
||||
const vague = (text.match(/大电池|快充|超大|高性能|旗舰|优秀|支持|其他/g) ?? []).length;
|
||||
return numbers * 6 + units * 3 + Math.min(text.length, 100) / 50 - vague * 2;
|
||||
}
|
||||
|
||||
function currentConfidence(device: PhoneDevice, path: string): Confidence {
|
||||
const meta = device.meta.find((item) => item.path === path);
|
||||
const syncedAt = device.specs.coolapk?.syncedAt;
|
||||
// 酷安同步会覆盖字段值;比同步时间更早的字段置信度已经失效,按酷安默认 high 处理
|
||||
if (meta && syncedAt && new Date(meta.capturedAt).getTime() < new Date(syncedAt).getTime()) {
|
||||
return 'high';
|
||||
}
|
||||
if (meta) return meta.confidence;
|
||||
return device.specs.coolapk ? 'high' : 'low';
|
||||
}
|
||||
|
||||
export type AiSuggestionAction = 'fill' | 'replace' | 'lower-info' | 'review';
|
||||
|
||||
export interface AiFieldJudgement {
|
||||
action: AiSuggestionAction;
|
||||
defaultSelected: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** 判断单个 AI 字段与当前库中值的差异,用于决定是否展示、默认是否勾选 */
|
||||
export function judgeAiField(device: PhoneDevice, field: AiLookupField): AiFieldJudgement | null {
|
||||
if (isBlankValue(field.value)) return null;
|
||||
|
||||
const current = getValueAtPath(device, field.path);
|
||||
if (isBlankValue(current)) {
|
||||
return { action: 'fill', defaultSelected: true, reason: '数据库为空,默认补全' };
|
||||
}
|
||||
if (aiValuesEquivalent(current, field.value)) return null;
|
||||
if (isRedundantReply(current, field.value)) return null;
|
||||
|
||||
const oldScore = informationScore(current);
|
||||
const newScore = informationScore(field.value);
|
||||
const oldConfidence = currentConfidence(device, field.path);
|
||||
const meta = device.meta.find((item) => item.path === field.path);
|
||||
|
||||
if (meta?.verified && CONFIDENCE_RANK[field.confidence] <= CONFIDENCE_RANK[oldConfidence]) {
|
||||
return { action: 'review', defaultSelected: false, reason: '现有值已核实,默认不覆盖' };
|
||||
}
|
||||
|
||||
if (newScore < oldScore) {
|
||||
if (CONFIDENCE_RANK[field.confidence] > CONFIDENCE_RANK[oldConfidence]) {
|
||||
return { action: 'replace', defaultSelected: true, reason: '信息量较少,但置信度更高' };
|
||||
}
|
||||
return { action: 'lower-info', defaultSelected: false, reason: '信息量少于当前值,默认不勾选' };
|
||||
}
|
||||
|
||||
if (CONFIDENCE_RANK[field.confidence] < CONFIDENCE_RANK[oldConfidence]) {
|
||||
return { action: 'review', defaultSelected: false, reason: '置信度低于当前值,需人工确认' };
|
||||
}
|
||||
|
||||
return { action: 'replace', defaultSelected: true, reason: '信息更新' };
|
||||
}
|
||||
|
||||
export interface PreparedAiFields {
|
||||
fields: AiLookupField[];
|
||||
selected: Record<string, boolean>;
|
||||
ignored: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 过滤 AI 结果:去掉空值、占位符、与当前一致的字段、重复字段;
|
||||
* 按「补空 → 替换 → 需人工确认」排序,并给出默认勾选状态。
|
||||
*/
|
||||
export function prepareAiFields(device: PhoneDevice, fields: AiLookupField[]): PreparedAiFields {
|
||||
const bestByPath = new Map<string, AiLookupField>();
|
||||
for (const field of fields) {
|
||||
if (isBlankValue(field.value)) continue;
|
||||
const current = bestByPath.get(field.path);
|
||||
if (!current) {
|
||||
bestByPath.set(field.path, field);
|
||||
continue;
|
||||
}
|
||||
const currentRank = CONFIDENCE_RANK[current.confidence];
|
||||
const nextRank = CONFIDENCE_RANK[field.confidence];
|
||||
if (
|
||||
nextRank > currentRank ||
|
||||
(nextRank === currentRank && informationScore(field.value) > informationScore(current.value))
|
||||
) {
|
||||
bestByPath.set(field.path, field);
|
||||
}
|
||||
}
|
||||
|
||||
const judged: { field: AiLookupField; judgement: AiFieldJudgement }[] = [];
|
||||
for (const field of bestByPath.values()) {
|
||||
const judgement = judgeAiField(device, field);
|
||||
if (!judgement) continue;
|
||||
judged.push({ field, judgement });
|
||||
}
|
||||
|
||||
const order: Record<AiSuggestionAction, number> = { fill: 0, replace: 1, 'lower-info': 2, review: 3 };
|
||||
judged.sort(
|
||||
(left, right) =>
|
||||
Number(right.judgement.defaultSelected) - Number(left.judgement.defaultSelected) ||
|
||||
order[left.judgement.action] - order[right.judgement.action] ||
|
||||
left.field.path.localeCompare(right.field.path),
|
||||
);
|
||||
|
||||
const selected: Record<string, boolean> = {};
|
||||
for (const item of judged) {
|
||||
selected[item.field.path] = item.judgement.defaultSelected;
|
||||
}
|
||||
|
||||
return {
|
||||
fields: judged.map((item) => item.field),
|
||||
selected,
|
||||
ignored: fields.length - judged.length,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user