{ "summary": { "total": 6, "paired": 6, "counts": { "improved": 2, "regressed": 2, "unchanged": 2, "excluded": 0 }, "reference_characters": 94, "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "baseline": { "macro_cer": 0.2238669590643275, "corpus_cer": 0.425531914893617, "character_errors": 40 }, "candidate": { "macro_cer": 0.12881562881562883, "corpus_cer": 0.07446808510638298, "character_errors": 7 }, "delta": { "macro_cer": -0.09505133024869866, "corpus_cer": -0.35106382978723405 } }, "samples": [ { "sample_id": "SYN-0001", "reference": "请把今天的会议纪要发给李明", "baseline": "请把今天的会议纪要发给李明", "candidate": "请把今天的会议纪要发给王芳", "baseline_metric": { "name": "CER", "value": 0.0, "substitutions": 0, "deletions": 0, "insertions": 0, "reference_length": 13, "normalized_reference": "请把今天的会议纪要发给李明", "normalized_hypothesis": "请把今天的会议纪要发给李明", "operations": [], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "完全一致" }, "candidate_metric": { "name": "CER", "value": 0.15384615384615385, "substitutions": 2, "deletions": 0, "insertions": 0, "reference_length": 13, "normalized_reference": "请把今天的会议纪要发给李明", "normalized_hypothesis": "请把今天的会议纪要发给王芳", "operations": [ { "op": "substitution", "reference_index": 11, "reference": "李", "hypothesis_index": 11, "hypothesis": "王" }, { "op": "substitution", "reference_index": 12, "reference": "明", "hypothesis_index": 12, "hypothesis": "芳" } ], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "存在字符级差异" }, "delta": 0.15384615384615385, "status": "regressed" }, { "sample_id": "SYN-0002", "reference": "请把订单 A1007 的数量改成 3 份", "baseline": "请把订单 A1007 的数量改成 3 ", "candidate": "请把订单 A1007 的数量改成 3 份", "baseline_metric": { "name": "CER", "value": 0.0625, "substitutions": 0, "deletions": 1, "insertions": 0, "reference_length": 16, "normalized_reference": "请把订单a1007的数量改成3份", "normalized_hypothesis": "请把订单a1007的数量改成3", "operations": [ { "op": "deletion", "reference_index": 15, "reference": "份", "hypothesis_index": "-", "hypothesis": "-" } ], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "存在字符级差异" }, "candidate_metric": { "name": "CER", "value": 0.0, "substitutions": 0, "deletions": 0, "insertions": 0, "reference_length": 16, "normalized_reference": "请把订单a1007的数量改成3份", "normalized_hypothesis": "请把订单a1007的数量改成3份", "operations": [], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "完全一致" }, "delta": -0.0625, "status": "improved" }, { "sample_id": "SYN-0003", "reference": "请把文件发给王芳", "baseline": "请把文件发给", "candidate": "请把文件发给", "baseline_metric": { "name": "CER", "value": 0.25, "substitutions": 0, "deletions": 2, "insertions": 0, "reference_length": 8, "normalized_reference": "请把文件发给王芳", "normalized_hypothesis": "请把文件发给", "operations": [ { "op": "deletion", "reference_index": 6, "reference": "王", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 7, "reference": "芳", "hypothesis_index": "-", "hypothesis": "-" } ], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "存在字符级差异" }, "candidate_metric": { "name": "CER", "value": 0.25, "substitutions": 0, "deletions": 2, "insertions": 0, "reference_length": 8, "normalized_reference": "请把文件发给王芳", "normalized_hypothesis": "请把文件发给", "operations": [ { "op": "deletion", "reference_index": 6, "reference": "王", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 7, "reference": "芳", "hypothesis_index": "-", "hypothesis": "-" } ], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "存在字符级差异" }, "delta": 0.0, "status": "unchanged" }, { "sample_id": "SYN-0004", "reference": "在今天下午的项目评审会上，请先介绍风险清单，再说明每项风险的负责人和预计解决时间。", "baseline": "在今", "candidate": "在今天下午的项目评审会上，请先介绍风险清单，再说明每项风险的负责人和预计解决时间。", "baseline_metric": { "name": "CER", "value": 0.9473684210526315, "substitutions": 0, "deletions": 36, "insertions": 0, "reference_length": 38, "normalized_reference": "在今天下午的项目评审会上请先介绍风险清单再说明每项风险的负责人和预计解决时间", "normalized_hypothesis": "在今", "operations": [ { "op": "deletion", "reference_index": 2, "reference": "天", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 3, "reference": "下", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 4, "reference": "午", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 5, "reference": "的", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 6, "reference": "项", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 7, "reference": "目", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 8, "reference": "评", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 9, "reference": "审", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 10, "reference": "会", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 11, "reference": "上", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 12, "reference": "请", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 13, "reference": "先", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 14, "reference": "介", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 15, "reference": "绍", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 16, "reference": "风", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 17, "reference": "险", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 18, "reference": "清", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 19, "reference": "单", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 20, "reference": "再", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 21, "reference": "说", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 22, "reference": "明", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 23, "reference": "每", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 24, "reference": "项", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 25, "reference": "风", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 26, "reference": "险", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 27, "reference": "的", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 28, "reference": "负", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 29, "reference": "责", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 30, "reference": "人", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 31, "reference": "和", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 32, "reference": "预", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 33, "reference": "计", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 34, "reference": "解", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 35, "reference": "决", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 36, "reference": "时", "hypothesis_index": "-", "hypothesis": "-" }, { "op": "deletion", "reference_index": 37, "reference": "间", "hypothesis_index": "-", "hypothesis": "-" } ], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "存在字符级差异" }, "candidate_metric": { "name": "CER", "value": 0.0, "substitutions": 0, "deletions": 0, "insertions": 0, "reference_length": 38, "normalized_reference": "在今天下午的项目评审会上请先介绍风险清单再说明每项风险的负责人和预计解决时间", "normalized_hypothesis": "在今天下午的项目评审会上请先介绍风险清单再说明每项风险的负责人和预计解决时间", "operations": [], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "完全一致" }, "delta": -0.9473684210526315, "status": "improved" }, { "sample_id": "SYN-0005", "reference": "把客厅空调打开", "baseline": "把客厅空调打开", "candidate": "把客厅空调关闭", "baseline_metric": { "name": "CER", "value": 0.0, "substitutions": 0, "deletions": 0, "insertions": 0, "reference_length": 7, "normalized_reference": "把客厅空调打开", "normalized_hypothesis": "把客厅空调打开", "operations": [], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "完全一致" }, "candidate_metric": { "name": "CER", "value": 0.2857142857142857, "substitutions": 2, "deletions": 0, "insertions": 0, "reference_length": 7, "normalized_reference": "把客厅空调打开", "normalized_hypothesis": "把客厅空调关闭", "operations": [ { "op": "substitution", "reference_index": 5, "reference": "打", "hypothesis_index": 5, "hypothesis": "关" }, { "op": "substitution", "reference_index": 6, "reference": "开", "hypothesis_index": 6, "hypothesis": "闭" } ], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "存在字符级差异" }, "delta": 0.2857142857142857, "status": "regressed" }, { "sample_id": "SYN-0006", "reference": "请查询明天北京的天气信息", "baseline": "请查询明天北京的天气信", "candidate": "请查询明天北京的天气信", "baseline_metric": { "name": "CER", "value": 0.08333333333333333, "substitutions": 0, "deletions": 1, "insertions": 0, "reference_length": 12, "normalized_reference": "请查询明天北京的天气信息", "normalized_hypothesis": "请查询明天北京的天气信", "operations": [ { "op": "deletion", "reference_index": 11, "reference": "息", "hypothesis_index": "-", "hypothesis": "-" } ], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "存在字符级差异" }, "candidate_metric": { "name": "CER", "value": 0.08333333333333333, "substitutions": 0, "deletions": 1, "insertions": 0, "reference_length": 12, "normalized_reference": "请查询明天北京的天气信息", "normalized_hypothesis": "请查询明天北京的天气信", "operations": [ { "op": "deletion", "reference_index": 11, "reference": "息", "hypothesis_index": "-", "hypothesis": "-" } ], "metric_version": "cer-v1", "normalization_version": "cer-normalize-v1", "applicable": true, "reason": null, "interpretation": "存在字符级差异" }, "delta": 0.0, "status": "unchanged" } ], "provenance": { "input_sha256": "27698b1cb28f2057b1c901d1c17dfec9c259e09b4562c02720c019b31763cf57", "baseline_name": "旧版", "candidate_name": "新版", "synthetic": true, "network_used": false } }