Commit c4ed1b5c authored by Data Governance Dev's avatar Data Governance Dev

feat(web2): 身份证合规扫描 — 多线程 + 实时进度接口

需求链路(按用户描述):
  1. 按当前数据库连接抽数据字典(复用 web.core.data_dict)
  2. 按「字段名 / 字段注释」关键字过滤命中列
  3. 每个命中列 SELECT DISTINCT → Python 校验(GB 11643-1999)
  4. 按违规值反查完整行(SELECT * WHERE col IN (...))
  5. 前端按表折叠渲染

架构亮点 —— 三端点异步:
  POST   /api/id-card/scan                     → 立即 202 {scan_id}(不阻塞)
  GET    /api/id-card/scan/{id}/progress       → {done, total, current_column, status}
  GET    /api/id-card/scan/{id}/result         → 完整 IdCardScanResponse / 202 / 500
  进度共享一个 in-memory ProgressTracker(thread-safe,10min TTL)

性能:
  ThreadPoolExecutor(max_workers=5) 并发跑每列
  每个 worker 自己开 DBConnection(oracledb/pymysql/dmPython 不能跨线程共享)
  实测 Oracle 84 列从单线程 221s → 多线程 115s(1.9x 加速)

方言适配(按 web2 库三态):
  MySQL  → LIMIT %s,%s 占位符
  达梦   → LIMIT ?,? 占位符
  Oracle → 子查询 ROWNUM <= N,:1 :2 :3 numbered bind(11g 无 LIMIT;? 占位符会 ORA-01036)

文件:
- web2/backend/idcard_validator.py  新增
    GB 11643-1999 §6 加权和 + mod 11 映射 + 15 位老号升级 18 位
    违规规则:EMPTY / LENGTH / CHARSET / CHECK_DIGIT / ADDRESS_PREFIX /
             BIRTH_DATE / ORDER_CODE
- web2/backend/routes/idcard_scan.py 新增
    异步三端点 + ProgressTracker + ThreadPoolExecutor 5 worker
- web2/src/config/idCardKeywords.js 新增
    默认关键字常量(前端离线兜底,与 web/configs/standards_match.yaml
    的 IND-001-a 保持一致)
    DEFAULT_NAME_KEYWORDS = [id_card, id_card_no, id_number, identity_card]
    DEFAULT_COMMENT_KEYWORDS = [身份证号]

踩坑见 work-logs/2026-08-17.md 第三、五、六章。
parent 97a82545
"""GB 11643-1999 身份证号码校验
依据:
GB 11643-1999《公民身份号码》§6 校验位计算
GB/T 7408 日期格式
GB/T 2260 地址码(最简校验:非 00 前缀即可,完整区划表留待外部数据接入)
校验项:
1. LENGTH 长度不是 18 位 / 15 位
2. CHARSET 字符集不合规(前 17 位非数字 / 第 18 位非 0-9 或 X)
3. CHECK_DIGIT 校验位计算错误(GB 11643 §6)
4. ADDRESS_PREFIX 地址码全 0 或前 2 位为 00
5. BIRTH_DATE 出生日期不是真实日期 / 晚于今天 / 早于 1900
6. ORDER_CODE 顺序码不在 001-999 范围
返回结构:
{"ok": bool, "violations": [rule_id, ...], "normalized": str | None}
- ok=True 表示合规;violations 为空
- normalized:如果是 15 位老号 → 升 18 位后的值;否则 None
"""
from __future__ import annotations
import datetime as _dt
# GB 11643-1999 §6 加权因子
_WEIGHTS = (7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2)
# mod 11 → 校验码映射
_CHECK_MAP = ("1", "0", "X", "9", "8", "7", "6", "5", "4", "3", "2")
# 合理出生日期区间(身份证制度从 1985 年实施,但出生日期可追溯)
_MIN_BIRTH_YEAR = 1900
_MAX_BIRTH_DATE = _dt.date.today()
def _is_all_same(s: str) -> bool:
return len(s) > 1 and len(set(s)) == 1
def _validate_18(code: str) -> tuple[bool, list[str], str | None]:
"""校验 18 位身份证号。返回 (ok, violations, normalized)"""
violations: list[str] = []
normalized = code
# 字符集
body, last = code[:17], code[17]
if not body.isdigit():
violations.append("CHARSET")
if not (last.isdigit() or last == "X"):
violations.append("CHARSET")
# 不再继续算校验位(last 不合法)
return bool(violations), violations, normalized
# 全相同
if _is_all_same(code):
violations.append("CHARSET") # 复用 CHARSET 槽位:占位符类
return False, violations, normalized
# 校验位
s = sum(int(body[i]) * _WEIGHTS[i] for i in range(17))
expected = _CHECK_MAP[s % 11]
if expected != last:
violations.append("CHECK_DIGIT")
# 地址码(前 6 位)
region = code[:6]
if region == "000000" or region[:2] == "00":
violations.append("ADDRESS_PREFIX")
# 出生日期(第 7-14 位)
try:
bd = _dt.date(int(code[6:10]), int(code[10:12]), int(code[12:14]))
if bd > _MAX_BIRTH_DATE:
violations.append("BIRTH_DATE")
elif bd.year < _MIN_BIRTH_YEAR:
violations.append("BIRTH_DATE")
except ValueError:
violations.append("BIRTH_DATE")
# 顺序码(15-17 位)
order = code[14:17]
if not order.isdigit() or not (1 <= int(order) <= 999):
violations.append("ORDER_CODE")
return (not violations), violations, normalized
def _lift_15_to_18(code15: str) -> str:
"""15 位老号升 18 位:插入 19 + 重算校验位
例: 750101 → 19750101 → 再算校验位
"""
if len(code15) != 15 or not code15.isdigit():
return code15
body18 = "19" + code15[:6] + code15[6:] # 19 + yyMMdd + 顺序码
s = sum(int(body18[i]) * _WEIGHTS[i] for i in range(17))
return body18 + _CHECK_MAP[s % 11]
def validate(value) -> dict:
"""对外主入口:校验一个身份证值,返回结构化结果
Args:
value: 任意值(None / str / 数字),内部统一转字符串
Returns:
{
"ok": bool, # 合规
"violations": [str], # 违反的规则 ID(CHARSET / LENGTH / CHECK_DIGIT / ...)
"length": int, # 输入字符串长度
"is_15_old": bool, # 是否 15 位老号
"normalized": str|None # 15 位老号升 18 位后的值;合规 18 位也是它本身
}
"""
if value is None:
return {"ok": False, "violations": ["EMPTY"], "length": 0, "is_15_old": False, "normalized": None}
s = str(value).strip()
if not s:
return {"ok": False, "violations": ["EMPTY"], "length": 0, "is_15_old": False, "normalized": None}
if _is_all_same(s) and len(s) >= 15:
return {"ok": False, "violations": ["CHARSET"], "length": len(s), "is_15_old": False, "normalized": None}
if len(s) == 15 and s.isdigit():
# 15 位老号:单独校验(没有校验位)
violations: list[str] = []
if region := s[:6]:
if region == "000000" or region[:2] == "00":
violations.append("ADDRESS_PREFIX")
try:
bd = _dt.date(1900 + int(s[6:8]), int(s[8:10]), int(s[10:12]))
if bd > _MAX_BIRTH_DATE or bd.year < _MIN_BIRTH_YEAR:
violations.append("BIRTH_DATE")
except ValueError:
violations.append("BIRTH_DATE")
order = s[12:15]
if not order.isdigit() or not (1 <= int(order) <= 999):
violations.append("ORDER_CODE")
# 15 位长度本身不算违规(合法老号)
return {
"ok": not violations,
"violations": violations,
"length": 15,
"is_15_old": True,
"normalized": _lift_15_to_18(s),
}
if len(s) != 18:
return {"ok": False, "violations": ["LENGTH"], "length": len(s), "is_15_old": False, "normalized": None}
ok, violations, normalized = _validate_18(s)
return {
"ok": ok,
"violations": violations,
"length": 18,
"is_15_old": False,
"normalized": normalized,
}
# ── 规则 ID → 中文描述(前端展示用)───────────────────────
VIOLATION_LABELS = {
"EMPTY": "空值",
"LENGTH": "长度不是 18 位",
"CHARSET": "字符集 / 占位符类不合规",
"CHECK_DIGIT": "校验位错误(GB 11643 §6)",
"ADDRESS_PREFIX": "地址码(前 6 位)不合规",
"BIRTH_DATE": "出生日期(第 7-14 位)不合规",
"ORDER_CODE": "顺序码(第 15-17 位)不合规",
}
\ No newline at end of file
This diff is collapsed.
/**
* 身份证字段匹配的默认关键字
*
* 来源:
* web/configs/standards_match.yaml → IND-001-a (GB 11643-1999 身份证号格式)
*
* 原 YAML 片段:
* IND-001-a:
* applies_to_fields: [id_card, id_card_no, id_number, identity_card]
* comment_keywords:
* - 身份证
*
* 设计:
* - 字段名用英文/下划线(数据库列名习惯)
* - 注释用中文/可读词(DDL COMMENT 字段)
* - 数组形式而非字符串,组件内 join(', ') 显示在输入框
* - 改这里 = 改 web/configs/standards_match.yaml 的 IND-001-a(保持两边一致)
*
* 后续:
* - 后端可以加 GET /api/id-card-keywords 直接读 YAML 返给前端,
* 这里作为前端离线兜底
*/
export const DEFAULT_NAME_KEYWORDS = [
'id_card',
'id_card_no',
'id_number',
'identity_card',
]
export const DEFAULT_COMMENT_KEYWORDS = [
'身份证号',
]
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment