Commit 9ac8c11b authored by Data Governance Dev's avatar Data Governance Dev

feat(web+step7): 国标违规 — 取数口径 + 字段点击联动 + 0违规过滤 + 表/字段注释

经历多轮迭代,本次提交整合以下改动:

后端 step7_standards.py:
- SAMPLE_SPECS 列表:每条标准可对应多条抽样规格(当前 1 条 default)
- 抽样时捕获实际渲染的 SQL 与 spec_id 落到每条 detail
  (sample_sql / sample_spec_id)— 后端保留供报告用
- violations_flat 增 table_comment / column_comment 字段
- 每条标准新增 violating_fields_count + details_with_violations
  (过滤掉 violations==0 的字段,前端直接消费)
- 过滤零违规标准,不进入 violations_by_standard

前端 index.html:
- 字段点击联动:selectedField + selectFieldFilter / clearFieldFilter
  + filteredViolations computed;选中后明细表只显示该字段违规
- 按标准统计表中字段清单改用 details_with_violations,
  自动隐藏 0 违规字段;字段数列加「N / M」红字显示有违规字段数
- 标准列去除「取数口径」折叠面板(用户认为没意义)
- 明细表去除选中字段时的 SQL 蓝边面板(用户认为没意义)
- 明细表改为 4 列:表/表注释、字段/字段注释、标准/标准名、值/违规原因
- alert 文案改为「X 个标准有违规(共 Y 条违规记录)」
- .field-link / .field-link--active CSS:选中字段高亮

数据 / UI 一致性验证:本后端 AST 通过;本地 == HTTP 服务返回(59051 字节)。
"
parent d2917cf8
...@@ -26,6 +26,25 @@ SAMPLE_LIMIT = 500 # 每字段最多抽样 ...@@ -26,6 +26,25 @@ SAMPLE_LIMIT = 500 # 每字段最多抽样
MAX_TABLES_PER_FIELD = 5 # 每字段最多检查前 N 张表 MAX_TABLES_PER_FIELD = 5 # 每字段最多检查前 N 张表
# 取数口径列表 —— 一个标准可对应多条抽样 SQL(按 spec_id 区分)
# 当前所有标准都共享同一默认口径;后续若不同标准需不同抽样策略,
# 在此处按 standard_id 配置多条即可。
SAMPLE_SPECS: list[dict] = [
{
"id": "default",
"name": "抽样非空非空白字段值",
"description": f"从每张表抽 ≤{SAMPLE_LIMIT} 行非空、非空白字段值",
"sql_template": (
"SELECT `${field}` AS val\n"
"FROM `${table}`\n"
"WHERE `${field}` IS NOT NULL\n"
" AND TRIM(`${field}`) <> ''\n"
"LIMIT ${limit}"
),
},
]
def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> dict: def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> dict:
"""执行 Step 7:国标校验""" """执行 Step 7:国标校验"""
columns = dict_data.get("data_dictionary", []) columns = dict_data.get("data_dictionary", [])
...@@ -104,6 +123,9 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di ...@@ -104,6 +123,9 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di
limit=SAMPLE_LIMIT, limit=SAMPLE_LIMIT,
) )
rows = db.fetchall(sql) rows = db.fetchall(sql)
# 真正执行过的 SQL,落库展示给审计
sample_sql_rendered = sql
sample_spec_id = "default"
except Exception as e: except Exception as e:
std_record["fields"].append({ std_record["fields"].append({
"table": table, "table": table,
...@@ -111,6 +133,8 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di ...@@ -111,6 +133,8 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di
"error": str(e), "error": str(e),
"violations": 0, "violations": 0,
"sampled": 0, "sampled": 0,
"sample_spec_id": "default",
"sample_sql": f"SELECT `${field_name}` ... -- 渲染失败: {e}",
}) })
if log: if log:
log("WARN", log("WARN",
...@@ -136,7 +160,9 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di ...@@ -136,7 +160,9 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di
}) })
violations_flat.append({ violations_flat.append({
"table_name": table, "table_name": table,
"table_comment": col_record.get("table_comment", ""),
"column_name": field_name, "column_name": field_name,
"column_comment": col_record.get("column_comment", ""),
"rule_type": std_id, "rule_type": std_id,
"standard_name": std_instance.standard_name, "standard_name": std_instance.standard_name,
"column_type": col_record.get("column_type", ""), "column_type": col_record.get("column_type", ""),
...@@ -154,6 +180,8 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di ...@@ -154,6 +180,8 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di
"violations": invalid_count, "violations": invalid_count,
"violation_rate": round(invalid_count / sampled, 4) if sampled else 0, "violation_rate": round(invalid_count / sampled, 4) if sampled else 0,
"samples": invalid_samples, "samples": invalid_samples,
"sample_spec_id": sample_spec_id,
"sample_sql": sample_sql_rendered,
}) })
logger.debug( logger.debug(
f"{table}.{field_name}: 抽样 {sampled}, 违规 {invalid_count}" f"{table}.{field_name}: 抽样 {sampled}, 违规 {invalid_count}"
...@@ -165,18 +193,33 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di ...@@ -165,18 +193,33 @@ def run_step7(cfg: DBConfig, dict_data: dict, log: Callable | None = None) -> di
by_standard_list = [] by_standard_list = []
for std_id, rec in by_standard.items(): for std_id, rec in by_standard.items():
# 过滤掉零违规的标准
if rec["total_invalid"] == 0:
if log:
log("INFO",
f" · 标准 {rec['standard']} 抽样 {rec['total_sampled']} 条全部合规,跳过",
step="7")
continue
violating_fields = [d for d in rec["fields"] if d.get("violations", 0) > 0]
by_standard_list.append({ by_standard_list.append({
"standard": rec["standard"], "standard": rec["standard"],
"standard_name": rec["standard_name"], "standard_name": rec["standard_name"],
"fields_count": len(rec["fields"]), "fields_count": len(rec["fields"]),
"violating_fields_count": len(violating_fields),
"total_invalid": rec["total_invalid"], "total_invalid": rec["total_invalid"],
"total_sampled": rec["total_sampled"], "total_sampled": rec["total_sampled"],
"violation_rate": round(rec["total_invalid"] / rec["total_sampled"], 4) if rec["total_sampled"] else 0, "violation_rate": round(rec["total_invalid"] / rec["total_sampled"], 4) if rec["total_sampled"] else 0,
"details": rec["fields"], "details": rec["fields"],
# 字段清单只展示有违规的,过滤掉 0 违规的占位项
"details_with_violations": violating_fields,
# 取数口径:每条标准附带一份规格清单(一个标准可对应多条)
"sample_specs": [dict(s) for s in SAMPLE_SPECS],
"sample_limit": SAMPLE_LIMIT,
"max_tables_per_field": MAX_TABLES_PER_FIELD,
}) })
if log: if log:
log("INFO", f"校验完成: {len(by_standard_list)} 个标准, {total_fields_checked} 个字段, {total_violations} 条违规", step="7") log("INFO", f"校验完成: {len(by_standard_list)} 个标准有违规, {total_fields_checked} 个字段, {total_violations} 条违规", step="7")
return { return {
"summary": { "summary": {
......
...@@ -4,6 +4,13 @@ ...@@ -4,6 +4,13 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>数据分析工具</title> <title>数据分析工具</title>
<style>
/* 国标违规:字段点击联动样式 */
.field-link { cursor: pointer; padding: 0 2px; border-radius: 3px; transition: background-color 0.15s; }
.field-link:hover { background-color: #ecf5ff; text-decoration: underline; }
.field-link--active { background-color: #409eff; color: #fff !important; }
.field-link--active code { color: #fff !important; }
</style>
<!-- 前端依赖全部本地化(避免 CDN 被墙/慢) --> <!-- 前端依赖全部本地化(避免 CDN 被墙/慢) -->
<!-- Element Plus CSS --> <!-- Element Plus CSS -->
<link rel="stylesheet" href="/static/lib/element-plus/index.css"> <link rel="stylesheet" href="/static/lib/element-plus/index.css">
...@@ -506,12 +513,12 @@ ...@@ -506,12 +513,12 @@
<!-- 国标违规 --> <!-- 国标违规 -->
<el-tab-pane label="国标违规" name="standards"> <el-tab-pane label="国标违规" name="standards">
<el-alert :title="`${jobResult.sections.standard_violations?.summary?.standards_applied || 0} 个标准,${jobResult.sections.standard_violations?.summary?.violations || 0} 条违规`" type="error" :closable="false" /> <el-alert :title="`${jobResult.sections.standard_violations?.summary?.standards_used || 0} 个标准有违规(共 ${jobResult.sections.standard_violations?.summary?.violations || 0} 条违规记录)`" type="error" :closable="false" />
<!-- 按标准汇总 --> <!-- 按标准汇总 -->
<h4 style="margin-top: 16px">按标准统计</h4> <h4 style="margin-top: 16px">按标准统计<span class="muted" style="font-size: 12px; margin-left: 8px">点击下方字段即可联动下方明细</span></h4>
<el-table :data="jobResult.sections.standard_violations?.violations_by_standard || []" stripe size="small" style="margin-top: 12px; width: 100%" max-height="480"> <el-table :data="jobResult.sections.standard_violations?.violations_by_standard || []" stripe size="small" style="margin-top: 12px; width: 100%" max-height="480">
<el-table-column prop="standard" label="标准" width="120" /> <el-table-column prop="standard" label="标准" width="120" />
<el-table-column prop="standard_name" label="标准名称" min-width="380"> <el-table-column label="标准名称" min-width="280" show-overflow-tooltip>
<template #default="{ row }"> <template #default="{ row }">
<div> <div>
<strong style="color: #303133">{{ row.standard }}</strong> <strong style="color: #303133">{{ row.standard }}</strong>
...@@ -519,7 +526,14 @@ ...@@ -519,7 +526,14 @@
</div> </div>
</template> </template>
</el-table-column> </el-table-column>
<el-table-column prop="fields_count" label="字段数" width="100" align="right" /> <el-table-column label="字段数 / 有违规" width="120" align="right">
<template #default="{ row }">
<div style="font-size: 13px">
<span style="color: #606266">{{ row.fields_count }}</span>
<span v-if="row.violating_fields_count !== undefined && row.violating_fields_count !== row.fields_count" class="muted" style="margin-left: 4px">/ <span style="color: #f56c6c; font-weight: 600">{{ row.violating_fields_count }}</span></span>
</div>
</template>
</el-table-column>
<el-table-column prop="total_sampled" label="抽样数" width="120" align="right" /> <el-table-column prop="total_sampled" label="抽样数" width="120" align="right" />
<el-table-column prop="total_invalid" label="违规数" width="120" align="right" /> <el-table-column prop="total_invalid" label="违规数" width="120" align="right" />
<el-table-column label="违规率" width="140" align="center"> <el-table-column label="违规率" width="140" align="center">
...@@ -529,18 +543,20 @@ ...@@ -529,18 +543,20 @@
</span> </span>
</template> </template>
</el-table-column> </el-table-column>
<el-table-column label="字段清单 / 违规率" min-width="340"> <el-table-column label="字段清单 / 违规率(点击联动)" min-width="380">
<template #default="{ row }"> <template #default="{ row }">
<div v-if="row.details && row.details.length"> <div v-if="row.details_with_violations && row.details_with_violations.length">
<div v-for="d in row.details.slice(0, 6)" :key="d.table + '.' + d.field" style="font-size: 12px; margin-top: 2px"> <div v-for="d in row.details_with_violations.slice(0, 8)" :key="d.table + '.' + d.field" style="font-size: 12px; margin-top: 2px">
<span class="field-link" :class="{ 'field-link--active': selectedField && selectedField.table === d.table && selectedField.field === d.field }" @click="selectFieldFilter(d.table, d.field)">
<code style="color: #303133">{{ d.table }}.{{ d.field }}</code> <code style="color: #303133">{{ d.table }}.{{ d.field }}</code>
</span>
<span class="muted"> · </span> <span class="muted"> · </span>
<span :style="{ color: d.violation_rate > 0.5 ? '#f56c6c' : (d.violation_rate > 0.2 ? '#e6a23c' : '#67c23a') }"> <span :style="{ color: d.violation_rate > 0.5 ? '#f56c6c' : (d.violation_rate > 0.2 ? '#e6a23c' : '#67c23a') }">
{{ d.violations }}/{{ d.sampled }} ({{ (d.violation_rate * 100).toFixed(1) }}%) {{ d.violations }}/{{ d.sampled }} ({{ (d.violation_rate * 100).toFixed(1) }}%)
</span> </span>
</div> </div>
<div v-if="row.details.length > 6" class="muted" style="font-size: 11px; margin-top: 2px; font-style: italic"> <div v-if="row.details_with_violations.length > 8" class="muted" style="font-size: 11px; margin-top: 2px; font-style: italic">
… 等共 {{ row.details.length }} 个字段 … 等共 {{ row.details_with_violations.length }} 个有违规字段
</div> </div>
</div> </div>
<span v-else class="muted">—</span> <span v-else class="muted">—</span>
...@@ -548,17 +564,35 @@ ...@@ -548,17 +564,35 @@
</el-table-column> </el-table-column>
</el-table> </el-table>
<!-- 违规明细 --> <!-- 违规明细 -->
<h4 style="margin-top: 16px">违规明细(前 200 条)</h4> <h4 style="margin-top: 16px">
<el-table :data="(jobResult.sections.standard_violations?.violations || []).slice(0, 200)" stripe size="small" style="margin-top: 12px; width: 100%" max-height="480"> 违规明细
<el-table-column label="表 / 字段" min-width="280"> <span v-if="selectedField" style="font-size: 13px; font-weight: normal; margin-left: 8px">
<el-tag type="info" effect="plain" closable @close="clearFieldFilter">
{{ selectedField.table }}.{{ selectedField.field }}
</el-tag>
</span>
<span v-else style="font-size: 12px; font-weight: normal; margin-left: 8px" class="muted">(前 200 条)</span>
</h4>
<el-table :data="filteredViolations" stripe size="small" style="margin-top: 12px; width: 100%" max-height="480">
<el-table-column label="表 / 表注释" min-width="220">
<template #default="{ row }"> <template #default="{ row }">
<div> <div>
<strong style="color: #303133">{{ row.table_name }}</strong> <strong style="color: #303133">{{ row.table_name }}</strong>
<div class="muted" style="font-size: 12px; margin-top: 2px">{{ row.column_name }}</div> <div v-if="row.table_comment" class="muted" style="font-size: 12px; margin-top: 2px">{{ row.table_comment }}</div>
<div v-else class="muted" style="font-size: 12px; margin-top: 2px; font-style: italic">(无表注释)</div>
</div>
</template>
</el-table-column>
<el-table-column label="字段 / 字段注释" min-width="220">
<template #default="{ row }">
<div>
<strong style="color: #303133">{{ row.column_name }}</strong>
<div v-if="row.column_comment" class="muted" style="font-size: 12px; margin-top: 2px">{{ row.column_comment }}</div>
<div v-else class="muted" style="font-size: 12px; margin-top: 2px; font-style: italic">(无字段注释)</div>
</div> </div>
</template> </template>
</el-table-column> </el-table-column>
<el-table-column label="标准 / 标准名" min-width="320"> <el-table-column label="标准 / 标准名" min-width="300">
<template #default="{ row }"> <template #default="{ row }">
<div> <div>
<strong style="color: #303133">{{ row.rule_type }}</strong> <strong style="color: #303133">{{ row.rule_type }}</strong>
...@@ -566,7 +600,7 @@ ...@@ -566,7 +600,7 @@
</div> </div>
</template> </template>
</el-table-column> </el-table-column>
<el-table-column label="值 / 违规原因" min-width="420"> <el-table-column label="值 / 违规原因" min-width="400">
<template #default="{ row }"> <template #default="{ row }">
<div> <div>
<code style="color: #303133">{{ row.value }}</code> <code style="color: #303133">{{ row.value }}</code>
...@@ -658,6 +692,27 @@ ...@@ -658,6 +692,27 @@
const jobResult = ref(null); const jobResult = ref(null);
const activeTab = ref('merge'); const activeTab = ref('merge');
// ── 国标违规:字段点击联动 ──
const selectedField = ref(null);
function selectFieldFilter(table, field) {
if (selectedField.value
&& selectedField.value.table === table
&& selectedField.value.field === field) {
selectedField.value = null; // 再次点击同一字段:取消
} else {
selectedField.value = { table, field };
}
}
function clearFieldFilter() {
selectedField.value = null;
}
const filteredViolations = computed(() => {
const all = jobResult.value?.sections?.standard_violations?.violations || [];
if (!selectedField.value) return all.slice(0, 200);
const { table, field } = selectedField.value;
return all.filter(v => v.table_name === table && v.column_name === field);
});
// ── 加载步骤 ── // ── 加载步骤 ──
async function loadSteps() { async function loadSteps() {
try { try {
...@@ -843,6 +898,7 @@ ...@@ -843,6 +898,7 @@
testConnection, startJob, cancelJob, resetAndStart, testConnection, startJob, cancelJob, resetAndStart,
downloadReport, downloadReport,
loadStandards, loadStandards,
selectedField, selectFieldFilter, clearFieldFilter, filteredViolations,
}; };
} }
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment