Skip to content

D
db-tools
Commit 323709a5 authored by Data Governance Dev's avatar Data Governance Dev
Browse files
Options

feat: 增量结果发布 + 每 tab 三状态徽章 + OverviewStats 必填字段兜底 

之前结果只在所有 step 完成后才返回，导致前端看不到中间进度。本轮改：

后端增量发布：
- orchestrator.run_governance_workflow 新增 on_section_update 回调
  回调在每步完成且有 section_key 时触发
- job_manager 新增 _sync_section_update 同步回调到 job.result['sections']
  （job.result 在任务结束时再被整体 result 替换）
- api/routes.py /jobs/{id}/result 取消'必须 completed'限制，
  running/failed/cancelled 状态也允许读取 sections

前端轮询 + tab 状态：
- pollStatus 每 1.5s 同时拉 status 与 result；
  loadResult 改为 merge sections（不整体替换 jobResult）
- TAB_STEP_MAP 映射 tab -> (step, sectionKey)
- tabStatus computed 按状态返回 'idle'|'running'|'done'
- TAB_BADGE 配置：info(非分析对象)/warning(分析中)/success(分析完成)
- 6 个 el-tab-pane 全部改用 #label slot，加 el-tag 徽章

bug 修复：OverviewStats 必填字段在运行中没数据导致 500：
- core/models.py: database/host/db_type 改为带默认值 ''
- api/routes.py: OverviewStats 构造用 try/except 兜底，失败降级到全零值

AST 全部通过；本地 == HTTP 服务返回一致。
parent 9ac8c11b
Show whitespace changes
Inline Side-by-side
Showing with 113 additions and 22 deletions
web/api/routes.py
View file @ 323709a5
......@@ -186,7 +186,7 @@ async def cancel_job(job_id: str):
return {"ok": True, "job_id": job_id}
# ── 任务结果 ──
# ── 任务结果（支持增量：运行中也可拿到已完成的 sections） ──
@router.get("/jobs/{job_id}/result", response_model=JobResultResponse)
async def get_job_result(job_id: str):
jm = JobManager.get_instance()
......@@ -194,20 +194,26 @@ async def get_job_result(job_id: str):
if not job:
logger.warning(f"GET /api/jobs/{job_id}/result 任务不存在")
raise HTTPException(status_code=404, detail="任务不存在")
if job.status != "completed":
logger.warning(f"GET /api/jobs/{job_id}/result 任务未完成（status={job.status}）")
raise HTTPException(status_code=400, detail=f"任务未完成（当前: {job.status}）")
sections = job.result.get("sections", {})
overview_dict = sections.get("overview", {})
# 不再要求 status == "completed"：运行中也允许前端拉取已完成的 sections
if job.status not in ("running", "completed", "failed", "cancelled"):
logger.warning(f"GET /api/jobs/{job_id}/result 任务状态异常（status={job.status}）")
raise HTTPException(status_code=400, detail=f"任务状态异常（当前: {job.status}）")
sections = job.result.get("sections", {}) if isinstance(job.result, dict) else {}
overview_dict = sections.get("overview", {}) or {}
# OverviewStats 必有字段（database/host/db_type）在运行中还没生成，
# 用 try/except 兜底：构造失败时降级到零值
try:
overview = OverviewStats(**{k: v for k, v in overview_dict.items() if k in OverviewStats.model_fields})
logger.info(f"GET /api/jobs/{job_id}/result 成功, sections={list(sections.keys())}")
except Exception:
overview = OverviewStats() # 全部走默认值
logger.info(f"GET /api/jobs/{job_id}/result 成功(status={job.status}), sections={list(sections.keys())}")
return JobResultResponse(
job_id=job_id,
status=job.status,
overview=overview,
sections=sections,
reports=job.result.get("reports", {}),
reports=job.result.get("reports", {}) if isinstance(job.result, dict) else {},
)
......
web/core/job_manager.py
View file @ 323709a5
......@@ -229,6 +229,7 @@ class JobManager:
on_log=lambda entry: self._sync_log(job, entry),
on_step_start=lambda num, title: self._sync_step_start(job, num, title),
on_step_done=lambda num, title, ok: self._sync_step_done(job, num, title, ok),
on_section_update=lambda section_key, data: self._sync_section_update(job, section_key, data),
cancel_event=job._cancel_event,
)
job.result = result
......@@ -278,6 +279,12 @@ class JobManager:
else:
job.steps_failed.append(num)
def _sync_section_update(self, job: Job, section_key: str, data: dict) -> None:
"""Step 完成时增量落盘：让 API 可以拿到部分 sections"""
if not isinstance(job.result, dict):
job.result = {}
job.result.setdefault("sections", {})[section_key] = data
# ── 查询 ──
def get(self, job_id: str) -> Optional[Job]:
return self.jobs.get(job_id)
......
web/core/models.py
View file @ 323709a5
......@@ -95,9 +95,9 @@ class StandardsListResponse(BaseModel):
# ── 治理结果（前端多级表格消费） ──
class OverviewStats(BaseModel):
database: str
host: str
db_type: str
database: str = ""
host: str = ""
db_type: str = ""
total_tables: int = 0
total_fields: int = 0
high_empty_fields: int = 0
......
web/core/orchestrator.py
View file @ 323709a5
......@@ -88,6 +88,7 @@ async def run_governance_workflow(
on_log: Callable[[dict], None] | None = None,
on_step_start: Callable[[int, str], None] | None = None,
on_step_done: Callable[[int, str, bool], None] | None = None,
on_section_update: Callable[[str, dict], None] | None = None,
cancel_event: Optional[asyncio.Event] = None,
) -> dict:
"""主入口：跑一轮治理，返回结构化结果。
......@@ -267,6 +268,9 @@ async def run_governance_workflow(
section_key = output.get("section_key")
if section_key:
sections[section_key] = output["data"]
if on_section_update:
try: on_section_update(section_key, output["data"])
except Exception: pass
if output.get("extras"):
step_outputs[step_name(num)] = output["extras"]
......
web/static/index.html
View file @ 323709a5
......@@ -231,7 +231,12 @@
<!-- 多级结果 -->
<el-tabs v-model="activeTab" type="border-card">
<!-- 表合并候选 -->
<el-tab-pane label="表合并候选" name="merge">
<el-tab-pane name="merge">
<template #label>
<span>表合并候选
<el-tag v-if="tabBadgeShow('merge')" :type="tabBadgeType('merge')" size="small" effect="plain" style="margin-left: 4px">{{ tabBadgeText('merge') }}</el-tag>
</span>
</template>
<el-table :data="jobResult.sections.merge_candidates?.merge_candidates || []" stripe size="small">
<el-table-column type="expand">
<template #default="{ row }">
......@@ -270,7 +275,12 @@
</el-tab-pane>
<!-- 冗余字段 -->
<el-tab-pane label="冗余字段" name="redundancy">
<el-tab-pane name="redundancy">
<template #label>
<span>冗余字段
<el-tag v-if="tabBadgeShow('redundancy')" :type="tabBadgeType('redundancy')" size="small" effect="plain" style="margin-left: 4px">{{ tabBadgeText('redundancy') }}</el-tag>
</span>
</template>
<el-alert type="info" :closable="false" style="margin-bottom: 12px"
:title="`共 ${(jobResult.sections.merge_candidates?.redundancy_fields || []).length} 个高频字段（出现在 ≥ 10 张表），LLM 已给出分类与处置建议；标为「LLM 失败」的需人工核对`" />
<el-table :data="jobResult.sections.merge_candidates?.redundancy_fields || []" stripe size="small">
......@@ -306,7 +316,12 @@
</el-tab-pane>
<!-- 大范围空字段 -->
<el-tab-pane label="大范围空字段" name="empty">
<el-tab-pane name="empty">
<template #label>
<span>大范围空字段
<el-tag v-if="tabBadgeShow('empty')" :type="tabBadgeType('empty')" size="small" effect="plain" style="margin-left: 4px">{{ tabBadgeText('empty') }}</el-tag>
</span>
</template>
<el-alert :title="`扫描 ${jobResult.sections.empty_fields?.summary?.total_tables_scanned || 0} 张表，发现 ${jobResult.sections.empty_fields?.summary?.high_empty_fields_count || 0} 个高空字段、${jobResult.sections.empty_fields?.summary?.mid_empty_fields_count || 0} 个中空字段，涉及 ${jobResult.sections.empty_fields?.summary?.tables_with_issues_count || 0} 张表`" type="info" :closable="false" />
<!-- 字段明细：把每个空字段摊开显示（表名 + 字段名 + 字段注释） -->
......@@ -448,7 +463,12 @@
</el-tab-pane>
<!-- 缺失注释字段 -->
<el-tab-pane label="缺失注释字段" name="missing">
<el-tab-pane name="missing">
<template #label>
<span>缺失注释字段
<el-tag v-if="tabBadgeShow('missing')" :type="tabBadgeType('missing')" size="small" effect="plain" style="margin-left: 4px">{{ tabBadgeText('missing') }}</el-tag>
</span>
</template>
<el-alert :title="`${jobResult.sections.missing_comments?.summary?.total_missing_comments || 0} 个字段缺注释，已推测 ${jobResult.sections.missing_comments?.summary?.predicted_count || 0} 个（其中 LLM 推测 ${jobResult.sections.missing_comments?.summary?.predicted_by_llm || 0} 个）`" type="info" :closable="false" />
<el-table :data="jobResult.sections.missing_comments?.predicted_comments || []" stripe size="small" style="margin-top: 12px">
<el-table-column prop="table_name" label="表名" width="180" fixed />
......@@ -467,7 +487,12 @@
</el-tab-pane>
<!-- 字段长度异常 -->
<el-tab-pane label="字段长度异常" name="length">
<el-tab-pane name="length">
<template #label>
<span>字段长度异常
<el-tag v-if="tabBadgeShow('length')" :type="tabBadgeType('length')" size="small" effect="plain" style="margin-left: 4px">{{ tabBadgeText('length') }}</el-tag>
</span>
</template>
<el-alert :title="`${jobResult.sections.length_issues?.summary?.total_issues || 0} 条异常，应用 ${jobResult.sections.length_issues?.summary?.rules_applied || 0} 条规则`" type="warning" :closable="false" />
<el-table :data="jobResult.sections.length_issues?.issues || []" stripe size="small" style="margin-top: 12px; width: 100%" max-height="560">
<el-table-column label="表名 / 表注释" width="260">
......@@ -512,7 +537,12 @@
</el-tab-pane>
<!-- 国标违规 -->
<el-tab-pane label="国标违规" name="standards">
<el-tab-pane name="standards">
<template #label>
<span>国标违规
<el-tag v-if="tabBadgeShow('standards')" :type="tabBadgeType('standards')" size="small" effect="plain" style="margin-left: 4px">{{ tabBadgeText('standards') }}</el-tag>
</span>
</template>
<el-alert :title="`${jobResult.sections.standard_violations?.summary?.standards_used || 0} 个标准有违规（共 ${jobResult.sections.standard_violations?.summary?.violations || 0} 条违规记录）`" type="error" :closable="false" />
<!-- 按标准汇总 -->
<h4 style="margin-top: 16px">按标准统计<span class="muted" style="font-size: 12px; margin-left: 8px">点击下方字段即可联动下方明细</span></h4>
......@@ -692,6 +722,43 @@
const jobResult = ref(null);
const activeTab = ref('merge');
// tab → (所属 step, 对应 sections 里的 key)
const TAB_STEP_MAP = {
merge: { step: 2, sectionKey: 'merge_candidates' },
redundancy: { step: 3, sectionKey: 'redundancy_fields' },
empty: { step: 4, sectionKey: 'empty_fields' },
missing: { step: 5, sectionKey: 'missing_comments' },
length: { step: 6, sectionKey: 'length_issues' },
standards: { step: 7, sectionKey: 'standard_violations' },
};
// 每 tab 实时状态：idle=非分析对象 / running=分析中 / done=分析完成
const tabStatus = computed(() => {
const planned = new Set(jobStatus.value?.steps_planned || []);
const sections = jobResult.value?.sections || {};
const out = {};
for (const tab of Object.keys(TAB_STEP_MAP)) {
const { step, sectionKey } = TAB_STEP_MAP[tab];
if (!planned.has(step)) {
out[tab] = 'idle';
} else if (sections[sectionKey]) {
out[tab] = 'done';
} else {
out[tab] = 'running';
}
}
return out;
});
// 各状态对应的标签配置
const TAB_BADGE = {
idle: { type: 'info', text: '非分析对象' },
running: { type: 'warning', text: '分析中' },
done: { type: 'success', text: '分析完成' },
};
function tabBadgeType(tab) { return TAB_BADGE[tabStatus.value[tab]]?.type || 'info'; }
function tabBadgeText(tab) { return TAB_BADGE[tabStatus.value[tab]]?.text || ''; }
function tabBadgeShow(tab) { return tabStatus.value[tab] !== 'idle'; }
// ── 国标违规：字段点击联动 ──
const selectedField = ref(null);
function selectFieldFilter(table, field) {
......@@ -798,14 +865,14 @@
try {
const r = await fetch(`/api/jobs/${jobId}`).then(r => r.json());
jobStatus.value = r;
// 不论 status：周期拉 result —— 运行中取增量 sections，完成时取全量+报告
await loadResult(jobId);
if (r.status === 'completed') {
clearInterval(pollTimer);
await loadResult(jobId);
ElementPlus.ElMessage.success('分析完成');
} else if (r.status === 'failed' || r.status === 'cancelled') {
clearInterval(pollTimer);
ElementPlus.ElMessage.warning(`任务${r.status === 'failed' ? '失败' : '已取消'}`);
if (r.status === 'completed') loadResult(jobId);
}
} catch (e) { console.error(e); }
}, 1500);
......@@ -814,7 +881,13 @@
async function loadResult(jobId) {
try {
const r = await fetch(`/api/jobs/${jobId}/result`).then(r => r.json());
jobResult.value = r;
// 合并 sections：保留已有，把后端返回的新/更新 sections 覆盖进去
const existing = jobResult.value?.sections || {};
const incoming = r.sections || {};
jobResult.value = {
...r,
sections: { ...existing, ...incoming }
};
} catch (e) { console.error(e); }
}
......@@ -895,6 +968,7 @@
currentJob, jobStatus, statusTagType,
logs, logExpanded, showStandardsDialog,
jobResult, activeTab, perTableStats, flatEmptyFields,
tabStatus, tabBadgeType, tabBadgeText, tabBadgeShow,
testConnection, startJob, cancelJob, resetAndStart,
downloadReport,
loadStandards,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment