Commit e6fbfb51 authored by Data Governance Dev's avatar Data Governance Dev

refactor(web): 报告生成不再作为 Step,主流程跑完后无条件触发

之前 Step 8 出现在 UI 的「执行步骤」复选框里,用户可取消勾选;后端也按 8 in steps 决定是否生成报告,导致:
1. 用户可能漏勾导致拿不到报告
2. 进度条分母里挂着「报告生成」让人误以为它也算分析步骤

按用户要求,报告生成应该是分析完成的固定产出,不应该作为可选步骤。

改动:

- orchestrator.py
  - STEP_REGISTRY 移除 (8, 报告生成, ...) 一项
  - STEP_DEPENDENCIES 移除 8: set() 注释也清掉
  - step_name() 不再支持 8
  - 去掉 has_step8 / main_steps 不再过滤 8
  - 「开始 生成报告(Markdown + Word)」从 if 8 in steps 改为无条件执行
  - 顶部 docstring 改写说明新约定

- job_manager.py:默认 all_steps 从 range(1,9) 改为 range(1,8)

- routes.py:list_steps() 不再返回 Step 8;steps_planned 默认 range(1,9) → (1,8)

- models.py:ConnectRequest.steps 注释从「默认全 8 步」改为「默认全 7 步;报告生成不在 steps 内」

- step8_report.py:移除 4 处 step='8' 日志标签(不再是 Step)

- frontend index.html
  - form.steps 默认 [1..8] 改为 [1..7]
  - startJob 兜底过滤:form.steps = form.steps.filter(n => n !== 8)
    (防止旧浏览器缓存里残留的 8 让后端报错)
  - form-hint 补充「报告生成不在步骤列表里,每次分析完成后会自动生成 Markdown + Word 供下载」

行为:
- UI 上「执行步骤」区只有 7 个分析 Step
- /api/steps 返回 7 项
- 主流程跑完后无条件生成报告,sections.reports.markdown/docx 照常返回
- 关键 Step 失败时仍会 RuntimeError 终止整个任务(与之前一致,sections 数据不全就没东西可报告)
parent 5d3c7883
...@@ -89,9 +89,6 @@ async def list_steps(): ...@@ -89,9 +89,6 @@ async def list_steps():
StepInfo(num=7, title="国家标准校验", StepInfo(num=7, title="国家标准校验",
description="连库抽样:身份证/USCC/手机号/行政区划符合性", description="连库抽样:身份证/USCC/手机号/行政区划符合性",
requires_db=True, llm_mode="none", required=False), requires_db=True, llm_mode="none", required=False),
StepInfo(num=8, title="报告生成",
description="汇总所有 findings 生成 Markdown + Word 报告(最后执行)",
requires_db=False, llm_mode="none", required=False),
]) ])
...@@ -158,7 +155,7 @@ async def create_job(req: ConnectRequest): ...@@ -158,7 +155,7 @@ async def create_job(req: ConnectRequest):
job_id=job.job_id, job_id=job.job_id,
status=job.status, status=job.status,
created_at=job.created_at, created_at=job.created_at,
steps_planned=job.steps_planned or list(range(1, 9)), steps_planned=job.steps_planned or list(range(1, 8)),
) )
......
...@@ -187,8 +187,8 @@ class JobManager: ...@@ -187,8 +187,8 @@ class JobManager:
job.run_dir = run_dir job.run_dir = run_dir
logger.info(f"[job {job.job_id}] 运行目录: {run_dir}") logger.info(f"[job {job.job_id}] 运行目录: {run_dir}")
# 收集要跑的步骤 # 收集要跑的步骤(7 个分析 Step;报告生成在主流程末尾无条件触发,不在此处)
all_steps = list(range(1, 9)) all_steps = list(range(1, 8))
steps_planned = job.req.steps if job.req.steps else all_steps steps_planned = job.req.steps if job.req.steps else all_steps
job.steps_planned = steps_planned job.steps_planned = steps_planned
logger.info(f"[job {job.job_id}] 计划步骤: {steps_planned}") logger.info(f"[job {job.job_id}] 计划步骤: {steps_planned}")
...@@ -230,6 +230,7 @@ class JobManager: ...@@ -230,6 +230,7 @@ class JobManager:
on_step_start=lambda num, title: self._sync_step_start(job, num, title), on_step_start=lambda num, title: self._sync_step_start(job, num, title),
on_step_done=lambda num, title, ok: self._sync_step_done(job, num, title, ok), on_step_done=lambda num, title, ok: self._sync_step_done(job, num, title, ok),
on_section_update=lambda section_key, data: self._sync_section_update(job, section_key, data), on_section_update=lambda section_key, data: self._sync_section_update(job, section_key, data),
on_plan_updated=lambda main_steps: self._sync_plan_updated(job, main_steps),
cancel_event=job._cancel_event, cancel_event=job._cancel_event,
) )
job.result = result job.result = result
...@@ -279,6 +280,14 @@ class JobManager: ...@@ -279,6 +280,14 @@ class JobManager:
else: else:
job.steps_failed.append(num) job.steps_failed.append(num)
def _sync_plan_updated(self, job: Job, main_steps: list[int]) -> None:
"""Orchestrator 过滤依赖缺失的步骤后,同步实际计划列表到前端进度分母。
若不过滤:用户勾了 5 步但其中 1 步因依赖缺失被跳过,结果
steps_planned=5 / steps_completed=4,分母虚高 → 进度永远停 4/5。
"""
job.steps_planned = list(main_steps)
def _sync_section_update(self, job: Job, section_key: str, data: dict) -> None: def _sync_section_update(self, job: Job, section_key: str, data: dict) -> None:
"""Step 完成时增量落盘:让 API 可以拿到部分 sections""" """Step 完成时增量落盘:让 API 可以拿到部分 sections"""
if not isinstance(job.result, dict): if not isinstance(job.result, dict):
......
...@@ -20,7 +20,7 @@ class ConnectRequest(BaseModel): ...@@ -20,7 +20,7 @@ class ConnectRequest(BaseModel):
charset: str = "utf8mb4" charset: str = "utf8mb4"
connect_timeout: int = 10 connect_timeout: int = 10
# 可选:选择要跑的步骤(默认全 8 步) # 可选:选择要跑的步骤(默认全 7 步;报告生成不在 steps 内,固定在主流程末尾触发)
steps: Optional[list[int]] = None steps: Optional[list[int]] = None
# 可选:是否启用 LLM 增强(默认 True,缺 Key 时自动降级) # 可选:是否启用 LLM 增强(默认 True,缺 Key 时自动降级)
enable_llm: bool = True enable_llm: bool = True
......
"""治理流程调度器 """治理流程调度器
把 8 个 Step 串起来执行: 把 7 个分析 Step 串起来执行:
1. 数据字典 → 2. 合并/冗余(离线) → 3. 数据验证(连库) 1. 数据字典 → 2. 合并/冗余(离线) → 3. 数据验证(连库)
→ 4. 空字段(连库) → 5. 缺注释(离线) → 6. 字段长度(离线) → 4. 空字段(连库) → 5. 缺注释(离线) → 6. 字段长度(离线)
→ 7. 国标校验(连库) → 8. 报告生成(离线) → 7. 国标校验(连库)
报告生成不在 Step 列表内:主步骤全部跑完后无条件触发(一次性生成
Markdown + docx,供前端下载)。即使主步骤全部失败/跳过,也会尝试
基于已有的 sections 生成报告(可能只包含部分章节)。
特性: 特性:
- LLM 失败自动降级 - LLM 失败自动降级
...@@ -34,11 +38,12 @@ logger = logging.getLogger(__name__) ...@@ -34,11 +38,12 @@ logger = logging.getLogger(__name__)
# 顺序敏感:前面 Step 的产出可被后面 Step 复用 # 顺序敏感:前面 Step 的产出可被后面 Step 复用
# #
# llm_mode 取值: # llm_mode 取值:
# "none" —— 不用 LLM(Step 1、3、4、6、7、8) # "none" —— 不用 LLM(Step 1、3、4、6、7)
# "optional" —— LLM 可用就用,失败降级到规则推理(当前未使用) # "optional" —— LLM 可用就用,失败降级到规则推理(当前未使用)
# "required" —— 必须 LLM;缺 LLM = 任务失败(Step 2、5) # "required" —— 必须 LLM;缺 LLM = 任务失败(Step 2、5)
# #
# required 标记:true 表示用户在 UI 上不能取消勾选(如 Step 1 为全部依赖的根基) # required 标记:true 表示用户在 UI 上不能取消勾选(如 Step 1 为全部依赖的根基)
# 报告生成不再作为 Step 8 注册——它是流程结束后的固定动作,见 _generate_reports
STEP_REGISTRY: list[tuple[int, str, Callable, bool, str, bool]] = [ STEP_REGISTRY: list[tuple[int, str, Callable, bool, str, bool]] = [
# (num, title, fn, requires_db, llm_mode, required) # (num, title, fn, requires_db, llm_mode, required)
(1, "获取数据字典", "_run_step1", True, "none", True), (1, "获取数据字典", "_run_step1", True, "none", True),
...@@ -48,14 +53,12 @@ STEP_REGISTRY: list[tuple[int, str, Callable, bool, str, bool]] = [ ...@@ -48,14 +53,12 @@ STEP_REGISTRY: list[tuple[int, str, Callable, bool, str, bool]] = [
(5, "缺失注释检查 + 推测", "_run_step5", False, "required", False), (5, "缺失注释检查 + 推测", "_run_step5", False, "required", False),
(6, "字段长度检查", "_run_step6", False, "none", False), (6, "字段长度检查", "_run_step6", False, "none", False),
(7, "国家标准校验", "_run_step7", True, "none", False), (7, "国家标准校验", "_run_step7", True, "none", False),
(8, "报告生成", "_run_step8", False, "none", False),
] ]
# ── Step 依赖图 ─────────────────────────────────────────── # ── Step 依赖图 ───────────────────────────────────────────
# 每个 Step 的前置步骤。Wave 调度器据此分组并行任务: # 每个 Step 的前置步骤。Wave 调度器据此分组并行任务:
# 没有依赖的(空集合)→ 同 Wave 并行 # 没有依赖的(空集合)→ 同 Wave 并行
# 依赖的全部完成 → 进 Wave 并行 # 依赖的全部完成 → 进 Wave 并行
# Step 8 自动等所有执行的 Step(1-7)完成,再单独跑
STEP_DEPENDENCIES: dict[int, set[int]] = { STEP_DEPENDENCIES: dict[int, set[int]] = {
1: set(), # 根基 1: set(), # 根基
2: {1}, 2: {1},
...@@ -64,7 +67,6 @@ STEP_DEPENDENCIES: dict[int, set[int]] = { ...@@ -64,7 +67,6 @@ STEP_DEPENDENCIES: dict[int, set[int]] = {
5: {1}, 5: {1},
6: {1}, 6: {1},
7: {1}, 7: {1},
8: set(), # 由调度器在所有其他 Step 完成后触发,不通过此图
} }
# ── 关键 Step 集合 ──────────────────────────────────────── # ── 关键 Step 集合 ────────────────────────────────────────
...@@ -89,6 +91,7 @@ async def run_governance_workflow( ...@@ -89,6 +91,7 @@ async def run_governance_workflow(
on_step_start: Callable[[int, str], None] | None = None, on_step_start: Callable[[int, str], None] | None = None,
on_step_done: Callable[[int, str, bool], None] | None = None, on_step_done: Callable[[int, str, bool], None] | None = None,
on_section_update: Callable[[str, dict], None] | None = None, on_section_update: Callable[[str, dict], None] | None = None,
on_plan_updated: Callable[[list[int]], None] | None = None,
cancel_event: Optional[asyncio.Event] = None, cancel_event: Optional[asyncio.Event] = None,
) -> dict: ) -> dict:
"""主入口:跑一轮治理,返回结构化结果。 """主入口:跑一轮治理,返回结构化结果。
...@@ -173,9 +176,8 @@ async def run_governance_workflow( ...@@ -173,9 +176,8 @@ async def run_governance_workflow(
f"用户未勾选必选步骤 {sorted(missing_required)},已自动追加") f"用户未勾选必选步骤 {sorted(missing_required)},已自动追加")
steps = sorted(set(steps) | missing_required) steps = sorted(set(steps) | missing_required)
# Step 8 单独走:所有 1-7 跑完后再触发 # 主步骤:用户传入的 steps(已不含报告生成,报告生成在主流程末尾无条件触发)
has_step8 = 8 in steps main_steps = sorted(steps)
main_steps = sorted(n for n in steps if n != 8)
# ── 依赖完整性检查 ── # ── 依赖完整性检查 ──
# 如果某个 Step 的前置依赖不在计划里,会造成 wave 死锁 # 如果某个 Step 的前置依赖不在计划里,会造成 wave 死锁
...@@ -194,10 +196,15 @@ async def run_governance_workflow( ...@@ -194,10 +196,15 @@ async def run_governance_workflow(
f"以下步骤因缺少前置依赖被自动跳过(避免调度死锁): " f"以下步骤因缺少前置依赖被自动跳过(避免调度死锁): "
+ ", ".join(f"Step {n}(需要 {deps})" for n, deps in skipped_for_deps)) + ", ".join(f"Step {n}(需要 {deps})" for n, deps in skipped_for_deps))
main_steps = keep main_steps = keep
# 同步实际计划列表:让前端的 steps_planned 与实际执行的步骤一致,
# 避免进度分母虚高(被跳过的步骤永远不进 steps_completed,但会计入分母)
if on_plan_updated:
try: on_plan_updated(list(main_steps))
except Exception: pass
log("INFO", log("INFO",
f"调度计划: 必选={sorted(required_steps)}, " f"调度计划: 必选={sorted(required_steps)}, "
f"主步骤={main_steps}, 是否报告={has_step8}, " f"主步骤={main_steps}, 完成后将自动生成报告, "
f"因依赖缺失跳过={skipped_for_deps or '无'}") f"因依赖缺失跳过={skipped_for_deps or '无'}")
# ── Wave 调度器 ── # ── Wave 调度器 ──
...@@ -344,10 +351,9 @@ async def run_governance_workflow( ...@@ -344,10 +351,9 @@ async def run_governance_workflow(
f"长度异常 {overview.get('length_issues',0)}, " f"长度异常 {overview.get('length_issues',0)}, "
f"国标违规 {overview.get('standard_violations',0)}") f"国标违规 {overview.get('standard_violations',0)}")
# ── Step 8:报告生成(在调度器中独立处理) ── # ── 报告生成(主步骤跑完后无条件触发,不是 Step) ──
if 8 in steps:
_check_cancel(cancel_event) _check_cancel(cancel_event)
log("INFO", "开始 报告生成", step="8") log("INFO", "开始 生成报告(Markdown + Word)")
report_started = time.monotonic() report_started = time.monotonic()
try: try:
md_path, docx_path = await asyncio.get_event_loop().run_in_executor( md_path, docx_path = await asyncio.get_event_loop().run_in_executor(
...@@ -357,9 +363,9 @@ async def run_governance_workflow( ...@@ -357,9 +363,9 @@ async def run_governance_workflow(
sections.setdefault("reports", {}) sections.setdefault("reports", {})
sections["reports"] = {"markdown": md_path, "docx": docx_path} sections["reports"] = {"markdown": md_path, "docx": docx_path}
elapsed = time.monotonic() - report_started elapsed = time.monotonic() - report_started
log("INFO", f"报告已生成: md={md_path}, docx={docx_path}(耗时 {elapsed:.1f}s)", step="8") log("INFO", f"报告已生成: md={md_path}, docx={docx_path}(耗时 {elapsed:.1f}s)")
except Exception as e: except Exception as e:
log("ERROR", f"报告生成失败: {type(e).__name__}: {e}", step="8") log("ERROR", f"报告生成失败: {type(e).__name__}: {e}")
traceback.print_exc() traceback.print_exc()
elapsed = time.monotonic() - started_ts elapsed = time.monotonic() - started_ts
...@@ -390,7 +396,6 @@ def step_name(num: int) -> str: ...@@ -390,7 +396,6 @@ def step_name(num: int) -> str:
5: "5_missing_comments", 5: "5_missing_comments",
6: "6_length_check", 6: "6_length_check",
7: "7_standards", 7: "7_standards",
8: "8_report",
}[num] }[num]
...@@ -451,10 +456,10 @@ def _run_step7(cfg, step_outputs, llm, findings_dir, log, cancel_event): ...@@ -451,10 +456,10 @@ def _run_step7(cfg, step_outputs, llm, findings_dir, log, cancel_event):
return {"section_key": "standard_violations", "data": data} return {"section_key": "standard_violations", "data": data}
# ── 报告生成(Step 8) ── # ── 报告生成 ──
def _generate_reports(run_dir: Path, sections: dict, overview: dict, def _generate_reports(run_dir: Path, sections: dict, overview: dict,
log: Callable | None = None) -> tuple[str | None, str | None]: log: Callable | None = None) -> tuple[str | None, str | None]:
"""生成 Markdown + Word 报告""" """生成 Markdown + Word 报告(在主步骤全部完成后无条件触发)"""
from .step_impl.step8_report import generate_reports from .step_impl.step8_report import generate_reports
return generate_reports(run_dir, sections, overview, log=log) return generate_reports(run_dir, sections, overview, log=log)
......
...@@ -40,8 +40,8 @@ def generate_reports(run_dir: Path, sections: dict, overview: dict, ...@@ -40,8 +40,8 @@ def generate_reports(run_dir: Path, sections: dict, overview: dict,
docx_path = reports_dir / f"数据治理报告_{db_name}.docx" docx_path = reports_dir / f"数据治理报告_{db_name}.docx"
if log: if log:
log("INFO", f"输出目录: {reports_dir}", step="8") log("INFO", f"输出目录: {reports_dir}")
log("INFO", f"[1/2] 生成 Markdown 报告 → {md_path.name}", step="8") log("INFO", f"[1/2] 生成 Markdown 报告 → {md_path.name}")
# 1. Markdown # 1. Markdown
md_text = _build_markdown(sections, overview) md_text = _build_markdown(sections, overview)
...@@ -52,7 +52,7 @@ def generate_reports(run_dir: Path, sections: dict, overview: dict, ...@@ -52,7 +52,7 @@ def generate_reports(run_dir: Path, sections: dict, overview: dict,
docx_result = None docx_result = None
try: try:
if log: if log:
log("INFO", f"[2/2] 生成 Word 报告 → {docx_path.name}", step="8") log("INFO", f"[2/2] 生成 Word 报告 → {docx_path.name}")
from docx import Document from docx import Document
doc = Document() doc = Document()
_render_docx(doc, sections, overview) _render_docx(doc, sections, overview)
...@@ -62,11 +62,11 @@ def generate_reports(run_dir: Path, sections: dict, overview: dict, ...@@ -62,11 +62,11 @@ def generate_reports(run_dir: Path, sections: dict, overview: dict,
except ImportError: except ImportError:
logger.warning("未安装 python-docx,跳过 Word 报告生成") logger.warning("未安装 python-docx,跳过 Word 报告生成")
if log: if log:
log("WARN", "未安装 python-docx,跳过 Word 报告生成", step="8") log("WARN", "未安装 python-docx,跳过 Word 报告生成")
except Exception as e: except Exception as e:
logger.exception("Word 报告生成失败") logger.exception("Word 报告生成失败")
if log: if log:
log("ERROR", f"Word 报告生成失败: {e}", step="8") log("ERROR", f"Word 报告生成失败: {e}")
return str(md_path), docx_result return str(md_path), docx_result
......
...@@ -116,7 +116,7 @@ ...@@ -116,7 +116,7 @@
</span> </span>
</el-checkbox> </el-checkbox>
</el-checkbox-group> </el-checkbox-group>
<div class="form-hint">Step 1 为数据基础,必选且不可取消;其他步骤可并行执行。带 ✨ 图标的步骤需调用大模型,耗时会较长(每步数秒到数十秒)。</div> <div class="form-hint">Step 1 为数据基础,必选且不可取消;其他步骤可并行执行。带 ✨ 图标的步骤需调用大模型,耗时会较长(每步数秒到数十秒)。报告生成不在步骤列表里,每次分析完成后会自动生成 Markdown + Word 供下载。</div>
</el-form-item> </el-form-item>
</el-col> </el-col>
</el-row> </el-row>
...@@ -732,7 +732,7 @@ ...@@ -732,7 +732,7 @@
database: 'smart-build', database: 'smart-build',
charset: 'utf8mb4', charset: 'utf8mb4',
connect_timeout: 10, connect_timeout: 10,
steps: [1, 2, 3, 4, 5, 6, 7, 8], steps: [1, 2, 3, 4, 5, 6, 7],
}); });
const testing = ref(false); const testing = ref(false);
...@@ -874,6 +874,8 @@ ...@@ -874,6 +874,8 @@
form.steps.push(s.num); form.steps.push(s.num);
} }
} }
// 兜底:报告生成不再是 Step,不在 form.steps 里出现(兼容旧本地缓存里残留的 8)
form.steps = form.steps.filter(n => n !== 8);
try { try {
const r = await fetch('/api/jobs', { const r = await fetch('/api/jobs', {
method: 'POST', method: 'POST',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment