Commit 0e9df863 authored by wangteng's avatar wangteng

功能优化升级,增加数据模拟,备份定时任务。

parent 53638176
......@@ -5,6 +5,14 @@
# 总体要求
- 每做一个任务就在工作记录中记录一次
# 数据库建表 / 改表约束(2026-09-30 起)
- **表和字段都必须写中文注释**,两处都要做:
1. ORM 模型(`web/backend/models/`):表注释写在 `__table_args__ = (..., {"comment": "..."})`,字段注释写在 `Column(..., comment="...")`——新库 create_all 时直接带上;
2. 注释登记表 `web/backend/db/schema_comments.py` 的 `SCHEMA_COMMENTS`——后端启动时由 `apply_mysql_comments()` 自动给**已存在的库**补齐/修正表和字段注释(幂等)。
- 新表必须注册进 `web/backend/models/__init__.py`,否则 create_all 会静默跳过建表。
- 给**已有表加字段**时,create_all 不会生效,必须在 `web/backend/db/database.py` 写幂等迁移函数(inspect 判断列存在 → `ALTER TABLE ADD COLUMN`),并在 `init_db()` 里调用;新列同样要登记 SCHEMA_COMMENTS。
- 发版会自动完成:重启后端 → create_all 建缺表 → 迁移函数补列 → 注释同步。正式库(192.168.50.10)无需手工动表。
# 服务器发版本运行
- `deploy/deploy_server.py`
......
......@@ -45,7 +45,9 @@ logger.info("DB: %s", engine.url.render_as_string(hide_password=True))
@asynccontextmanager
async def _lifespan(_: FastAPI):
await asyncio.get_event_loop().run_in_executor(None, start_scheduler)
await asyncio.get_event_loop().run_in_executor(None, start_backup_scheduler)
yield
stop_backup_scheduler()
stop_scheduler()
......@@ -81,6 +83,7 @@ from web.backend.routers.table_design import router as table_design_router # no
from web.backend.routers.llm_config import router as llm_config_router # noqa: E402
from web.backend.routers.table_prefs import router as table_prefs_router # noqa: E402
from web.backend.routers.analysis import router as analysis_router, start_scheduler, stop_scheduler # noqa: E402
from web.backend.routers.operations import start_backup_scheduler, stop_backup_scheduler # noqa: E402
app.include_router(task_groups_router, prefix="/api")
app.include_router(tasks_router, prefix="/api")
app.include_router(db_router, prefix="/api")
......
......@@ -40,14 +40,13 @@ _SYSTEM_PROMPT = "你是 Python 程序员,擅长写简短的数据校验函数
_DESCRIPTION_SYSTEM_PROMPT = "你是数据质量规则专家,擅长用一句简洁的话说明数据校验规则。"
_DESCRIPTION_TEMPLATE = """用一句中文生成 30-40 字规则说明,只输出说明本身。
_DESCRIPTION_TEMPLATE = """用一句简洁、完整的中文说明下面的数据校验规则,只输出说明本身。
不限定字数:以意思完整为准,关键格式 / 范围 / 排除项都要讲到,能省的字省掉即可。
规则名称:{name}
优先依据:{prompt}
现有说明:{desc}
规则类型:{rule_type}
仅保留生成规则所需的关键格式或范围,不要补充额外校验细节。"""
_MAX_AI_DESCRIPTION_LENGTH = 40
不要补充规则未提及的校验细节。"""
# ── User Prompt ─────────────────────────────────────────
......@@ -232,9 +231,9 @@ def gen_rule_description(name: str, desc: str, rule_type: str, prompt: str = "")
description = description.replace("\n", " ").strip().strip('"')
if not description:
return "", "AI 未生成规则说明"
# 兜底截断,避免模型未遵守长度要求时撑大规则列表。
description = description[:_MAX_AI_DESCRIPTION_LENGTH].rstrip(",、;:。 ")
return description, "已生成简短规则说明(30-40字)"
# 不截断字数(用户 2026-09-29:限制字数会把意思截断;DB 列是 Text 无长度约束,
# 长度交给提示词里的「以意思完整为准、能省则省」约束)。
return description, "已生成规则说明"
# ── 主入口 ─────────────────────────────────────────────
......
......@@ -238,7 +238,10 @@ class LLMClient:
{b for b in (initial_budget, max(initial_budget * 2, 2048), 8192) if b <= 8192}
))
for attempt, max_tokens in enumerate(token_budgets, start=1):
kwargs = {**base_kwargs, "max_tokens": max_tokens}
# 思考型模型每一跳都可能超 30s(内部代理生成速度慢,实测 1024-2048
# 额度的思考+输出也要 30s+),统一给 90s 下限;配置了更大值则以配置为准。
attempt_timeout = max(self.cfg.timeout, 90)
kwargs = {**base_kwargs, "max_tokens": max_tokens, "timeout": attempt_timeout}
msg = self._provider.messages.create(**kwargs)
for block in msg.content:
text = getattr(block, "text", None)
......@@ -325,10 +328,10 @@ def _format_api_error(exc: Exception) -> str:
if hint:
return f"API Error: 请求被拒绝 ({status}) · {detail} ({hint})"
return f"API Error: 请求失败 ({status}) · {detail}"
if isinstance(exc, APITimeoutError):
return f"API Error: 请求超时(模型响应时间过长,可调大 llm 配置 timeout)· {exc_msg}"
if isinstance(exc, APIConnectionError):
return f"API Error: 网络连接失败 · {exc_msg}"
if isinstance(exc, APITimeoutError):
return f"API Error: 请求超时 · {exc_msg}"
except ImportError:
pass
......
......@@ -19,6 +19,10 @@ def record_operation(
tables: Iterable[str],
filename: str = "",
execute_setting: str = "",
trigger_type: str = "manual",
backup_mode: str = "",
task_id: int | None = None,
task_name: str = "",
) -> OperationRecord:
"""保存一次已成功完成的操作;密码等敏感请求参数永不写入历史。
......@@ -36,6 +40,10 @@ def record_operation(
operation=operation,
source_id=source.id if source else None,
source_name=source.name if source else "",
trigger_type=trigger_type or "manual",
backup_mode=backup_mode or "",
task_id=task_id,
task_name=(task_name or "")[:120],
database_name=database or "",
scope=scope or "",
tables_json=tables_json,
......@@ -46,3 +54,15 @@ def record_operation(
db.commit()
db.refresh(row)
return row
def update_operation(db: Session, record_id: int, *, status: str, execute_setting: str = "", error_message: str = ""):
row = db.get(OperationRecord, record_id)
if row is None:
return None
row.status = status
row.execute_setting = (execute_setting or "")[:400]
row.error_message = (error_message or "")[:1000]
db.commit()
db.refresh(row)
return row
......@@ -5,7 +5,7 @@ from contextlib import contextmanager
import json
from pathlib import Path
from sqlalchemy import create_engine, event
from sqlalchemy import create_engine, event, inspect
from sqlalchemy.orm import sessionmaker, Session, declarative_base
from web.backend.config import APP_ENV, DATA_DIR, DATABASE_CONNECT_ARGS, DATABASE_URL
......@@ -66,6 +66,8 @@ def init_db() -> None:
"""初始化当前环境的 MySQL 表,并保留 SQLite 旧库升级兼容。"""
from web.backend.db import seed
Base.metadata.create_all(engine)
_migrate_operation_record_error()
_migrate_operation_record_context()
if IS_SQLITE:
# 一次性迁移:补齐 connection_preset 的 UNIQUE(db_type, name) 约束
# 历史背景:connection_preset 最初是手动跑 schema.sql 片段建的(仅 7 张主表里没有它),
......@@ -100,14 +102,48 @@ def init_db() -> None:
if changed:
logger.info("[init_db] 已更新 %s 项 MySQL 中文注释", changed)
# 灌种子(每个 seed 函数内部判重,可重复调用)
# 2026-09-29:不再灌示例数据源 —— 测试/正式环境每次发版都会多出 3 条
# 127.0.0.1 占位数据源(seed_data_sources 保留函数但不再调用)
seed.seed_task_groups()
seed.seed_connection_presets()
seed.seed_data_sources()
_migrate_llm_config_from_file()
_migrate_backup_config_to_database()
print(f"[init_db] 初始化完成:{engine.url.render_as_string(hide_password=True)}")
def _migrate_operation_record_error() -> None:
"""Add async operation error text to existing SQLite installations."""
inspector = inspect(engine)
if "operation_record" not in inspector.get_table_names():
return
cols = {column["name"] for column in inspector.get_columns("operation_record")}
if "error_message" in cols:
return
with engine.begin() as conn:
if IS_SQLITE:
conn.exec_driver_sql("ALTER TABLE operation_record ADD COLUMN error_message TEXT NULL")
else:
conn.exec_driver_sql("ALTER TABLE operation_record ADD COLUMN error_message TEXT NULL")
def _migrate_operation_record_context() -> None:
"""补齐操作记录的触发方式、备份模式和任务关联字段。"""
inspector = inspect(engine)
if "operation_record" not in inspector.get_table_names():
return
existing = {column["name"] for column in inspector.get_columns("operation_record")}
columns = (
("trigger_type", "VARCHAR(20) NOT NULL DEFAULT 'manual'"),
("backup_mode", "VARCHAR(20) NOT NULL DEFAULT ''"),
("task_id", "INTEGER NULL"),
("task_name", "VARCHAR(120) NOT NULL DEFAULT ''"),
)
with engine.begin() as conn:
for name, definition in columns:
if name not in existing:
conn.exec_driver_sql(f"ALTER TABLE operation_record ADD COLUMN {name} {definition}")
def _migrate_connection_preset_unique() -> None:
"""确保 connection_preset 表有显式 UNIQUE INDEX uq_connection_preset_db_type_name。
......
......@@ -198,6 +198,10 @@ CREATE TABLE operation_record (
id INTEGER PRIMARY KEY AUTOINCREMENT,
operation TEXT NOT NULL,
source_id INTEGER REFERENCES data_source(id) ON DELETE SET NULL,
trigger_type TEXT NOT NULL DEFAULT 'manual',
backup_mode TEXT NOT NULL DEFAULT '',
task_id INTEGER,
task_name TEXT NOT NULL DEFAULT '',
source_name TEXT NOT NULL DEFAULT '',
database_name TEXT NOT NULL DEFAULT '',
scope TEXT NOT NULL DEFAULT '',
......@@ -205,10 +209,29 @@ CREATE TABLE operation_record (
filename TEXT NOT NULL DEFAULT '',
execute_setting TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT '已完成',
error_message TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX idx_operation_record_operation_created ON operation_record(operation, created_at DESC);
-- 9.1) backup_task · 定时备份任务配置
CREATE TABLE backup_task (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
source_id INTEGER NOT NULL REFERENCES data_source(id) ON DELETE CASCADE,
database_name TEXT NOT NULL DEFAULT '',
schema_name TEXT,
scope TEXT NOT NULL DEFAULT 'database',
tables_json TEXT NOT NULL DEFAULT '[]',
backup_mode TEXT NOT NULL DEFAULT 'full',
include_schema INTEGER NOT NULL DEFAULT 1,
include_data INTEGER NOT NULL DEFAULT 1,
cron_expression TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- 10) table_design_config · 表设计校验配置与最近扫描结果
CREATE TABLE table_design_config (
id INTEGER PRIMARY KEY AUTOINCREMENT,
......
......@@ -44,7 +44,7 @@ SCHEMA_COMMENTS: Dict[str, dict] = {
},
"operation_record": {
"table": "数据库操作记录",
"columns": {"id": "主键", "operation": "操作类型", "source_id": "数据源ID", "source_name": "数据源名称", "database_name": "数据库名称", "scope": "操作范围", "tables_json": "数据表列表JSON", "filename": "备份文件名", "execute_setting": "执行设置", "status": "执行状态", "created_at": "创建时间"},
"columns": {"id": "主键", "operation": "操作类型", "source_id": "数据源ID", "source_name": "数据源名称", "database_name": "数据库名称", "scope": "操作范围", "tables_json": "数据表列表JSON", "filename": "备份文件名", "execute_setting": "执行设置", "status": "执行状态", "trigger_type": "触发方式:manual 手动 / scheduled 定时", "backup_mode": "备份方式:full 全量 / incremental 增量", "task_id": "关联备份任务ID(无则手动操作)", "task_name": "关联备份任务名称", "error_message": "失败原因", "created_at": "创建时间"},
},
"table_design_config": {
"table": "表设计校验配置",
......@@ -83,4 +83,26 @@ SCHEMA_COMMENTS: Dict[str, dict] = {
"table": "系统参数配置",
"columns": {"setting_key": "配置参数键", "setting_value": "配置参数值", "updated_at": "更新时间"},
},
"backup_task": {
"table": "定时备份任务配置",
"columns": {
"id": "主键", "name": "任务名称", "source_id": "数据源ID", "database_name": "数据库/Schema名称",
"schema_name": "Schema名称", "scope": "备份范围:database 整库 / tables 指定表",
"tables_json": "指定数据表清单JSON", "backup_mode": "备份方式:full 全量 / incremental 增量",
"include_schema": "是否备份表结构:1 是 0 否", "include_data": "是否备份表数据:1 是 0 否",
"cron_expression": "Cron执行表达式(标准5段)", "enabled": "是否启用:1 启用 0 停用",
"created_at": "创建时间", "updated_at": "更新时间",
},
},
"table_pref": {
"table": "列表页表格偏好(表设置:列宽/列显隐/列对齐/列顺序/行高/线框粗细)",
"columns": {
"id": "主键", "page_key": "页面标识", "col_key": "列标识(表头文字;__page__ 为页面级偏好行)",
"width": "列宽 px;NULL=未自定义", "hidden": "是否隐藏:0 显示 1 隐藏",
"align": "列对齐:left/center/right;NULL=默认", "ord": "列顺序(0 起);NULL=原始位置",
"row_height": "行高 px(36/40/48);仅页面级行使用",
"border_width": "线框粗细档位(0=细 1=中 2=粗);仅页面级行使用",
"updated_at": "最近更新时间",
},
},
}
......@@ -18,5 +18,6 @@ from .table_design_config import TableDesignConfig
from .llm_config import LLMConfigRecord
from .system_config import SystemConfig
from .analysis import AnalysisTask, AnalysisRun, AnalysisTableSnapshot
from .backup_task import BackupTask
__all__ = ["ConnectionPreset", "Field", "Rule", "RuleLibrary", "FieldRule", "Task", "TaskGroup", "DataSource", "OperationRecord", "TableDesignConfig", "LLMConfigRecord", "SystemConfig", "TablePref", "AnalysisTask", "AnalysisRun", "AnalysisTableSnapshot"]
__all__ = ["ConnectionPreset", "Field", "Rule", "RuleLibrary", "FieldRule", "Task", "TaskGroup", "DataSource", "OperationRecord", "TableDesignConfig", "LLMConfigRecord", "SystemConfig", "TablePref", "AnalysisTask", "AnalysisRun", "AnalysisTableSnapshot", "BackupTask"]
"""Persistent scheduled backup configurations."""
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
from sqlalchemy.sql import func
from web.backend.db.database import Base
class BackupTask(Base):
__tablename__ = "backup_task"
__table_args__ = (
# 表注释(新库建表时写入 MySQL COMMENT;已存在的表由 apply_mysql_comments 启动时同步)
{"comment": "定时备份任务配置"},
)
id = Column(Integer, primary_key=True, autoincrement=True, comment="主键")
name = Column(String(120), nullable=False, comment="任务名称")
source_id = Column(Integer, ForeignKey("data_source.id", ondelete="CASCADE"), nullable=False, comment="数据源ID")
database_name = Column(String(255), nullable=False, default="", comment="数据库/Schema名称")
schema_name = Column(String(255), nullable=True, comment="Schema名称")
scope = Column(String(20), nullable=False, default="database", comment="备份范围:database 整库 / tables 指定表")
tables_json = Column(Text, nullable=False, default="[]", comment="指定数据表清单JSON")
backup_mode = Column(String(20), nullable=False, default="full", comment="备份方式:full 全量 / incremental 增量")
include_schema = Column(Integer, nullable=False, default=1, comment="是否备份表结构:1 是 0 否")
include_data = Column(Integer, nullable=False, default=1, comment="是否备份表数据:1 是 0 否")
cron_expression = Column(String(100), nullable=False, comment="Cron执行表达式(标准5段)")
enabled = Column(Integer, nullable=False, default=1, comment="是否启用:1 启用 0 停用")
created_at = Column(DateTime, nullable=False, server_default=func.current_timestamp(), comment="创建时间")
updated_at = Column(DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), comment="更新时间")
def to_dict(self, next_run_at=None):
try:
import json
tables = json.loads(self.tables_json or "[]")
except Exception:
tables = []
return {"id": self.id, "name": self.name, "source_id": self.source_id,
"database": self.database_name, "schema": self.schema_name,
"scope": self.scope, "tables": tables, "backup_mode": self.backup_mode,
"include_schema": bool(self.include_schema), "include_data": bool(self.include_data),
"cron_expression": self.cron_expression, "enabled": bool(self.enabled),
"created_at": self.created_at, "next_run_at": next_run_at}
......@@ -16,6 +16,11 @@ class OperationRecord(Base):
id = Column(Integer, primary_key=True, autoincrement=True)
operation = Column(String(20), nullable=False)
source_id = Column(Integer, ForeignKey("data_source.id", ondelete="SET NULL"), nullable=True)
# 记录来源和备份模式,保证历史列表可以区分手动/自动、全量/增量。
trigger_type = Column(String(20), nullable=False, default="manual", server_default="manual")
backup_mode = Column(String(20), nullable=False, default="", server_default="")
task_id = Column(Integer, nullable=True)
task_name = Column(String(120), nullable=False, default="", server_default="")
source_name = Column(String(100), nullable=False, default="")
database_name = Column(String(255), nullable=False, default="")
scope = Column(String(20), nullable=False, default="")
......@@ -23,6 +28,7 @@ class OperationRecord(Base):
filename = Column(String(255), nullable=False, default="")
execute_setting = Column(Text, nullable=False, default="")
status = Column(String(20), nullable=False, default="已完成")
error_message = Column(Text, nullable=True, default="")
created_at = Column(DateTime, nullable=False, server_default=func.current_timestamp())
def to_dict(self) -> dict:
......@@ -35,11 +41,16 @@ class OperationRecord(Base):
"operation": self.operation,
"source_id": self.source_id,
"source_name": self.source_name,
"trigger_type": self.trigger_type or "manual",
"backup_mode": self.backup_mode or "",
"task_id": self.task_id,
"task_name": self.task_name or "",
"database": self.database_name,
"scope": self.scope,
"tables": tables if isinstance(tables, list) else [],
"filename": self.filename,
"execute_setting": self.execute_setting,
"status": self.status,
"error_message": self.error_message or "",
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
}
......@@ -271,6 +271,32 @@ def _snapshots(db: Session, run_id: int) -> dict[str, AnalysisTableSnapshot]:
.filter(AnalysisTableSnapshot.run_id == run_id).all()}
def _column_changes(old_cols: dict[str, dict], current_cols: dict[str, dict]) -> list[dict]:
"""Compare column properties; a shifted position after add/drop is not a modification."""
common = old_cols.keys() & current_cols.keys()
def relative_order(cols: dict[str, dict]) -> dict[str, int]:
names = sorted(common, key=lambda name: (int(cols[name].get("ordinal_position") or 0), name))
return {name: index for index, name in enumerate(names)}
old_order, current_order = relative_order(old_cols), relative_order(current_cols)
changes = []
for col_name in sorted(old_cols.keys() | current_cols.keys()):
left, right = old_cols.get(col_name), current_cols.get(col_name)
if left is None:
changes.append({"column": col_name, "kind": "added", "before": None, "after": right})
elif right is None:
changes.append({"column": col_name, "kind": "removed", "before": left, "after": None})
else:
fields = [key for key in _COLUMNS if key not in ("column_name", "ordinal_position")
and left.get(key) != right.get(key)]
if old_order[col_name] != current_order[col_name]:
fields.append("ordinal_position")
if fields:
changes.append({"column": col_name, "kind": "changed", "before": left,
"after": right, "changed_fields": fields})
return changes
def _report(db: Session, run: AnalysisRun) -> dict:
previous = (db.query(AnalysisRun).filter(AnalysisRun.task_id == run.task_id,
AnalysisRun.status == "success", AnalysisRun.id < run.id)
......@@ -282,15 +308,7 @@ def _report(db: Session, run: AnalysisRun) -> dict:
now, before = current.get(name), old.get(name)
current_cols = {c["column_name"]: c for c in json.loads(now.columns_json)} if now else {}
old_cols = {c["column_name"]: c for c in json.loads(before.columns_json)} if before else {}
changes = []
for col_name in (sorted(set(current_cols) | set(old_cols)) if previous else []):
left, right = old_cols.get(col_name), current_cols.get(col_name)
if left is None:
changes.append({"column": col_name, "kind": "added", "before": None, "after": right})
elif right is None:
changes.append({"column": col_name, "kind": "removed", "before": left, "after": None})
elif left != right:
changes.append({"column": col_name, "kind": "changed", "before": left, "after": right})
changes = _column_changes(old_cols, current_cols) if previous else []
tables.append({"table_name": name, "change": "baseline" if previous is None else "added" if before is None else "removed" if now is None else
"changed" if changes or now.table_comment != before.table_comment else "unchanged",
"table_comment_before": before.table_comment if before else None,
......@@ -360,7 +378,7 @@ def _column_def(col: dict, dialect: str) -> str:
"""列定义片段:类型 + NOT NULL + 默认值(注释由调用方按方言拼)。"""
ctype = str(col.get("column_type") or col.get("data_type") or "VARCHAR(255)")
definition = f"{quote_ident(str(col.get('column_name') or ''), dialect)} {ctype}"
if str(col.get("is_nullable") or "YES").upper() == "NO":
if str(col.get("is_nullable") or "YES").upper() in ("NO", "N"):
definition += " NOT NULL"
default = col.get("column_default")
if default is not None and str(default).strip() != "":
......@@ -460,7 +478,16 @@ def analyze_table(run_id: int, table_name: str, db: Session = Depends(get_sessio
change_lines = []
for change in (row.get("column_changes") or [])[:60]:
if change["kind"] == "changed" and change.get("before") and change.get("after"):
detail = f"({change['before'].get('column_type')} → {change['after'].get('column_type')})"
before, after = change["before"], change["after"]
labels = {"column_type": "类型", "data_type": "数据类型", "char_max_length": "字符长度",
"numeric_precision": "数值精度", "numeric_scale": "小数位数",
"is_nullable": "可空", "column_default": "默认值", "column_comment": "注释",
"extra": "附加属性", "ordinal_position": "顺序"}
fields = [key for key in change.get("changed_fields", [])
if key == "column_type" or "column_type" not in change["changed_fields"]
or key not in ("data_type", "char_max_length", "numeric_precision", "numeric_scale")]
detail = "(" + ";".join(
f"{labels[key]}:{before.get(key)} → {after.get(key)}" for key in fields) + ")"
else:
detail = ""
label = {"added": "新增", "removed": "删除", "changed": "修改"}.get(change["kind"], change["kind"])
......
This diff is collapsed.
This diff is collapsed.
......@@ -444,9 +444,19 @@ def start_query(req: StartQueryRequest, db: Session = Depends(get_session)):
count_row = dbc.fetchone(count_sql)
total_rows = int((count_row or {}).get("n") or 0)
if schema:
# 表设置需要能选择未配置校验规则的字段。规则字段仍排在前面,
# 其余字段补入 field_list,并加入显式 SELECT,避免前端只能看到问题字段。
# Oracle 的 conn.db 是 service name,不能拿来当 OWNER;未填写
# schema 时使用连接用户读取当前用户表的字段元数据。
meta_schema = schema
if not meta_schema:
if cfg.db_type == "oracle":
meta_schema = ((conn or {}).get("user") or "").strip()
else:
meta_schema = ((conn or {}).get("db") or "").strip()
if meta_schema:
try:
col_meta = dbc.list_columns(schema, task.source_table) or []
col_meta = dbc.list_columns(meta_schema, task.source_table) or []
comments_by_key: dict[str, str] = {
(m.get("column_name") or "").lower(): (m.get("column_comment") or "")
for m in col_meta
......@@ -456,6 +466,22 @@ def start_query(req: StartQueryRequest, db: Session = Depends(get_session)):
c = comments_by_key.get(f_out["field_key"].lower(), "")
if c:
f_out["field_comment"] = c
existing = {f["field_key"].lower() for f in field_list_out}
for meta in col_meta:
field_key = (meta.get("column_name") or "").strip()
if not field_key or field_key.lower() in existing:
continue
field_list_out.append({
"id": None,
"field_key": field_key,
"field_comment": meta.get("column_comment") or "",
"show_default": False,
"ord": len(field_list_out),
"rules": 0,
"rule_list": [],
})
col_names.append(field_key)
existing.add(field_key.lower())
except Exception as e:
logger.warning(f"[queries] 获取字段注释失败,跳过:{type(e).__name__}: {e}")
except Exception as e:
......
import unittest
from web.backend.routers.analysis import _column_changes, _generate_sql
def column(name, position, column_type="VARCHAR2(1)", default=None):
return {"column_name": name, "ordinal_position": position,
"column_type": column_type, "data_type": "VARCHAR2",
"char_max_length": 1, "numeric_precision": None, "numeric_scale": None,
"is_nullable": "N", "column_default": default,
"column_comment": None, "extra": None}
class AnalysisReportDiffTests(unittest.TestCase):
def test_removal_does_not_modify_shifted_columns_or_generate_modify_sql(self):
old = {c["column_name"]: c for c in
[column("FIRST", 1), column("REMOVED", 2), column("ISHJTS", 3)]}
current = {c["column_name"]: c for c in
[column("FIRST", 1), column("ISHJTS", 2)]}
changes = _column_changes(old, current)
self.assertEqual([(item["column"], item["kind"]) for item in changes],
[("REMOVED", "removed")])
sql = _generate_sql({"table_name": "GD_GRDK", "columns": current,
"column_changes": changes}, "oracle")["diff_sql"]
self.assertIn('DROP COLUMN "REMOVED"', sql)
self.assertNotIn("MODIFY", sql)
def test_real_property_change_is_reported(self):
old = {"ISHJTS": column("ISHJTS", 1, default="'0'")}
current = {"ISHJTS": column("ISHJTS", 1, default="'1'")}
changes = _column_changes(old, current)
self.assertEqual(changes[0]["kind"], "changed")
self.assertEqual(changes[0]["changed_fields"], ["column_default"])
def test_reorder_of_existing_columns_is_reported(self):
old = {"A": column("A", 1), "B": column("B", 2)}
current = {"A": column("A", 2), "B": column("B", 1)}
changes = _column_changes(old, current)
self.assertEqual([item["changed_fields"] for item in changes],
[["ordinal_position"], ["ordinal_position"]])
......@@ -7,6 +7,11 @@ export const listOperationRecords = (operation) => http.get('/backup/records', {
export const importOperationRecords = (operation, records) => http.post(`/backup/records/import?operation=${encodeURIComponent(operation)}`, { records })
export const createBackup = (sourceId, payload) => http.post(`/data-sources/${sourceId}/backup`, payload)
export const restoreBackup = (payload) => http.post('/backup/restore', payload)
export const listBackupTasks = () => http.get('/backup/tasks')
export const createBackupTask = (payload) => http.post('/backup/tasks', payload)
export const updateBackupTask = (id, payload) => http.put(`/backup/tasks/${id}`, payload)
export const deleteBackupTask = (id) => http.delete(`/backup/tasks/${id}`)
export const runBackupTask = (id) => http.post(`/backup/tasks/${id}/run`)
export const encryptBackup = (payload) => http.post('/backup/encrypt', payload)
export const decryptBackup = (payload) => http.post('/backup/decrypt', payload)
export const encryptData = (payload) => http.post('/backup/data-encrypt', payload)
......
......@@ -5,3 +5,4 @@ export const createDataSource = (payload) => http.post('/data-sources', payload)
export const updateDataSource = (id, payload) => http.put(`/data-sources/${id}`, payload)
export const deleteDataSource = (id) => http.delete(`/data-sources/${id}`)
export const testDataSource = (id) => http.post(`/data-sources/${id}/test`)
export const insertSimulatedData = (id, payload) => http.post(`/data-sources/${id}/simulate/insert`, payload)
......@@ -59,6 +59,7 @@
</el-menu-item>
<el-sub-menu index="backup">
<template #title><el-icon><Download /></el-icon><span>备份</span></template>
<el-menu-item index="/backup/tasks"><template #title>任务</template></el-menu-item>
<el-menu-item index="/backup"><template #title>备份</template></el-menu-item>
<el-menu-item index="/restore"><template #title>还原</template></el-menu-item>
<el-menu-item index="/clear-tables"><template #title>清表</template></el-menu-item>
......@@ -74,13 +75,17 @@
<el-menu-item index="/validation"><template #title>数据</template></el-menu-item>
</el-sub-menu>
<el-sub-menu index="analysis">
<template #title><el-icon><TrendCharts /></el-icon><span>分析</span></template>
<template #title><el-icon><TrendCharts /></el-icon><span>监测</span></template>
<el-menu-item index="/analysis/tasks"><template #title>任务</template></el-menu-item>
<el-menu-item index="/analysis/reports"><template #title>报告</template></el-menu-item>
</el-sub-menu>
<el-sub-menu index="data">
<template #title><el-icon><DataAnalysis /></el-icon><span>数据</span></template>
<el-menu-item index="/data/simulation"><template #title>模拟</template></el-menu-item>
<el-menu-item index="/config/data-sources"><template #title>数据源</template></el-menu-item>
</el-sub-menu>
<el-sub-menu index="config">
<template #title><el-icon><Setting /></el-icon><span>配置</span></template>
<el-menu-item index="/config/data-sources"><template #title>数据源</template></el-menu-item>
<el-menu-item index="/config/rule-config"><template #title>规则</template></el-menu-item>
<el-menu-item index="/config/system"><template #title>配置</template></el-menu-item>
<el-menu-item index="/config/model"><template #title>模型</template></el-menu-item>
......@@ -118,11 +123,12 @@ import { listTaskGroups } from '@/api/taskGroups'
import { fontScaleOptions, useUiPrefs } from '@/composables/useUiPrefs'
import brandLogoDark from '@/assets/brand-logo-dark.png'
import brandLogoLight from '@/assets/brand-logo-light.png'
import { DataAnalysis } from '@element-plus/icons-vue'
const route = useRoute()
const router = useRouter()
const sidebarCollapsed = ref(false)
const openGroups = ref(['backup', 'crypto', 'config', 'validation', 'analysis'])
const openGroups = ref(['backup', 'crypto', 'config', 'validation', 'analysis', 'data'])
// ── UI 偏好(主题 / 字号,见 composables/useUiPrefs.js)────
const { theme, fontScale, currentScaleLabel, themeTip, themeIcon, setFontScale, toggleTheme } = useUiPrefs()
......
......@@ -5,6 +5,7 @@ import DataQualityView from '@/views/DataQualityView.vue'
import TaskConfigView from '@/views/TaskConfigView.vue'
import DataSourceView from '@/views/DataSourceView.vue'
import OperationView from '@/views/OperationView.vue'
import BackupTasksView from '@/views/BackupTasksView.vue'
import ClearTablesView from '@/views/ClearTablesView.vue'
import SystemConfigView from '@/views/SystemConfigView.vue'
import ModelConfigView from '@/views/ModelConfigView.vue'
......@@ -15,6 +16,7 @@ import HomeView from '@/views/HomeView.vue'
import AnalysisTasksView from '@/views/AnalysisTasksView.vue'
import AnalysisReportsView from '@/views/AnalysisReportsView.vue'
import AnalysisReportDetailView from '@/views/AnalysisReportDetailView.vue'
import DataSimulationView from '@/views/DataSimulationView.vue'
const routes = [
{
......@@ -47,7 +49,9 @@ const routes = [
{ path: 'config/data-sources', name: 'DataSources', component: DataSourceView, meta: { title: '数据源' } },
{ path: 'config/system', name: 'SystemConfig', component: SystemConfigView, meta: { title: '系统配置' } },
{ path: 'config/model', name: 'ModelConfig', component: ModelConfigView, meta: { title: '模型配置' } },
{ path: 'data/simulation', name: 'DataSimulation', component: DataSimulationView, meta: { title: '数据模拟' } },
{ path: 'backup', name: 'Backup', component: OperationView, props: { type: 'backup' }, meta: { title: '备份' } },
{ path: 'backup/tasks', name: 'BackupTasks', component: BackupTasksView, meta: { title: '任务' } },
{ path: 'restore', name: 'Restore', component: OperationView, props: { type: 'restore' }, meta: { title: '还原' } },
{ path: 'clear-tables', name: 'ClearTables', component: ClearTablesView, meta: { title: '清表' } },
{ path: 'encrypt', name: 'Encrypt', component: OperationView, props: { type: 'encrypt' }, meta: { title: '加密' } },
......
......@@ -57,6 +57,9 @@
font-size: var(--font-size-base);
font-weight: 400;
}
.list-shell-page .filter-form .table-settings-item {
margin-left: auto;
}
/* ---- 主操作组(重置 / + 新建 / 危险操作):按钮长度对齐参考项目(min-width 72,内容自适应) ---- */
.filter-main-actions .el-form-item__content {
......
......@@ -26,8 +26,7 @@
<div v-for="change in row.column_changes" :key="change.column" class="change-line">
<el-tag :type="changeTag(change.kind)" size="small">{{ changeLabel(change.kind) }}</el-tag>
<b>{{ change.column }}</b>
<span v-if="change.kind === 'changed'">{{ change.before?.column_type }} → {{ change.after?.column_type }}
<template v-if="commentDiff(change)">,注释:{{ change.before?.column_comment || '(空)' }} → {{ change.after?.column_comment || '(空)' }}</template></span>
<span v-if="change.kind === 'changed'">{{ changeDetails(change) }}</span>
<span v-else-if="change.kind === 'added'">{{ change.after?.column_type }}{{ change.after?.column_comment ? `,注释:${change.after.column_comment}` : '' }}</span>
</div>
</div>
......@@ -41,7 +40,7 @@
<tbody>
<tr v-for="col in columnList" :key="col.column_name">
<td>{{ col.column_name }}</td><td>{{ col.column_type }}</td>
<td>{{ col.is_nullable === 'NO' ? '否' : '是' }}</td>
<td>{{ isNotNull(col.is_nullable) ? '否' : '是' }}</td>
<td>{{ col.column_default ?? '—' }}</td>
<td>{{ col.column_comment || '—' }}</td>
</tr>
......@@ -97,7 +96,26 @@ const number = (v) => v === null || v === undefined ? '—' : Number(v).toLocale
const delta = (v) => v === null || v === undefined ? '基线' : `${v > 0 ? '+' : ''}${Number(v).toLocaleString()}`
const changeLabel = (k) => ({ added: '新增', removed: '删除', changed: '修改' }[k] || k)
const changeTag = (k) => ({ added: 'success', removed: 'danger', changed: 'warning' }[k] || 'info')
const commentDiff = (c) => (c.before?.column_comment || '') !== (c.after?.column_comment || '')
const isNotNull = (value) => ['NO', 'N'].includes(String(value || '').toUpperCase())
function changeDetails(change) {
const before = change.before || {}; const after = change.after || {}
const fields = change.changed_fields || []
const parts = []
const show = (label, key) => parts.push(`${label}:${before[key] ?? '(空)'} → ${after[key] ?? '(空)'}`)
if (fields.includes('column_type')) show('类型', 'column_type')
else {
for (const [key, label] of [['data_type', '数据类型'], ['char_max_length', '字符长度'],
['numeric_precision', '数值精度'], ['numeric_scale', '小数位数']]) {
if (fields.includes(key)) show(label, key)
}
}
if (fields.includes('is_nullable')) parts.push(`可空:${isNotNull(before.is_nullable) ? '否' : '是'} → ${isNotNull(after.is_nullable) ? '否' : '是'}`)
for (const [key, label] of [['column_default', '默认值'], ['column_comment', '注释'],
['extra', '附加属性'], ['ordinal_position', '顺序']]) {
if (fields.includes(key)) show(label, key)
}
return parts.join(',')
}
async function loadDetail() {
if (!Number.isFinite(runId) || !tableName) { error.value = '缺少报告参数'; loading.value = false; return }
......
This diff is collapsed.
......@@ -32,6 +32,13 @@
@click="onCancel"
>{{ cancelling ? '中断中…' : '中断' }}</el-button>
</el-form-item>
<el-form-item v-if="resultFields.length" class="table-settings-item">
<TableSettings
:state="tableSettingsState"
@visible="onFieldVisible"
@reorder="onFieldReorder"
/>
</el-form-item>
</el-form>
</section>
......@@ -67,6 +74,7 @@
:rows="rows"
:append-mode="true"
:row-key="rowKeyFn"
:visible-fields="visibleFieldKeys"
@explain-issue="onExplainIssue"
/>
</article>
......@@ -87,6 +95,7 @@ import { listTasks } from '@/api/tasks'
import { startQuery, pullQuery, cancelQuery } from '@/api/queries'
import { exportToExcel } from '@/utils/excel'
import ResultTable from '@/components/ResultTable.vue'
import TableSettings from '@/components/TableSettings.vue'
import ExplainIssueDialog from '@/components/ExplainIssueDialog.vue'
const route = useRoute()
......@@ -149,6 +158,8 @@ function takeSnapshot(typeKey) {
taskOptions: taskOptions.value,
rows: rows.value,
resultFields: resultFields.value,
visibleFieldKeys: visibleFieldKeys.value,
tableSettingsState: tableSettingsState.value,
lastResult: lastResult.value || partial,
// 续跑所需(2026-09-24 流式版):新 session 会全表重扫,已收到的行靠
// __row_index > lastRowIndex 去重(旧分页版按 receivedPages 页号去重已删)
......@@ -169,6 +180,8 @@ function applySnapshot(snap) {
taskOptions.value = []
rows.value = []
resultFields.value = []
visibleFieldKeys.value = []
tableSettingsState.value = { columns: [], rowHeight: 40, borderWidth: 1, colPrefs: {} }
lastResult.value = null
scannedCount.value = 0
totalRows.value = 0
......@@ -179,6 +192,15 @@ function applySnapshot(snap) {
taskOptions.value = snap.taskOptions
rows.value = snap.rows
resultFields.value = snap.resultFields
visibleFieldKeys.value = snap.visibleFieldKeys?.length
? snap.visibleFieldKeys
: resultFields.value.filter((f) => (f.rules || 0) > 0 || f.showDefault !== false).map((f) => f.key)
tableSettingsState.value = snap.tableSettingsState || {
columns: resultFields.value.map((f) => ({ key: f.key, label: f.cn })),
rowHeight: 40,
borderWidth: 1,
colPrefs: Object.fromEntries(resultFields.value.map((f) => [f.key, { visible: visibleFieldKeys.value.includes(f.key) }])),
}
lastResult.value = snap.lastResult
scannedCount.value = snap.lastResult?.scanned || 0
// 恢复 in-flight 进度(fire-and-forget 启动续跑)
......@@ -237,6 +259,8 @@ const lastResult = ref(null) // 最近一次查询结
const queryError = ref('')
const rows = ref([]) // 喂给 ResultTable 的行
const resultFields = ref([])
const visibleFieldKeys = ref([])
const tableSettingsState = ref({ columns: [], rowHeight: 40, borderWidth: 1, colPrefs: {} })
// 本端刚发过 cancel(onCancel / abortInFlight / unmount)→ 之后 /pull 的 404 静默退出
// (非本端取消的 404 = 会话被别人取消 / TTL 过期,要给用户提示)
......@@ -318,6 +342,10 @@ async function pollLoop(ac, { lastRowIndex = -1 } = {}) {
queryError.value = `扫描失败:${res.error}`
try { await cancelQuery(sessionId.value) } catch (_) { /* best-effort 释放 */ }
} else {
// done 只表示「扫描线程结束」,不表示缓冲区已取干:扫描比轮询快时缓冲区
// 还有积压(2026-09-29 线上 87187 vs 69478 就是这么丢的)。满额包 → 立即
// 续拉;尾包(<PULL_CHUNK_MAX)才说明取干,此时定稿。
if (res.bad_rows.length >= PULL_CHUNK_MAX) continue
// 定稿:bad_rows 用 bad_total(含截断丢弃部分),rows 是实际展示的
lastResult.value = { scanned: res.scanned, bad_rows: res.bad_total, rows: rows.value }
if (res.truncated) {
......@@ -408,8 +436,21 @@ async function onQuery() {
comment: f.field_comment || '',
rules: f.rules || 0,
ruleList: f.rule_list || [],
showDefault: f.show_default !== false,
// 有规则的字段始终默认展示,避免历史配置 show_default=false 时把问题字段隐藏;
// 未配置规则的源表字段仍按 show_default(后端补入字段默认 false)。
showDefault: (f.rules || 0) > 0 || f.show_default !== false,
}))
visibleFieldKeys.value = resultFields.value
.filter((f) => (f.rules || 0) > 0 || f.showDefault !== false)
.map((f) => f.key)
tableSettingsState.value = {
columns: resultFields.value.map((f) => ({ key: f.key, label: f.cn })),
rowHeight: 40,
borderWidth: 1,
colPrefs: Object.fromEntries(
resultFields.value.map((f) => [f.key, { visible: visibleFieldKeys.value.includes(f.key) }]),
),
}
if (start.total_rows === 0) {
// COUNT=0:表里没数据
lastResult.value = { scanned: 0, bad_rows: 0, rows: [] }
......@@ -454,6 +495,23 @@ async function onCancel() {
// 直接塞进 explainContext → 打开 ExplainIssueDialog 自动调 LLM
const explainDialogOpen = ref(false)
const explainContext = ref(null)
function onFieldVisible({ key, visible }) {
const next = visibleFieldKeys.value.filter((k) => k !== key)
if (visible) next.push(key)
// 按表设置当前列顺序重建可见列,勾选字段时不重置用户已调整的顺序。
const orderedKeys = tableSettingsState.value.columns.map((c) => c.key)
visibleFieldKeys.value = orderedKeys.filter((k) => next.includes(k))
tableSettingsState.value.colPrefs[key] = { ...(tableSettingsState.value.colPrefs[key] || {}), visible }
}
function onFieldReorder({ from, to }) {
const keys = tableSettingsState.value.columns.map((c) => c.key)
const a = keys.indexOf(from), b = keys.indexOf(to)
if (a < 0 || b < 0) return
keys.splice(a, 1); keys.splice(b, 0, from)
tableSettingsState.value.columns = keys.map((k) => resultFields.value.find((f) => f.key === k)).filter(Boolean).map((f) => ({ key: f.key, label: f.cn }))
visibleFieldKeys.value = keys.filter((k) => visibleFieldKeys.value.includes(k))
}
function onExplainIssue(payload) {
explainContext.value = payload
explainDialogOpen.value = true
......@@ -469,7 +527,7 @@ async function onExport() {
return
}
const exportCols = [
...resultFields.value.filter((c) => c.key !== '__reason' && c.showDefault !== false),
...resultFields.value.filter((c) => c.key !== '__reason' && visibleFieldKeys.value.includes(c.key)),
{ key: '__reason', cn: '备注' },
]
if (exportCols.length <= 1) {
......
This diff is collapsed.
This diff is collapsed.
......@@ -47,7 +47,7 @@
<el-form-item label="规则名称" required><el-input v-model="form.name" placeholder="如:身份证格式校验" /></el-form-item>
<el-form-item label="规则类型"><el-radio-group v-model="form.rule_type"><el-radio-button label="regex">正则</el-radio-button><el-radio-button label="number">数值</el-radio-button><el-radio-button label="date">日期</el-radio-button><el-radio-button label="string">字符串</el-radio-button></el-radio-group></el-form-item>
<el-form-item label="AI生成提示词"><el-input v-model="form.ai_prompt" type="textarea" :rows="3" maxlength="2000" show-word-limit placeholder="如:校验 18 位身份证号" /></el-form-item>
<el-form-item label="规则说明" required><el-input v-model="form.description" type="textarea" :rows="4" maxlength="500" show-word-limit placeholder="30-40 字说明,可手动修改" /></el-form-item>
<el-form-item label="规则说明" required><el-input v-model="form.description" type="textarea" :rows="4" maxlength="500" show-word-limit placeholder="简洁且意思完整的说明,可手动修改" /></el-form-item>
<el-form-item :label="form.rule_type === 'regex' ? '正则表达式' : '校验代码'" required>
<el-input v-model="ruleContent" type="textarea" :rows="form.rule_type === 'regex' ? 2 : 8" :placeholder="form.rule_type === 'regex' ? '^.+$' : 'def check(value):\n return True'" />
<div class="expression-actions">
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment