|
@@ -0,0 +1,340 @@
|
|
|
|
|
+"""Backfill the first detected cycle's sample_index range into wave_file.
|
|
|
|
|
+
|
|
|
|
|
+The original full re-run encoded the first complete cycle of every file but
|
|
|
|
|
+never wrote back where that cycle starts and ends. This script re-detects the
|
|
|
|
|
+first complete cycle for every pressure file (rpm > 0) and records the real
|
|
|
|
|
+``sample_index`` bounds into two columns:
|
|
|
|
|
+
|
|
|
|
|
+* ``cycle_start`` / ``cycle_end`` >= 0 : first complete cycle, half-open
|
|
|
|
|
+ ``[cycle_start, cycle_end)`` in actual sample_index values;
|
|
|
|
|
+* ``-1`` : no complete cycle found (matches the
|
|
|
|
|
+ ``tspluse_status = -1`` convention);
|
|
|
|
|
+* ``NULL`` : not yet processed.
|
|
|
|
|
+
|
|
|
|
|
+It reuses predict.py's adaptive multi-stage batch reading (STAGE_LIMITS =
|
|
|
|
|
+3500 -> 7000 -> 14000, parallel reader connections, tuple watermark) but needs
|
|
|
|
|
+no model: only the trigger channel is used, so it runs on CPU only. The fetch
|
|
|
|
|
+also keeps the real ``sample_index`` per row because ``detect_cycles`` returns
|
|
|
|
|
+array offsets, which differ from database indices for sparse columns.
|
|
|
|
|
+
|
|
|
|
|
+Only rows still holding ``cycle_start IS NULL`` are selected, so the
|
|
|
|
|
+``(sample_time, id)`` watermark is naturally idempotent: a file already
|
|
|
|
|
+written (0/-1) is never re-selected on resume. Writes are flushed in one
|
|
|
|
|
+``CASE id`` UPDATE per ``--write-batch`` rows.
|
|
|
|
|
+
|
|
|
|
|
+Usage:
|
|
|
|
|
+ conda activate tspulse
|
|
|
|
|
+ python detect_cycle_index.py [--start-time ""] [--start-id 0]
|
|
|
|
|
+ [--batch 100] [--read-workers 4] [--limit 0]
|
|
|
|
|
+ [--report-every 100] [--write-batch 100]
|
|
|
|
|
+ [--dry-run]
|
|
|
|
|
+"""
|
|
|
|
|
+
|
|
|
|
|
+import argparse
|
|
|
|
|
+import sys
|
|
|
|
|
+from concurrent.futures import ThreadPoolExecutor
|
|
|
|
|
+from pathlib import Path
|
|
|
|
|
+
|
|
|
|
|
+import numpy as np
|
|
|
|
|
+
|
|
|
|
|
+BACKEND = Path(__file__).resolve().parents[1]
|
|
|
|
|
+sys.path.insert(0, str(BACKEND))
|
|
|
|
|
+
|
|
|
|
|
+from app.algorithms.cycles import detect_cycles # noqa: E402
|
|
|
|
|
+import predict # noqa: E402
|
|
|
|
|
+from predict import MEASUREMENT_TYPE, STAGE_LIMITS # noqa: E402
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _fetch_chunk(connection, file_ids: list[int], limit: int) -> dict[int, np.ndarray]:
|
|
|
|
|
+ """Fetch the first ``limit`` rows of a chunk of files in one query.
|
|
|
|
|
+
|
|
|
|
|
+ Returns {file_id: (n, 4) array} with columns
|
|
|
|
|
+ [array_offset(0..n-1), signal_value, second_value, sample_index].
|
|
|
|
|
+ """
|
|
|
|
|
+ if not file_ids:
|
|
|
|
|
+ return {}
|
|
|
|
|
+ placeholders = ",".join(["%s"] * len(file_ids))
|
|
|
|
|
+ with connection.cursor() as cursor:
|
|
|
|
|
+ cursor.execute(
|
|
|
|
|
+ f"""
|
|
|
|
|
+ SELECT wave_file_id,
|
|
|
|
|
+ CAST(signal_value AS FLOAT) AS sig,
|
|
|
|
|
+ CAST(second_value AS FLOAT) AS sec,
|
|
|
|
|
+ sample_index
|
|
|
|
|
+ FROM wave_sample
|
|
|
|
|
+ WHERE wave_file_id IN ({placeholders}) AND sample_index < %s
|
|
|
|
|
+ ORDER BY wave_file_id, sample_index
|
|
|
|
|
+ """,
|
|
|
|
|
+ (*file_ids, limit),
|
|
|
|
|
+ )
|
|
|
|
|
+ rows = cursor.fetchall()
|
|
|
|
|
+ grouped: dict[int, tuple[list[float], list[float], list[int]]] = {}
|
|
|
|
|
+ for row in rows:
|
|
|
|
|
+ signal, second, index = grouped.setdefault(row["wave_file_id"], ([], [], []))
|
|
|
|
|
+ signal.append(float(row["sig"]))
|
|
|
|
|
+ second.append(float(row["sec"]) if row["sec"] is not None else np.nan)
|
|
|
|
|
+ index.append(int(row["sample_index"]))
|
|
|
|
|
+ out: dict[int, np.ndarray] = {}
|
|
|
|
|
+ for file_id, (signal, second, index) in grouped.items():
|
|
|
|
|
+ count = len(signal)
|
|
|
|
|
+ out[file_id] = np.column_stack(
|
|
|
|
|
+ [
|
|
|
|
|
+ np.arange(count, dtype=float),
|
|
|
|
|
+ np.asarray(signal),
|
|
|
|
|
+ np.asarray(second),
|
|
|
|
|
+ np.asarray(index, dtype=float),
|
|
|
|
|
+ ]
|
|
|
|
|
+ )
|
|
|
|
|
+ return out
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def read_batch_samples(
|
|
|
|
|
+ reader_connections: list,
|
|
|
|
|
+ file_ids: list[int],
|
|
|
|
|
+ workers: int,
|
|
|
|
|
+ limit: int,
|
|
|
|
|
+) -> dict[int, np.ndarray]:
|
|
|
|
|
+ """Fetch the first ``limit`` rows of every file in one round-trip."""
|
|
|
|
|
+ if not file_ids:
|
|
|
|
|
+ return {}
|
|
|
|
|
+ if workers <= 1 or len(file_ids) <= workers:
|
|
|
|
|
+ return _fetch_chunk(reader_connections[0], file_ids, limit)
|
|
|
|
|
+ chunks = np.array_split(file_ids, min(workers, len(file_ids)))
|
|
|
|
|
+ with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
|
|
|
|
|
+ futures = [
|
|
|
|
|
+ executor.submit(_fetch_chunk, reader_connections[i], chunk.tolist(), limit)
|
|
|
|
|
+ for i, chunk in enumerate(chunks)
|
|
|
|
|
+ ]
|
|
|
|
|
+ merged: dict[int, np.ndarray] = {}
|
|
|
|
|
+ for future in futures:
|
|
|
|
|
+ merged.update(future.result())
|
|
|
|
|
+ return merged
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def first_cycle_bounds(samples: np.ndarray) -> tuple[int, int] | None:
|
|
|
|
|
+ """Return ``(cycle_start, cycle_end)`` in real sample_index, half-open."""
|
|
|
|
|
+ cycles, _ = detect_cycles(samples)
|
|
|
|
|
+ if not cycles:
|
|
|
|
|
+ return None
|
|
|
|
|
+ cycle = cycles[0]
|
|
|
|
|
+ return (int(samples[cycle.start_offset, 3]), int(samples[cycle.end_offset, 3]))
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def resolve_batch_bounds(
|
|
|
|
|
+ rows: list[dict],
|
|
|
|
|
+ reader_connections: list,
|
|
|
|
|
+ workers: int,
|
|
|
|
|
+ first_stage: int = 0,
|
|
|
|
|
+) -> tuple[dict[int, tuple[int, int] | None], int]:
|
|
|
|
|
+ """Adaptively detect the first cycle per file, escalating window sizes.
|
|
|
|
|
+
|
|
|
|
|
+ Returns (``{file_id: (start, end) | None}``, error_count). None means no
|
|
|
|
|
+ complete cycle was found at any window (-> -1).
|
|
|
|
|
+ """
|
|
|
|
|
+ errors = 0
|
|
|
|
|
+ resolved: dict[int, tuple[int, int] | None] = {}
|
|
|
|
|
+ stage_files: dict[int, dict] = {int(row["id"]): row for row in rows}
|
|
|
|
|
+ for stage in range(first_stage, len(STAGE_LIMITS)):
|
|
|
|
|
+ if not stage_files:
|
|
|
|
|
+ break
|
|
|
|
|
+ limit = STAGE_LIMITS[stage]
|
|
|
|
|
+ samples_map = read_batch_samples(reader_connections, list(stage_files), workers, limit)
|
|
|
|
|
+ next_pending: dict[int, dict] = {}
|
|
|
|
|
+ for file_id, row in stage_files.items():
|
|
|
|
|
+ samples = samples_map.get(file_id)
|
|
|
|
|
+ if samples is None or len(samples) == 0:
|
|
|
|
|
+ resolved[file_id] = None
|
|
|
|
|
+ continue
|
|
|
|
|
+ try:
|
|
|
|
|
+ bounds = first_cycle_bounds(samples)
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ errors += 1
|
|
|
|
|
+ resolved[file_id] = None
|
|
|
|
|
+ continue
|
|
|
|
|
+ if bounds is None and int(row.get("sample_count") or 0) > limit:
|
|
|
|
|
+ next_pending[file_id] = row
|
|
|
|
|
+ else:
|
|
|
|
|
+ resolved[file_id] = bounds
|
|
|
|
|
+ stage_files = next_pending
|
|
|
|
|
+ for file_id in stage_files:
|
|
|
|
|
+ resolved[file_id] = None
|
|
|
|
|
+ return resolved, errors
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def flush_bounds(connection, buffer: list[tuple[int, int, int]]) -> None:
|
|
|
|
|
+ """Write pending (file_id, cycle_start, cycle_end) rows in one UPDATE."""
|
|
|
|
|
+ if not buffer:
|
|
|
|
|
+ return
|
|
|
|
|
+ case_start = " ".join("WHEN %s THEN %s" for _ in buffer)
|
|
|
|
|
+ case_end = " ".join("WHEN %s THEN %s" for _ in buffer)
|
|
|
|
|
+ placeholders = ",".join(["%s"] * len(buffer))
|
|
|
|
|
+ params_start: list[int] = []
|
|
|
|
|
+ params_end: list[int] = []
|
|
|
|
|
+ for file_id, start, end in buffer:
|
|
|
|
|
+ params_start.extend([file_id, start])
|
|
|
|
|
+ params_end.extend([file_id, end])
|
|
|
|
|
+ with connection.cursor() as cursor:
|
|
|
|
|
+ cursor.execute(
|
|
|
|
|
+ f"UPDATE wave_file SET cycle_start = CASE id {case_start} END, "
|
|
|
|
|
+ f"cycle_end = CASE id {case_end} END "
|
|
|
|
|
+ f"WHERE id IN ({placeholders})",
|
|
|
|
|
+ tuple(params_start + params_end + [file_id for file_id, _, _ in buffer]),
|
|
|
|
|
+ )
|
|
|
|
|
+ buffer.clear()
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def fetch_batch(
|
|
|
|
|
+ connection,
|
|
|
|
|
+ watermark_time: object | None,
|
|
|
|
|
+ watermark_id: int,
|
|
|
|
|
+ batch: int,
|
|
|
|
|
+) -> list[dict]:
|
|
|
|
|
+ with connection.cursor() as cursor:
|
|
|
|
|
+ if watermark_time is None:
|
|
|
|
|
+ cursor.execute(
|
|
|
|
|
+ """
|
|
|
|
|
+ SELECT id, point_name, measurement_type, sample_time, sample_count
|
|
|
|
|
+ FROM wave_file
|
|
|
|
|
+ WHERE rpm > 0 AND measurement_type = %s AND cycle_start IS NULL
|
|
|
|
|
+ ORDER BY sample_time ASC, id ASC
|
|
|
|
|
+ LIMIT %s
|
|
|
|
|
+ """,
|
|
|
|
|
+ (MEASUREMENT_TYPE, batch),
|
|
|
|
|
+ )
|
|
|
|
|
+ else:
|
|
|
|
|
+ cursor.execute(
|
|
|
|
|
+ """
|
|
|
|
|
+ SELECT id, point_name, measurement_type, sample_time, sample_count
|
|
|
|
|
+ FROM wave_file
|
|
|
|
|
+ WHERE rpm > 0 AND measurement_type = %s AND cycle_start IS NULL
|
|
|
|
|
+ AND (sample_time > %s OR (sample_time = %s AND id > %s))
|
|
|
|
|
+ ORDER BY sample_time ASC, id ASC
|
|
|
|
|
+ LIMIT %s
|
|
|
|
|
+ """,
|
|
|
|
|
+ (MEASUREMENT_TYPE, watermark_time, watermark_time, watermark_id, batch),
|
|
|
|
|
+ )
|
|
|
|
|
+ return cursor.fetchall()
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def main() -> int:
|
|
|
|
|
+ parser = argparse.ArgumentParser(description="回写首个完整周期的 sample_index 边界")
|
|
|
|
|
+ parser.add_argument("--start-time", type=str, default="",
|
|
|
|
|
+ help="起始水位时间(含),留空从头开始")
|
|
|
|
|
+ parser.add_argument("--start-id", type=int, default=0,
|
|
|
|
|
+ help="起始水位 id(含)")
|
|
|
|
|
+ parser.add_argument("--batch", type=int, default=100, help="每轮读取的文件数")
|
|
|
|
|
+ parser.add_argument("--read-workers", type=int, default=4, help="并行读样本的连接数")
|
|
|
|
|
+ parser.add_argument("--limit", type=int, default=0, help="最多处理文件数,0 表示不限")
|
|
|
|
|
+ parser.add_argument("--report-every", type=int, default=100, help="每 N 个文件打印一次汇总")
|
|
|
|
|
+ parser.add_argument("--write-batch", type=int, default=100,
|
|
|
|
|
+ help="回写攒满 N 条才批量写一次")
|
|
|
|
|
+ parser.add_argument("--dry-run", action="store_true", help="只检测不回写数据库")
|
|
|
|
|
+ args = parser.parse_args()
|
|
|
|
|
+
|
|
|
|
|
+ start_time = args.start_time.strip()
|
|
|
|
|
+ watermark_time: object | None = start_time or None
|
|
|
|
|
+ watermark_id = args.start_id
|
|
|
|
|
+ total = 0
|
|
|
|
|
+ stats = {"有周期": 0, "无周期": 0}
|
|
|
|
|
+ errors = 0
|
|
|
|
|
+ last_reported = 0
|
|
|
|
|
+ write_buffer: list[tuple[int, int, int]] = []
|
|
|
|
|
+
|
|
|
|
|
+ read_workers = max(1, min(args.read_workers, args.batch))
|
|
|
|
|
+ reader_connections = [predict.get_connection() for _ in range(read_workers)]
|
|
|
|
|
+
|
|
|
|
|
+ def flush(connection) -> None:
|
|
|
|
|
+ if args.dry_run:
|
|
|
|
|
+ write_buffer.clear()
|
|
|
|
|
+ else:
|
|
|
|
|
+ flush_bounds(connection, write_buffer)
|
|
|
|
|
+
|
|
|
|
|
+ def summary_text() -> str:
|
|
|
|
|
+ return (
|
|
|
|
|
+ f"已处理 {total}, 有周期 {stats['有周期']}, 无周期 {stats['无周期']}, "
|
|
|
|
|
+ f"错误 {errors}, 水位 sample_time={watermark_time} id={watermark_id}"
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ print(f"水位起点: sample_time={start_time or '从头'}, id={args.start_id}, "
|
|
|
|
|
+ f"批大小: {args.batch}, 读并行: {read_workers} 连接, 窗口: {STAGE_LIMITS}"
|
|
|
|
|
+ + (", 干跑(不回写)" if args.dry_run else ""))
|
|
|
|
|
+
|
|
|
|
|
+ try:
|
|
|
|
|
+ while True:
|
|
|
|
|
+ connection = predict.get_connection()
|
|
|
|
|
+ try:
|
|
|
|
|
+ rows = fetch_batch(connection, watermark_time, watermark_id, args.batch)
|
|
|
|
|
+ if not rows:
|
|
|
|
|
+ flush(connection)
|
|
|
|
|
+ connection.close()
|
|
|
|
|
+ print(f"无待处理文件,完成 [{summary_text()}]")
|
|
|
|
|
+ return 0
|
|
|
|
|
+
|
|
|
|
|
+ resolved, batch_errors = resolve_batch_bounds(
|
|
|
|
|
+ rows, reader_connections, read_workers,
|
|
|
|
|
+ )
|
|
|
|
|
+ errors += batch_errors
|
|
|
|
|
+
|
|
|
|
|
+ for row in rows:
|
|
|
|
|
+ file_id = int(row["id"])
|
|
|
|
|
+ total += 1
|
|
|
|
|
+ bounds = resolved[file_id]
|
|
|
|
|
+ if bounds is None:
|
|
|
|
|
+ stats["无周期"] += 1
|
|
|
|
|
+ write_buffer.append((file_id, -1, -1))
|
|
|
|
|
+ else:
|
|
|
|
|
+ stats["有周期"] += 1
|
|
|
|
|
+ write_buffer.append((file_id, bounds[0], bounds[1]))
|
|
|
|
|
+ if len(write_buffer) >= args.write_batch:
|
|
|
|
|
+ flush(connection)
|
|
|
|
|
+
|
|
|
|
|
+ if total // args.report_every > last_reported:
|
|
|
|
|
+ last_reported = total // args.report_every
|
|
|
|
|
+ print(f" 汇总: {summary_text()}", flush=True)
|
|
|
|
|
+
|
|
|
|
|
+ if args.limit and total >= args.limit:
|
|
|
|
|
+ flush(connection)
|
|
|
|
|
+ watermark_time = rows[-1]["sample_time"]
|
|
|
|
|
+ watermark_id = int(rows[-1]["id"])
|
|
|
|
|
+ connection.close()
|
|
|
|
|
+ print(f"已达 --limit={args.limit},退出")
|
|
|
|
|
+ print(f" 汇总: {summary_text()}")
|
|
|
|
|
+ return 0
|
|
|
|
|
+
|
|
|
|
|
+ flush(connection)
|
|
|
|
|
+ watermark_time = rows[-1]["sample_time"]
|
|
|
|
|
+ watermark_id = int(rows[-1]["id"])
|
|
|
|
|
+ finally:
|
|
|
|
|
+ try:
|
|
|
|
|
+ connection.close()
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+
|
|
|
|
|
+ except KeyboardInterrupt:
|
|
|
|
|
+ if write_buffer and not args.dry_run:
|
|
|
|
|
+ try:
|
|
|
|
|
+ with predict.get_connection() as connection:
|
|
|
|
|
+ flush_bounds(connection, write_buffer)
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+ for connection in reader_connections:
|
|
|
|
|
+ try:
|
|
|
|
|
+ connection.close()
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+ print(f"\n已中断。{summary_text()}")
|
|
|
|
|
+ if watermark_time is not None:
|
|
|
|
|
+ print(f"续跑命令: python detect_cycle_index.py --start-time \"{watermark_time}\" "
|
|
|
|
|
+ f"--start-id {watermark_id}")
|
|
|
|
|
+ return 0
|
|
|
|
|
+
|
|
|
|
|
+ for connection in reader_connections:
|
|
|
|
|
+ try:
|
|
|
|
|
+ connection.close()
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+ return 0
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+if __name__ == "__main__":
|
|
|
|
|
+ sys.exit(main())
|