Files
cantabile-ha-addons/src/edge/cantabile_edge/trainhist.py
T

282 lines
15 KiB
Python

# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""The box's own training history (docs/methods.md B10): what its nightly
training (boxtrain.py) learns from, kept in
the agent's SQLite (<state dir>/training.db), rolling.
slots 15-minute means per asset the trainer uses (load.power,
pv.power, the planned devices' power, room.<r>.temp, the
heating's power, the outdoor thermometer, the grid meter and
the battery; the carriers': heat.power, the presence sources
room.<r>.occupied and person.<slug>.home, the chargers' plug
state, 0/1 each), from the agent's own 1-minute windows: per slot
the mean of the windows' means, as the cloud's trainer reads
telemetry (time_bucket('15 minutes') of avg(mean)). A slot the
agent did not see can be filled once from Home Assistant's
statistics (fill_from_stats: 5-minute where HA still has them,
hourly before), never over the agent's own. 13 months (the
training weighs recent weeks more and last year's same season
still: fcmodels.recency_weights); compact: an asset's number,
the slot's number, the sum and count (WITHOUT ROWID; ~20 bytes
a row).
weather hourly records (pvforecast.Hour's fields) per kind: observed
(the cloud's DWD observations shipped in the model bundle,
else Home Assistant's recorded weather entity) and forecast
(the last forecast made before the hour was over: Home
Assistant's, the cloud's), each with its source. 13 months.
fc_slots per series, slot and source the last forecast made before the
slot began (the box's own model, the cloud's series, the cloud's
model on the box, the hour-of-week, the coordinator's week
mean): the live error per source against the meter. 14 days.
Nothing here leaves the site except the error figures the plan carries
(controller.public_plan "ml")."""
from __future__ import annotations
import math
import os
import re
import sqlite3
import threading
from datetime import datetime, timezone
SLOT_S = 900
KEEP_DAYS = 396 # 13 months
KEEP_FC_DAYS = 14
# the assets the trainer reads (building/trainlib.py: SITE_SERIES, ROOM_INPUTS,
# CONTROLLED; trainer.load_site), and the meter for the member's net draw
ASSETS = {"load.power", "pv.power", "weather.outside_temp", "heat.power", "heatpump.power", "heating.power",
"ac.power", "cooling.power", "ev.power", "grid.import_power", "grid.export_power", "battery.power",
"ev.plugged", "dhw.heater_power"}
# a room's temperature and presence, a person at home, a part's power, a charger's plug state and power
ASSET_RE = re.compile(r"^(room\.[^.]+\.(temp|occupied)|person\.[^.]+\.home|(heatpump|heating|ac|cooling|ev)\.[0-9]+\.power"
r"|ev\.[0-9]+\.plugged|ev[0-9]+\.(plugged|power))$")
OBSERVED_PREF = ("cloud", "ha") # an observed hour: DWD's (with irradiation) before HA's entity
FORECAST_PREF = ("ha", "cloud") # a forecast hour: as the box forecasts (localml.merge_hours: HA's first)
def wanted(asset: str) -> bool:
return asset in ASSETS or bool(ASSET_RE.match(asset))
def _finite(v) -> bool:
return isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)
class Store:
"""The training history. Thread-safe (the nightly job reads it in a
worker thread while the agent's loop writes): one connection, a lock."""
def __init__(self, path: str):
self.path = path
self.lock = threading.Lock()
self.db = sqlite3.connect(path, isolation_level=None, check_same_thread=False)
self.db.execute("PRAGMA journal_mode=WAL")
self.db.execute("PRAGMA synchronous=NORMAL")
self.db.execute("CREATE TABLE IF NOT EXISTS assets (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE)")
# slot: the quarter hour's number (unix / 900); src: 0 the agent, 1 HA's 5-minute, 2 HA's hourly statistics
self.db.execute("CREATE TABLE IF NOT EXISTS slots (aid INTEGER NOT NULL, slot INTEGER NOT NULL, sum REAL NOT NULL, "
"n INTEGER NOT NULL, src INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (aid, slot)) WITHOUT ROWID")
self._ids = {name: i for i, name in self.db.execute("SELECT id, name FROM assets")}
self.db.execute("CREATE TABLE IF NOT EXISTS weather (end_ts INTEGER NOT NULL, kind TEXT NOT NULL, source TEXT NOT NULL, "
"solar REAL, cloud REAL, temp REAL, issued INTEGER, PRIMARY KEY (end_ts, kind, source))")
# the wind (m/s; the heat model's, methods.md B13), 2026-11-06: an older file gets the column
if "wind" not in {r[1] for r in self.db.execute("PRAGMA table_info(weather)")}:
self.db.execute("ALTER TABLE weather ADD COLUMN wind REAL")
self.db.execute("CREATE TABLE IF NOT EXISTS fc_slots (series TEXT NOT NULL, slot INTEGER NOT NULL, source TEXT NOT NULL, "
"value REAL NOT NULL, issued INTEGER NOT NULL, PRIMARY KEY (series, slot, source))")
self.db.execute("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
def _aid(self, name: str) -> int:
i = self._ids.get(name)
if i is None:
self.db.execute("INSERT OR IGNORE INTO assets (name) VALUES (?)", (name,))
i = self._ids[name] = self.db.execute("SELECT id FROM assets WHERE name = ?", (name,)).fetchone()[0]
return i
def close(self):
with self.lock:
self.db.close()
# -- the agent's own samples -------------------------------------------------
def add_window(self, start: float, records: list[dict]) -> int:
"""A closed 1-minute window (agent.close_window's records): each
wanted gauge's mean into its 15-minute slot (by the window's start)."""
slot = int(start // SLOT_S)
recs = [(r["asset"], float(r["mean"])) for r in records
if r.get("kind") == "gauge" and _finite(r.get("mean")) and wanted(r.get("asset", ""))]
if not recs:
return 0
with self.lock:
rows = [(self._aid(a), slot, v) for a, v in recs]
self.db.executemany(
"INSERT INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, 0) ON CONFLICT (aid, slot) "
"DO UPDATE SET sum = CASE WHEN src = 0 THEN sum + excluded.sum ELSE excluded.sum END, "
"n = CASE WHEN src = 0 THEN n + 1 ELSE 1 END, src = 0", rows)
return len(rows)
def fill(self, asset: str, values: dict[int, float], src: str) -> int:
"""Slots the agent did not see (Home Assistant's statistics): never
over a slot that has a value."""
code = {"agent": 0, "ha-5minute": 1, "ha-hour": 2}.get(src, 2)
with self.lock:
aid = self._aid(asset)
rows = [(aid, int(k) // SLOT_S, float(v), code) for k, v in values.items() if _finite(v) and int(k) % SLOT_S == 0]
before = self.db.total_changes
self.db.executemany("INSERT OR IGNORE INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, ?)", rows)
return self.db.total_changes - before
def assets(self) -> list[str]:
with self.lock:
return [a for (a,) in self.db.execute("SELECT name FROM assets WHERE id IN (SELECT DISTINCT aid FROM slots)")]
def series(self, since: int, until: int, assets=None) -> dict[str, dict[int, float]]:
"""{asset: {slot: mean}} for slots in [since, until)."""
lo, hi = -(-int(since) // SLOT_S), -(-int(until) // SLOT_S)
with self.lock:
names = {i: n for n, i in self._ids.items()}
out: dict[str, dict[int, float]] = {}
for aid, name in names.items():
if assets is not None and name not in assets:
continue
rows = self.db.execute("SELECT slot, sum / n FROM slots WHERE aid = ? AND slot >= ? AND slot < ?",
(aid, lo, hi)).fetchall()
if rows:
out[name] = {int(s) * SLOT_S: float(v) for s, v in rows}
return out
def span(self) -> tuple[int | None, int | None, int]:
"""The first and last slot kept and how many (all assets)."""
with self.lock:
a, b, n = self.db.execute("SELECT min(slot), max(slot), count(*) FROM slots").fetchone()
return (a * SLOT_S if a is not None else None), (b * SLOT_S if b is not None else None), n
# -- the weather -------------------------------------------------------------
def put_weather(self, kind: str, source: str, hours: list, issued: float | None = None) -> int:
"""Hourly records (pvforecast.Hour: end, solar_kwh_m2, cloud_pct,
temp_c, wind_ms). A forecast only for hours not over when it was
made, the newest such kept; an observation replaces what was there."""
rows = []
for h in hours:
end = int(h.end.timestamp())
if end % 3600:
continue
if kind == "forecast" and (issued is None or issued >= end):
continue
if h.solar_kwh_m2 is None and h.cloud_pct is None and h.temp_c is None:
continue
rows.append((end, kind, source, h.solar_kwh_m2, h.cloud_pct, h.temp_c, int(issued) if issued else None,
getattr(h, "wind_ms", None)))
with self.lock:
self.db.executemany("INSERT OR REPLACE INTO weather (end_ts, kind, source, solar, cloud, temp, issued, wind) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)", rows)
return len(rows)
def weather_hours(self, since: int, until: int, kind: str) -> list:
"""One record per hour ending in (since, until] of this kind, by the
kind's preferred source (OBSERVED_PREF, FORECAST_PREF)."""
from .pvforecast import Hour
pref = OBSERVED_PREF if kind == "observed" else FORECAST_PREF
with self.lock:
rows = self.db.execute("SELECT end_ts, source, solar, cloud, temp, wind FROM weather WHERE kind = ? AND end_ts > ? "
"AND end_ts <= ?", (kind, int(since), int(until))).fetchall()
by: dict[int, tuple] = {}
for end, src, sol, cl, tc, wd in rows:
rank = pref.index(src) if src in pref else len(pref)
if end not in by or rank < by[end][0]:
by[end] = (rank, sol, cl, tc, wd)
return [Hour(datetime.fromtimestamp(e, timezone.utc), *by[e][1:]) for e in sorted(by)]
def training_hours(self, since: int, until: int) -> list:
"""The weather the models learn from: observed, else (an hour never
observed) the last forecast for it (wxfetch.hours, prefer observed)."""
obs = {int(h.end.timestamp()): h for h in self.weather_hours(since, until, "observed")}
for h in self.weather_hours(since, until, "forecast"):
obs.setdefault(int(h.end.timestamp()), h)
return [obs[k] for k in sorted(obs)]
def missing_observed(self, since: int, until: int) -> list[tuple[int, int]]:
"""Spans of hours without an observation (for Home Assistant's history)."""
with self.lock:
have = {e for (e,) in self.db.execute("SELECT end_ts FROM weather WHERE kind = 'observed' AND end_ts > ? "
"AND end_ts <= ?", (int(since), int(until)))}
spans, cur = [], None
for e in range(int(since) // 3600 * 3600 + 3600, int(until) + 1, 3600):
if e in have:
if cur:
spans.append(tuple(cur))
cur = None
elif cur is None:
cur = [e - 3600, e]
else:
cur[1] = e
if cur:
spans.append(tuple(cur))
return spans
# -- the forecasts per source -------------------------------------------------
def put_forecasts(self, series: str, start: int, cands: dict[str, list], issued: float, first: int = 1,
steps: int = 96) -> int:
"""Per source its values for the slots start + k·900, k in
[first, steps): the slot not begun yet (the quarter under way is
not a forecast any more)."""
rows = []
for src, vals in cands.items():
for k in range(first, min(steps, len(vals or []))):
v = vals[k]
if _finite(v):
rows.append((series, int(start) + k * SLOT_S, src, float(v), int(issued)))
with self.lock:
self.db.executemany("INSERT OR REPLACE INTO fc_slots (series, slot, source, value, issued) VALUES (?, ?, ?, ?, ?)",
rows)
return len(rows)
def forecasts(self, series: str, since: int, until: int) -> dict[str, dict[int, float]]:
with self.lock:
rows = self.db.execute("SELECT source, slot, value FROM fc_slots WHERE series = ? AND slot >= ? AND slot < ?",
(series, int(since), int(until))).fetchall()
out: dict[str, dict[int, float]] = {}
for src, s, v in rows:
out.setdefault(src, {})[int(s)] = float(v)
return out
# -- meta, pruning ---------------------------------------------------------------
def get(self, key: str) -> str | None:
with self.lock:
row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
return row[0] if row else None
def set(self, key: str, value: str):
with self.lock:
self.db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, str(value)))
def prune(self, now: float, days: int = KEEP_DAYS):
cut = int(now) - days * 86400
with self.lock:
self.db.execute("DELETE FROM slots WHERE slot < ?", (cut // SLOT_S,))
self.db.execute("DELETE FROM weather WHERE end_ts < ?", (cut,))
self.db.execute("DELETE FROM fc_slots WHERE slot < ?", (int(now) - KEEP_FC_DAYS * 86400,))
def size_bytes(self) -> int:
try:
return sum(os.path.getsize(self.path + s) for s in ("", "-wal") if os.path.exists(self.path + s))
except OSError:
return 0
def stats_to_slots(rows: list[dict], period: str) -> dict[int, float]:
"""Home Assistant's statistics rows ({"start": unix, "mean"}) as
15-minute values: 5-minute means averaged per slot, an hourly mean held
for its four slots (a step, not a shape: noted where used)."""
out: dict[int, list[float]] = {}
for r in rows:
v = r.get("mean")
if not _finite(v):
continue
s = int(r["start"])
if period == "hour":
for q in range(4):
out.setdefault(s // 3600 * 3600 + q * SLOT_S, []).append(float(v))
else:
out.setdefault(s // SLOT_S * SLOT_S, []).append(float(v))
return {k: sum(v) / len(v) for k, v in out.items()}