# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved. """The box's own training history (docs/methods.md B10): what its nightly training (boxtrain.py) learns from, kept in the agent's SQLite (/training.db), rolling. slots 15-minute means per asset the trainer uses (load.power, pv.power, the planned devices' power, room..temp, the heating's power, the outdoor thermometer, the grid meter and the battery; the carriers': heat.power, the presence sources room..occupied and person..home, the chargers' plug state, 0/1 each), from the agent's own 1-minute windows: per slot the mean of the windows' means, as the cloud's trainer reads telemetry (time_bucket('15 minutes') of avg(mean)). A slot the agent did not see can be filled once from Home Assistant's statistics (fill_from_stats: 5-minute where HA still has them, hourly before), never over the agent's own. 13 months (the training weighs recent weeks more and last year's same season still: fcmodels.recency_weights); compact: an asset's number, the slot's number, the sum and count (WITHOUT ROWID; ~20 bytes a row). weather hourly records (pvforecast.Hour's fields) per kind: observed (the cloud's DWD observations shipped in the model bundle, else Home Assistant's recorded weather entity) and forecast (the last forecast made before the hour was over: Home Assistant's, the cloud's), each with its source. 13 months. fc_slots per series, slot and source the last forecast made before the slot began (the box's own model, the cloud's series, the cloud's model on the box, the hour-of-week, the coordinator's week mean): the live error per source against the meter. 14 days. Nothing here leaves the site except the error figures the plan carries (controller.public_plan "ml").""" from __future__ import annotations import math import os import re import sqlite3 import threading from datetime import datetime, timezone SLOT_S = 900 KEEP_DAYS = 396 # 13 months KEEP_FC_DAYS = 14 # the assets the trainer reads (building/trainlib.py: SITE_SERIES, ROOM_INPUTS, # CONTROLLED; trainer.load_site), and the meter for the member's net draw ASSETS = {"load.power", "pv.power", "weather.outside_temp", "heat.power", "heatpump.power", "heating.power", "ac.power", "cooling.power", "ev.power", "grid.import_power", "grid.export_power", "battery.power", "ev.plugged", "dhw.heater_power"} # a room's temperature and presence, a person at home, a part's power, a charger's plug state and power ASSET_RE = re.compile(r"^(room\.[^.]+\.(temp|occupied)|person\.[^.]+\.home|(heatpump|heating|ac|cooling|ev)\.[0-9]+\.power" r"|ev\.[0-9]+\.plugged|ev[0-9]+\.(plugged|power))$") OBSERVED_PREF = ("cloud", "ha") # an observed hour: DWD's (with irradiation) before HA's entity FORECAST_PREF = ("ha", "cloud") # a forecast hour: as the box forecasts (localml.merge_hours: HA's first) def wanted(asset: str) -> bool: return asset in ASSETS or bool(ASSET_RE.match(asset)) def _finite(v) -> bool: return isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v) class Store: """The training history. Thread-safe (the nightly job reads it in a worker thread while the agent's loop writes): one connection, a lock.""" def __init__(self, path: str): self.path = path self.lock = threading.Lock() self.db = sqlite3.connect(path, isolation_level=None, check_same_thread=False) self.db.execute("PRAGMA journal_mode=WAL") self.db.execute("PRAGMA synchronous=NORMAL") self.db.execute("CREATE TABLE IF NOT EXISTS assets (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE)") # slot: the quarter hour's number (unix / 900); src: 0 the agent, 1 HA's 5-minute, 2 HA's hourly statistics self.db.execute("CREATE TABLE IF NOT EXISTS slots (aid INTEGER NOT NULL, slot INTEGER NOT NULL, sum REAL NOT NULL, " "n INTEGER NOT NULL, src INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (aid, slot)) WITHOUT ROWID") self._ids = {name: i for i, name in self.db.execute("SELECT id, name FROM assets")} self.db.execute("CREATE TABLE IF NOT EXISTS weather (end_ts INTEGER NOT NULL, kind TEXT NOT NULL, source TEXT NOT NULL, " "solar REAL, cloud REAL, temp REAL, issued INTEGER, PRIMARY KEY (end_ts, kind, source))") # the wind (m/s; the heat model's, methods.md B13), 2026-11-06: an older file gets the column if "wind" not in {r[1] for r in self.db.execute("PRAGMA table_info(weather)")}: self.db.execute("ALTER TABLE weather ADD COLUMN wind REAL") self.db.execute("CREATE TABLE IF NOT EXISTS fc_slots (series TEXT NOT NULL, slot INTEGER NOT NULL, source TEXT NOT NULL, " "value REAL NOT NULL, issued INTEGER NOT NULL, PRIMARY KEY (series, slot, source))") self.db.execute("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)") def _aid(self, name: str) -> int: i = self._ids.get(name) if i is None: self.db.execute("INSERT OR IGNORE INTO assets (name) VALUES (?)", (name,)) i = self._ids[name] = self.db.execute("SELECT id FROM assets WHERE name = ?", (name,)).fetchone()[0] return i def close(self): with self.lock: self.db.close() # -- the agent's own samples ------------------------------------------------- def add_window(self, start: float, records: list[dict]) -> int: """A closed 1-minute window (agent.close_window's records): each wanted gauge's mean into its 15-minute slot (by the window's start).""" slot = int(start // SLOT_S) recs = [(r["asset"], float(r["mean"])) for r in records if r.get("kind") == "gauge" and _finite(r.get("mean")) and wanted(r.get("asset", ""))] if not recs: return 0 with self.lock: rows = [(self._aid(a), slot, v) for a, v in recs] self.db.executemany( "INSERT INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, 0) ON CONFLICT (aid, slot) " "DO UPDATE SET sum = CASE WHEN src = 0 THEN sum + excluded.sum ELSE excluded.sum END, " "n = CASE WHEN src = 0 THEN n + 1 ELSE 1 END, src = 0", rows) return len(rows) def fill(self, asset: str, values: dict[int, float], src: str) -> int: """Slots the agent did not see (Home Assistant's statistics): never over a slot that has a value.""" code = {"agent": 0, "ha-5minute": 1, "ha-hour": 2}.get(src, 2) with self.lock: aid = self._aid(asset) rows = [(aid, int(k) // SLOT_S, float(v), code) for k, v in values.items() if _finite(v) and int(k) % SLOT_S == 0] before = self.db.total_changes self.db.executemany("INSERT OR IGNORE INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, ?)", rows) return self.db.total_changes - before def assets(self) -> list[str]: with self.lock: return [a for (a,) in self.db.execute("SELECT name FROM assets WHERE id IN (SELECT DISTINCT aid FROM slots)")] def series(self, since: int, until: int, assets=None) -> dict[str, dict[int, float]]: """{asset: {slot: mean}} for slots in [since, until).""" lo, hi = -(-int(since) // SLOT_S), -(-int(until) // SLOT_S) with self.lock: names = {i: n for n, i in self._ids.items()} out: dict[str, dict[int, float]] = {} for aid, name in names.items(): if assets is not None and name not in assets: continue rows = self.db.execute("SELECT slot, sum / n FROM slots WHERE aid = ? AND slot >= ? AND slot < ?", (aid, lo, hi)).fetchall() if rows: out[name] = {int(s) * SLOT_S: float(v) for s, v in rows} return out def span(self) -> tuple[int | None, int | None, int]: """The first and last slot kept and how many (all assets).""" with self.lock: a, b, n = self.db.execute("SELECT min(slot), max(slot), count(*) FROM slots").fetchone() return (a * SLOT_S if a is not None else None), (b * SLOT_S if b is not None else None), n # -- the weather ------------------------------------------------------------- def put_weather(self, kind: str, source: str, hours: list, issued: float | None = None) -> int: """Hourly records (pvforecast.Hour: end, solar_kwh_m2, cloud_pct, temp_c, wind_ms). A forecast only for hours not over when it was made, the newest such kept; an observation replaces what was there.""" rows = [] for h in hours: end = int(h.end.timestamp()) if end % 3600: continue if kind == "forecast" and (issued is None or issued >= end): continue if h.solar_kwh_m2 is None and h.cloud_pct is None and h.temp_c is None: continue rows.append((end, kind, source, h.solar_kwh_m2, h.cloud_pct, h.temp_c, int(issued) if issued else None, getattr(h, "wind_ms", None))) with self.lock: self.db.executemany("INSERT OR REPLACE INTO weather (end_ts, kind, source, solar, cloud, temp, issued, wind) " "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", rows) return len(rows) def weather_hours(self, since: int, until: int, kind: str) -> list: """One record per hour ending in (since, until] of this kind, by the kind's preferred source (OBSERVED_PREF, FORECAST_PREF).""" from .pvforecast import Hour pref = OBSERVED_PREF if kind == "observed" else FORECAST_PREF with self.lock: rows = self.db.execute("SELECT end_ts, source, solar, cloud, temp, wind FROM weather WHERE kind = ? AND end_ts > ? " "AND end_ts <= ?", (kind, int(since), int(until))).fetchall() by: dict[int, tuple] = {} for end, src, sol, cl, tc, wd in rows: rank = pref.index(src) if src in pref else len(pref) if end not in by or rank < by[end][0]: by[end] = (rank, sol, cl, tc, wd) return [Hour(datetime.fromtimestamp(e, timezone.utc), *by[e][1:]) for e in sorted(by)] def training_hours(self, since: int, until: int) -> list: """The weather the models learn from: observed, else (an hour never observed) the last forecast for it (wxfetch.hours, prefer observed).""" obs = {int(h.end.timestamp()): h for h in self.weather_hours(since, until, "observed")} for h in self.weather_hours(since, until, "forecast"): obs.setdefault(int(h.end.timestamp()), h) return [obs[k] for k in sorted(obs)] def missing_observed(self, since: int, until: int) -> list[tuple[int, int]]: """Spans of hours without an observation (for Home Assistant's history).""" with self.lock: have = {e for (e,) in self.db.execute("SELECT end_ts FROM weather WHERE kind = 'observed' AND end_ts > ? " "AND end_ts <= ?", (int(since), int(until)))} spans, cur = [], None for e in range(int(since) // 3600 * 3600 + 3600, int(until) + 1, 3600): if e in have: if cur: spans.append(tuple(cur)) cur = None elif cur is None: cur = [e - 3600, e] else: cur[1] = e if cur: spans.append(tuple(cur)) return spans # -- the forecasts per source ------------------------------------------------- def put_forecasts(self, series: str, start: int, cands: dict[str, list], issued: float, first: int = 1, steps: int = 96) -> int: """Per source its values for the slots start + k·900, k in [first, steps): the slot not begun yet (the quarter under way is not a forecast any more).""" rows = [] for src, vals in cands.items(): for k in range(first, min(steps, len(vals or []))): v = vals[k] if _finite(v): rows.append((series, int(start) + k * SLOT_S, src, float(v), int(issued))) with self.lock: self.db.executemany("INSERT OR REPLACE INTO fc_slots (series, slot, source, value, issued) VALUES (?, ?, ?, ?, ?)", rows) return len(rows) def forecasts(self, series: str, since: int, until: int) -> dict[str, dict[int, float]]: with self.lock: rows = self.db.execute("SELECT source, slot, value FROM fc_slots WHERE series = ? AND slot >= ? AND slot < ?", (series, int(since), int(until))).fetchall() out: dict[str, dict[int, float]] = {} for src, s, v in rows: out.setdefault(src, {})[int(s)] = float(v) return out # -- meta, pruning --------------------------------------------------------------- def get(self, key: str) -> str | None: with self.lock: row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone() return row[0] if row else None def set(self, key: str, value: str): with self.lock: self.db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, str(value))) def prune(self, now: float, days: int = KEEP_DAYS): cut = int(now) - days * 86400 with self.lock: self.db.execute("DELETE FROM slots WHERE slot < ?", (cut // SLOT_S,)) self.db.execute("DELETE FROM weather WHERE end_ts < ?", (cut,)) self.db.execute("DELETE FROM fc_slots WHERE slot < ?", (int(now) - KEEP_FC_DAYS * 86400,)) def size_bytes(self) -> int: try: return sum(os.path.getsize(self.path + s) for s in ("", "-wal") if os.path.exists(self.path + s)) except OSError: return 0 def stats_to_slots(rows: list[dict], period: str) -> dict[int, float]: """Home Assistant's statistics rows ({"start": unix, "mean"}) as 15-minute values: 5-minute means averaged per slot, an hourly mean held for its four slots (a step, not a shape: noted where used).""" out: dict[int, list[float]] = {} for r in rows: v = r.get("mean") if not _finite(v): continue s = int(r["start"]) if period == "hour": for q in range(4): out.setdefault(s // 3600 * 3600 + q * SLOT_S, []).append(float(v)) else: out.setdefault(s // SLOT_S * SLOT_S, []).append(float(v)) return {k: sum(v) / len(v) for k, v in out.items()}