cantabile_edge, cantabile_modbus, cantabile_eebus 0.1.676: full source
This commit is contained in:
1 parent
3edd015039
commit
d48b9aff87
32 files changed
+4633
-133
No files matched your search
@@ -0,0 +1,272 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The box's own training history (docs/methods.md B10): what its nightly
|
||||
training (boxtrain.py) learns from, kept in
|
||||
the agent's SQLite (<state dir>/training.db), rolling.
|
||||
|
||||
slots 15-minute means per asset the trainer uses (load.power,
|
||||
pv.power, the planned devices' power, room.<r>.temp, the
|
||||
heating's power, the outdoor thermometer, the grid meter and
|
||||
the battery), from the agent's own 1-minute windows: per slot
|
||||
the mean of the windows' means, as the cloud's trainer reads
|
||||
telemetry (time_bucket('15 minutes') of avg(mean)). A slot the
|
||||
agent did not see can be filled once from Home Assistant's
|
||||
statistics (fill_from_stats: 5-minute where HA still has them,
|
||||
hourly before), never over the agent's own. 13 months (the
|
||||
training weighs recent weeks more and last year's same season
|
||||
still: fcmodels.recency_weights); compact: an asset's number,
|
||||
the slot's number, the sum and count (WITHOUT ROWID; ~20 bytes
|
||||
a row).
|
||||
weather hourly records (pvforecast.Hour's fields) per kind: observed
|
||||
(the cloud's DWD observations shipped in the model bundle,
|
||||
else Home Assistant's recorded weather entity) and forecast
|
||||
(the last forecast made before the hour was over: Home
|
||||
Assistant's, the cloud's), each with its source. 13 months.
|
||||
fc_slots per series, slot and source the last forecast made before the
|
||||
slot began (the box's own model, the cloud's series, the cloud's
|
||||
model on the box, the hour-of-week, the coordinator's week
|
||||
mean): the live error per source against the meter. 14 days.
|
||||
|
||||
Nothing here leaves the site except the error figures the plan carries
|
||||
(controller.public_plan "ml")."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
|
||||
SLOT_S = 900
|
||||
KEEP_DAYS = 396 # 13 months
|
||||
KEEP_FC_DAYS = 14
|
||||
# the assets the trainer reads (building/trainlib.py: SITE_SERIES, ROOM_INPUTS,
|
||||
# CONTROLLED; trainer.load_site), and the meter for the member's net draw
|
||||
ASSETS = {"load.power", "pv.power", "weather.outside_temp", "heat.power", "heatpump.power", "heating.power",
|
||||
"ac.power", "cooling.power", "ev.power", "grid.import_power", "grid.export_power", "battery.power"}
|
||||
ASSET_RE = re.compile(r"^(room\.[^.]+\.temp|(heatpump|heating|ac|cooling|ev)\.[0-9]+\.power)$")
|
||||
OBSERVED_PREF = ("cloud", "ha") # an observed hour: DWD's (with irradiation) before HA's entity
|
||||
FORECAST_PREF = ("ha", "cloud") # a forecast hour: as the box forecasts (localml.merge_hours: HA's first)
|
||||
|
||||
|
||||
def wanted(asset: str) -> bool:
|
||||
return asset in ASSETS or bool(ASSET_RE.match(asset))
|
||||
|
||||
|
||||
def _finite(v) -> bool:
|
||||
return isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)
|
||||
|
||||
|
||||
class Store:
|
||||
"""The training history. Thread-safe (the nightly job reads it in a
|
||||
worker thread while the agent's loop writes): one connection, a lock."""
|
||||
|
||||
def __init__(self, path: str):
|
||||
self.path = path
|
||||
self.lock = threading.Lock()
|
||||
self.db = sqlite3.connect(path, isolation_level=None, check_same_thread=False)
|
||||
self.db.execute("PRAGMA journal_mode=WAL")
|
||||
self.db.execute("PRAGMA synchronous=NORMAL")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS assets (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE)")
|
||||
# slot: the quarter hour's number (unix / 900); src: 0 the agent, 1 HA's 5-minute, 2 HA's hourly statistics
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS slots (aid INTEGER NOT NULL, slot INTEGER NOT NULL, sum REAL NOT NULL, "
|
||||
"n INTEGER NOT NULL, src INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (aid, slot)) WITHOUT ROWID")
|
||||
self._ids = {name: i for i, name in self.db.execute("SELECT id, name FROM assets")}
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS weather (end_ts INTEGER NOT NULL, kind TEXT NOT NULL, source TEXT NOT NULL, "
|
||||
"solar REAL, cloud REAL, temp REAL, issued INTEGER, PRIMARY KEY (end_ts, kind, source))")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS fc_slots (series TEXT NOT NULL, slot INTEGER NOT NULL, source TEXT NOT NULL, "
|
||||
"value REAL NOT NULL, issued INTEGER NOT NULL, PRIMARY KEY (series, slot, source))")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
||||
|
||||
def _aid(self, name: str) -> int:
|
||||
i = self._ids.get(name)
|
||||
if i is None:
|
||||
self.db.execute("INSERT OR IGNORE INTO assets (name) VALUES (?)", (name,))
|
||||
i = self._ids[name] = self.db.execute("SELECT id FROM assets WHERE name = ?", (name,)).fetchone()[0]
|
||||
return i
|
||||
|
||||
def close(self):
|
||||
with self.lock:
|
||||
self.db.close()
|
||||
|
||||
# -- the agent's own samples -------------------------------------------------
|
||||
def add_window(self, start: float, records: list[dict]) -> int:
|
||||
"""A closed 1-minute window (agent.close_window's records): each
|
||||
wanted gauge's mean into its 15-minute slot (by the window's start)."""
|
||||
slot = int(start // SLOT_S)
|
||||
recs = [(r["asset"], float(r["mean"])) for r in records
|
||||
if r.get("kind") == "gauge" and _finite(r.get("mean")) and wanted(r.get("asset", ""))]
|
||||
if not recs:
|
||||
return 0
|
||||
with self.lock:
|
||||
rows = [(self._aid(a), slot, v) for a, v in recs]
|
||||
self.db.executemany(
|
||||
"INSERT INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, 0) ON CONFLICT (aid, slot) "
|
||||
"DO UPDATE SET sum = CASE WHEN src = 0 THEN sum + excluded.sum ELSE excluded.sum END, "
|
||||
"n = CASE WHEN src = 0 THEN n + 1 ELSE 1 END, src = 0", rows)
|
||||
return len(rows)
|
||||
|
||||
def fill(self, asset: str, values: dict[int, float], src: str) -> int:
|
||||
"""Slots the agent did not see (Home Assistant's statistics): never
|
||||
over a slot that has a value."""
|
||||
code = {"agent": 0, "ha-5minute": 1, "ha-hour": 2}.get(src, 2)
|
||||
with self.lock:
|
||||
aid = self._aid(asset)
|
||||
rows = [(aid, int(k) // SLOT_S, float(v), code) for k, v in values.items() if _finite(v) and int(k) % SLOT_S == 0]
|
||||
before = self.db.total_changes
|
||||
self.db.executemany("INSERT OR IGNORE INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, ?)", rows)
|
||||
return self.db.total_changes - before
|
||||
|
||||
def assets(self) -> list[str]:
|
||||
with self.lock:
|
||||
return [a for (a,) in self.db.execute("SELECT name FROM assets WHERE id IN (SELECT DISTINCT aid FROM slots)")]
|
||||
|
||||
def series(self, since: int, until: int, assets=None) -> dict[str, dict[int, float]]:
|
||||
"""{asset: {slot: mean}} for slots in [since, until)."""
|
||||
lo, hi = -(-int(since) // SLOT_S), -(-int(until) // SLOT_S)
|
||||
with self.lock:
|
||||
names = {i: n for n, i in self._ids.items()}
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for aid, name in names.items():
|
||||
if assets is not None and name not in assets:
|
||||
continue
|
||||
rows = self.db.execute("SELECT slot, sum / n FROM slots WHERE aid = ? AND slot >= ? AND slot < ?",
|
||||
(aid, lo, hi)).fetchall()
|
||||
if rows:
|
||||
out[name] = {int(s) * SLOT_S: float(v) for s, v in rows}
|
||||
return out
|
||||
|
||||
def span(self) -> tuple[int | None, int | None, int]:
|
||||
"""The first and last slot kept and how many (all assets)."""
|
||||
with self.lock:
|
||||
a, b, n = self.db.execute("SELECT min(slot), max(slot), count(*) FROM slots").fetchone()
|
||||
return (a * SLOT_S if a is not None else None), (b * SLOT_S if b is not None else None), n
|
||||
|
||||
# -- the weather -------------------------------------------------------------
|
||||
def put_weather(self, kind: str, source: str, hours: list, issued: float | None = None) -> int:
|
||||
"""Hourly records (pvforecast.Hour: end, solar_kwh_m2, cloud_pct,
|
||||
temp_c). A forecast only for hours not over when it was made, the
|
||||
newest such kept; an observation replaces what was there."""
|
||||
rows = []
|
||||
for h in hours:
|
||||
end = int(h.end.timestamp())
|
||||
if end % 3600:
|
||||
continue
|
||||
if kind == "forecast" and (issued is None or issued >= end):
|
||||
continue
|
||||
if h.solar_kwh_m2 is None and h.cloud_pct is None and h.temp_c is None:
|
||||
continue
|
||||
rows.append((end, kind, source, h.solar_kwh_m2, h.cloud_pct, h.temp_c, int(issued) if issued else None))
|
||||
with self.lock:
|
||||
self.db.executemany("INSERT OR REPLACE INTO weather (end_ts, kind, source, solar, cloud, temp, issued) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?)", rows)
|
||||
return len(rows)
|
||||
|
||||
def weather_hours(self, since: int, until: int, kind: str) -> list:
|
||||
"""One record per hour ending in (since, until] of this kind, by the
|
||||
kind's preferred source (OBSERVED_PREF, FORECAST_PREF)."""
|
||||
from .pvforecast import Hour
|
||||
pref = OBSERVED_PREF if kind == "observed" else FORECAST_PREF
|
||||
with self.lock:
|
||||
rows = self.db.execute("SELECT end_ts, source, solar, cloud, temp FROM weather WHERE kind = ? AND end_ts > ? "
|
||||
"AND end_ts <= ?", (kind, int(since), int(until))).fetchall()
|
||||
by: dict[int, tuple] = {}
|
||||
for end, src, sol, cl, tc in rows:
|
||||
rank = pref.index(src) if src in pref else len(pref)
|
||||
if end not in by or rank < by[end][0]:
|
||||
by[end] = (rank, sol, cl, tc)
|
||||
return [Hour(datetime.fromtimestamp(e, timezone.utc), *by[e][1:]) for e in sorted(by)]
|
||||
|
||||
def training_hours(self, since: int, until: int) -> list:
|
||||
"""The weather the models learn from: observed, else (an hour never
|
||||
observed) the last forecast for it (wxfetch.hours, prefer observed)."""
|
||||
obs = {int(h.end.timestamp()): h for h in self.weather_hours(since, until, "observed")}
|
||||
for h in self.weather_hours(since, until, "forecast"):
|
||||
obs.setdefault(int(h.end.timestamp()), h)
|
||||
return [obs[k] for k in sorted(obs)]
|
||||
|
||||
def missing_observed(self, since: int, until: int) -> list[tuple[int, int]]:
|
||||
"""Spans of hours without an observation (for Home Assistant's history)."""
|
||||
with self.lock:
|
||||
have = {e for (e,) in self.db.execute("SELECT end_ts FROM weather WHERE kind = 'observed' AND end_ts > ? "
|
||||
"AND end_ts <= ?", (int(since), int(until)))}
|
||||
spans, cur = [], None
|
||||
for e in range(int(since) // 3600 * 3600 + 3600, int(until) + 1, 3600):
|
||||
if e in have:
|
||||
if cur:
|
||||
spans.append(tuple(cur))
|
||||
cur = None
|
||||
elif cur is None:
|
||||
cur = [e - 3600, e]
|
||||
else:
|
||||
cur[1] = e
|
||||
if cur:
|
||||
spans.append(tuple(cur))
|
||||
return spans
|
||||
|
||||
# -- the forecasts per source -------------------------------------------------
|
||||
def put_forecasts(self, series: str, start: int, cands: dict[str, list], issued: float, first: int = 1,
|
||||
steps: int = 96) -> int:
|
||||
"""Per source its values for the slots start + k·900, k in
|
||||
[first, steps): the slot not begun yet (the quarter under way is
|
||||
not a forecast any more)."""
|
||||
rows = []
|
||||
for src, vals in cands.items():
|
||||
for k in range(first, min(steps, len(vals or []))):
|
||||
v = vals[k]
|
||||
if _finite(v):
|
||||
rows.append((series, int(start) + k * SLOT_S, src, float(v), int(issued)))
|
||||
with self.lock:
|
||||
self.db.executemany("INSERT OR REPLACE INTO fc_slots (series, slot, source, value, issued) VALUES (?, ?, ?, ?, ?)",
|
||||
rows)
|
||||
return len(rows)
|
||||
|
||||
def forecasts(self, series: str, since: int, until: int) -> dict[str, dict[int, float]]:
|
||||
with self.lock:
|
||||
rows = self.db.execute("SELECT source, slot, value FROM fc_slots WHERE series = ? AND slot >= ? AND slot < ?",
|
||||
(series, int(since), int(until))).fetchall()
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for src, s, v in rows:
|
||||
out.setdefault(src, {})[int(s)] = float(v)
|
||||
return out
|
||||
|
||||
# -- meta, pruning ---------------------------------------------------------------
|
||||
def get(self, key: str) -> str | None:
|
||||
with self.lock:
|
||||
row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def set(self, key: str, value: str):
|
||||
with self.lock:
|
||||
self.db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, str(value)))
|
||||
|
||||
def prune(self, now: float, days: int = KEEP_DAYS):
|
||||
cut = int(now) - days * 86400
|
||||
with self.lock:
|
||||
self.db.execute("DELETE FROM slots WHERE slot < ?", (cut // SLOT_S,))
|
||||
self.db.execute("DELETE FROM weather WHERE end_ts < ?", (cut,))
|
||||
self.db.execute("DELETE FROM fc_slots WHERE slot < ?", (int(now) - KEEP_FC_DAYS * 86400,))
|
||||
|
||||
def size_bytes(self) -> int:
|
||||
try:
|
||||
return sum(os.path.getsize(self.path + s) for s in ("", "-wal") if os.path.exists(self.path + s))
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def stats_to_slots(rows: list[dict], period: str) -> dict[int, float]:
|
||||
"""Home Assistant's statistics rows ({"start": unix, "mean"}) as
|
||||
15-minute values: 5-minute means averaged per slot, an hourly mean held
|
||||
for its four slots (a step, not a shape: noted where used)."""
|
||||
out: dict[int, list[float]] = {}
|
||||
for r in rows:
|
||||
v = r.get("mean")
|
||||
if not _finite(v):
|
||||
continue
|
||||
s = int(r["start"])
|
||||
if period == "hour":
|
||||
for q in range(4):
|
||||
out.setdefault(s // 3600 * 3600 + q * SLOT_S, []).append(float(v))
|
||||
else:
|
||||
out.setdefault(s // SLOT_S * SLOT_S, []).append(float(v))
|
||||
return {k: sum(v) / len(v) for k, v in out.items()}
|
||||
Reference in new issue
Block a user