cantabile_edge, cantabile_modbus, cantabile_eebus 0.1.676: full source

This commit is contained in:
Setti Solutions S.R.L committed 2026-11-04 10:33:14 +02:00
1 parent 3edd015039
commit d48b9aff87
32 files changed
+4617 -117

No files matched your search

+1 -1
View File
@@ -22,7 +22,7 @@ Settings → Add-ons → Add-on store → ⋮ → Repositories, add
`https://git.setti.solutions/Daniel/cantabile-ha-addons`, then install the
add-on. The Supervisor pulls the prebuilt images
(`git.setti.solutions/daniel/cantabile-ha-<edge|modbus|eebus>-<arch>:<version>`,
named by `image:` in each `config.yaml`); this version: `0.1.650`, arch amd64.
named by `image:` in each `config.yaml`); this version: `0.1.676`, arch amd64.
## Layout
+1 -1
View File
@@ -3,7 +3,7 @@
# the Hub's, wrapped by addon-run (Dockerfile). The image is prebuilt,
# per arch (README.md: building it from src/).
name: Cantabile Edge Agent
version: "0.1.650"
version: "0.1.676"
slug: cantabile_edge
description: Plans your home's energy with Cantabile and, with your consent, runs the devices you choose.
url: https://cantabile.cloud
+1 -1
View File
@@ -4,7 +4,7 @@
# Assistant that isn't a Cantabile Hub. Devices are paired on its page
# (ingress), by their SKI.
name: Cantabile EEBUS
version: "0.1.650"
version: "0.1.676"
slug: cantabile_eebus
description: Pairs with EEBUS devices (the grid operator's control box, heat pumps, wallboxes, inverters), shows them in Home Assistant under Cantabile's names and hands the grid operator's §14a limit to the Cantabile agent; every control with its protection.
url: https://cantabile.cloud
+1 -1
View File
@@ -4,7 +4,7 @@
# Home Assistant that isn't a Cantabile Hub. Devices come from the options;
# the page (ingress) shows them and each control's protection.
name: Cantabile Modbus bridge
version: "0.1.650"
version: "0.1.676"
slug: cantabile_modbus
description: Reads inverters, meters, relays and circuits by Cantabile's device profiles (Modbus, or Home Assistant's own entities) and shows them in Home Assistant, each control with its protection.
url: https://cantabile.cloud
+3
View File
@@ -17,6 +17,9 @@ RUN CGO_ENABLED=0 GOARCH=$TARGETARCH GOAMD64=v1 go build -trimpath -ldflags="-s
FROM python:3.12.12-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
# LightGBM's OpenMP runtime: the box trains its own models (boxtrain.py,
# methods.md B10), as the cloud's trainer image does (building/Dockerfile)
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && rm -rf /var/lib/apt/lists/*
WORKDIR /src
COPY pyproject.toml ./
COPY cantabile_edge ./cantabile_edge
+47 -11
View File
@@ -15,7 +15,7 @@ import logging
import os
import time
from . import areas, ha, propose, sinks
from . import areas, ha, propose, sinks, trainhist
from .announce import Announcer
from .consent import Consent
from .controller import Controller
@@ -62,6 +62,15 @@ class Agent:
self.control = Controller(cfg, mapping, self.consent, sink) if mapping.controls else None
if self.control is not None and hasattr(sink, "on_command"):
sink.on_command = self.control.command
# the box's own training history (trainhist.py): its own 1-minute means,
# with or without the share consent: they never leave the box (nothing
# of it goes upward without that consent); only the consented values
# go to the cloud (self.agg)
self.train_agg = None
self.tparts: dict[str, float | None] = {}
if self.control is not None and getattr(self.control, "boxml", None) is not None:
self.control.boxml.mapping = mapping # HA's statistics into its history (boxtrain.fill_from_ha)
self.train_agg = Aggregator(cfg.interval_s)
# the grid operator's orders (EEBUS LPC, gridorder.py): the controller's,
# or, on a site without controls, checked and told to the cloud only
self.gridorders = self.control.gridorders if self.control is not None else GridOrders(cfg.state_dir)
@@ -169,9 +178,15 @@ class Agent:
return self.last_sent > 0
def sample(self, entity_id: str, state: str | None, ts: float):
if not self.consent.share:
share = self.consent.share # without it nothing is collected to send
local = self.train_agg is not None
if not share and not local:
return # the owner has not (or no longer) agreed
for a in self.mapping.consented_all(entity_id): # none: not mapped or no consent, never leaves
for a in self.mapping.of_entity.get(entity_id, []):
up = share and a.consent # to the cloud: the site's and the asset's consent
mine = local and trainhist.wanted(a.asset) # the box's own training: never leaves the box
if not (up or mine):
continue
raw = value_of(a, state, self.attrs.get(entity_id))
v = number(raw)
if v is not None and not a.equals:
@@ -180,24 +195,33 @@ class Agent:
v = scaled(convert(v, unit, a.unit), a)
if v is None:
log.warning("%s: unit %s cannot become %s; value dropped", a.asset, unit, a.unit)
self.agg.sample(a.asset, a.kind, v, ts)
total = self.mapping.part_total.get(a.asset)
if up:
self.agg.sample(a.asset, a.kind, v, ts)
if total:
self.sample_total(self.mapping.totals[total], a.asset, v, ts)
if local and a.kind == "gauge" and (mine or (total and trainhist.wanted(total))):
if mine:
self.train_agg.sample(a.asset, a.kind, v, ts)
if total and trainhist.wanted(total):
self.sample_total(self.mapping.totals[total], a.asset, v, ts, self.train_agg, self.tparts, False)
def sample_total(self, t, part: str, v: float | None, ts: float):
def sample_total(self, t, part: str, v: float | None, ts: float, agg=None, store=None, consented: bool = True):
"""A total of parts (pv.power from pv.1.power, pv.2.power: mapping.py)
from its consented parts' last values: their sum (the mean for % and
°C); unknown while a part is (a counter then waits: a missing part
must not look like a meter reset)."""
self.parts[part] = v
parts = [p for p in t.parts if getattr(self.mapping.by_asset.get(p), "consent", False)]
vals = [self.parts.get(p) for p in parts]
must not look like a meter reset). agg, store, consented: the box's
own training history's (every part, consented or not)."""
agg = self.agg if agg is None else agg
store = self.parts if store is None else store
store[part] = v
parts = [p for p in t.parts if not consented or getattr(self.mapping.by_asset.get(p), "consent", False)]
vals = [store.get(p) for p in parts]
if not vals or None in vals:
if t.kind == "gauge":
self.agg.sample(t.asset, t.kind, None, ts)
agg.sample(t.asset, t.kind, None, ts)
return
self.agg.sample(t.asset, t.kind, sum(vals) / (len(vals) if t.mean else 1), ts)
agg.sample(t.asset, t.kind, sum(vals) / (len(vals) if t.mean else 1), ts)
def on_entities(self, ev: dict):
now = self.clock()
@@ -229,6 +253,9 @@ class Agent:
except Exception as e: # never keeps the agent from reading
log.warning("mapping: %s", e)
ids = list(dict.fromkeys(a.entity for a in self.mapping.assets if a.consent)) # one entity may feed several
if self.train_agg is not None: # and the box's own training's (never sent: sample())
ids = list(dict.fromkeys(ids + [a.entity for a in self.mapping.assets if trainhist.wanted(a.asset)
or trainhist.wanted(self.mapping.part_total.get(a.asset, ""))]))
await client.subscribe_entities(ids, self.on_entities)
log.info("subscribed to %d consented entities (mapping v%d)", len(ids), self.mapping.version)
if self.control is not None:
@@ -258,6 +285,13 @@ class Agent:
def close_window(self, end: float):
start = self.agg.start
records = self.agg.flush(end)
if self.train_agg is not None:
tstart, trecs = self.train_agg.start, self.train_agg.flush(end)
try: # the box's own training history (trainhist.py): 15-minute means, kept 13 months
if trecs:
self.control.boxml.store.add_window(tstart, trecs)
except Exception as e:
log.warning("training history: %s", e)
if not records:
return
seq = self.queue.put("tel", {
@@ -327,6 +361,8 @@ class Agent:
async def run(self):
self.started = time.time()
self.agg.begin(self.started)
if self.train_agg is not None:
self.train_agg.begin(self.started)
if self.cfg.mode == "off":
log.info("mode off: not reading Home Assistant")
await self.stop.wait()
+238 -8
View File
@@ -3,8 +3,10 @@
"Cantabile" (one per site) under HA's MQTT integration, in the area
"Cantabile" (suggested_area, in HA's language: names.py), with
availability (offline when the agent stops: MQTT's last will, or said
before a clean stop). Read only: nothing in HA changes the agent; the
owner's consent is given in the box's setup wizard and panel only (sensitive).
before a clean stop). Nothing in HA changes the agent but the owner's
preferences (below); the owner's consent is given
in the box's setup wizard and panel only (sensitive),
and the mode only from the console.
What it shows (what goes between the cloud or the
building and the home, and what the agent does to each device):
@@ -36,6 +38,19 @@ building and the home, and what the agent does to each device):
actuator its contract target vs its state now;
sensors only, unavailable when the bundle is
stale (cantabile/<site>/market/availability)
the owner's preferences (preferences.py; every mode but off; entity
category config, usable in automations; HA's MQTT integration has no
time or date entity: times are text in the preferences' text forms):
text pref_<room>_schedule "Mo-Fr 22:00-06:00 21 must; ..." (empty: none)
number pref_<room>_boost_c, pref_<room>_boost_h, switch pref_<room>_boost
switch pref_away, text pref_away_until ("2026-11-10"; empty: until off)
text pref_<car>_ready "Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct"
number pref_<car>_capacity (kWh, for %), text pref_<car>_needed (one-off)
number pref_<battery>_reserve (%), text pref_<battery>_reserve_blocks
sensor pref_<control>_result, preferences (advisory and auto, with the
plan): per preference met or not and each must's cost
Commands on cantabile/<site>/prefs/<object>/set (the owner's, "Home
Assistant"); states on cantabile/<site>/prefs and /prefs/result.
On a building's own box (flats.json, a signed `flats` bundle from the
cloud: bundles.py) each flat is a device of its own ("Flat", via the
building's Cantabile device, area "Flats"): its grid power, its
@@ -64,7 +79,8 @@ log = logging.getLogger(__name__)
INTERVAL = 30 # state at least this often (seconds)
RETIRED = (("switch", "share"), ("sensor", "queued"), ("sensor", "last_sent"), ("sensor", "mode"))
RECEIVED = 10 # commands listed in HA (sensor received)
COMMANDS = ("mode", "control", "release", "limits", "tariff", "model", "building")
COMMANDS = ("mode", "control", "release", "limits", "tariff", "model", "building", "preferences")
PREF_TOPIC = re.compile(r"^cantabile/[^/]+/prefs/([^/]+)/set$")
MAX_FLATS = 200
@@ -86,9 +102,15 @@ class Announcer:
self.sent: dict[tuple, str] = {} # (node, component, object) -> config sent (JSON)
ctl = getattr(agent, "control", None)
self.bundles = ctl.bundles if ctl is not None else Bundles(agent.cfg.state_dir, site)
self.pref_cmds: dict[str, tuple[str, str]] = {} # object -> (control id, what): the commands HA may send
if ctl is not None:
ctl.on_plan.append(self._planned)
ctl.on_received.append(self._received)
if hasattr(ctl, "on_prefs"):
ctl.on_prefs.append(self._prefs_changed)
async def _prefs_changed(self):
self.changed.set()
@property
def lang(self) -> str:
@@ -394,6 +416,193 @@ class Announcer:
"last": a.get("last"), "shadow": True}
return out
# -- the owner's preferences (preferences.py) -------------------------------
def _pref_controls(self):
ctl = getattr(self.agent, "control", None)
if ctl is None or not hasattr(ctl, "prefs") or ctl.mode() == "off":
return None, []
return ctl, [c for c in ctl.controls if c.kind in ("heating", "cooling", "ev", "battery")]
def _prefs_objects(self) -> dict:
"""The preferences' entities: settings in every mode but off, their
results with the plan (advisory, auto)."""
ctl, cs = self._pref_controls()
if not cs:
self.pref_cmds = {}
return {}
lang, dev, top = self.lang, self._device(), f"{self.base}/prefs"
common = {"state_topic": top, "entity_category": "config", **self._avail(), "device": dev}
out, cmds = {}, {}
def add(comp, obj, cid, what, name, **conf):
cmds[obj] = (cid, what)
out[(comp, obj)] = {"name": name, "unique_id": f"{self.uid}_{obj}", "command_topic": f"{top}/{obj}/set",
"value_template": "{{ value_json['%s'] }}" % obj, **common, **conf}
rooms = [c for c in cs if c.kind in ("heating", "cooling")]
for c in rooms:
lo, hi = (16.0, 32.0) if c.kind == "cooling" else (5.0, 30.0)
k, n = _slug(c.id), c.label()
add("text", f"pref_{k}_schedule", c.id, "schedule", names.name("pref_schedule_cool" if c.kind == "cooling"
else "pref_schedule", lang, n),
max=255, mode="text", icon="mdi:calendar-clock")
add("number", f"pref_{k}_boost_c", c.id, "boost_c", names.name("pref_boost_c", lang, n), min=lo, max=hi,
step=0.5, mode="box", unit_of_measurement="°C", icon="mdi:thermometer-chevron-up")
add("number", f"pref_{k}_boost_h", c.id, "boost_h", names.name("pref_boost_h", lang, n), min=0.5, max=24,
step=0.5, mode="box", unit_of_measurement="h", icon="mdi:timer-outline")
add("switch", f"pref_{k}_boost", c.id, "boost", names.name("pref_boost", lang, n),
payload_on="ON", payload_off="OFF", state_on="ON", state_off="OFF", icon="mdi:fire")
if rooms:
add("switch", "pref_away", "", "away", names.name("pref_away", lang),
payload_on="ON", payload_off="OFF", state_on="ON", state_off="OFF", icon="mdi:home-export-outline")
add("text", "pref_away_until", "", "away_until", names.name("pref_away_until", lang), max=40, mode="text",
icon="mdi:calendar-arrow-right")
for c in cs:
k, n = _slug(c.id), c.label()
if c.kind == "ev":
add("text", f"pref_{k}_ready", c.id, "ready", names.name("pref_ready", lang, n), max=255, mode="text",
icon="mdi:car-clock")
add("number", f"pref_{k}_capacity", c.id, "capacity", names.name("pref_capacity", lang, n), min=1,
max=250, step=0.5, mode="box", unit_of_measurement="kWh", icon="mdi:car-battery")
add("text", f"pref_{k}_needed", c.id, "needed", names.name("pref_needed", lang, n), max=80, mode="text",
icon="mdi:car-arrow-right")
elif c.kind == "battery":
add("number", f"pref_{k}_reserve", c.id, "reserve", names.name("pref_reserve", lang, n), min=0, max=100,
step=1, mode="box", unit_of_measurement="%", icon="mdi:battery-lock")
add("text", f"pref_{k}_reserve_blocks", c.id, "reserve_blocks", names.name("pref_reserve_blocks", lang, n),
max=255, mode="text", icon="mdi:battery-clock")
self.pref_cmds = cmds
if ctl.mode() in ("advisory", "auto"):
res = {"state_topic": f"{top}/result", "json_attributes_topic": f"{top}/result", **self._avail(), "device": dev}
out[("sensor", "preferences")] = {
"name": names.name("pref_result", lang), "unique_id": f"{self.uid}_preferences", "icon": "mdi:clipboard-check",
"value_template": "{{ value_json.summary }}",
"json_attributes_template": "{{ {'results': value_json.lines, 'last_error': value_json.last_error} | tojson }}",
**res}
for c in cs:
k = _slug(c.id)
out[("sensor", f"pref_{k}_result")] = {
"name": names.name("pref_result_of", lang, c.label()), "unique_id": f"{self.uid}_pref_{k}_result",
"icon": "mdi:clipboard-check-outline",
"value_template": "{{ value_json.controls['%s'].state }}" % c.id,
"json_attributes_template": "{{ value_json.controls['%s'] | tojson }}" % c.id, **res}
return out
def prefs_state(self) -> dict:
"""The settings' values as HA shows them (cantabile/<site>/prefs)."""
ctl, cs = self._pref_controls()
if not cs:
return {}
from . import preferences as pm
p, out = ctl.prefs, {}
act = p.active_overrides()
for c in cs:
k = _slug(c.id)
if c.kind in ("heating", "cooling"):
out[f"pref_{k}_schedule"] = pm.room_text(p.value(f"room:{c.id}"))
b = next((o for o in act if o["kind"] == "boost" and o.get("control") == c.id), None)
out[f"pref_{k}_boost_c"] = b["c"] if b else self._boost(p, c.id)["c"]
out[f"pref_{k}_boost_h"] = p.ha.get(f"boost_h:{c.id}", 2.0)
out[f"pref_{k}_boost"] = "ON" if b else "OFF"
elif c.kind == "ev":
v = p.value(f"ev:{c.id}") or {}
out[f"pref_{k}_ready"] = pm.ev_text(v)
out[f"pref_{k}_capacity"] = v.get("capacity_kwh")
car = next((o for o in act if o["kind"] == "car" and o.get("control") == c.id), None)
out[f"pref_{k}_needed"] = pm.car_text(car, p.tz) if car else ""
elif c.kind == "battery":
v = p.value(f"battery:{c.id}") or {}
out[f"pref_{k}_reserve"] = v.get("reserve_pct", 0.0)
out[f"pref_{k}_reserve_blocks"] = pm.battery_text(v)
if any(c.kind in ("heating", "cooling") for c in cs):
away = next((o for o in act if o["kind"] == "away"), None)
out["pref_away"] = "ON" if away else "OFF"
out["pref_away_until"] = p.ha.get("away_until", "")
return out
def prefs_result(self) -> dict:
"""The plan's feedback per control (cantabile/<site>/prefs/result)."""
ctl, cs = self._pref_controls()
p = ctl.prefs if ctl is not None else None
fb = p.feedback if p is not None else []
per = {}
for c in cs:
mine = [f for f in fb if f.get("control") == c.id]
musts = [f for f in mine if f.get("how") == "must" and f.get("met") is not None]
state = "none" if not mine else "not met" if any(f.get("met") is False for f in mine) else \
"met" if any(f.get("met") for f in mine) else "open"
per[c.id] = {"state": state, "lines": [f["text"] for f in mine],
"cost_eur_day": round(sum(f.get("cost_eur_day") or 0.0 for f in musts), 2) if musts else None,
"results": mine}
bad = [f for f in fb if f.get("met") is False]
last = (p.errors[-1] if p is not None and p.errors else None)
return {"summary": "none" if not fb else f"{len(bad)} not met" if bad else "met",
"lines": [f["text"] for f in fb], "controls": per,
"last_error": f"{last['item']}: {last['why']}" if last else ""}
async def pref_message(self, obj: str, payload: bytes):
"""A preference set in HA (its entity's command): as the owner's."""
ctl = getattr(self.agent, "control", None)
if ctl is None or obj not in self.pref_cmds:
return
cid, what = self.pref_cmds[obj]
text = payload.decode(errors="replace").strip()
p, by = ctl.prefs, "Home Assistant"
msg = None
if what == "schedule":
msg = {"set": {"item": f"room:{cid}", "value": text or None}}
elif what in ("boost_c", "boost_h"):
try:
p.ha[f"{what}:{cid}"] = float(text)
except ValueError:
return
p.save()
b = next((o for o in p.active_overrides() if o["kind"] == "boost" and o.get("control") == cid), None)
if b is None:
self.changed.set()
return
msg = {"override": self._boost(p, cid)}
elif what == "boost":
msg = {"override": self._boost(p, cid)} if text.upper() == "ON" else {"cancel": {"kind": "boost", "control": cid}}
elif what == "away":
msg = {"override": {"kind": "away", "until_text": p.ha.get("away_until", "")}} if text.upper() == "ON" \
else {"cancel": {"kind": "away", "control": ""}}
elif what == "away_until":
p.ha["away_until"] = text[:40]
p.save()
if not any(o["kind"] == "away" for o in p.active_overrides()):
self.changed.set()
return
msg = {"override": {"kind": "away", "until_text": text}}
elif what in ("ready", "capacity"):
v = dict(p.value(f"ev:{cid}") or {"ready": [], "capacity_kwh": None})
if what == "capacity":
v["capacity_kwh"] = text if text not in ("", "None") else None
msg = {"set": {"item": f"ev:{cid}", "value": v}}
else:
msg = {"set": {"item": f"ev:{cid}", "value": text}} if text else \
{"set": {"item": f"ev:{cid}", "value": dict(v, ready=[]) if v.get("capacity_kwh") else None}}
elif what == "needed":
msg = {"override": {"kind": "car", "control": cid, "text": text}}
elif what in ("reserve", "reserve_blocks"):
v = dict(p.value(f"battery:{cid}") or {"reserve_pct": 0.0, "blocks": []})
if what == "reserve":
v["reserve_pct"] = text
msg = {"set": {"item": f"battery:{cid}", "value": v}}
else:
msg = {"set": {"item": f"battery:{cid}", "value": text}} if text else \
{"set": {"item": f"battery:{cid}", "value": dict(v, blocks=[])}}
if msg is not None:
await ctl.prefs_change(msg, "owner", by)
self.changed.set()
def _boost(self, p, cid) -> dict:
"""A boost with HA's numbers (their defaults: comfort + 1 °C, 2 h)."""
ctl = self.agent.control
c = next((c for c in ctl.controls if c.id == cid), None)
default = float(c.p("comfort_c")) + (-1.0 if c is not None and c.kind == "cooling" else 1.0) if c else 21.0
return {"kind": "boost", "control": cid, "c": p.ha.get(f"boost_c:{cid}", default),
"hours": p.ha.get(f"boost_h:{cid}", 2.0)}
def objects(self) -> dict:
"""Every discovery config HA should have now: (node, component,
object) -> config; node is the site's or a flat's."""
@@ -401,7 +610,7 @@ class Announcer:
tariff, model = self.bundles.tariff_now(now), self.bundles.model_info()
out = {}
for (comp, obj), conf in {**self._configs(), **self._plan_objects(),
**self._exchange_objects(tariff, model),
**self._exchange_objects(tariff, model), **self._prefs_objects(),
**self._market_objects(self.bundles.market_now(now))}.items():
out[(self.uid, comp, obj)] = conf
out.update(self._flat_objects(self.bundles.flat_list()))
@@ -495,6 +704,11 @@ class Announcer:
await c.publish(f"{self.base}/model", json.dumps(model), retain=True)
if ctl is not None and ctl.mode() != "off":
await c.publish(f"{self.base}/actuators", json.dumps(ctl.actuators(), default=str), retain=True)
prefs = self.prefs_state()
if prefs:
await c.publish(f"{self.base}/prefs", json.dumps(prefs, default=str), retain=True)
if ctl.mode() in ("advisory", "auto"):
await c.publish(f"{self.base}/prefs/result", json.dumps(self.prefs_result(), default=str), retain=True)
if ctl is not None and ctl.received:
# kept (retained) across restarts until new ones come
await c.publish(f"{self.base}/received", json.dumps({"commands": self.received_list(ctl)}), retain=True)
@@ -540,19 +754,35 @@ class Announcer:
async def _grid_orders(self, c):
"""The EEBUS bridge's grid order (gridorder.py), to the
controller: the grid operator's limit as a higher-rank decision."""
controller: the grid operator's limit as a higher-rank decision; and
the preferences HA sets (pref_message)."""
take = getattr(self.agent, "grid_order", None)
if take is None:
return
prefs = getattr(getattr(self.agent, "control", None), "prefs", None) is not None
def on(topic, payload):
if topic == gridorder.TOPIC and payload:
if topic == gridorder.TOPIC and payload and take is not None:
asyncio.ensure_future(take(payload))
return
m = PREF_TOPIC.match(topic)
if m and topic.startswith(f"{self.base}/prefs/") and prefs:
asyncio.ensure_future(self._pref_safe(m.group(1), payload))
c.on_message = on
if take is not None:
try:
await c.subscribe(gridorder.TOPIC)
except Exception as e:
log.info("mqtt: grid orders not subscribed: %s", e)
if prefs:
try:
await c.subscribe(f"{self.base}/prefs/+/set")
except Exception as e:
log.info("mqtt: preferences not subscribed: %s", e)
async def _pref_safe(self, obj, payload):
try:
await self.pref_message(obj, payload)
except Exception as e:
log.warning("preferences from HA (%s): %s", obj, e)
def _password(self) -> str:
if self.password_file:
+34
View File
@@ -56,6 +56,40 @@ async def backfill(client: HAClient, mapping, queue, since: datetime, until: dat
return batches
async def collect(client: HAClient, mapping, since: datetime, until: datetime, period: str = "hour",
want=None, consented: bool = True) -> dict[str, list[dict]]:
"""HA's statistics of the gauges (want(asset): which; consented: only
those with consent) as {asset: [{"start": unix, "mean"}]} in the assets'
units: for the box's own training history (trainhist.py, every mapped
gauge: nothing of it is queued or sent). A total of parts (pv.power from
pv.1.power, ...) has no entity of its own: left out."""
assets = {a.entity: a for a in mapping.assets
if (a.consent or not consented) and a.kind == "gauge" and not a.attribute and not a.equals
and (want is None or want(a.asset))}
if not assets:
return {}
meta = await client.send("recorder/list_statistic_ids", statistic_type="mean")
units = {m["statistic_id"]: m.get("statistics_unit_of_measurement") for m in meta}
ids = [e for e in assets if e in units]
out: dict[str, list[dict]] = {}
start = since
while start < until and ids:
end = min(start + CHUNK, until)
res = await client.send("recorder/statistics_during_period", timeout=300, start_time=start.isoformat(),
end_time=end.isoformat(), statistic_ids=ids, period=period, types=["mean"])
for eid, rows in (res or {}).items():
a, unit = assets.get(eid), units.get(eid)
if a is None:
continue
for r in rows:
v = number(r.get("mean"))
v = None if v is None else scaled(convert(v, unit, a.unit), a)
if v is not None:
out.setdefault(a.asset, []).append({"start": _ts(r["start"]), "mean": v})
start = end
return out
def parse_since(s: str) -> datetime:
"""An ISO date/time (UTC unless it says otherwise) or days back, e.g. 30d."""
if s.endswith("d") and s[:-1].isdigit():
+809
View File
@@ -0,0 +1,809 @@
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""The box trains its own models (docs/methods.md B10; step 2 of the
distributed models).
Nightly (cfg.train_at, default 03:30 in Home Assistant's time zone) from
the agent's own training history (trainhist.py, 13 months, recent weeks weighing more): per series
(load, pv, and base_load where the box plans devices) the cloud's own code
(trainlib.train_series: how, gbm, pvphys; byte-identical copies of
building/'s trainlib.py, fcmodels.py, pvcalib.py, rcfit.py) fitted on the
window before the held-out one and scored on it, on the weather as it was
forecast where the box kept it (else observed, and the run says so); the
rooms' R and C (trainlib.fit_rooms).
Never in the controller's way: each series is a child process (python -m
cantabile_edge.boxtrain <job> <result>) at the lowest priority (nice 19,
one LightGBM thread), one at a time, killed past its wall time
(train_series_s) or its resident memory (train_mem_mb), not started while
less than train_min_free_mb is available; the agent's loop only awaits
it. The data are read in a worker thread.
Which model the box plans on (inputs._model_series), per step: the box's
own model where it is in force, else the cloud's series, else the cloud's
model on the box (localml.py), else the hour-of-week. In force means: on
the night's held-out window it beat the model the box had (the champion;
trainlib's rule) AND the cloud's model in force (the bundle's "models",
scored on the same window): the cloud's model is the standing benchmark
until the cloud's teacher model exists. Without a cloud model to compare,
a box model other than the hour-of-week that beats the hour-of-week. A
model in force whose live error over 7 days (the meter against what was
planned on, 2 days at least) is worse than the cloud's series' leaves
force until it wins again.
Upward (the plan's "ml", controller.public_plan): the forecasts the box
planned on (hourly), the held-out errors per kind, the live error per
source, and the training's own figures (seconds, peak memory)."""
from __future__ import annotations
import asyncio
import json
import logging
import math
import os
import sys
import time
from dataclasses import asdict
from datetime import datetime, timedelta, timezone
from zoneinfo import ZoneInfo
import numpy as np
from . import fcmodels, localml, pvforecast, trainhist
log = logging.getLogger(__name__)
SLOT_S = 900
NAMES = {"load": "load_w", "pv": "pv_w", "base_load": "base_load_w"} # the bundle's names (boxmodel.NAMES)
SERIES = {v: k for k, v in NAMES.items()}
CTL_KINDS = ("heating", "cooling", "ev") # trainlib.CONTROLLED_POWER's keys
ROOM_DAYS = 42 # the rooms on the cloud's window (trainer: 6 weeks)
LIVE_DAYS = 7
LIVE_MIN = 2 * 96 # live errors count from 2 days of slots
RUNS_KEPT = 30
PUBLISH_EVERY_S = 3600 # the forecasts upward at most hourly (the plan comes every 15 min)
def _f(v):
return None if v is None or (isinstance(v, float) and not math.isfinite(v)) else float(v)
def _clean(v):
"""JSON without NaN, numpy scalars or private keys (lgbtrees' cache)."""
if isinstance(v, float):
return v if math.isfinite(v) else None
if isinstance(v, dict):
return {k: _clean(x) for k, x in v.items() if not str(k).startswith("_")}
if isinstance(v, (list, tuple)):
return [_clean(x) for x in v]
if isinstance(v, np.generic):
return _clean(v.item())
if isinstance(v, np.ndarray):
return _clean(v.tolist())
return v
def parse_hhmm(s: str) -> tuple[int, int]:
try:
h, m = (int(x) for x in str(s).split(":")[:2])
if 0 <= h < 24 and 0 <= m < 60:
return h, m
except ValueError:
pass
return 3, 30
def next_run(now: float, at: str, tz: str) -> float:
"""The next train_at (local) after now."""
zone = ZoneInfo(tz or "UTC")
h, m = parse_hhmm(at)
d = datetime.fromtimestamp(now, timezone.utc).astimezone(zone)
t = d.replace(hour=h, minute=m, second=0, microsecond=0)
if t.timestamp() <= now:
t = (d + timedelta(days=1)).replace(hour=h, minute=m, second=0, microsecond=0)
return t.timestamp()
def mem_available_mb() -> float | None:
try:
with open("/proc/meminfo") as f:
for line in f:
if line.startswith("MemAvailable:"):
return int(line.split()[1]) / 1024.0
except (OSError, ValueError):
pass
return None
def rss_mb(pid: int) -> float | None:
try:
with open(f"/proc/{pid}/status") as f:
for line in f:
if line.startswith("VmRSS:"):
return int(line.split()[1]) / 1024.0
except (OSError, ValueError):
pass
return None
def hours_json(hours: list) -> list:
return [[int(h.end.timestamp()), h.solar_kwh_m2, h.cloud_pct, h.temp_c] for h in hours]
def hours_from(rows) -> list:
return [pvforecast.Hour(datetime.fromtimestamp(int(r[0]), timezone.utc), r[1], r[2], r[3]) for r in rows or []]
def arrays(series: dict[int, float], t: np.ndarray) -> np.ndarray:
return np.array([series.get(int(k), np.nan) for k in t.tolist()], dtype=float)
def net_series(raw: dict[str, np.ndarray], n: int) -> np.ndarray | None:
"""A member's draw per slot (the coordinator's core.member_use_w): grid
import less export where metered, else load less PV and battery."""
if "grid.import_power" in raw:
exp = raw.get("grid.export_power")
return raw["grid.import_power"] - (np.nan_to_num(exp) if exp is not None else 0.0)
if "load.power" in raw:
out = raw["load.power"].copy()
for a in ("pv.power", "battery.power"):
if a in raw:
out = out - np.nan_to_num(raw[a])
return out
return None
def score(m: dict | None, key: str) -> float | None:
v = (m or {}).get(key)
return float(v) if isinstance(v, (int, float)) and math.isfinite(v) else None
# ---- the child process: one job -------------------------------------------------
def run_job(job: dict) -> dict:
"""One series (or the rooms), as the cloud trains it (trainlib). Pure:
the parent wrote everything it needs."""
from . import trainlib
t0 = time.monotonic()
t = np.asarray(job["t"], dtype=np.int64)
s = job["site"]
site = fcmodels.Site(s.get("tz") or "UTC", s.get("lat"), s.get("lon"), s.get("kwp"), s.get("tilt", 30.0),
s.get("azimuth", 180.0), s.get("pvgis_monthly"), s.get("state"))
w = fcmodels.weather(t, hours_from(job.get("hours")), site.lat, site.lon)
out: dict = {"series": job["series"]}
if job["series"] == "rooms":
assets = {a: np.array([np.nan if v is None else v for v in vals], dtype=float) for a, vals in job["assets"].items()}
fits = trainlib.fit_rooms(t, assets, w)
tout, p, ghi = trainlib.room_inputs(assets, w)
for room, f in fits.items():
c = (job.get("cloud_rooms") or {}).get(room)
theta = assets.get(f"room.{room}.temp")
if c and p is not None and theta is not None and c.get("power", "thermal") == f.get("power"):
from . import rcfit
b = rcfit.DT_H / c["c_kwh_per_k"]
gain, solar = c.get("gain_w") or 0.0, c.get("solar_m2") or 0.0
gain = sum(gain) / len(gain) if isinstance(gain, list) else gain
f["cloud_rmse_k"] = rcfit.sim_rmse(theta, tout, p / 1000.0, np.nan_to_num(ghi) / 1000.0, c["r_k_per_kw"],
c["c_kwh_per_k"], solar * b, gain * b / 1000.0)
out["rooms"] = _clean(fits)
else:
series = job["series"]
y = np.array([np.nan if v is None else v for v in job["y"]], dtype=float)
wt = fcmodels.weather(t, hours_from(job["test_hours"]), site.lat, site.lon) if job.get("test_hours") else None
res = trainlib.train_series(series, t, y, w, site, job.get("champion"), w_test=wt)
key = trainlib.score_key(res)
cloud = trainlib.score_params(job["cloud"], series, t, y, w, site, wt) if job.get("cloud") else None
out.update(key=key, window=list(res.window), n_train=res.n_train, n_test=res.n_test, note=res.note,
champion=res.champion, baseline=res.baseline, cloud=cloud,
runs=[{"kind": r.kind, "metrics": r.metrics, "note": r.note, "chosen": r.chosen} for r in res.runs],
chosen=res.chosen.kind if res.chosen else None,
params=fcmodels.box_params(res.chosen.params) if res.chosen else None)
if job.get("want_predictions") and res.chosen: # the parity test
out["predictions"] = fcmodels.predict(res.chosen.params, series, t, w, site).tolist()
out["seconds"] = round(time.monotonic() - t0, 2)
try: # this process's own peak (VmHWM: getrusage's maxrss keeps the parent's across exec)
with open("/proc/self/status") as f:
out["peak_mb"] = round(int(f.read().split("VmHWM:")[1].split()[0]) / 1024.0, 1)
except (OSError, IndexError, ValueError):
pass
return _clean(out)
def child_main(argv: list[str]) -> int:
try:
os.nice(19)
except (OSError, AttributeError):
pass
with open(argv[0]) as f:
job = json.load(f)
out = run_job(job)
tmp = argv[1] + ".tmp"
with open(tmp, "w") as f:
json.dump(out, f, separators=(",", ":"))
os.replace(tmp, argv[1])
return 0
# ---- the box's own models and the nightly job --------------------------------------
class StoreHistory:
"""inputs.History's interface over one series of the training history
(the lag of the box's own models: the right series, not the agent's
base load for all)."""
def __init__(self, name: str, values: dict[int, float]):
self.slots = {name: {k: [v, 1] for k, v in values.items()}}
def mean(self, name: str, slot: int):
s = self.slots.get(name, {}).get(slot)
return s[0] if s else None
class BoxTraining:
def __init__(self, cfg, store: trainhist.Store, bundles, kinds=None, clock=time.time, child_cmd=None):
self.cfg, self.store, self.bundles, self.clock = cfg, store, bundles, clock
self.kinds = kinds or (lambda: set()) # the planned devices' kinds (CTL_KINDS) now
self.path = os.path.join(cfg.state_dir, "boxmodels.json")
self.child_cmd = child_cmd or [sys.executable, "-m", "cantabile_edge.boxtrain"]
self.state = self._load()
self.running = False
self.progress = ""
self._models: dict = {} # (name, trained_at) -> localml.Model
self._fc_cache: dict = {} # (name, start) -> forecast
self._hist_cache: dict = {}
self.last_published = 0.0
self.enabled = bool(getattr(cfg, "train", True))
# LightGBM is imported in the child only (the agent's own memory stays
# as it was); here only whether it is there at all
import importlib.util
self.lightgbm = importlib.util.find_spec("lightgbm") is not None
if not self.lightgbm:
log.warning("LightGBM not in this image: the box's gbm fits fail, the other kinds train")
# -- the state ---------------------------------------------------------------
def _load(self) -> dict:
try:
with open(self.path) as f:
d = json.load(f)
if isinstance(d, dict) and d.get("v") == 1:
d.setdefault("series", {})
d.setdefault("rooms", {})
d.setdefault("runs", [])
return d
except (OSError, ValueError):
pass
return {"v": 1, "series": {}, "rooms": {}, "runs": [], "last": None}
def save(self):
tmp = self.path + ".tmp"
with open(tmp, "w") as f:
json.dump(_clean(self.state), f, separators=(",", ":"))
os.replace(tmp, self.path)
def in_force(self, series: str) -> dict | None:
e = self.state["series"].get(series)
return e if e and e.get("in_force") and e.get("params") else None
# -- what the planner asks -------------------------------------------------------
def model(self, name: str):
"""The box's own model for a bundle series name (load_w, pv_w,
base_load_w) as localml runs it, or None (none in force)."""
e = self.in_force(SERIES.get(name, ""))
if e is None:
return None
key = (name, e.get("trained_at"))
m = self._models.get(key)
if m is None:
try:
m = localml.Model(name, {"params": e["params"], "site": e.get("site") or {}})
except (ValueError, TypeError, KeyError) as ex:
log.warning("the box's own %s model cannot run: %s", name, ex)
e["in_force"], e["why"] = False, f"cannot run: {ex}"[:200]
return None
self._models = {k: v for k, v in self._models.items() if k[0] != name}
self._models[key] = m
return m
def history_for(self, series: str, start: int) -> StoreHistory:
"""The series' 8 days before start from the training history."""
key = (series, start)
h = self._hist_cache.get(key)
if h is None:
since = start - 8 * 86400
raw = {a: v for a, v in self.store.series(since, start).items()}
t = np.arange(since, start, SLOT_S, dtype=np.int64)
arr = self.actual(series, t, {a: arrays(v, t) for a, v in raw.items()})
vals = {} if arr is None else {int(k): float(v) for k, v in zip(t.tolist(), arr.tolist()) if not math.isnan(v)}
h = StoreHistory(localml.HISTORY[NAMES.get(series, "load_w")], vals)
self._hist_cache = {key: h}
return h
def forecast(self, name: str, start: int, steps: int, ha_forecast, weather_entity: str = ""):
"""The box's own forecast of a series (localml.predict on its own
model, HA's weather, else the cloud's bundle weather): (values with
None, info) or None without a model in force. Once per quarter hour."""
m = self.model(name)
if m is None:
return None
e = self.in_force(SERIES[name])
key = (name, start, e.get("trained_at"))
hit = self._fc_cache.get(key)
if hit is None:
b = self.bundles.model.get() if self.bundles is not None else None
b = b if isinstance(b, dict) and b.get("site") == getattr(self.bundles, "site", None) else {}
try:
hit = localml.predict(m, [start + k * SLOT_S for k in range(steps)], self.history_for(SERIES[name], start),
ha_forecast, b.get("weather"), weather_entity, b.get("issued"))
except Exception as ex: # never stops a plan: the next source then
log.warning("the box's own %s model: %s", name, ex)
hit = ([None] * steps, {"kind": m.kind, "weather": "", "steps": 0, "error": str(ex)[:200]})
self._fc_cache = {k: v for k, v in self._fc_cache.items() if k[1] == start}
self._fc_cache[key] = hit
return hit
def correct(self, series: str, start: int, values: list) -> tuple[list, float]:
"""The intraday correction (fcmodels.intraday) of what the box plans
on: the last 3 hours' forecast as planned on ("used", before any
correction) against the meter. series: plan (the load) or pv."""
since = start - int(fcmodels.INTRADAY_WINDOW_H * 3600)
fc = self.store.forecasts(series, since, start).get("used")
if not fc:
return list(values), 0.0
t = np.arange(since, start, SLOT_S, dtype=np.int64)
raw = {a: arrays(v, t) for a, v in self.store.series(since, start).items()}
y = self.actual(series, t, raw)
if y is None:
return list(values), 0.0
out, e = fcmodels.intraday("pv" if series == "pv" else "load", np.asarray(values, dtype=float), arrays(fc, t), y)
return [float(v) for v in out], e
def room_rc(self, control_id: str, area: str = "", only_one: bool = False) -> dict | None:
"""A room the box fitted itself and put in force (it simulated the
room better than the cloud's fit on the same data, or the cloud has
none): as bundles.room_rc gives the cloud's."""
rooms = {k: v for k, v in (self.state.get("rooms") or {}).items() if v.get("in_force")}
import re
key = next((k for k in (control_id, re.sub(r"_(heating|cooling|heat|cool)$", "", control_id or ""), area or "")
if k and k in rooms), None)
if key is None and only_one and len(rooms) == 1:
key = next(iter(rooms))
if key is None:
return None
r = rooms[key]
return {"r_k_per_kw": float(r["r_k_per_kw"]), "c_kwh_per_k": float(r["c_kwh_per_k"]),
"power": r.get("power") or "thermal", "gain_w": float(r.get("gain_w") or 0.0),
"solar_m2": min(max(float(r.get("solar_m2") or 0.0), 0.0), 50.0)}
# -- the series and their actual values -------------------------------------------
def train_series_list(self) -> list[str]:
out = ["load", "pv"]
if self.kinds():
out.insert(1, "base_load")
return out
def actual(self, series: str, t: np.ndarray, raw: dict[str, np.ndarray]) -> np.ndarray | None:
"""What was metered for a series: load.power, pv.power, the base
load (trainlib.base_load with the planned devices' kinds), the
member's net draw; for "plan" the series the box plans on."""
from . import trainlib
if series == "plan":
series = "base_load" if self.kinds() else "load"
if series == "base_load":
return trainlib.base_load(raw, set(self.kinds()))
if series == "net":
return net_series(raw, len(t))
a = trainlib.SITE_SERIES.get(series)
return raw.get(a) if a else None
# -- live errors ----------------------------------------------------------------
def record(self, series: str, start: int, cands: dict[str, list], now: float):
"""The forecasts per source the box had for the next 24 h (the
live error against the meter, trainhist fc_slots). series: plan
(the load the box plans on), pv or net."""
try:
self.store.put_forecasts(series, start, {k: v for k, v in cands.items() if v}, now)
except Exception as e: # never stops a plan
log.warning("forecasts per source: %s", e)
def week_mean(self, series: str, start: int, steps: int = 96) -> list[float | None]:
"""The coordinator's member forecast (methods.md B7) as the box
sees it: the same quarter hour's mean over the last 7 days."""
since = start - 7 * 86400
t = np.arange(since, start, SLOT_S, dtype=np.int64)
raw = {a: arrays(v, t) for a, v in self.store.series(since, start).items()}
y = self.actual(series, t, raw)
if y is None:
return [None] * steps
by: dict[int, list[float]] = {}
for ts, v in zip(t.tolist(), y.tolist()):
if not math.isnan(v):
by.setdefault((ts % 86400) // SLOT_S, []).append(v)
return [(sum(by[q]) / len(by[q])) if (q := ((start + k * SLOT_S) % 86400) // SLOT_S) in by else None
for k in range(steps)]
def live_errors(self, series: str, now: float, days: int = LIVE_DAYS) -> dict[str, dict]:
"""Per source the error of what was forecast against the meter
over the last days: {"nmae", "mae", "n"}."""
until = int(now // SLOT_S * SLOT_S)
since = until - days * 86400
fc = self.store.forecasts(series, since, until)
if not fc:
return {}
t = np.arange(since, until, SLOT_S, dtype=np.int64)
raw = {a: arrays(v, t) for a, v in self.store.series(since, until).items()}
y = self.actual(series, t, raw)
if y is None:
return {}
out = {}
for src, vals in fc.items():
p = arrays(vals, t)
m = fcmodels.metrics(y, p)
if m["n"]:
out[src] = {"nmae": _f(m["nmae"]), "mae": _f(m["mae"]), "n": m["n"]}
return out
# -- the nightly job ----------------------------------------------------------------
async def run(self, stop: asyncio.Event, tz=lambda: "UTC", location=lambda: {}, client=lambda: None,
weather_entity=lambda: ""):
"""Each night at train_at: train (never more than once at a time)."""
if not self.enabled:
log.info("the box's own training is off (train: false)")
await stop.wait()
return
while not stop.is_set():
wait = max(1.0, next_run(self.clock(), self.cfg.train_at, tz()) - self.clock())
try:
await asyncio.wait_for(stop.wait(), wait)
return
except asyncio.TimeoutError:
pass
try:
await self.train_once(tz(), location(), client(), weather_entity())
except Exception as e: # the next night again
log.exception("the box's training: %s", e)
def site(self, tz: str, location: dict) -> fcmodels.Site:
"""Where and what: the cloud's model's site where the bundle has one
(its kWp, orientation and PVGIS), else Home Assistant's location."""
for name in ("pv_w", "load_w", "base_load_w"):
m = self.bundles.box_model(name) if self.bundles is not None and hasattr(self.bundles, "box_model") else None
if m is not None and m.site.lat is not None:
return fcmodels.Site(tz or m.site.tz, m.site.lat, m.site.lon, m.site.kwp, m.site.tilt, m.site.azimuth,
m.site.pvgis_monthly)
return fcmodels.Site(tz or "UTC", (location or {}).get("latitude"), (location or {}).get("longitude"))
def metered_since(self) -> datetime | None:
v = (getattr(self.cfg, "pv_metered_since", "") or "").strip()
if not v:
return None
try:
t = datetime.fromisoformat(v.replace("Z", "+00:00"))
except ValueError:
log.warning("pv_metered_since %r: not ISO 8601", v)
return None
return t if t.tzinfo else t.replace(tzinfo=timezone.utc)
def prepare(self, now: float, tz: str, location: dict) -> tuple[list[dict], dict]:
"""The jobs (in a worker thread: SQLite and numpy, no await)."""
from . import trainlib
until = int(now // SLOT_S * SLOT_S)
since = until - int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS)) * 86400
t = np.arange(since, until, SLOT_S, dtype=np.int64)
stored = self.store.series(since, until)
raw = {a: arrays(v, t) for a, v in stored.items()}
site = self.site(tz, location)
obs = self.store.training_hours(since - 3600, until + 3600)
fc = self.store.weather_hours(since - 3600, until + 3600, "forecast")
w = fcmodels.weather(t, obs, site.lat, site.lon)
pv_since = self.metered_since()
jobs, info = [], {"slots": int(sum((~np.isnan(v)).sum() for v in raw.values())), "weather_hours": len(obs),
"forecast_hours": len(fc), "site": asdict(site)}
for series in self.train_series_list():
y = self.actual(series, t, raw)
if y is None or not (~np.isnan(y)).any():
continue
y = trainlib.cut_before(series, t, y, pv_since)
own = self.state["series"].get(series) or {}
champ = own.get("params")
if champ is not None and pv_since is not None and series in trainlib.SINCE_SERIES \
and (own.get("window") or [0])[0] and own["window"][0] < pv_since.timestamp():
champ = None # learned from PV before the cut: any new model replaces it
cloud = self.bundles.box_model(NAMES[series]) if self.bundles is not None and hasattr(self.bundles, "box_model") else None
w_test, on = trainlib.held_out_weather(t, y, w, fc, site.lat, site.lon)
jobs.append({"series": series, "t": t.tolist(), "y": [None if math.isnan(v) else float(v) for v in y.tolist()],
"hours": hours_json(obs), "test_hours": hours_json(w_test.hours) if w_test is not None else None,
"site": asdict(site), "champion": champ, "scored_on": on,
"cloud": _clean(cloud.params) if cloud is not None else None,
"cloud_model": ((self.bundles.model.get() or {}).get("models") or {}).get(NAMES[series], {}).get("model")
if cloud is not None else None})
r0 = int(np.searchsorted(t, until - ROOM_DAYS * 86400))
rooms_raw = {a: v[r0:] for a, v in raw.items() if a.startswith("room.") or a in trainlib.ROOM_INPUTS}
if any(a.startswith("room.") for a in rooms_raw):
m = self.bundles.model.get() if self.bundles is not None else None
cloud_rooms = (m or {}).get("rooms") if isinstance(m, dict) and m.get("site") == getattr(self.bundles, "site", None) else {}
jobs.append({"series": "rooms", "t": t[r0:].tolist(), "site": asdict(site),
"hours": hours_json(obs), "cloud_rooms": cloud_rooms or {},
"assets": {a: [None if math.isnan(x) else float(x) for x in v.tolist()] for a, v in rooms_raw.items()}})
return jobs, info
async def child(self, job: dict) -> tuple[dict | None, dict]:
"""One job in a child process within the caps: (result or None, how
it went: seconds, peak_mb, killed)."""
d = self.cfg.state_dir
jp, op = os.path.join(d, "train-job.json"), os.path.join(d, "train-result.json")
await asyncio.to_thread(_write_json, jp, job)
try:
os.unlink(op)
except OSError:
pass
env = dict(os.environ, OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1", MKL_NUM_THREADS="1")
cap_s = float(getattr(self.cfg, "train_series_s", 300.0))
cap_mb = float(getattr(self.cfg, "train_mem_mb", 600))
t0 = time.monotonic()
proc = await asyncio.create_subprocess_exec(*self.child_cmd, jp, op, env=env, stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.PIPE)
peak, killed = 0.0, ""
err_task = asyncio.ensure_future(proc.stderr.read(64 << 10))
while True:
try:
await asyncio.wait_for(proc.wait(), 0.25)
break
except asyncio.TimeoutError:
pass
r = rss_mb(proc.pid)
if r is not None:
peak = max(peak, r)
if r is not None and r > cap_mb:
killed = f"over its memory cap ({r:.0f} MB > {cap_mb:.0f} MB)"
elif time.monotonic() - t0 > cap_s:
killed = f"over its wall time ({cap_s:.0f} s)"
if killed:
proc.kill()
await proc.wait()
break
err = b""
try:
err = await asyncio.wait_for(err_task, 2)
except (asyncio.TimeoutError, Exception):
pass
how = {"seconds": round(time.monotonic() - t0, 2), "peak_mb": round(peak, 1), "killed": killed}
res = None
if not killed:
try:
with open(op) as f:
res = json.load(f)
how["peak_mb"] = max(how["peak_mb"], float(res.get("peak_mb") or 0.0))
except (OSError, ValueError):
how["error"] = (err.decode(errors="replace").strip().splitlines() or [f"exit {proc.returncode}"])[-1][:300]
for p in (jp, op):
try:
os.unlink(p)
except OSError:
pass
return res, how
async def train_once(self, tz: str = "UTC", location: dict | None = None, client=None, weather_entity: str = "") -> dict:
"""One night's training; the summary (also in state["last"])."""
if self.running:
return {"skipped": "already running"}
self.running = True
t_start = self.clock()
summary: dict = {"at": int(t_start), "series": {}, "rooms": None, "notes": []}
try:
if client is not None:
for what, fn in (("statistics", self.fill_from_ha), ("weather", self.weather_from_ha)):
try:
n = await fn(client, weather_entity) if what == "weather" else await fn(client)
if n:
summary["notes"].append(f"{n} {what} slots/hours from Home Assistant")
except Exception as e: # HA's history is a help, not a need
log.info("training: %s from Home Assistant: %s", what, e)
jobs, info = await asyncio.to_thread(self.prepare, t_start, tz, location or {})
summary["data"] = info
for job in jobs:
free = mem_available_mb()
need = float(getattr(self.cfg, "train_min_free_mb", 300))
name = job["series"]
if free is not None and free < need:
summary["notes"].append(f"{name}: not trained, {free:.0f} MB available (at least {need:.0f})")
continue
self.progress = name
res, how = await self.child(job)
if name == "rooms":
summary["rooms"] = {"how": how, **self.take_rooms(res, how)}
else:
summary["series"][name] = self.take(name, job, res, how, t_start)
self.state["last"] = summary
self.state["runs"] = (self.state.get("runs", []) + [_runlog(summary)])[-RUNS_KEPT:]
await asyncio.to_thread(self.save)
await asyncio.to_thread(self.store.prune, self.clock(), int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS)))
log.info("the box's training: %s in %.0f s", ", ".join(f"{k} {v.get('result')}" for k, v in summary["series"].items())
or "nothing to train", self.clock() - t_start)
finally:
self.running, self.progress = False, ""
self._models, self._fc_cache = {}, {}
return summary
def take(self, series: str, job: dict, res: dict | None, how: dict, now: float) -> dict:
"""A series' result: the new model in force only if it beats the
box's model (trainlib) and the cloud's (score on the same window)."""
out = {"how": how, "scored_on": job.get("scored_on", "")}
if res is None:
out["result"] = "failed: " + (how.get("killed") or how.get("error") or "no result")
return out
key = res.get("key") or "nmae"
runs = {r["kind"]: score(r.get("metrics"), key) for r in res.get("runs") or [] if r.get("metrics")}
cloud = score(res.get("cloud"), key)
own_now = score(res.get("champion"), key)
out.update(key=key, runs=runs, cloud=cloud, champion=own_now, n_test=res.get("n_test"), note=res.get("note", ""))
cur = self.state["series"].get(series)
chosen = res.get("chosen")
def beats(kind, v) -> bool:
"""In force: better than the cloud's model on the window; without
one, any kind but the hour-of-week (it beat the hour-of-week)."""
return (cloud is None and kind != "how") or (cloud is not None and v is not None and v < cloud)
def vs() -> str:
return f"the cloud's model {cloud:.3g}" if cloud is not None else "no cloud model on the box to compare"
new = runs.get(chosen) if chosen else None
if chosen and res.get("params") and (beats(chosen, new) or cur is None):
# a new model: it beat the box's own (trainlib) and, to be in force, the cloud's
cur = {"kind": chosen, "params": res["params"], "site": job["site"], "trained_at": int(now),
"window": res.get("window"), "score": new, "key": key, "in_force": beats(chosen, new)}
self.state["series"][series] = cur
out["result"] = (("new model in force: " if cur["in_force"] else "trained, not in force: ")
+ f"{chosen} {key} {new:.3g}, {vs()}"
+ (f", the box's own {own_now:.3g}" if own_now is not None else ""))
elif cur is not None:
# the box's model stays (nothing beat it, or the new one did not beat
# the cloud's): in force while it beats the cloud's on this window
stays = own_now if own_now is not None else cur.get("score")
cur["in_force"] = beats(cur.get("kind"), stays)
if own_now is not None:
cur["score"] = own_now
out["result"] = (f"the box's {cur.get('kind')} stays" + (f" ({key} {stays:.3g}, {vs()})" if stays is not None else "")
+ ("" if cur["in_force"] else ": not in force"))
if chosen and new is not None:
out["result"] += f"; the new {chosen} ({new:.3g}) does not beat {vs()}"
else:
out["result"] = "nothing trained: " + (res.get("note") or "no model")
if cur is not None:
cur.update(cloud=cloud, runs=runs, scored_on=job.get("scored_on", ""), n_test=res.get("n_test"),
checked_at=int(now), cloud_model=job.get("cloud_model"))
# the live error decides too: worse than the cloud's series over a week, out of force
live = self.live_errors("plan" if series == ("base_load" if self.kinds() else "load") else series, now)
mine, theirs = live.get("own"), live.get("cloud")
if cur.get("in_force") and mine and theirs and mine["n"] >= LIVE_MIN and theirs["n"] >= LIVE_MIN \
and mine["nmae"] is not None and theirs["nmae"] is not None and mine["nmae"] > theirs["nmae"]:
cur["in_force"] = False
out["result"] += (f"; out of force: live nMAE {mine['nmae']:.3g} vs the cloud's series "
f"{theirs['nmae']:.3g} over {LIVE_DAYS} days")
cur["why"] = out["result"][:300]
out["in_force"] = cur["in_force"]
return out
def take_rooms(self, res: dict | None, how: dict) -> dict:
if res is None:
return {"result": "failed: " + (how.get("killed") or how.get("error") or "no result")}
out = {}
rooms = self.state.setdefault("rooms", {})
for room, f in (res.get("rooms") or {}).items():
fitted = f.get("status") == "fitted"
cloud = f.get("cloud_rmse_k")
better = fitted and (cloud is None or (f.get("rmse_k") is not None and f["rmse_k"] < cloud))
f["in_force"] = bool(better)
f["why"] = ("fitted, simulates the room better than " + (f"the cloud's fit ({f['rmse_k']} K vs {cloud:.3f} K)"
if cloud is not None else "the defaults (no cloud fit to compare)")) if better else \
(f"fitted, but the cloud's fit simulates it better ({cloud:.3f} K vs {f.get('rmse_k')} K)" if fitted
else f.get("note", "default"))
rooms[room] = f
out[room] = {"in_force": f["in_force"], "why": f["why"]}
return {"result": out}
# -- Home Assistant's history ------------------------------------------------------
async def fill_from_ha(self, client) -> int:
"""Slots the agent never saw, from HA's statistics (backfill.collect),
once a day: 5-minute means for HA's last 10 days, hourly before."""
mapping = getattr(self, "mapping", None)
if mapping is None:
return 0
from . import backfill
now = datetime.fromtimestamp(self.clock(), timezone.utc)
days = int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS))
n = 0
for period, a, b in (("hour", now - timedelta(days=days), now - timedelta(days=9)),
("5minute", now - timedelta(days=9), now)):
got = await backfill.collect(client, mapping, a, b, period, trainhist.wanted, consented=False)
for asset, rows in got.items():
n += await asyncio.to_thread(self.store.fill, asset, trainhist.stats_to_slots(rows, period),
"ha-" + period)
return n
async def weather_from_ha(self, client, entity: str) -> int:
"""Observed hours the box lacks, from HA's recorded weather entity
(temperature and cloud cover at each hour's end; no irradiation)."""
if not entity:
return 0
now = self.clock()
days = int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS))
spans = await asyncio.to_thread(self.store.missing_observed, now - days * 86400, now)
if not spans:
return 0
a = datetime.fromtimestamp(spans[0][0], timezone.utc)
res = await client.send("history/history_during_period", timeout=120, start_time=a.isoformat(),
end_time=datetime.fromtimestamp(now, timezone.utc).isoformat(), entity_ids=[entity],
minimal_response=False, no_attributes=False, significant_changes_only=False)
hours = ha_history_hours((res or {}).get(entity) or [], spans)
return await asyncio.to_thread(self.store.put_weather, "observed", "ha", hours, now)
# -- upward ------------------------------------------------------------------------
def status(self, now: float, forecasts: dict | None = None) -> dict:
"""The plan's "ml": the training's figures and the error per source
(and, at most hourly, the forecasts the box planned on)."""
series = {}
for s, e in (self.state.get("series") or {}).items():
series[s] = {"kind": e.get("kind"), "in_force": bool(e.get("in_force")), "trained_at": e.get("trained_at"),
"checked_at": e.get("checked_at"), "key": e.get("key"), "held_out": e.get("runs") or {},
"held_out_box": e.get("score"), "held_out_cloud": e.get("cloud"), "scored_on": e.get("scored_on", ""),
"n_test": e.get("n_test"), "why": (e.get("why") or "")[:300]}
live = {}
for s in ("plan", "pv", "net"):
le = self.live_errors(s, now)
if le:
live[s] = le
last = self.state.get("last") or {}
out = {"v": 1, "series": series, "live": live, "live_days": LIVE_DAYS,
"planned": "base_load" if self.kinds() else "load",
"rooms": {k: {"in_force": bool(v.get("in_force")), "r_k_per_kw": v.get("r_k_per_kw"),
"c_kwh_per_k": v.get("c_kwh_per_k"), "rmse_k": v.get("rmse_k"),
"cloud_rmse_k": v.get("cloud_rmse_k")} for k, v in (self.state.get("rooms") or {}).items()},
"train": {"enabled": self.enabled, "lightgbm": self.lightgbm, "at": getattr(self.cfg, "train_at", ""),
"last": last.get("at"), "running": self.running,
"per_series": {k: v.get("how") for k, v in (last.get("series") or {}).items()},
"notes": (last.get("notes") or [])[:5]}}
if forecasts and now - self.last_published >= PUBLISH_EVERY_S:
out["forecasts"] = forecasts
self.last_published = now
return _clean(out)
def _write_json(path: str, d: dict):
tmp = path + ".tmp"
with open(tmp, "w") as f:
json.dump(d, f, separators=(",", ":"))
os.replace(tmp, path)
def _runlog(summary: dict) -> dict:
return {"at": summary.get("at"), "series": {k: {"result": v.get("result", "")[:200], "how": v.get("how")}
for k, v in (summary.get("series") or {}).items()}}
def ha_history_hours(states: list[dict], spans: list[tuple[int, int]]) -> list:
"""A weather entity's recorded states → per hour end in the spans the
temperature and cloud cover in force then (the last state before)."""
pts = []
for s in states:
a = s.get("a") or s.get("attributes") or {}
ts = s.get("lu") or s.get("lc") or s.get("last_updated") or s.get("last_changed")
try:
ts = float(ts) if isinstance(ts, (int, float)) else datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp()
except ValueError:
continue
temp, cloud = a.get("temperature"), a.get("cloud_coverage")
pts.append((ts, float(temp) if isinstance(temp, (int, float)) else None,
float(cloud) if isinstance(cloud, (int, float)) else None))
pts.sort()
out, j = [], -1
for a, b in spans:
for end in range(a + 3600, b + 1, 3600):
while j + 1 < len(pts) and pts[j + 1][0] <= end:
j += 1
if j >= 0 and (pts[j][1] is not None or pts[j][2] is not None) and end - pts[j][0] <= 3 * 3600:
out.append(pvforecast.Hour(datetime.fromtimestamp(end, timezone.utc), None, pts[j][2], pts[j][1]))
return out
if __name__ == "__main__":
sys.exit(child_main(sys.argv[1:]))
+13
View File
@@ -440,6 +440,19 @@ class Bundles:
out.append((float(f["values"][d.weekday() * 24 + d.hour]), "profile"))
return out
def profile(self, name: str, start: int, step_s: int, steps: int, tz: str) -> list[float] | None:
"""The model's hour-of-week profile of a series for every step (the
live error per source, boxtrain.py), or None."""
m = self.model.get()
if m is None or m.get("site") not in (self.site, "*"):
return None
f = (m.get("forecasts") or {}).get(name)
if f is None:
return None
zone = ZoneInfo(tz or "UTC")
return [float(f["values"][d.weekday() * 24 + d.hour]) for d in
(datetime.fromtimestamp(start + k * step_s, timezone.utc).astimezone(zone) for k in range(steps))]
def box_model(self, name: str):
"""The bundle's own model for a series (localml.Model), or None: no
model bundle for this site, none for the series, or one the box
+17
View File
@@ -62,6 +62,19 @@ class Config:
override_hold_s: int = 7200 # hands off a control the owner changed by hand
min_switch_s: int = 600 # anti-cycling: a switch stays on/off at least this long
max_actions_per_hour: int = 12 # per control
# The box trains its own models (boxtrain.py, methods.md B10): nightly at
# train_at (local time, HA's time zone), one series at a time in a child
# process at the lowest priority, each at most train_series_s and
# train_mem_mb (resident) and only with train_min_free_mb available; the
# history kept and trained on: train_days. pv_metered_since (ISO 8601, UTC; as the
# cloud's PV_METERED_SINCE): metered PV before it does not count.
train: bool = True
train_at: str = "03:30"
train_series_s: float = 300.0
train_mem_mb: int = 600
train_min_free_mb: int = 300
train_days: int = 396 # 13 months: recent weeks weigh more (fcmodels.recency_weights)
pv_metered_since: str = ""
def _read_secret(path: str) -> str:
@@ -94,6 +107,10 @@ def load(path: str | None, require_site: bool = True) -> Config:
cfg.dashboards = False
if env.get("CANTABILE_PROPOSE") in ("1", "true", "on"):
cfg.propose = True
if env.get("CANTABILE_TRAIN") in ("0", "false", "off"):
cfg.train = False
cfg.train_at = env.get("CANTABILE_TRAIN_AT", cfg.train_at)
cfg.pv_metered_since = env.get("CANTABILE_PV_METERED_SINCE", cfg.pv_metered_since)
if token_file:
cfg.ha_token = _read_secret(token_file)
# Inside a Supervisor add-on the Supervisor proxies Core's API and hands
+154 -5
View File
@@ -24,6 +24,15 @@ Commands (from the console through the link, which checked the signature):
"floor"?, "limit"? (empty: none), "until"?}
building {"v", "id", "building", "level", "control", "value", "from",
"until", "why"}: the building's requests (building.py)
preferences {"msg": JSON of preferences.apply_msg's {"set": {"item",
"value"}}, "level": "installer|operator", "by"}: an installer's or
Cantabile's setting within the owner's limits (preferences.py);
the owner's own come from Home Assistant and the panel
The owner's preferences (preferences.py): into every model state (a room's
schedule, a car's ready-by, a battery's reserve, the overrides), a plan at
once after a change (not held back by steady.py), their feedback and the
settings with the plan to the cloud (public_plan "preferences").
"""
import asyncio
import json
@@ -41,8 +50,10 @@ from .executor import Executor, building_msg
from .gridorder import GridOrders
from .guard import Guard
from .policy import Policy
from .preferences import Preferences
from .switchover import Switchover
from .inputs import Collector, History
from . import localml
log = logging.getLogger(__name__)
@@ -73,6 +84,7 @@ class Mode:
class Controller:
def __init__(self, cfg, mapping, consent, sink, clock=time.time):
self.cfg, self.mapping, self.sink, self.clock = cfg, mapping, sink, clock
self.consent = consent
self.controls = mapping.controls
self.inputs = mapping.inputs
self.mode = Mode(cfg.state_dir, cfg.mode)
@@ -108,6 +120,25 @@ class Controller:
self.on_plan = [] # callbacks (the announcer) when a plan is made
self.received: list[dict] = [] # the last commands and bundles (shown in HA, announce.py)
self.on_received = [] # callbacks (the announcer) for each
self.prefs = Preferences(cfg.state_dir, clock) # the owner's preferences (preferences.py)
self.on_prefs = [] # callbacks (the announcer) when they change
try:
self.prefs.write_view(self.controls, self.mode())
except Exception as e:
log.warning("preferences view: %s", e)
# the box trains its own models (boxtrain.py, methods.md B10): its
# history, the nightly job, and its models for the planner (bundles.own)
self.boxml = None
if getattr(cfg, "train", True):
try:
from .boxtrain import BoxTraining
from .trainhist import Store
self.boxml = BoxTraining(cfg, Store(os.path.join(cfg.state_dir, "training.db")), self.bundles,
kinds=self.planned_kinds, clock=clock)
self.bundles.own = self.boxml
except Exception as e: # never stops the controller
log.warning("the box's own training is off: %s", e)
self.boxml = None
# -- Home Assistant ----------------------------------------------------
def outside_entity(self) -> str:
@@ -181,6 +212,7 @@ class Controller:
try:
cfg = await client.send("get_config")
self.tz = (cfg or {}).get("time_zone") or "UTC"
self.prefs.tz = self.tz
lat, lon = (cfg or {}).get("latitude"), (cfg or {}).get("longitude")
if isinstance(lat, (int, float)) and isinstance(lon, (int, float)):
self.location = {"latitude": float(lat), "longitude": float(lon)}
@@ -215,6 +247,14 @@ class Controller:
value = st.get("s")
self.guard.observe(c.id, eid, value, now)
def planned_kinds(self) -> set[str]:
"""The kinds of the devices the box plans whose power is inside the
load (heating, cooling, EV): its base load is trained where any is
metered (inputs.controlled_power), as the planner uses it."""
if not (getattr(self.inputs, "controlled_power", None) or []):
return set()
return {c.kind for c in self.controls if c.kind in ("heating", "cooling", "ev")}
# -- plans ---------------------------------------------------------------
def model_state(self) -> dict:
now = self.clock()
@@ -223,13 +263,15 @@ class Controller:
# and of a thermostat that heats and cools only its active side
self.switch_sides()
controls = [c for c in self.controls if self.policy.plannable(c) and self.switchover.active(c)]
return inputs_mod.model_state(now, controls, self.inputs, self.col, self.bundles,
state = inputs_mod.model_state(now, controls, self.inputs, self.col, self.bundles,
self.history, self.tz, self.forecast, self.guard.extra_limits,
building=self.building.for_model(start, inputs_mod.STEP_S, inputs_mod.STEPS),
location=self.location, shade_ctx=self.shade_ctx(),
grid_order=self.gridorders.for_model(start, inputs_mod.STEP_S, inputs_mod.STEPS),
grid_max_w=getattr(self.cfg, "grid_max_w", None),
ev_phases=self.ev_phases())
# the owner's preferences per step (preferences.py)
return self.prefs.apply(state, {c.id: c for c in controls}, self.col, now)
async def make_plan(self) -> dict | None:
self.forecast = []
@@ -239,6 +281,7 @@ class Controller:
except Exception as e:
log.info("weather forecast: %s", e)
state = self.model_state()
cands, net_load = state.pop("_cands", None) or {}, state.pop("_net_load", None)
plan = await solve(state, self.cfg.solve_timeout_s)
if "error" in plan:
self.plan_error = plan["error"]
@@ -247,12 +290,25 @@ class Controller:
calm, self.calm = self.calm, False
if calm:
steady.carry_quarter(plan, self.plan, self.clock())
# changes that do not matter stay out of it (deadband, hold, gain: steady.py)
# what the plan does for each preference, and what each must costs
self.prefs.take_plan(plan, self.clock())
# changes that do not matter stay out of it (deadband, hold, gain: steady.py);
# the owner's preference changes are not held back
steady.stabilise(plan, state, [c for c in self.controls if c.id in plan.get("devices", {})], self.col,
self.guard.last_change, self.clock())
self.guard.last_change, self.clock(), free=self.prefs.free())
self.prefs.write_view(self.controls, self.mode())
plan["mode"] = self.mode()
plan["id"] = f"{self.cfg.site_id[:8]}-{plan['made']}"
plan["model"] = self.bundles.model_id()
if self.boxml is not None:
try: # SQLite and numpy: in a worker thread, the loop goes on
ml = await asyncio.to_thread(self.ml_after_plan, state, plan, cands, net_load)
# upward only with the owner's share consent: without it the box
# trains for itself and nothing of it leaves (no forecasts, no errors)
if getattr(self.consent, "share", False):
plan["ml"] = ml
except Exception as e:
log.warning("the box's model figures: %s", e)
self.plan, self.plan_error = plan, ""
self.executor.plan = plan
self.executor.inputs_ok = self.inputs_fresh()
@@ -276,7 +332,82 @@ class Controller:
# for the building (shed_w/grid_w per step, 24 h): absent from older agents
"flex": plan.get("flex", {}),
# the market's contracts in the plan (shadow): planned kWh and saving vs the grid
"market": plan.get("market", [])}
"market": plan.get("market", []),
# the owner's preferences: per level who set what and when, the
# overrides, and per preference met or not and each must's cost
"preferences": self.prefs.public(),
# the box's own models (boxtrain.py): held-out and live errors per source,
# hourly the forecasts it planned on; absent from older agents
**({"ml": plan["ml"]} if plan.get("ml") else {})}
# -- the owner's preferences (preferences.py) -------------------------------
async def prefs_change(self, msg: dict, level: str, by: str) -> tuple[bool, str]:
"""A change of the preferences (Home Assistant's entities and the
panel: the owner's; the console: an installer's or operator's): a
plan with it at once (not held back: steady.py), shown in HA."""
ok, why = self.prefs.apply_msg(msg, {c.id: c for c in self.controls}, self.tz, level, by)
log.info("preferences (%s, %s): %s %s", by, level, "applied" if ok else "refused", why)
await self._prefs_changed()
return ok, why
async def _prefs_changed(self):
self.prefs.write_view(self.controls, self.mode())
self.wake.set()
for cb in self.on_prefs:
try:
await cb()
except Exception as e:
log.warning("preferences: %s", e)
async def _prefs_watch(self, stop):
"""The panel's changes (prefs-inbox/), and overrides that expire."""
while not stop.is_set():
try:
n = self.prefs.take_inbox({c.id: c for c in self.controls}, self.tz)
if n or self.prefs.prune(self.clock()):
await self._prefs_changed()
except Exception as e:
log.warning("preferences: %s", e)
try:
await asyncio.wait_for(stop.wait(), PREFS_POLL_S)
except asyncio.TimeoutError:
pass
def ml_after_plan(self, state: dict, plan: dict, cands: dict, net_load) -> dict:
"""After a plan (in a worker thread): the weather the box forecast
with and each source's forecast kept for the training and the live
error (trainhist.py), and the figures for the plan's "ml"
(boxtrain.status), with, at most hourly, the forecasts it planned on:
the load (as planned: the base load where it plans devices), PV and
the member's net draw (the plan's import less export where it acts,
auto; else the metered load's forecast less PV) for the building."""
b, now, start = self.boxml, self.clock(), int(state["start"])
st = b.store
if self.forecast:
st.put_weather("forecast", "ha", localml.ha_hours(self.forecast), now)
m = self.bundles.model.get()
if isinstance(m, dict) and m.get("site") == self.cfg.site_id:
if m.get("weather"):
st.put_weather("forecast", "cloud", localml.cloud_hours(m["weather"]), m.get("issued"))
if m.get("weather_past"):
st.put_weather("observed", "cloud", localml.cloud_hours(m["weather_past"]), now)
for key, series in (("load", "plan"), ("pv", "pv")):
if cands.get(key):
b.record(series, start, cands[key], now)
auto = plan.get("mode") == "auto"
net = (plan.get("flex") or {}).get("net_w") if auto else \
([round(lo - pv) for lo, pv in zip(net_load, state["pv_w"])] if net_load else None)
if net:
b.record("net", start, {"own": net, "week_mean": b.week_mean("net", start, len(net))}, now)
fsrc = (state.get("sources") or {}).get("forecast") or {}
ups = {k: {"start": start, "step_s": state["step_s"], "values": [round(float(v), 1) for v in state[f"{k}_w"]],
"source": (fsrc.get(k) or {}).get("source") or (state.get("sources") or {}).get(k, "")[:80]}
for k in ("load", "pv")}
ups["load"]["series"] = "base_load" if b.kinds() else "load"
if net:
ups["net"] = {"start": start, "step_s": state["step_s"], "values": [float(v) for v in net],
"source": "plan" if auto else "forecast"}
return b.status(now, ups)
def inputs_fresh(self) -> bool:
"""The plan's room temperatures and the battery's state have a value
@@ -341,7 +472,12 @@ class Controller:
await stop.wait()
return
tasks = [asyncio.create_task(self._planner(stop)), asyncio.create_task(self._actor(stop)),
asyncio.create_task(self._bundle_watch(stop))]
asyncio.create_task(self._bundle_watch(stop)), asyncio.create_task(self._prefs_watch(stop))]
if self.boxml is not None:
# the nightly training (boxtrain.py): child processes, the loop only awaits them
tasks.append(asyncio.create_task(self.boxml.run(
stop, tz=lambda: self.tz, location=lambda: self.location, client=lambda: self.client,
weather_entity=lambda: self.inputs.weather or "")))
await stop.wait()
await asyncio.gather(*tasks, return_exceptions=True)
@@ -472,6 +608,18 @@ class Controller:
return ok, msg
if cmd == "building":
return await self.building_command(args, now)
if cmd == "preferences":
# the console's: an installer's or Cantabile's, within the owner's limits
level = args.get("level", "")
if level not in ("installer", "operator"):
return False, "level: installer or operator (the owner's own come from Home Assistant or the panel)"
try:
msg = json.loads(args.get("msg") or "{}")
except ValueError:
return False, "msg: JSON"
if not isinstance(msg, dict) or "set" not in msg:
return False, "msg: {\"set\": {\"item\", \"value\"}} (overrides are the owner's)"
return await self.prefs_change(msg, level, f"{level}:{str(args.get('by') or 'console')[:60]}")
return False, f"unknown command {cmd!r}"
async def building_command(self, args: dict, now: float) -> tuple[bool, str]:
@@ -668,6 +816,7 @@ async def solve(state: dict, timeout: float) -> dict:
PLAN_AT_S = 5.0 # a plan this long after a quarter hour: its first step is the quarter just begun
PREFS_POLL_S = 2.0 # how often the panel's preference changes are looked for
def plan_wait(now: float, interval: float, at: float = PLAN_AT_S) -> float:
+218 -32
View File
@@ -10,15 +10,21 @@ metered). The kinds:
week (168 values; the box's fallback format, ml/model_bundle.py).
The baseline every other kind has to beat.
gbm LightGBM on calendar (local hour, weekday, a day off: weekend or
a German public holiday), the hour-of-week value, the same slot a
a German public holiday, the special day's type as a categorical
feature, the yearly rhythm: day of year as sine and cosine, the
day's daylight hours), the hour-of-week value, the same slot a
week before, the weather (temperature, irradiance, cloud cover)
and, for PV, the sun and the physical prior. Small (150 trees, 15
leaves, one thread): hundreds of sites a night.
leaves, one thread): hundreds of sites a night. Recent weeks weigh
more (recency_weights); a model's own feature list is kept with
it, so a model trained before a feature was added still runs.
pvphys PV only: the physical irradiance model (pvforecast.py, shared
with the Mieterstrom pricing) times the site's calibration
(pvcalib.py).
A model is a JSON-able dict ("params"); fit_* makes it, predict() uses it.
intraday() carries the last hours' error against the meter into the next
hours (the box every 15 minutes; methods.md B10).
The weather per slot comes from hourly records (pvforecast.Hour: the hour
ENDING at `end`, solar in kWh/m² over that hour, cloud %, temp °C)."""
from __future__ import annotations
@@ -45,8 +51,23 @@ GBM_PARAMS = dict(objective="regression", learning_rate=0.05, num_leaves=15, min
bagging_freq=1, feature_fraction=0.9, lambda_l2=1.0, num_threads=1, verbose=-1, seed=1)
GBM_ROUNDS = 150
MIN_GBM_SLOTS = 96 # a day of data at least
LOAD_FEATURES = ["hod", "dow", "off", "how", "lag7", "temp", "ghi", "cloud", "hdd"]
PV_FEATURES = ["hod", "how", "lag7", "temp", "ghi", "cloud", "elev", "az", "phys"]
LOAD_FEATURES = ["hod", "dow", "off", "how", "lag7", "temp", "ghi", "cloud", "hdd", "special", "doy_sin", "doy_cos",
"daylight"]
PV_FEATURES = ["hod", "how", "lag7", "temp", "ghi", "cloud", "elev", "az", "phys", "doy_sin", "doy_cos", "daylight"]
FEATURES_V1 = {"load": ["hod", "dow", "off", "how", "lag7", "temp", "ghi", "cloud", "hdd"], # before 2026-11-04
"pv": ["hod", "how", "lag7", "temp", "ghi", "cloud", "elev", "az", "phys"]}
CATEGORICAL = ("special",)
# linear leaves (LightGBM linear_tree: each leaf a linear model on its
# branch's features, so the forecast keeps rising with heating degrees past
# the coldest day trained on): for these series only where it measured better
# (measured 2026-11-04, building's synthetic site: a cold snap 12 K below the
# coldest day trained on, held-out nMAE 29.5 % -> 6.4 %; an ordinary winter
# week 3.8 % -> 3.3 %; methods.md B10)
LINEAR_SERIES: tuple = ("load", "base_load")
# recency (methods.md B10): a slot's weight max(0.5^(age/30 d), 0.25 x the
# season's likeness (cos of the age over a year, last year's same weeks
# count), 0.05)
HALF_LIFE_D, SEASON_W, FLOOR_W = 30.0, 0.25, 0.05
# ---- the calendar -------------------------------------------------------------
@@ -74,12 +95,91 @@ def holidays_de(year: int) -> frozenset:
return frozenset(fixed + moving)
def calendar(t: np.ndarray, tz: str) -> dict[str, np.ndarray]:
# The special days' types (one categorical feature, "special"; the trees
# combine it with the hour of day). A type the site has under 2 days of in
# its training data counts as normal (fit_gbm: params["special_kept"]): the
# cloud's teacher model will give such a type its shape later.
SPECIAL = ("normal", "christmas_eve", "christmas_day", "boxing_day", "new_years_eve", "new_years_day", "easter",
"carnival", "bridge_day", "school_holiday", "dst_change", "holiday")
SPECIAL_CODE = {k: i for i, k in enumerate(SPECIAL)}
# carnival Monday and Tuesday where it is kept (a site's state unknown: everywhere)
CARNIVAL_STATES = frozenset({"NW", "RP", "HE", "BW", "BY", "SL"})
# School holidays per German state: {state: [(first day, last day), ...]}.
# A hook: no compact source in the repository yet (the KMK's yearly tables);
# empty, so no day is typed school_holiday until it is filled.
SCHOOL_HOLIDAYS: dict[str, list[tuple[date, date]]] = {}
def _last_sunday(year: int, month: int) -> date:
d = date(year, month + 1, 1) - timedelta(days=1) if month < 12 else date(year, 12, 31)
return d - timedelta(days=(d.weekday() + 1) % 7)
@lru_cache(maxsize=64)
def special_days(year: int, state: str | None = None) -> dict:
"""{date: type code} for the year's special days (SPECIAL); the rest
normal."""
e = _easter(year)
hol = holidays_de(year) | holidays_de(year - 1) | holidays_de(year + 1)
out: dict[date, int] = {}
if state in SCHOOL_HOLIDAYS:
for a, b in SCHOOL_HOLIDAYS[state]:
d = a
while d <= b:
if d.year == year:
out[d] = SPECIAL_CODE["school_holiday"]
d += timedelta(days=1)
for m in (3, 10):
out[_last_sunday(year, m)] = SPECIAL_CODE["dst_change"]
d = date(year, 1, 1)
while d.year == year: # bridge days: a working day between a holiday and a weekend
if d.weekday() < 5 and d not in hol:
prev, nxt = d - timedelta(days=1), d + timedelta(days=1)
off_prev, off_next = prev.weekday() >= 5 or prev in hol, nxt.weekday() >= 5 or nxt in hol
if (prev in hol and off_next) or (nxt in hol and off_prev):
out[d] = SPECIAL_CODE["bridge_day"]
d += timedelta(days=1)
for x in holidays_de(year):
out[x] = SPECIAL_CODE["holiday"]
if state is None or state in CARNIVAL_STATES:
out[e - timedelta(days=48)] = out[e - timedelta(days=47)] = SPECIAL_CODE["carnival"]
for x in (e - timedelta(days=2), e, e + timedelta(days=1)):
out[x] = SPECIAL_CODE["easter"]
out.update({date(year, 12, 24): SPECIAL_CODE["christmas_eve"], date(year, 12, 25): SPECIAL_CODE["christmas_day"],
date(year, 12, 26): SPECIAL_CODE["boxing_day"], date(year, 12, 31): SPECIAL_CODE["new_years_eve"],
date(year, 1, 1): SPECIAL_CODE["new_years_day"]})
return out
def daylight_h(doy: np.ndarray, lat: float | None) -> np.ndarray:
"""The day's hours of daylight at a latitude (the sun's declination by
day of year; polar day and night clipped), NaN without a location."""
if lat is None:
return np.full(len(doy), np.nan)
decl = np.radians(23.44) * np.sin(2 * np.pi * (284 + doy) / 365.0)
x = np.clip(-np.tan(np.radians(lat)) * np.tan(decl), -1.0, 1.0)
return 24.0 * np.degrees(np.arccos(x)) / 180.0
def recency_weights(t: np.ndarray, end: int | None = None) -> np.ndarray:
"""A slot's weight by its age before end (the last slot): recent weeks
weigh most, last year's same season still counts (HALF_LIFE_D,
SEASON_W, FLOOR_W)."""
if not len(t):
return np.zeros(0)
end = int(t[-1]) if end is None else int(end)
age = (end - t.astype(float)) / 86400.0
return np.maximum.reduce([0.5 ** (age / HALF_LIFE_D), SEASON_W * np.maximum(np.cos(2 * np.pi * age / 365.25), 0.0),
np.full(len(t), FLOOR_W)])
def calendar(t: np.ndarray, tz: str, state: str | None = None) -> dict[str, np.ndarray]:
"""Per slot: local hour of day (fractional), weekday (Monday 0), hour of
the week, day of year, and whether it is a day off."""
the week, day of year, whether it is a day off, and the special day's
type (SPECIAL's code)."""
zone = ZoneInfo(tz or "UTC")
n = len(t)
hod, dow, how, doy, off = (np.zeros(n) for _ in range(5))
hod, dow, how, doy, off, special = (np.zeros(n) for _ in range(6))
cache: dict[int, tuple] = {}
for i, ts in enumerate(t.tolist()):
hour_key = int(ts) // 3600
@@ -87,12 +187,13 @@ def calendar(t: np.ndarray, tz: str) -> dict[str, np.ndarray]:
if c is None:
d = datetime.fromtimestamp(hour_key * 3600, timezone.utc).astimezone(zone)
c = (d.hour, d.weekday(), d.timetuple().tm_yday,
1.0 if d.weekday() >= 5 or d.date() in holidays_de(d.year) else 0.0)
1.0 if d.weekday() >= 5 or d.date() in holidays_de(d.year) else 0.0,
special_days(d.year, state).get(d.date(), 0))
cache[hour_key] = c
h, wd, dy, of = c
h, wd, dy, of, sp = c
hod[i] = h + (int(ts) % 3600) / 3600.0
dow[i], how[i], doy[i], off[i] = wd, wd * 24 + h, dy, of
return {"hod": hod, "dow": dow, "how": how, "doy": doy, "off": off}
dow[i], how[i], doy[i], off[i], special[i] = wd, wd * 24 + h, dy, of, sp
return {"hod": hod, "dow": dow, "how": how, "doy": doy, "off": off, "special": special}
# ---- the weather per slot -------------------------------------------------------
@@ -183,14 +284,16 @@ def metrics(y: np.ndarray, p: np.ndarray) -> dict:
# ---- how: the hour-of-week profile -----------------------------------------------
def fit_how(t: np.ndarray, y: np.ndarray, tz: str) -> dict:
def fit_how(t: np.ndarray, y: np.ndarray, tz: str, weights: np.ndarray | None = None) -> dict:
"""weights: per slot (recency_weights); None: all alike."""
cal = calendar(t, tz)
vals: list[float | None] = []
have = ~np.isnan(y)
overall = float(y[have].mean()) if have.any() else 0.0
wt = np.ones(len(y)) if weights is None else np.asarray(weights, dtype=float)
overall = float(np.average(y[have], weights=wt[have])) if have.any() and wt[have].sum() > 0 else 0.0
for h in range(168):
m = have & (cal["how"] == h)
vals.append(float(y[m].mean()) if m.any() else None)
vals.append(float(np.average(y[m], weights=wt[m])) if m.any() and wt[m].sum() > 0 else None)
out = []
for h, v in enumerate(vals):
if v is None: # the same hour on the other weekdays, else the mean
@@ -217,27 +320,65 @@ def _lag7(t: np.ndarray, y: np.ndarray, hist_t: np.ndarray | None, hist_y: np.nd
def features(series: str, t: np.ndarray, w: Weather, how_p: dict, tz: str, lag7: np.ndarray,
phys: np.ndarray | None) -> tuple[np.ndarray, list[str]]:
cal = calendar(t, tz)
phys: np.ndarray | None, names: list[str] | None = None, lat: float | None = None,
state: str | None = None, kept: list | None = None) -> tuple[np.ndarray, list[str]]:
"""The feature matrix; names: a model's own list (default: the
series' current one). kept: the special days' types the model learnt
(the others count as normal)."""
cal = calendar(t, tz, state)
special = cal["special"]
if kept is not None:
special = np.where(np.isin(special, np.asarray(list(kept) + [0], dtype=float)), special, 0.0)
ang = 2 * np.pi * (cal["doy"] - 1) / 365.25
cols = {"hod": cal["hod"], "dow": cal["dow"], "off": cal["off"], "doy": cal["doy"],
"how": how_values(how_p, t, tz), "lag7": lag7, "temp": w.temp, "ghi": w.ghi, "cloud": w.cloud,
"hdd": np.maximum(15.0 - w.temp, 0.0), "elev": w.elev, "az": w.az,
"phys": phys if phys is not None else np.full(len(t), np.nan)}
names = PV_FEATURES if series == "pv" else LOAD_FEATURES
"phys": phys if phys is not None else np.full(len(t), np.nan), "special": special,
"doy_sin": np.sin(ang), "doy_cos": np.cos(ang), "daylight": daylight_h(cal["doy"], lat)}
names = list(names) if names is not None else (PV_FEATURES if series == "pv" else LOAD_FEATURES)
return np.column_stack([cols[c] for c in names]), names
def special_kept(t: np.ndarray, y: np.ndarray, tz: str, state: str | None = None, at_least: int = 2) -> list[int]:
"""The special days' types with at least 2 metered days in (t, y)."""
cal = calendar(t, tz, state)
have = ~np.isnan(y)
days: dict[int, set] = {}
# a local day by its local midnight (a day spans two UTC dates)
for sp, ts, h in zip(cal["special"][have].tolist(), t[have].tolist(), cal["hod"][have].tolist()):
if sp:
days.setdefault(int(sp), set()).add(round((int(ts) - h * 3600) / 86400))
return sorted(k for k, v in days.items() if len(v) >= at_least)
def fit_gbm(series: str, t: np.ndarray, y: np.ndarray, w: Weather, tz: str, lag7: np.ndarray,
phys: np.ndarray | None) -> dict | None:
phys: np.ndarray | None, lat: float | None = None, state: str | None = None,
weights: np.ndarray | None = None, linear: bool | None = None,
names: list[str] | None = None) -> dict | None:
"""weights: per slot (recency_weights; None: all alike); linear: leaves
with linear models (default: series in LINEAR_SERIES); names: the
features (default: the series' current list)."""
import lightgbm as lgb
have = ~np.isnan(y)
if have.sum() < MIN_GBM_SLOTS:
return None
how_p = fit_how(t, y, tz)
X, names = features(series, t, w, how_p, tz, lag7, phys)
m = lgb.train(GBM_PARAMS, lgb.Dataset(X[have], y[have], feature_name=names), num_boost_round=GBM_ROUNDS)
return {"kind": "gbm", "model": m.model_to_string(), "features": names, "how": how_p["values"], "tz": tz,
how_p = fit_how(t, y, tz, weights)
kept = special_kept(t, y, tz, state)
X, names = features(series, t, w, how_p, tz, lag7, phys, names, lat, state, kept)
params = dict(GBM_PARAMS)
if linear if linear is not None else series in LINEAR_SERIES:
params["linear_tree"] = True
cat = [c for c in CATEGORICAL if c in names]
ds = lgb.Dataset(X[have], y[have], feature_name=names, categorical_feature=cat or "auto",
weight=None if weights is None else np.asarray(weights, dtype=float)[have])
m = lgb.train(params, ds, num_boost_round=GBM_ROUNDS)
out = {"kind": "gbm", "model": m.model_to_string(), "features": names, "how": how_p["values"], "tz": tz,
"cap_w": float(np.nanpercentile(y[have], 99.9)) * 1.2 if have.any() else None}
if "special" in names:
out["special_kept"] = kept
if params.get("linear_tree"):
out["linear"] = True
return out
@lru_cache(maxsize=256)
@@ -269,11 +410,13 @@ def box_params(params: dict) -> dict:
def predict_gbm(params: dict, series: str, t: np.ndarray, w: Weather, lag7: np.ndarray,
phys: np.ndarray | None) -> np.ndarray:
phys: np.ndarray | None, lat: float | None = None, state: str | None = None) -> np.ndarray:
tz = params.get("tz", "UTC")
X, names = features(series, t, w, {"values": params["how"]}, tz, lag7, phys)
if names != params["features"]:
raise ValueError("gbm: the features changed since it was trained")
names = params["features"]
if not isinstance(names, list) or any(n not in FEATURE_NAMES for n in names):
raise ValueError("gbm: features this code does not know")
X, names = features(series, t, w, {"values": params["how"]}, tz, lag7, phys, names, lat, state,
params.get("special_kept"))
p = np.asarray(_predictor(params).predict(X), dtype=float)
p = np.maximum(p, 0.0)
if params.get("cap_w"):
@@ -283,6 +426,9 @@ def predict_gbm(params: dict, series: str, t: np.ndarray, w: Weather, lag7: np.n
return p
FEATURE_NAMES = frozenset(LOAD_FEATURES) | frozenset(PV_FEATURES) | {"doy"}
# ---- one entry point -----------------------------------------------------------
@dataclass
@@ -296,6 +442,7 @@ class Site:
tilt: float = 30.0
azimuth: float = 180.0
pvgis_monthly: list | None = None # PVGIS's kWh per kWp per month (pvcalib.pvgis_monthly)
state: str | None = None # the German state (NW, BY, ...): carnival, school holidays; None: unknown
def phys_prior(t: np.ndarray, w: Weather, s: Site) -> np.ndarray | None:
@@ -309,12 +456,13 @@ def phys_prior(t: np.ndarray, w: Weather, s: Site) -> np.ndarray | None:
def fit(kind: str, series: str, t: np.ndarray, y: np.ndarray, w: Weather, s: Site,
hist_t: np.ndarray | None = None, hist_y: np.ndarray | None = None) -> dict | None:
"""A model of this kind on (t, y), or None where the kind cannot be made
(too little data, PV physics without a location or weather)."""
(too little data, PV physics without a location or weather). Recent
slots weigh more (recency_weights; pvcalib has its own)."""
if kind == "how":
return fit_how(t, y, s.tz) if (~np.isnan(y)).any() else None
return fit_how(t, y, s.tz, recency_weights(t)) if (~np.isnan(y)).any() else None
if kind == "gbm":
prior = phys_prior(t, w, s) if series == "pv" else None
return fit_gbm(series, t, y, w, s.tz, _lag7(t, y, hist_t, hist_y), prior)
return fit_gbm(series, t, y, w, s.tz, _lag7(t, y, hist_t, hist_y), prior, s.lat, s.state, recency_weights(t))
if kind == "pvphys":
if series != "pv":
return None
@@ -339,9 +487,9 @@ def predict(params: dict, series: str, t: np.ndarray, w: Weather, s: Site,
if kind == "gbm":
prior = phys_prior(t, w, s) if series == "pv" else None
nan = np.full(len(t), np.nan)
return predict_gbm(params, series, t, w, _lag7(t, nan, hist_t, hist_y), prior)
return predict_gbm(params, series, t, w, _lag7(t, nan, hist_t, hist_y), prior, s.lat, s.state)
if kind == "pvphys":
ss = Site(s.tz, s.lat, s.lon, s.kwp, params.get("tilt", s.tilt), params.get("azimuth", s.azimuth))
ss = Site(s.tz, s.lat, s.lon, s.kwp, params.get("tilt", s.tilt), params.get("azimuth", s.azimuth), state=s.state)
prior = phys_prior(t, w, ss)
if prior is None:
return np.full(len(t), np.nan)
@@ -374,3 +522,41 @@ def choose(challengers: dict[str, float | None], champion: float | None) -> str
if champion is not None and math.isfinite(champion) and scored[best] >= champion:
return None
return best
# ---- the intraday correction ------------------------------------------------------
INTRADAY_WINDOW_H = 3.0 # the recent error over the last 3 hours,
INTRADAY_RECENT_H = 1.0 # the latest weighing most (half-life 1 h: an evening that starts at 18:00 shows by 19:00)
INTRADAY_HALF_LIFE_H = 2.0 # fading with the step ahead
INTRADAY_PV_RATIO = (0.5, 1.5) # PV: the metered over the forecast energy, capped
def intraday(series: str, pred: np.ndarray, past_pred: np.ndarray, past_y: np.ndarray,
half_life_h: float = INTRADAY_HALF_LIFE_H) -> tuple[np.ndarray, float]:
"""The forecast for the next steps (pred, the first the slot under way
or the next) moved by the last hours' error against the meter
(past_pred, past_y: the same slots, NaN where unknown; at least an hour
of both; the latest weighing most, half-life 1 h), fading with the step
ahead (half-life 2 h): load additive (W),
PV multiplicative (the ratio of energies, capped 0.5..1.5; none at
night). Returns the corrected forecast and the error carried (W, or the
PV ratio less 1). It catches any unusual day: a special day, guests,
illness."""
pred = np.asarray(pred, dtype=float)
past_pred, past_y = np.asarray(past_pred, dtype=float), np.asarray(past_y, dtype=float)
m = ~(np.isnan(past_pred) | np.isnan(past_y))
if m.sum() < 4:
return pred, 0.0
decay = 0.5 ** (np.arange(len(pred)) * (SLOT_S / 3600.0) / half_life_h)
# the past slots' weights: the latest most (the last slot ends where pred starts)
age_h = (len(past_y) - np.arange(len(past_y)) - 0.5) * (SLOT_S / 3600.0)
wt = np.where(m, 0.5 ** (age_h / INTRADAY_RECENT_H), 0.0)
if series == "pv":
e_pred, e_y = float(np.sum(wt * np.nan_to_num(past_pred))), float(np.sum(wt * np.nan_to_num(past_y)))
if e_pred < 50.0 * float(wt.sum()): # under 50 W on average: night or dusk, nothing to scale
return pred, 0.0
r = min(max(e_y / e_pred, INTRADAY_PV_RATIO[0]), INTRADAY_PV_RATIO[1]) - 1.0
return np.maximum(pred * (1.0 + r * decay), 0.0), r
e = float(np.sum(wt * np.nan_to_num(past_y - past_pred)) / wt.sum())
return np.maximum(pred + e * decay, 0.0), e
+1
View File
@@ -37,6 +37,7 @@ READ_ONLY = frozenset({
"config/area_registry/list",
"recorder/statistics_during_period",
"recorder/list_statistic_ids",
"history/history_during_period", # the weather entity's past, for the box's own training (boxtrain.py)
"ping",
})
+91 -9
View File
@@ -163,37 +163,74 @@ def base_load(col: Collector, inputs) -> float | None:
return max(0.0, sum(vals) - ctl)
FC_SOURCES = {"own": "the box's own model", "cloud": "the cloud's series", "box": "the cloud's model on the box",
"hour_of_week": "the hour-of-week profile"}
def _model_series(m, start, bundles, history, tz, box) -> tuple[list[float], dict] | None:
"""The model bundle's series m for the horizon: the cloud's forecast
where its series covers a step, else the box's own model (localml.py;
"""A model series m for the horizon, per step the best fresh source
(methods.md B10): the box's own model where it is in force and has a
value (boxtrain.py: bundles.own), else the cloud's forecast where its
series covers the step, else the cloud's model on the box (localml.py;
box: HA's weather forecast and its entity) where it has a value, else
the hour-of-week profile; and how many steps each gave."""
the hour-of-week profile; how many steps each gave, the source that
gave most ("source"), and every source's values ("_cands": the live
error per source, taken out before the plan)."""
trainer = getattr(bundles, "own", None) if bundles is not None else None
mine = None
if trainer is not None and box is not None:
try:
mine = trainer.forecast(m, start, STEPS, box[0], box[1])
except Exception as e: # never stops a plan
log.warning("the box's own %s model: %s", m, e)
if not hasattr(bundles, "forecast_parts"):
s = bundles.forecast(m, start, STEP_S, STEPS, tz)
return None if s is None else (s, {})
parts = bundles.forecast_parts(m, start, STEP_S, STEPS, tz)
if parts is None:
# no cloud model for it: the box's own alone, where it covers the horizon
if mine is None or any(v is None for v in mine[0]):
return None
vals = [float(v) for v in mine[0]]
return vals, {"series": m, "own": STEPS, "cloud": 0, "box": 0, "hour_of_week": 0, "own_model": mine[1].get("kind"),
"source": "own", "_cands": {"own": vals}}
vals = [v for v, _ in parts]
info = {"series": m, "cloud": sum(src == "series" for _, src in parts), "box": 0, "hour_of_week": 0}
info = {"series": m, "own": 0, "cloud": 0, "box": 0, "hour_of_week": 0}
cloud_series = [v if src == "series" else None for v, src in parts]
own = None
if box is not None and info["cloud"] < STEPS:
if box is not None and (trainer is not None or not all(v is not None for v in cloud_series)):
own = bundles.box_forecast(m, start, STEPS, history, box[0], box[1])
for k, (v, src) in enumerate(parts):
if src == "series":
continue
o = mine[0][k] if mine is not None else None
b = own[0][k] if own is not None else None
if b is not None:
if o is not None:
vals[k] = o
info["own"] += 1
elif src == "series":
info["cloud"] += 1
elif b is not None:
vals[k] = b
info["box"] += 1
else:
info["hour_of_week"] += 1
if mine is not None:
info["own_model"] = mine[1].get("kind")
if mine[1].get("weather"):
info["own_weather"] = mine[1]["weather"]
if own is not None:
info["box_model"] = own[1].get("kind")
if own[1].get("weather"):
info["box_weather"] = own[1]["weather"]
if own[1].get("error"):
info["box_error"] = own[1]["error"]
info["source"] = max(("own", "cloud", "box", "hour_of_week"), key=lambda k: info[k])
if trainer is not None:
info["_cands"] = {"own": list(mine[0]) if mine is not None else None, "cloud": cloud_series,
"cloud_model": list(own[0]) if own is not None else None,
"hour_of_week": (bundles.profile(m, start, STEP_S, STEPS, tz) if hasattr(bundles, "profile")
else [v if src == "profile" else None for v, src in parts])}
if not info["own"]:
info.pop("own")
return vals, info
@@ -207,6 +244,9 @@ def _series(name, start, bundles, history, tz, now_value, default, model_names=N
if got is not None:
s, n = got
src = "model" if m == name else f"model ({m})"
if n.get("own"):
src += (f"; the box's own {n.get('own_model')} model (trained on the box) for {n['own']} of {STEPS} steps"
f" (weather: {n.get('own_weather') or 'none'})")
if n.get("box"):
src += (f"; the box's own {n.get('box_model')} model for {n['box']} of {STEPS} steps"
f" (weather: {n.get('box_weather') or 'none'})")
@@ -326,6 +366,39 @@ def model_state(now: float, controls, inputs, col: Collector, bundles, history,
bundles.forecast("load_w", start, STEP_S, 1, tz) is not None:
load_src += " (the model's load_w includes the planned devices: not used)"
pv, pv_src = _series("pv_w", start, bundles, history, tz, None, 0.0, None, box, fc_src["pv"])
# the forecasts per source for the live error (boxtrain.py), not for the plan
cands = {k: v.pop("_cands") for k, v in fc_src.items() if "_cands" in v}
net_load = None
intraday = {}
trainer = getattr(bundles, "own", None)
if trainer is not None:
# the intraday correction (fcmodels.intraday, methods.md B10): the last
# 3 hours' error of what was planned on against the meter, carried into
# the next hours and fading (half-life 2 h); load in W, PV as a ratio
for key, series in (("load", "plan"), ("pv", "pv")):
base = load if key == "load" else pv
c = cands.setdefault(key, {})
c["used"] = list(base)
try:
new, e = trainer.correct(series, start, base)
except Exception as ex: # never stops a plan
log.warning("intraday correction %s: %s", key, ex)
continue
c["corrected"] = list(new)
intraday[key] = round(float(e), 3)
if key == "load":
load = new
else:
pv = new
if trainer is not None:
# the house as metered (devices in), for the member's net draw the box
# publishes upward where it does not act itself (controller.forecasts_up)
if ctl:
nl: dict = {}
got = _series("load_w", start, bundles, history, tz, None, 500.0, ("load_w",), box, nl)
net_load = got[0] if got[1].startswith("model") else None
else:
net_load = load
t_out = _hourly(weather_forecast or [], "temperature", start, STEPS)
t_src = "weather forecast"
measured = col.value(inputs.outside_temp) if inputs.outside_temp else None
@@ -390,6 +463,7 @@ def model_state(now: float, controls, inputs, col: Collector, bundles, history,
"sources": {"load": load_src, "pv": pv_src, "t_out": t_src, "model": bundles.model_id(),
"sky": "weather forecast" if clouds else "clear", "rooms": rooms_src, "ghi": ghi_src,
"ev_phases": phases_src,
**({"intraday": intraday} if intraday else {}),
# per series the steps from the cloud's forecast, the box's own model
# (its kind and weather) and the hour-of-week profile (methods.md B9)
"forecast": {k: v for k, v in fc_src.items() if v}},
@@ -398,6 +472,9 @@ def model_state(now: float, controls, inputs, col: Collector, bundles, history,
"market": bundles.market_for_model(start, STEP_S, STEPS, now) if hasattr(bundles, "market_for_model") else []}
if grid_max_w:
out["grid_max_w"] = float(grid_max_w)
if cands or net_load is not None:
out["_cands"] = cands # taken out by the controller before the plan
out["_net_load"] = net_load
# a shade's moves by rules on the sun and its room (shades.py); the
# optimiser takes them as they are
from . import shades
@@ -437,6 +514,11 @@ def _room_rc(c, params: dict, bundles, only_one: bool, now: float) -> str:
heat is cop * P). Into params; where they came from."""
if "r_k_per_kw" in c.params or "c_kwh_per_k" in c.params:
return "mapping"
# the box's own fit where it is in force (boxtrain.py), else the cloud's
own = getattr(bundles, "own", None)
rc = own.room_rc(c.id, str(c.params.get("area") or ""), only_one) if own is not None else None
src = "box model" if rc is not None else "model"
if rc is None:
get = getattr(bundles, "room_rc", None)
rc = get(c.id, str(c.params.get("area") or ""), only_one, now) if get else None
if rc is None:
@@ -456,7 +538,7 @@ def _room_rc(c, params: dict, bundles, only_one: bool, now: float) -> str:
params["internal_gain_w"] = gain
if "solar_m2" not in c.params:
params["solar_m2"] = solar
return "model"
return src
def _quantities(c) -> list[str]:
+129 -26
View File
@@ -17,7 +17,19 @@ are.
Trees(enc, n_features).predict(rows) sums the leaves in LightGBM's order
with its decision rule (tree.h NumericalDecision), so it gives LightGBM's
raw score to the last bit (measured 2026-11-04: max
relative difference 0.0 over 192 slots)."""
relative difference 0.0 over 192 slots).
lgbm-trees-v2 (2026-11-04, methods.md B10), written only for a booster
that needs it (a categorical split or linear leaves; else v1, which older
agents read): per tree uint16 L, uint8 flags (bit 0: linear leaves); the
L-1 nodes as v1, decision bit 0 set for a categorical split, whose
threshold is then the index of its category set; uint16 the number of
sets, each uint8 n and n uint16 categories (those going left); the L
leaf values; with linear leaves per leaf float64 constant, uint8 n and n
(uint8 feature, float64 coefficient). Categorical rule (tree.h
CategoricalDecision): NaN or a negative value right, else left if the
integer is in the set. A linear leaf: constant plus the coefficients
times the values in LightGBM's order; a NaN among them: the leaf's value."""
from __future__ import annotations
import base64
@@ -26,6 +38,7 @@ import struct
import zlib
ENCODING = "lgbm-trees-v1"
ENCODING_V2 = "lgbm-trees-v2"
NODE = struct.Struct("<BBhhd")
MISSING = {"None": 0, "Zero": 1, "NaN": 2}
ZERO = 1.0000000180025095e-35 # LightGBM's kZeroThreshold (1e-35f)
@@ -36,44 +49,86 @@ def _ref(n: dict) -> int:
return int(n["split_index"]) if "split_index" in n else ~int(n.get("leaf_index", 0))
def pack_tree(root: dict) -> bytes:
"""One tree (dump_model()'s tree_info[i]["tree_structure"])."""
if "split_index" not in root: # no split: one leaf
return struct.pack("<Hd", 1, float(root["leaf_value"]))
def _walk(root: dict) -> tuple[dict[int, dict], dict[int, dict]]:
nodes: dict[int, dict] = {}
leaves: dict[int, float] = {}
leaves: dict[int, dict] = {}
stack = [root]
while stack:
n = stack.pop()
if "split_index" in n:
if n.get("decision_type", "<=") != "<=":
raise ValueError("a categorical split: not supported")
if n.get("decision_type", "<=") not in ("<=", "=="):
raise ValueError(f"a split {n.get('decision_type')!r}: not supported")
nodes[int(n["split_index"])] = n
stack += [n["left_child"], n["right_child"]]
else:
leaves[int(n["leaf_index"])] = float(n["leaf_value"])
leaves[int(n.get("leaf_index", 0))] = n
k = len(leaves)
if sorted(nodes) != list(range(k - 1)) or sorted(leaves) != list(range(k)):
raise ValueError("a tree's node or leaf indices are not contiguous")
return nodes, leaves
def _needs_v2(dump: dict) -> bool:
for t in dump["tree_info"]:
nodes, leaves = _walk(t["tree_structure"])
if any(n.get("decision_type") == "==" for n in nodes.values()) or any("leaf_coeff" in x for x in leaves.values()):
return True
return False
def pack_tree(root: dict, v2: bool = False) -> bytes:
"""One tree (dump_model()'s tree_info[i]["tree_structure"])."""
if "split_index" not in root and not v2: # no split: one leaf
return struct.pack("<Hd", 1, float(root["leaf_value"]))
nodes, leaves = _walk(root)
k = len(leaves)
linear = v2 and any("leaf_coeff" in x for x in leaves.values())
out = [struct.pack("<H", k)]
if v2:
out.append(struct.pack("<B", 1 if linear else 0))
cats: list[list[int]] = []
for i in range(k - 1):
n = nodes[i]
dec = (2 if n.get("default_left") else 0) | (MISSING[n.get("missing_type", "None")] << 2)
out.append(NODE.pack(int(n["split_feature"]), dec, _ref(n["left_child"]), _ref(n["right_child"]),
float(n["threshold"])))
out.append(struct.pack(f"<{k}d", *(leaves[i] for i in range(k))))
threshold = n["threshold"]
if n.get("decision_type") == "==":
if not v2:
raise ValueError("a categorical split: lgbm-trees-v2")
dec |= 1
cats.append([int(c) for c in str(threshold).split("||") if c != ""])
threshold = len(cats) - 1
out.append(NODE.pack(int(n["split_feature"]), dec, _ref(n["left_child"]), _ref(n["right_child"]), float(threshold)))
if v2:
out.append(struct.pack("<H", len(cats)))
for c in cats:
if len(c) > 255 or any(not 0 <= x < 65536 for x in c):
raise ValueError("a categorical split: at most 255 categories below 65536")
out.append(struct.pack(f"<B{len(c)}H", len(c), *c))
out.append(struct.pack(f"<{k}d", *(float(leaves[i]["leaf_value"]) for i in range(k))))
if linear:
for i in range(k):
x = leaves[i]
feats, coeff = [int(f) for f in x.get("leaf_features") or []], [float(c) for c in x.get("leaf_coeff") or []]
if len(feats) != len(coeff) or len(feats) > 255:
raise ValueError("a linear leaf")
out.append(struct.pack("<dB", float(x.get("leaf_const", 0.0)), len(feats)))
for f, c in zip(feats, coeff):
out.append(struct.pack("<Bd", f, c))
return b"".join(out)
def pack(dump: dict) -> dict:
"""A booster's dump_model() as lgbm-trees-v1."""
"""A booster's dump_model() as lgbm-trees-v1, or v2 where it has
categorical splits or linear leaves."""
if dump.get("num_class", 1) != 1 or dump.get("num_tree_per_iteration", 1) != 1:
raise ValueError("one tree per round only")
obj = str(dump.get("objective") or "").split(" ")[0]
if obj != "regression":
raise ValueError(f"objective {obj!r}: regression only (no output transform)")
raw = b"".join(pack_tree(t["tree_structure"]) for t in dump["tree_info"])
return {"enc": ENCODING, "n": len(dump["tree_info"]), "data": base64.b64encode(zlib.compress(raw, 9)).decode()}
v2 = _needs_v2(dump)
raw = b"".join(pack_tree(t["tree_structure"], v2) for t in dump["tree_info"])
return {"enc": ENCODING_V2 if v2 else ENCODING, "n": len(dump["tree_info"]),
"data": base64.b64encode(zlib.compress(raw, 9)).decode()}
class Trees:
@@ -81,8 +136,9 @@ class Trees:
leaf (no loops), features in range, finite leaves."""
def __init__(self, enc: dict, n_features: int):
if not isinstance(enc, dict) or enc.get("enc") != ENCODING:
raise ValueError(f"trees: {ENCODING} only")
if not isinstance(enc, dict) or enc.get("enc") not in (ENCODING, ENCODING_V2):
raise ValueError(f"trees: {ENCODING} or {ENCODING_V2} only")
v2 = enc["enc"] == ENCODING_V2
z = zlib.decompressobj()
try:
raw = z.decompress(base64.b64decode(str(enc.get("data")), validate=True), MAX_RAW)
@@ -100,6 +156,13 @@ class Trees:
off += 2
if not 1 <= k <= MAX_LEAVES:
raise ValueError("trees: leaves")
linear = False
if v2:
(flags,) = struct.unpack_from("<B", raw, off)
off += 1
if flags & ~1:
raise ValueError("trees: flags")
linear = bool(flags & 1)
nodes = []
for i in range(k - 1):
f, dec, left, right, threshold = NODE.unpack_from(raw, off)
@@ -107,14 +170,44 @@ class Trees:
for c in (left, right):
if not (i < c < k - 1 or -k <= c < 0):
raise ValueError("trees: a child out of range")
if f >= n_features or dec & 1 or (dec >> 2) & 3 > 2 or math.isnan(threshold):
if f >= n_features or (dec & 1 and not v2) or (dec >> 2) & 3 > 2 or math.isnan(threshold):
raise ValueError("trees: a split")
nodes.append((f, threshold, bool(dec & 2), (dec >> 2) & 3, left, right))
nodes.append([f, threshold, bool(dec & 2), (dec >> 2) & 3, left, right, bool(dec & 1)])
if v2:
(ncat,) = struct.unpack_from("<H", raw, off)
off += 2
sets = []
for _ in range(ncat):
(m,) = struct.unpack_from("<B", raw, off)
sets.append(frozenset(struct.unpack_from(f"<{m}H", raw, off + 1)))
off += 1 + 2 * m
for nd in nodes:
if nd[6]:
if not 0 <= nd[1] < len(sets) or nd[1] != int(nd[1]):
raise ValueError("trees: a category set")
nd[1] = sets[int(nd[1])]
nodes = [tuple(nd) for nd in nodes]
leaves = struct.unpack_from(f"<{k}d", raw, off)
off += 8 * k
if not all(math.isfinite(v) for v in leaves):
raise ValueError("trees: a leaf")
self.trees.append((nodes, leaves))
lin = None
if linear:
lin = []
for _ in range(k):
const, m = struct.unpack_from("<dB", raw, off)
off += 9
terms = []
for _ in range(m):
f, c = struct.unpack_from("<Bd", raw, off)
off += 9
if f >= n_features or not math.isfinite(c):
raise ValueError("trees: a linear leaf")
terms.append((f, c))
if not math.isfinite(const):
raise ValueError("trees: a linear leaf")
lin.append((const, terms))
self.trees.append((nodes, leaves, lin))
if len(self.trees) > MAX_TREES:
raise ValueError("trees: too many")
except struct.error as e:
@@ -128,21 +221,31 @@ class Trees:
split knows NaN; the split's missing values (zero or NaN, by its
type) go its default way; the leaves summed in order."""
total = 0.0
for nodes, leaves in self.trees:
if not nodes:
total += leaves[0]
continue
i = 0
for nodes, leaves, lin in self.trees:
i = 0 if nodes else ~0
while i >= 0:
f, threshold, default_left, missing, left, right = nodes[i]
f, threshold, default_left, missing, left, right, cat = nodes[i]
v = x[f]
if cat:
i = right if v != v or v < 0 or int(v) not in threshold else left
continue
if v != v and missing != 2:
v = 0.0
if (missing == 1 and -ZERO <= v <= ZERO) or (missing == 2 and v != v):
i = left if default_left else right
else:
i = left if v <= threshold else right
if lin is None:
total += leaves[~i]
continue
out = lin[~i][0]
for f, c in lin[~i][1]:
v = x[f]
if v != v:
out = leaves[~i]
break
out += c * v
total += out
return total
def predict(self, rows) -> list[float]:
+7 -3
View File
@@ -45,7 +45,7 @@ SERIES = {"load_w": "load", "pv_w": "pv", "base_load_w": "base_load"} # the bu
# is the base load already (the house without the planned devices), which
# is the load where the box plans none (the only case its load_w is used)
HISTORY = {"load_w": "load_w", "base_load_w": "load_w", "pv_w": "pv_w"}
SITE_FIELDS = {"tz": str, "lat": float, "lon": float, "kwp": float, "tilt": float, "azimuth": float}
SITE_FIELDS = {"tz": str, "lat": float, "lon": float, "kwp": float, "tilt": float, "azimuth": float, "state": str}
def _num(v) -> bool:
@@ -86,8 +86,12 @@ class Model:
if not _numbers(p.get("values"), 168):
raise ValueError("how: 168 numbers")
elif self.kind == "gbm":
want = fcmodels.PV_FEATURES if self.series == "pv" else fcmodels.LOAD_FEATURES
if p.get("features") != want or not _numbers(p.get("how"), 168) or "model" in p:
# the features it was trained with: today's, or those before the
# special days and the yearly rhythm (fcmodels.FEATURES_V1)
key = "pv" if self.series == "pv" else "load"
want = fcmodels.PV_FEATURES if key == "pv" else fcmodels.LOAD_FEATURES
if p.get("features") not in (want, fcmodels.FEATURES_V1[key]) or not _numbers(p.get("how"), 168) \
or "model" in p or not all(isinstance(k, int) for k in p.get("special_kept") or []):
raise ValueError(f"gbm: the features {', '.join(want)}, 168 hour-of-week values, trees")
if p.get("cap_w") is not None and not _num(p["cap_w"]):
raise ValueError("gbm: cap_w")
+15
View File
@@ -171,6 +171,21 @@ NAMES = {
"control": {"en": "Control", "de": "Gerät"},
"value": {"en": "Value", "de": "Wert"},
"why": {"en": "Why", "de": "Warum"},
# the owner's preferences (preferences.py; announce.py)
"pref_schedule": {"en": "{0}: minimum temperatures", "de": "{0}: Mindesttemperaturen"},
"pref_schedule_cool": {"en": "{0}: maximum temperatures", "de": "{0}: Höchsttemperaturen"},
"pref_boost_c": {"en": "{0}: boost temperature", "de": "{0}: Boost-Temperatur"},
"pref_boost_h": {"en": "{0}: boost hours", "de": "{0}: Boost-Dauer"},
"pref_boost": {"en": "{0}: boost now", "de": "{0}: Boost jetzt"},
"pref_away": {"en": "Away", "de": "Abwesend"},
"pref_away_until": {"en": "Away until", "de": "Abwesend bis"},
"pref_ready": {"en": "{0}: ready by", "de": "{0}: fertig bis"},
"pref_capacity": {"en": "{0}: battery capacity", "de": "{0}: Akkukapazität"},
"pref_needed": {"en": "{0}: needed once at", "de": "{0}: einmalig gebraucht um"},
"pref_reserve": {"en": "{0}: reserve", "de": "{0}: Reserve"},
"pref_reserve_blocks": {"en": "{0}: reserve by time", "de": "{0}: Reserve nach Uhrzeit"},
"pref_result": {"en": "Preferences", "de": "Präferenzen"},
"pref_result_of": {"en": "{0}: preferences", "de": "{0}: Präferenzen"},
}
+264 -11
View File
@@ -48,7 +48,18 @@ The building's requests (building.py, state["building"]): a load limit on
the grid import and a thermostat cap are constraints with a penalty (binding
far above any price, flexible with a price), a shift request a soft target
(the window's import lowered by its kWh against the house's own net load).
The owner's preferences (preferences.py, a device's "pref"): a room's
target per step from its schedule (in place of the occupied hours) at the
comfort cost, and a "must" as a floor (heating) or limit (cooling) with the
penalty MUST_EUR_K, inside the bounds and under a building's thermostat cap; a
car's ready-by time and energy (a must at EV_MUST_EUR_KWH, a prefer valued
at its max_ct per kWh with min_kwh at EV_SHORT_EUR_KWH); a battery's
reserve as its floor per step. The plan's "prefs" says per preference
whether it is met; with state["prefs_cost"] its "prefs_cost" the cost of
each must (pref_costs: the plan made again without it).
"""
import copy
import json
import math
import sys
@@ -67,6 +78,13 @@ COMFORT_EUR_KH = 2.0 # EUR per K and hour below (heating) or above (cooling
# while someone is home: above any price (0.30 let a room cool by 1.3 K
# at a 40 ct peak); pre-heating in cheap hours still pays
EV_SHORT_EUR_KWH = 1.0 # EUR per kWh the car lacks at departure
EV_MUST_EUR_KWH = 5.0 # EUR per kWh a car lacks at a "must" ready-by (preferences.py): above any price,
# below a building's binding load limit (50) and the grid operator's (500)
MUST_EUR_K = 20.0 # EUR per K and slot across a room's "must" (preferences.py): 80 EUR per K·h, so the
# heat a room holds (C kWh/K) is not worth letting it fall even at 3 EUR/kWh; it stays
# inside the bounds and under a building's cap, so those still win
COST_SOLVE_S = 8.0 # s at most per extra plan for a must's cost (pref_costs)
COST_MAX = 6 # musts costed per plan at most
LIGHT_EUR = 0.002 # EUR per step for a closed cover while someone is home in daylight
CHANGE_EUR_KW = 0.002 # EUR per kW a device's power changes from one step to the next: smooth
# plans (equal prices would otherwise give any order, and a charger or
@@ -245,7 +263,13 @@ class Model:
soc0 = cap * min(max(float(st["soc_pct"]), 0.0), 100.0) / 100.0
mean_price = sum(self.s["price_import"]) / self.T
soc, prev, ch, dis = [], soc0, [], []
pref = d.get("pref") # the owner's reserve per step (preferences.py)
floors = []
for t in range(self.T):
lo_t = lo_pct
if pref and not (pref.get("off") is not None and pref["group"][t] == pref["off"]):
lo_t = min(max(lo_pct, pref["floor_pct"][t]), hi_pct)
floors.append(lo_t)
c = self.var(0.0, max_ch)
x = self.var(0.0, max_dis)
z = self.var(0.0, 1.0, integer=True)
@@ -255,7 +279,7 @@ class Model:
under = self.var(cost=PENALTY_LIMIT / cap * 100) # Wh below the floor, per % of the battery
over = self.var(cost=PENALTY_LIMIT / cap * 100)
self.h.addConstr(e - prev - self.dt * eff * c + self.dt / eff * x == 0)
self.h.addConstr(e + under >= lo_pct / 100.0 * cap)
self.h.addConstr(e + under >= lo_t / 100.0 * cap)
self.h.addConstr(e - over <= hi_pct / 100.0 * cap)
self.flows[t] += [(-1, c), (1, x)]
soc.append(e)
@@ -267,7 +291,7 @@ class Model:
self.h.addConstr(prev + short >= min(soc0, hi_pct / 100.0 * cap))
self.smooth([c - x for c, x in zip(ch, dis)])
self.devices[d["id"]] = ("battery", {"ch": ch, "dis": dis, "soc": soc, "cap": cap, "max_dis": max_dis,
"floor_wh": lo_pct / 100.0 * cap})
"floor_wh": lo_pct / 100.0 * cap, "floors": floors})
def room(self, d, cooling: bool, gains):
p, st, b = d["params"], d["state"], d["bounds"]
@@ -284,6 +308,8 @@ class Model:
internal = self.hourly(p.get("internal_gain_w") or 0.0)
solar = float(p.get("solar_m2") or 0.0)
ghi = self.s.get("ghi_w_m2") or [0.0] * self.T
pref = d.get("pref") # the owner's schedule and overrides (preferences.py)
cap = (self.s.get("building") or {}).get("thermostat_cap")
for t in range(self.T):
if switched:
u = self.var(0.0, 1.0, integer=True)
@@ -302,7 +328,23 @@ class Model:
self.h.addConstr(tt - (1 - a) * prev - sign * self.dt / c * cop / 1000.0 * pw
- self.dt / c / 1000.0 * gain == a * self.s["t_out"][t] + self.dt / c / 1000.0 * fixed)
occ = self.s["occupied"][t + 1] if t + 1 < self.T else self.s["occupied"][t]
target = min(max(comfort if occ else other, lo), hi)
want, must = (comfort if occ else other), None
if pref:
want, must = pref["target"][t], pref["must"][t]
if pref.get("off") is not None and pref["group"][t] == pref["off"]:
want, must = pref["base"][t], None # pref_costs: the plan without this group
target = min(max(want, lo), hi)
capped = bool(cap) and cap["from_step"] <= t < cap["to_step"]
if must is not None:
# a bound inside the bounds in force, and under a building's cap
mv = min(max(must, lo), hi)
if capped:
mv = max(mv, cap["value"]) if cooling else min(mv, cap["value"])
cross = self.var(cost=MUST_EUR_K)
if cooling:
self.h.addConstr(tt - cross <= mv)
else:
self.h.addConstr(tt + cross >= mv)
miss = self.var(cost=COMFORT_EUR_KH * self.dt)
under = self.var(cost=PENALTY_LIMIT)
over = self.var(cost=PENALTY_LIMIT)
@@ -312,8 +354,7 @@ class Model:
self.h.addConstr(tt + miss >= target)
self.h.addConstr(tt + under >= lo)
self.h.addConstr(tt - over <= hi)
cap = (self.s.get("building") or {}).get("thermostat_cap")
if cap and cap["from_step"] <= t < cap["to_step"]:
if capped:
miss_cap = self.var(cost=cap["penalty"] * self.dt)
if cooling:
self.h.addConstr(tt + miss_cap >= cap["value"]) # cooling: not below the cap
@@ -345,6 +386,11 @@ class Model:
imin = max(float(p["min_current_a"]), b["current_a"][0])
dep = departure_step(self.s, str(st.get("departure") or p["departure"]))
need = float(st["need_wh"]) if st.get("need_wh") is not None else float(p["need_wh"])
pref = d.get("pref") # the owner's ready-by (preferences.py)
if pref:
dep, need = int(pref["dep_step"]), float(pref["need_wh"])
if pref.get("off"):
need = 0.0
powers, total = [], Lin()
for t in range(self.T):
if t >= dep or imax < imin:
@@ -357,10 +403,21 @@ class Model:
self.flows[t].append((-1, pw))
powers.append(pw)
total += self.dt * pw
short = self.var(cost=EV_SHORT_EUR_KWH / 1000.0)
if pref and pref["how"] == "prefer":
# a kWh is worth max_ct to the owner (without one: today's), min_kwh at any price
worth = EV_SHORT_EUR_KWH if pref.get("max_ct") is None else max(0.0, pref["max_ct"]) / 100.0
short = self.var(cost=worth / 1000.0)
self.h.addConstr(total + short >= need)
mwh = min(float(pref.get("min_wh") or 0.0), need)
if mwh > 0:
short_min = self.var(cost=EV_SHORT_EUR_KWH / 1000.0)
self.h.addConstr(total + short_min >= mwh)
else:
short = self.var(cost=(EV_MUST_EUR_KWH if pref else EV_SHORT_EUR_KWH) / 1000.0)
self.h.addConstr(total + short >= need)
self.smooth([p for p in powers if p is not None])
self.devices[d["id"]] = ("ev", {"power": powers, "volts": volts, "dep": dep})
self.devices[d["id"]] = ("ev", {"power": powers, "volts": volts, "dep": dep,
"pmax": imax * volts if imax >= imin else 0.0})
def cover(self, d, room_ids):
p, b = d["params"], d["bounds"]
@@ -463,16 +520,198 @@ def plan(s: dict, time_limit: float = 30.0) -> dict:
if e is not None and (not entries or _key(entries[-1]) != _key(e)):
entries.append({"t": start + t * step, **e})
devices[d["id"]] = entries
energy = sum(m.eur_wh(t, pi[t]) * x(m.gin[t]) - m.eur_wh(t, s["price_export"][t]) * x(m.gout[t])
for t in range(T))
market = market_plan(s, m, x)
energy += sum(c["cost_eur"] for c in market)
energy = energy_of(s, m, x, market)
out = {"v": 1, "made": int(time.time()), "start": start, "step_s": step, "status": status,
"solve_s": round(time.time() - t0, 2), "devices": devices, "flex": flexibility(m, x),
"cost_eur": round(energy, 2), "baseline_eur": round(baseline(s, m, x), 2),
"price_note": s.get("price_note", ""), "sources": s.get("sources", {})}
if market:
out["market"] = market
if any(d.get("pref") for d in s["devices"]):
out["prefs"] = pref_feedback(s, m, x)
if s.get("prefs_cost"):
out["prefs_cost"] = pref_costs(s, energy, t0 + time_limit)
return out
def energy_of(s, m, x, market=None) -> float:
"""The plan's cost for power (EUR): the grid's import less its feed-in,
and the market's contracts."""
pi, pe = s["price_import"], s["price_export"]
e = sum(m.eur_wh(t, pi[t]) * x(m.gin[t]) - m.eur_wh(t, pe[t]) * x(m.gout[t]) for t in range(m.T))
return e + sum(c["cost_eur"] for c in (market if market is not None else market_plan(s, m, x)))
def _when(s, ts: float) -> str:
return datetime.fromtimestamp(ts, timezone.utc).astimezone(ZoneInfo(s.get("tz") or "UTC")).strftime("%a %H:%M")
TOL_K = 0.3 # a room's target counts as met within this
TOL_PCT = 0.5 # a battery's reserve
TOL_WH = 50.0 # a car's energy
def pref_feedback(s, m, x) -> list[dict]:
"""Per preference (a room's block, boost or away; a car's next ready-by;
a battery's reserve): met or not in the plan, and in a few words why.
Fields: control, group, kind, how, label, met (None: not in the
plan's 48 h, or not plugged in), detail, and per kind the numbers."""
out = []
start, step, T = s["start"], s["step_s"], m.T
for d in s["devices"]:
pref = d.get("pref")
if not pref:
continue
kind, v = m.devices.get(d["id"], (d["kind"], None))
if kind in ("heating", "cooling") and v:
out += _room_feedback(s, d, pref, v, x, kind == "cooling")
elif kind == "battery" and v:
for gi, g in enumerate(pref["groups"]):
ts = [t for t in range(T) if pref["group"][t] == gi]
f = {"control": d["id"], "group": g["group"], "kind": "battery", "how": "must", "label": g["text"]}
if not ts:
out.append(dict(f, met=None, detail="not in the next 48 h"))
continue
soc = [100.0 * x(v["soc"][t]) / v["cap"] for t in ts]
worst = min(range(len(ts)), key=lambda k: soc[k] - v["floors"][ts[k]])
miss = v["floors"][ts[worst]] - soc[worst]
met = miss <= TOL_PCT
f.update(met=met, min_pct=round(min(soc), 1), floor_pct=round(v["floors"][ts[worst]], 1))
if not met:
f["detail"] = f"{soc[worst]:.0f} % at {_when(s, start + (ts[worst] + 1) * step)}"
if d["state"].get("soc_pct") is not None and float(d["state"]["soc_pct"]) < v["floors"][ts[0]]:
f["detail"] += f" (starts at {float(d['state']['soc_pct']):.0f} %)"
out.append(f)
elif kind == "ev":
out.append(_ev_feedback(s, d, pref, v, x, m))
return out
def _room_feedback(s, d, pref, v, x, cooling) -> list[dict]:
out = []
start, step, T = s["start"], s["step_s"], len(pref["target"])
lo, hi, lo_by, hi_by = (d["bounds"]["temp_c"] + ["device", "device"])[:4]
cap = (s.get("building") or {}).get("thermostat_cap")
sign = -1.0 if cooling else 1.0
for gi, g in enumerate(pref["groups"]):
ts = [t for t in range(T) if pref["group"][t] == gi]
f = {"control": d["id"], "group": g["group"], "kind": g["kind"], "how": g["how"], "label": g["text"]}
if not ts:
out.append(dict(f, met=None, detail="not in the next 48 h"))
continue
notes = []
want = []
for t in ts:
w0 = pref["must"][t] if g["how"] == "must" and pref["must"][t] is not None else pref["target"][t]
w = min(max(w0, lo), hi)
by = (hi_by if not cooling else lo_by) if sign * (w0 - w) > 1e-9 else None
if cap and cap["from_step"] <= t < cap["to_step"]:
wc = max(w, cap["value"]) if cooling else min(w, cap["value"])
if wc != w:
by, w = "building", wc
if by and by not in notes:
notes.append(by)
want.append(w)
temps = [x(v["temp"][t]) for t in ts]
gap = [sign * (w - tt) for w, tt in zip(want, temps)]
k = max(range(len(ts)), key=lambda i: gap[i])
met = gap[k] <= TOL_K
f.update(met=met, first=start + ts[0] * step, worst_c=round(temps[k], 1), want_c=round(want[k], 1))
bits = []
if notes:
held = min(want) if not cooling else max(want)
bits.append(f"held at {held:g} °C by the {notes[0]}'s limit")
f["held_by"] = notes[0]
if not met:
bits.append(f"{temps[k]:.1f} °C at {_when(s, start + (ts[k] + 1) * step)}")
if bits:
f["detail"] = "; ".join(bits)
out.append(f)
return out
def _ev_feedback(s, d, pref, v, x, m) -> dict:
start, step = s["start"], s["step_s"]
f = {"control": d["id"], "group": pref["group"], "kind": "ev", "how": pref["how"],
"label": f"car {pref['text']}", "at": pref["at"], "need_kwh": round(pref["need_wh"] / 1000.0, 2)}
plug = f"; plugged in at {_when(s, pref['plugged_at'])}" if pref.get("plugged_at") else ""
if pref.get("note"):
f["note"] = pref["note"]
if v is None:
return dict(f, met=None if pref["how"] == "prefer" else False, detail="not plugged in")
pw = [x(p) if p is not None else 0.0 for p in v["power"]]
dep = min(v["dep"], len(pw))
wh = sum(m.dt * p for p in pw[:dep])
short = max(0.0, pref["need_wh"] - wh)
pi = s["price_import"]
f["planned_kwh"] = round(wh / 1000.0, 2)
f["charge_eur"] = round(sum(m.eur_wh(t, pi[t]) * pw[t] for t in range(dep)), 2)
reached = ""
if pref.get("soc_pct") is not None and pref.get("capacity_wh"):
pct = min(100.0, float(pref["soc_pct"]) + 100.0 * wh / pref["capacity_wh"])
f["reached_pct"] = round(pct, 0)
reached = f"{pct:.0f} %"
else:
reached = f"{wh / 1000.0:.1f} kWh"
when = _when(s, pref["at"])
if pref.get("beyond"):
f.update(met=None, detail=f"{reached} by the end of the plan (the ready-by is later){plug}")
return f
if pref["how"] == "prefer":
f["met"] = short <= max(TOL_WH, 0.01 * pref["need_wh"])
f["detail"] = f"{reached} by {when}" + ("" if f["met"] else
f" ({short / 1000.0:.1f} kWh less: dearer than the price wanted)") + plug
return f
f["met"] = short <= max(TOL_WH, 0.01 * pref["need_wh"])
if f["met"]:
f["detail"] = f"{reached} by {when}{plug}"
return f
# a must it cannot meet: by how much, and when it could be ready at full power from now
f["short_kwh"] = round(short / 1000.0, 2)
rate = v.get("pmax") or 0.0
if rate > 0:
n = math.ceil(pref["need_wh"] / (rate * m.dt) - 1e-9)
f["ready_at"] = int(start + n * step)
can = f"; ready at full power by {_when(s, f['ready_at'])}"
else:
can = "; it may not charge now (a limit holds it at 0 A)"
f["detail"] = f"{reached} by {when}, short by {short / 1000.0:.1f} kWh{can}{plug}"
return f
def pref_costs(s, energy: float, deadline: float) -> dict:
"""Each must's cost (EUR a day): the plan's cost for power less that of
the plan made again without it (a room's block at its setback, a
battery's reserve at its floor before); a car's must: what its charging
costs (no second plan). At most COST_MAX, each within COST_SOLVE_S and
the time left."""
out = {}
days = s["steps"] * s["step_s"] / 86400.0
n = 0
for i, d in enumerate(s["devices"]):
pref = d.get("pref")
if not pref or d["kind"] not in ("heating", "cooling", "battery"):
continue
for gi, g in enumerate(pref["groups"]):
if g["how"] != "must" or gi not in pref["group"]:
continue
left = deadline - time.time() - 1.0
if n >= COST_MAX or left < 1.0:
return out
n += 1
s2 = copy.deepcopy(s)
s2["prefs_cost"] = False
s2["devices"][i]["pref"]["off"] = gi
try:
m2 = Model(s2)
m2.solve(min(COST_SOLVE_S, left))
val = m2.h.getSolution().col_value
e2 = energy_of(s2, m2, lambda v: val[v.index])
except Exception: # noqa: BLE001 - a cost is not worth failing the plan
continue
c = (energy - e2) / days
out[g["group"]] = round(c if abs(c) >= 0.005 else 0.0, 2)
return out
@@ -546,7 +785,9 @@ def flexibility(m, x) -> dict:
for t in range(min(n, len(cols or []))):
if cols[t] is not None and hasattr(cols[t], "index"): # an EV's steps after departure are empty
shed[t] += max(x(cols[t]), 0.0)
return {"grid_w": grid, "shed_w": [round(min(sh, g)) for sh, g in zip(shed, grid)]}
# net_w: import less export, the member's draw as planned (the box's forecast of it upward, boxtrain.py)
return {"grid_w": grid, "shed_w": [round(min(sh, g)) for sh, g in zip(shed, grid)],
"net_w": [round(x(m.gin[t]) - x(m.gout[t])) for t in range(n)]}
def _charge_why(pi, t) -> str:
@@ -607,6 +848,14 @@ def _entry(d, kind, v, t, x, s, pi, grid_w=(1e9, 1e9)):
temp = x(v["temp"][t])
pw = x(v["power"][t])
ahead = s["occupied"][min(t + 4, s["steps"] - 1)]
pref = d.get("pref")
if pref:
# a schedule: "someone home" is a target above the setback ahead
# (below the setup, cooling); its target in place of comfort/setback
sign = -1 if kind == "cooling" else 1
tg, k0 = pref["target"], max(t - 1, 0)
occ = sign * (tg[k0] - pref["base"][k0]) > 0
ahead = any(sign * (tg[k] - tg[k0]) > 0 for k in range(t, min(t + 4, len(tg))))
if kind == "heating" and v["switched"]:
on = pw > 0.5 * v["pmax"]
why = ("pre-heat before someone is home" if on and not occ and ahead else
@@ -630,6 +879,10 @@ def _entry(d, kind, v, t, x, s, pi, grid_w=(1e9, 1e9)):
occ1 = s["occupied"][min(t + 1, s["steps"] - 1)]
p = d["params"]
want = float(p["comfort_c"]) if occ1 else float(p["setback_c"])
if pref:
want = float(pref["target"][t])
if pref["must"][t] is not None:
want = max(want, float(pref["must"][t]))
floor_t = min(max(math.floor(want * 2) / 2, math.ceil(lo * 2) / 2), math.floor(hi * 2) / 2)
return {"set": min(target, floor_t), "why": "setback: away or night" if not occ else "warm enough"}
return {"set": target, "why": f"pre-heat while power is cheap ({pi[t]:.1f} ct)" if not occ and ahead
+975
View File
@@ -0,0 +1,975 @@
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""The owner's preferences for the plan ("I may want
my car to be fully charged in any case at 8 AM ... I want my home to be min
21 degree in night, 19 degree in day").
Per control, kept in <state dir>/preferences.json:
room:<id> a heating room's minimum / a cooling room's maximum per
weekday and time: blocks {days, from, to, c, how}
ev:<id> ready-by times per weekday {days, at, pct | kwh, how,
max_ct, min_kwh}, and the car's capacity_kwh (for %)
battery:<id> a reserve (min state of charge, %) and optional blocks
{days, from, to, pct}: the owner's floor
and overrides that win over them until they expire:
boost {control, c, from, until} a room to c (a must) for some hours
away {from, until} every room at its setback/setup
car {control, at, pct | kwh, how} the car needed at a one-off time
"must": a bound (a floor or limit in the plan: MUST_EUR_K, EV_MUST_EUR_KWH,
inside the bounds in force: the grid operator's, the building's and the
owner's limits still win). "prefer": today's comfort cost (rooms), or a
value per kWh (a car: charge where a kWh costs at most max_ct, and at least
min_kwh at any price). A room without a schedule keeps today's occupied
hours (inputs.py) and comfort/setback targets.
Who sets them, and who wins (limits.py's spirit): the owner (Home
Assistant, the Hub's panel: level "owner"), an installer or Cantabile's
operators (the console, level "installer" / "operator": only within the
owner's limits, never above them). Each level's setting is kept; the
highest level's is in force, so the owner's own changes win and an
installer's comes back when the owner clears theirs. Overrides are the
owner's only. Every change is logged with who, when and what.
Where they come in: Home Assistant's entities (announce.py, the text
formats below), the panel (an inbox directory, prefs-inbox/, the panel
writes one JSON file per change; the agent writes preferences-view.json
for it), the console (a signed command "preferences"). They go to the
cloud with the plan (controller.public_plan), with the feedback per
preference: met or not, and the cost of each must (the plan made again
without it: optimizer.pref_costs).
Text formats (an HA text entity holds 255 characters), blocks separated by
";": days are Mo Tu We Th Fr Sa Su (or Mo Di Mi Do Fr Sa So), ranges
"Mo-Fr", lists "Sa,Su", or daily / weekdays / weekend.
room Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19; Sa,Su 07:00-23:00 20
ev Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct min 5kWh
battery daily 17:00-21:00 50%
car 2026-11-05 07:00 80% must (or "07:00 80%": the next 07:00)
away 2026-11-10 (back that day; or with a time)
A block from 22:00 to 06:00 belongs to the day it starts. Times are the
home's wall clock (HA's time zone), so a schedule holds across the change
to and from summer time.
"""
from __future__ import annotations
import json
import logging
import math
import os
import re
import time
import uuid
from datetime import date, datetime, timedelta, timezone
from zoneinfo import ZoneInfo
log = logging.getLogger(__name__)
DAYS = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
DAY_NAMES = {"mo": 0, "mon": 0, "monday": 0, "montag": 0,
"tu": 1, "tue": 1, "tuesday": 1, "di": 1, "dienstag": 1,
"we": 2, "wed": 2, "wednesday": 2, "mi": 2, "mittwoch": 2,
"th": 3, "thu": 3, "thursday": 3, "do": 3, "donnerstag": 3,
"fr": 4, "fri": 4, "friday": 4, "freitag": 4,
"sa": 5, "sat": 5, "saturday": 5, "samstag": 5,
"su": 6, "sun": 6, "sunday": 6, "so": 6, "sonntag": 6}
DAY_SETS = {"daily": list(range(7)), "all": list(range(7)), "täglich": list(range(7)), "taeglich": list(range(7)),
"weekdays": list(range(5)), "werktags": list(range(5)), "weekend": [5, 6], "wochenende": [5, 6]}
HOW = ("must", "prefer")
HOW_WORDS = {"must": "must", "muss": "must", "prefer": "prefer", "bevorzugt": "prefer", "gern": "prefer"}
LEVELS = ("operator", "installer", "owner") # low to high: the highest set is in force
RANK = {lvl: i + 1 for i, lvl in enumerate(LEVELS)}
KINDS = {"heating": "room", "cooling": "room", "ev": "ev", "battery": "battery"}
RANGE_C = {"heating": (5.0, 30.0), "cooling": (16.0, 32.0)} # controls.RANGE
MAX_BLOCKS = 16
MAX_OVERRIDES = 20
MAX_LOG = 200
AWAY_MAX_S = 60 * 86400 # away without a date: at most this long
BOOST_MAX_H = 24
COST_EVERY_S = 3600 # the musts' costs are made again at most this often (unless something changed)
HHMM = re.compile(r"^([01]?\d|2[0-4])(?::([0-5]\d))?$")
# -- small parsers -------------------------------------------------------------
def hhmm(v, end=False) -> str:
""""7", "07:00", "7:30" -> "07:00"/"07:30"; "24:00" only as an end."""
m = HHMM.match(str(v).strip())
if not m:
raise ValueError(f"{v!r} is not a time (HH:MM)")
h, mi = int(m.group(1)), int(m.group(2) or 0)
if h == 24 and (mi or not end):
raise ValueError(f"{v!r}: 24:00 only as the end of a block")
return f"{h:02d}:{mi:02d}"
def minutes(s: str) -> int:
h, m = s.split(":")
return int(h) * 60 + int(m)
def parse_days(v) -> list[int]:
""""Mo-Fr", "Sa,Su", "daily", [0, 1], "mon-fri" -> sorted weekday numbers (Mon 0)."""
if isinstance(v, (list, tuple)):
out = sorted({int(x) for x in v})
if not out or out[0] < 0 or out[-1] > 6:
raise ValueError("days: weekday numbers 0 (Monday) to 6 (Sunday)")
return out
s = str(v).strip().lower()
if s in DAY_SETS:
return list(DAY_SETS[s])
out = set()
for part in re.split(r"[,/ ]+", s):
if not part:
continue
if part in DAY_SETS:
out.update(DAY_SETS[part])
continue
a, _, b = part.partition("-")
if a not in DAY_NAMES or (b and b not in DAY_NAMES):
raise ValueError(f"{part!r} is not a day (Mo Tu We Th Fr Sa Su, a range Mo-Fr, or daily)")
i, j = DAY_NAMES[a], DAY_NAMES[b] if b else DAY_NAMES[a]
k = i
while True:
out.add(k)
if k == j:
break
k = (k + 1) % 7
if not out:
raise ValueError("days: none given")
return sorted(out)
def days_text(days: list[int]) -> str:
d = sorted(set(days))
if d == list(range(7)):
return "daily"
runs, cur = [], [d[0]]
for x in d[1:]:
if x == cur[-1] + 1:
cur.append(x)
else:
runs.append(cur)
cur = [x]
runs.append(cur)
return ",".join(DAYS[r[0]][:2] if len(r) == 1 else f"{DAYS[r[0]][:2]}-{DAYS[r[-1]][:2]}" if len(r) > 2
else f"{DAYS[r[0]][:2]},{DAYS[r[1]][:2]}" for r in runs)
def _num(v, what, lo, hi) -> float:
try:
x = float(str(v).replace(",", ".").strip())
except (TypeError, ValueError):
raise ValueError(f"{what}: {v!r} is not a number") from None
if not math.isfinite(x) or not lo <= x <= hi:
raise ValueError(f"{what}: {x:g} is outside {lo:g}..{hi:g}")
return x
def _how(v, default="prefer") -> str:
h = HOW_WORDS.get(str(v or default).strip().lower())
if h is None:
raise ValueError(f"how: must or prefer, not {v!r}")
return h
def _fmt(x: float) -> str:
return f"{x:g}"
# -- checked values (every source comes through these) --------------------------
def check_room(v: dict, kind: str) -> dict:
lo, hi = RANGE_C["cooling" if kind == "cooling" else "heating"]
blocks = (v or {}).get("blocks")
if not isinstance(blocks, list) or len(blocks) > MAX_BLOCKS:
raise ValueError(f"blocks: a list of at most {MAX_BLOCKS}")
out = []
for b in blocks:
out.append({"days": parse_days(b.get("days", "daily")), "from": hhmm(b.get("from", "00:00")),
"to": hhmm(b.get("to", "24:00"), end=True),
"c": _num(b.get("c"), "temperature", lo, hi), "how": _how(b.get("how"))})
return {"blocks": out}
def check_ev(v: dict) -> dict:
v = v or {}
ready = v.get("ready") or []
if not isinstance(ready, list) or len(ready) > MAX_BLOCKS:
raise ValueError(f"ready: a list of at most {MAX_BLOCKS}")
out = {"ready": [], "capacity_kwh": None}
if v.get("capacity_kwh") not in (None, ""):
out["capacity_kwh"] = _num(v["capacity_kwh"], "capacity", 1, 250)
for r in ready:
out["ready"].append(_ev_target(r, {"days": parse_days(r.get("days", "daily")), "at": hhmm(r.get("at", ""))}))
return out
def _ev_target(r: dict, into: dict) -> dict:
pct, kwh = r.get("pct"), r.get("kwh")
if (pct in (None, "")) == (kwh in (None, "")):
raise ValueError("a car's target: % or kWh, one of them")
into["pct"] = _num(pct, "target %", 1, 100) if pct not in (None, "") else None
into["kwh"] = _num(kwh, "target kWh", 0.1, 250) if kwh not in (None, "") else None
into["how"] = _how(r.get("how"))
into["max_ct"] = _num(r["max_ct"], "max price", -100, 500) if r.get("max_ct") not in (None, "") else None
into["min_kwh"] = _num(r["min_kwh"], "min kWh", 0, 250) if r.get("min_kwh") not in (None, "") else None
if into["how"] == "must":
into["max_ct"] = into["min_kwh"] = None # a must charges at any price
return into
def check_battery(v: dict) -> dict:
v = v or {}
blocks = v.get("blocks") or []
if not isinstance(blocks, list) or len(blocks) > MAX_BLOCKS:
raise ValueError(f"blocks: a list of at most {MAX_BLOCKS}")
out = {"reserve_pct": _num(v.get("reserve_pct", 0) or 0, "reserve", 0, 100), "blocks": []}
for b in blocks:
out["blocks"].append({"days": parse_days(b.get("days", "daily")), "from": hhmm(b.get("from", "00:00")),
"to": hhmm(b.get("to", "24:00"), end=True), "pct": _num(b.get("pct"), "reserve", 0, 100)})
return out
def check(key: str, value: dict, kind: str) -> dict:
what = key.split(":", 1)[0]
if what == "room":
return check_room(value, kind)
if what == "ev":
return check_ev(value)
if what == "battery":
return check_battery(value)
raise ValueError(f"no preference {key!r}")
# -- text forms (Home Assistant's text entities) -----------------------------------
def _split(text: str) -> list[list[str]]:
return [p.split() for p in re.split(r"[;\n]+", str(text or "")) if p.strip()]
def _days_and_rest(words: list[str]) -> tuple[list[int], list[str]]:
"""A block's leading days, else daily."""
try:
return parse_days(words[0]), words[1:]
except ValueError:
return list(range(7)), words
def _range(w: str) -> tuple[str, str]:
a, sep, b = w.partition("-")
if not sep:
raise ValueError(f"{w!r}: a time range, e.g. 22:00-06:00")
return hhmm(a), hhmm(b, end=True)
def parse_room_text(text: str, kind: str) -> dict:
blocks = []
for words in _split(text):
days, rest = _days_and_rest(words)
if len(rest) < 2:
raise ValueError(f"{' '.join(words)!r}: days, a time range and a temperature (e.g. Mo-Fr 22:00-06:00 21 must)")
a, b = _range(rest[0])
c = rest[1].strip("≥≤>=<°Cc")
how = rest[2] if len(rest) > 2 else "prefer"
blocks.append({"days": days, "from": a, "to": b, "c": c, "how": how})
return check_room({"blocks": blocks}, kind)
def room_text(v: dict | None) -> str:
return "; ".join(f"{days_text(b['days'])} {b['from']}-{b['to']} {_fmt(b['c'])}" + (" must" if b["how"] == "must" else "")
for b in (v or {}).get("blocks") or [])
def _target_words(words: list[str]) -> dict:
""""80%", "20kWh", "20 kWh", "must", "max 25ct", "<25ct", "min 5kWh"."""
out, i = {}, 0
ws = [w.lower() for w in words]
while i < len(ws):
w = ws[i]
nxt = ws[i + 1] if i + 1 < len(ws) else ""
if w in HOW_WORDS:
out["how"] = w
elif w.endswith("%"):
out["pct"] = w[:-1]
elif w.endswith("kwh") and w[:-3]:
out["kwh"] = w[:-3]
elif nxt == "kwh" and "pct" not in out and "kwh" not in out:
out["kwh"] = w
i += 1
elif w in ("max", "<", "bis") or (w.startswith("<") and w[1:]):
v = w[1:] if w.startswith("<") and w[1:] else nxt
if not (w.startswith("<") and w[1:]):
i += 1
out["max_ct"] = v.removesuffix("ct")
elif w == "min":
v = nxt
i += 1
if v.endswith("kwh"):
out["min_kwh"] = v[:-3]
elif i + 1 < len(ws) and ws[i + 1] == "kwh":
out["min_kwh"] = v
i += 1
else:
out["min_kwh"] = v
else:
raise ValueError(f"{w!r}: not understood (a target like 80% or 20kWh, must/prefer, max 25ct, min 5kWh)")
i += 1
return out
def parse_ev_text(text: str, capacity_kwh=None) -> dict:
ready = []
for words in _split(text):
days, rest = _days_and_rest(words)
if not rest:
raise ValueError(f"{' '.join(words)!r}: a time and a target (e.g. Mo-Fr 08:00 100% must)")
ready.append({"days": days, "at": hhmm(rest[0]), **_target_words(rest[1:])})
return check_ev({"ready": ready, "capacity_kwh": capacity_kwh})
def _target_text(r: dict) -> str:
s = f"{_fmt(r['pct'])}%" if r.get("pct") is not None else f"{_fmt(r['kwh'])}kWh"
s += " must" if r["how"] == "must" else ""
if r.get("max_ct") is not None:
s += f" max {_fmt(r['max_ct'])}ct"
if r.get("min_kwh") is not None:
s += f" min {_fmt(r['min_kwh'])}kWh"
return s
def ev_text(v: dict | None) -> str:
return "; ".join(f"{days_text(r['days'])} {r['at']} {_target_text(r)}" for r in (v or {}).get("ready") or [])
def parse_battery_text(text: str, reserve_pct=0.0) -> dict:
blocks = []
for words in _split(text):
days, rest = _days_and_rest(words)
if len(rest) < 2:
raise ValueError(f"{' '.join(words)!r}: days, a time range and a reserve (e.g. daily 17:00-21:00 50%)")
a, b = _range(rest[0])
blocks.append({"days": days, "from": a, "to": b, "pct": rest[1].rstrip("%")})
return check_battery({"reserve_pct": reserve_pct, "blocks": blocks})
def battery_text(v: dict | None) -> str:
return "; ".join(f"{days_text(b['days'])} {b['from']}-{b['to']} {_fmt(b['pct'])}%" for b in (v or {}).get("blocks") or [])
def local_when(text: str, zone: ZoneInfo, now: float, default_time: str | None = None) -> float:
""""2026-11-05 07:00", "2026-11-05" (default_time, else 00:00), "07:00"
(the next one) -> unix time, in the home's time zone."""
words = str(text).split()
if not words:
raise ValueError("a date or time is needed")
d, t = None, None
for w in words:
if re.match(r"^\d{4}-\d{2}-\d{2}$", w):
d = date.fromisoformat(w)
else:
t = hhmm(w)
t = t or default_time or "00:00"
h, mi = (int(x) for x in t.split(":"))
if d is None:
n = datetime.fromtimestamp(now, zone)
cand = _wall(n.date(), h, mi, zone)
if cand <= now:
cand = _wall(n.date() + timedelta(days=1), h, mi, zone)
return cand
return _wall(d, h, mi, zone)
def _wall(d: date, h: int, mi: int, zone: ZoneInfo) -> float:
"""A wall-clock time's unix time; one that does not exist (the hour
skipped in spring) moves on by the gap, as a clock does (02:30 is 03:30
summer time: PEP 495's fold 0 takes the offset before the change); one
that comes twice (autumn) is the first."""
naive = datetime(d.year, d.month, d.day, h % 24, mi) + timedelta(days=h // 24)
return naive.replace(tzinfo=zone).timestamp()
def parse_car_text(text: str, zone: ZoneInfo, now: float) -> dict | None:
"""The one-off "car needed" text: None when empty (cancel)."""
words = str(text or "").split()
if not words:
return None
when = [w for w in words if re.match(r"^\d{4}-\d{2}-\d{2}$", w) or HHMM.match(w)]
rest = [w for w in words if w not in when]
if not when:
raise ValueError("a time (and a date): e.g. 2026-11-05 07:00 80% must")
at = local_when(" ".join(when), zone, now, default_time="07:00")
tgt = _target_words(rest)
tgt.setdefault("how", "must")
if "pct" not in tgt and "kwh" not in tgt:
tgt["pct"] = 100
return _ev_target(tgt, {"kind": "car", "at": at})
# -- the store --------------------------------------------------------------------
class Preferences:
def __init__(self, state_dir: str, clock=time.time):
self.dir, self.clock = state_dir, clock
self.tz = "UTC" # the home's (HA's), for times in words
self.path = os.path.join(state_dir, "preferences.json")
self.inbox = os.path.join(state_dir, "prefs-inbox")
self.view_path = os.path.join(state_dir, "preferences-view.json")
self.items: dict[str, dict[str, dict]] = {} # key -> level -> {"value", "by", "at"}
self.overrides: list[dict] = []
self.ha: dict = {} # HA's helper values: boost_c/boost_h per room, away_until text
self.log: list[dict] = []
self.changed_at = 0.0 # last change (the plan's costs, steady's free controls)
self.dirty: set[str] = set() # controls changed since the last plan (steady.py: not held back)
self.costs: dict = {"at": 0, "of": 0.0, "eur_day": {}} # the musts' costs, as of a plan
self.feedback: list[dict] = []
self.errors: list[dict] = [] # the last refused changes (the panel shows them)
try:
with open(self.path) as f:
raw = json.load(f)
self.items = {k: {lvl: e for lvl, e in v.items() if lvl in RANK} for k, v in (raw.get("items") or {}).items()}
self.overrides = list(raw.get("overrides") or [])
self.ha = dict(raw.get("ha") or {})
self.log = list(raw.get("log") or [])[-MAX_LOG:]
self.costs = raw.get("costs") or self.costs
self.changed_at = float(raw.get("changed_at") or 0)
except (OSError, ValueError, AttributeError):
pass
def save(self):
os.makedirs(self.dir, exist_ok=True)
tmp = self.path + ".tmp"
with open(tmp, "w") as f:
json.dump({"v": 1, "items": self.items, "overrides": self.overrides, "ha": self.ha, "log": self.log,
"costs": self.costs, "changed_at": self.changed_at}, f)
os.replace(tmp, self.path)
# -- reading -------------------------------------------------------------
def effective(self, key: str) -> dict | None:
"""The setting in force: the highest level's ({"value", "by", "at", "level"})."""
per = self.items.get(key) or {}
for lvl in reversed(LEVELS):
if lvl in per:
return dict(per[lvl], level=lvl)
return None
def value(self, key: str) -> dict | None:
e = self.effective(key)
return e["value"] if e else None
def active_overrides(self, now: float | None = None) -> list[dict]:
now = self.clock() if now is None else now
return [o for o in self.overrides if _end(o) > now]
def prune(self, now: float | None = None) -> bool:
"""Expired overrides out; True when one went (a plan without it)."""
now = self.clock() if now is None else now
keep = [o for o in self.overrides if _end(o) > now]
gone = [o for o in self.overrides if _end(o) <= now]
if not gone:
return False
self.overrides = keep
for o in gone:
self._log(now, o.get("by", ""), "owner", f"override:{o['kind']}", "expired", o.get("control", ""))
if o.get("control"):
self.dirty.add(o["control"])
self.save()
return True
# -- changing --------------------------------------------------------------
def _log(self, now, by, level, key, what, control=""):
self.log = (self.log + [{"at": int(now), "by": by, "level": level, "item": key, "what": str(what)[:200],
**({"control": control} if control else {})}])[-MAX_LOG:]
def _refused(self, now, by, key, why):
self.errors = (self.errors + [{"at": int(now), "by": by, "item": key, "why": why}])[-10:]
return False, why
def set(self, key: str, value: dict | None, by: str, level: str, control, now: float | None = None) -> tuple[bool, str]:
"""A setting at a level (None: that level's cleared). control: the
control it is for (its kind; its owner limits for lower levels)."""
now = self.clock() if now is None else now
if level not in RANK:
return self._refused(now, by, key, f"level: one of {', '.join(LEVELS)}")
what, _, cid = key.partition(":")
if control is None or control.id != cid or KINDS.get(control.kind) != what:
return self._refused(now, by, key, f"no {what} control {cid!r}")
if value is None:
if level in (self.items.get(key) or {}):
del self.items[key][level]
if not self.items[key]:
del self.items[key]
self._log(now, by, level, key, "cleared", cid)
else:
try:
value = check(key, value, control.kind)
if level != "owner":
within_owner(key, value, control)
except ValueError as e:
return self._refused(now, by, key, str(e))
self.items.setdefault(key, {})[level] = {"value": value, "by": by, "at": int(now)}
self._log(now, by, level, key, describe(key, value, control.kind), cid)
self.changed_at = now
self.dirty.add(cid)
self.save()
top = self.effective(key)
note = "" if top is None or top["level"] == level else f" (the {top['level']}'s own setting is in force)"
return True, f"{key}: {'cleared' if value is None else describe(key, value, control.kind)}{note}"
def add_override(self, o: dict, by: str, controls: dict, now: float | None = None) -> tuple[bool, str]:
"""An override (the owner's): boost, away or car; one of each kind
per control (a new one replaces it)."""
now = self.clock() if now is None else now
kind = o.get("kind")
try:
if kind == "boost":
c = controls.get(o.get("control", ""))
if c is None or c.kind not in ("heating", "cooling"):
raise ValueError(f"no room {o.get('control')!r}")
lo, hi = RANGE_C[c.kind]
hours = _num(o.get("hours", 2), "hours", 0.25, BOOST_MAX_H)
o = {"kind": "boost", "control": c.id, "c": _num(o.get("c"), "temperature", lo, hi),
"from": int(now), "until": int(o.get("until") or now + hours * 3600)}
elif kind == "away":
until = int(o.get("until") or now + AWAY_MAX_S)
if until <= now or until > now + AWAY_MAX_S:
raise ValueError("away: until a time within the next 60 days")
o = {"kind": "away", "from": int(o.get("from") or now), "until": until}
elif kind == "car":
c = controls.get(o.get("control", ""))
if c is None or c.kind != "ev":
raise ValueError(f"no car charger {o.get('control')!r}")
at = int(o.get("at") or 0)
if not now < at <= now + 7 * 86400:
raise ValueError("car needed: a time within the next 7 days")
o = _ev_target(dict(o, how=o.get("how") or "must"), {"kind": "car", "control": c.id, "at": at})
else:
raise ValueError("override: boost, away or car")
except (ValueError, TypeError) as e:
return self._refused(now, by, f"override:{kind}", str(e))
o.update(id=uuid.uuid4().hex[:8], by=by, set_at=int(now))
self.overrides = [x for x in self.overrides if not (x["kind"] == o["kind"] and x.get("control") == o.get("control"))]
if len(self.overrides) >= MAX_OVERRIDES:
return self._refused(now, by, f"override:{kind}", f"at most {MAX_OVERRIDES} overrides")
self.overrides.append(o)
self._log(now, by, "owner", f"override:{kind}", override_words(o, self.tz), o.get("control", ""))
self._touch(now, o)
return True, override_words(o, self.tz)
def cancel_override(self, kind: str, control: str, by: str, now: float | None = None) -> tuple[bool, str]:
now = self.clock() if now is None else now
gone = [x for x in self.overrides if x["kind"] == kind and x.get("control", "") == (control or "")]
if not gone:
return True, "no such override"
self.overrides = [x for x in self.overrides if x not in gone]
self._log(now, by, "owner", f"override:{kind}", "cancelled", control)
for o in gone:
self._touch(now, o)
return True, f"{kind} cancelled"
def _touch(self, now, o):
self.changed_at = now
if o.get("control"):
self.dirty.add(o["control"])
else:
self.dirty.add("*")
self.save()
def free(self) -> set[str]:
"""Controls whose next plan is taken as it is (steady.py), once."""
d, self.dirty = self.dirty, set()
return d
# -- the panel's inbox ---------------------------------------------------------
def take_inbox(self, controls: dict, tz: str) -> int:
"""The panel's changes (prefs-inbox/*.json, one per change, in name
order): {"by", "set": {"item", "value" | null}} | {"override": {...}}
| {"cancel": {"kind", "control"}}; each applied as the owner's."""
try:
names = sorted(n for n in os.listdir(self.inbox) if n.endswith(".json"))
except OSError:
return 0
n = 0
for name in names:
p = os.path.join(self.inbox, name)
try:
with open(p) as f:
msg = json.load(f)
except (OSError, ValueError):
msg = None
try:
os.remove(p)
except OSError:
pass
if isinstance(msg, dict):
self.apply_msg(msg, controls, tz, "owner", "panel:" + str(msg.get("by") or "owner")[:40])
n += 1
return n
def apply_msg(self, msg: dict, controls: dict, tz: str, level: str, by: str) -> tuple[bool, str]:
"""One change in the inbox's form (the panel's, the console's)."""
now = self.clock()
zone = ZoneInfo(tz or "UTC")
if "set" in msg:
s = msg["set"] or {}
key = str(s.get("item", ""))
val = s.get("value")
if isinstance(val, str): # a text form
try:
val = from_text(key, val, controls.get(key.partition(":")[2]),
(self.value(key) or {}) if key.startswith(("ev:", "battery:")) else {})
except ValueError as e:
return self._refused(now, by, key, str(e))
return self.set(key, val, by, level, controls.get(key.partition(":")[2]), now)
if level != "owner":
return self._refused(now, by, "override", "overrides are the owner's (Home Assistant, the panel)")
if "override" in msg:
o = dict(msg["override"] or {})
if o.get("kind") == "car" and isinstance(o.get("text"), str):
try:
car = parse_car_text(o["text"], zone, now)
except ValueError as e:
return self._refused(now, by, "override:car", str(e))
if car is None:
return self.cancel_override("car", o.get("control", ""), by, now)
o = dict(car, control=o.get("control", ""))
if o.get("kind") == "away" and isinstance(o.get("until_text"), str) and o["until_text"].strip():
try:
o["until"] = local_when(o["until_text"], zone, now)
except ValueError as e:
return self._refused(now, by, "override:away", str(e))
return self.add_override(o, by, controls, now)
if "cancel" in msg:
c = msg["cancel"] or {}
return self.cancel_override(str(c.get("kind", "")), str(c.get("control", "")), by, now)
return self._refused(now, by, "", "set, override or cancel")
# -- into the plan ---------------------------------------------------------------
def apply(self, state: dict, controls: dict, col=None, now: float | None = None) -> dict:
"""The model state with each control's preferences per step
(d["pref"]: optimizer.py). Controls without any keep today's."""
now = self.clock() if now is None else now
self.prune(now)
zone = ZoneInfo(state.get("tz") or "UTC")
T, start, step = state["steps"], state["start"], state["step_s"]
ends = [_local(start + (t + 1) * step, zone) for t in range(T)] # a room's or battery's value at a step's end
ovs = self.active_overrides(now)
for d in state["devices"]:
c = controls.get(d["id"])
if c is None:
continue
if d["kind"] in ("heating", "cooling"):
p = room_series(d, self.value(f"room:{d['id']}"), ovs, state, ends)
elif d["kind"] == "ev":
p = ev_need(d, self.value(f"ev:{d['id']}"), ovs, state, zone, now, col, c)
elif d["kind"] == "battery":
p = battery_series(d, self.value(f"battery:{d['id']}"), ends)
else:
p = None
if p is not None:
d["pref"] = p
state["prefs_cost"] = bool(any(d.get("pref") for d in state["devices"])) and (
self.changed_at > self.costs.get("of", 0) or now - self.costs.get("at", 0) >= COST_EVERY_S)
return state
def take_plan(self, plan: dict, now: float | None = None):
"""The plan's feedback (optimizer.pref_feedback), with the musts'
costs: the plan's own when it made them, else the last ones."""
now = self.clock() if now is None else now
made = plan.get("prefs_cost")
if made is not None:
self.costs = {"at": int(now), "of": max(self.changed_at, 0.0), "eur_day": made}
try:
self.save()
except OSError as e:
log.warning("preferences: %s", e)
eur = self.costs.get("eur_day") or {}
fb = []
for f in plan.get("prefs") or []:
f = dict(f)
if f.get("how") == "must" and f.get("group") in eur:
f["cost_eur_day"] = eur[f["group"]]
f["cost_as_of"] = self.costs.get("at")
f["text"] = feedback_words(f)
fb.append(f)
self.feedback = fb
plan["prefs"] = fb
plan.pop("prefs_cost", None)
return fb
# -- out ---------------------------------------------------------------------------
def public(self) -> dict:
"""For the cloud (with the plan) and the panel: the settings per
level with who and when, the overrides in force, the feedback, the
last changes."""
items = {}
for key, per in self.items.items():
top = self.effective(key)
items[key] = {"in_force": top["level"] if top else None,
"levels": {lvl: {"value": e["value"], "by": e["by"], "at": e["at"],
"text": text_of(key, e["value"])} for lvl, e in per.items()}}
return {"v": 1, "items": items, "overrides": self.active_overrides(), "feedback": self.feedback,
"log": self.log[-20:], "changed_at": int(self.changed_at)}
def write_view(self, controls, mode: str = ""):
"""preferences-view.json for the Hub's panel: the controls (id,
kind, name, the owner's limits), and public()."""
cl = []
for c in controls:
if c.kind not in KINDS:
continue
q = {"heating": "temp_c", "cooling": "temp_c", "battery": "soc_pct", "ev": "current_a"}[c.kind]
b = c.bounds(q)
cl.append({"id": c.id, "kind": c.kind, "name": c.label(), "bounds": [b.lo, b.hi],
"setback_c": _p(c, "setback_c"), "setup_c": _p(c, "setup_c"), "comfort_c": _p(c, "comfort_c")})
v = dict(self.public(), controls=cl, mode=mode, errors=self.errors, ha=self.ha, made=int(self.clock()))
try:
tmp = self.view_path + ".tmp"
with open(tmp, "w") as f:
json.dump(v, f)
os.replace(tmp, self.view_path)
except OSError as e:
log.warning("preferences view: %s", e)
def _p(c, k):
try:
return float(c.p(k))
except (KeyError, TypeError, ValueError):
return None
def _end(o: dict) -> float:
return float(o.get("until") or o.get("at") or 0)
def _local(t: float, zone: ZoneInfo) -> tuple[int, int]:
d = datetime.fromtimestamp(t, timezone.utc).astimezone(zone)
return d.weekday(), d.hour * 60 + d.minute
def in_block(b: dict, wd: int, m: int) -> bool:
"""A block {days, from, to} covers the wall-clock (weekday, minute)? A
block past midnight belongs to the day it starts; from == to: all day."""
a, z = minutes(b["from"]), minutes(b["to"])
if a < z:
return wd in b["days"] and a <= m < z
if a == z:
return wd in b["days"]
return (wd in b["days"] and m >= a) or ((wd - 1) % 7 in b["days"] and m < z)
def within_owner(key: str, value: dict, c):
"""An installer's or operator's setting within the owner's limits (the
mapping's owner level), never above them."""
what = key.split(":", 1)[0]
owner = lambda q: (c.limits.get(q) or {}).get("owner") or (None, None) # noqa: E731
if what == "room":
lo, hi = owner("temp_c")
for b in value["blocks"]:
if (lo is not None and b["c"] < lo) or (hi is not None and b["c"] > hi):
raise ValueError(f"{_fmt(b['c'])} °C is outside the owner's limits "
f"{_fmt(lo) if lo is not None else '-'}..{_fmt(hi) if hi is not None else '-'} °C")
elif what == "battery":
lo, hi = owner("soc_pct")
for p in [value["reserve_pct"]] + [b["pct"] for b in value["blocks"]]:
if (lo is not None and p < lo) or (hi is not None and p > hi):
raise ValueError(f"a reserve of {_fmt(p)} % is outside the owner's limits")
# -- per step ----------------------------------------------------------------------
def room_series(d: dict, value: dict | None, ovs: list[dict], state: dict, ends) -> dict | None:
"""A room's target per step (its temperature at the step's end), the
must's bound (None: none), and the group (a block, a boost, away) it
comes from. base: what it would be without the group (setback/setup;
today's occupied hours without a schedule)."""
cooling = d["kind"] == "cooling"
p = d["params"]
other = float(p["setup_c"] if cooling else p["setback_c"])
comfort = float(p["comfort_c"])
blocks = (value or {}).get("blocks") or []
mine = [o for o in ovs if (o["kind"] == "boost" and o.get("control") == d["id"]) or o["kind"] == "away"]
if not blocks and not mine:
return None
T, start, step = state["steps"], state["start"], state["step_s"]
occ = state.get("occupied") or [1] * T
groups = [{"group": f"{d['id']}:{i}", "kind": "block", "how": b["how"], "c": b["c"],
"text": f"{days_text(b['days'])} {b['from']}-{b['to']} {'≤' if cooling else '≥'} {_fmt(b['c'])} °C"}
for i, b in enumerate(blocks)]
gi = {}
for o in mine:
gi[o["id"]] = len(groups)
groups.append({"group": f"{d['id']}:{o['kind']}", "kind": o["kind"], "how": "must" if o["kind"] == "boost" else "prefer",
"c": o.get("c", other), "text": override_words(o, state.get("tz")), "until": o.get("until")})
target, must, group, base = [], [], [], []
pick = min if cooling else max
for t in range(T):
wd, m = ends[t]
tend = start + (t + 1) * step
b0 = other if blocks else (comfort if occ[min(t + 1, T - 1)] else other)
base.append(b0)
boost = next((o for o in mine if o["kind"] == "boost" and o["from"] <= tend < o["until"]), None)
away = next((o for o in mine if o["kind"] == "away" and o.get("from", 0) <= tend < o["until"]), None)
if boost is not None:
target.append(boost["c"])
must.append(boost["c"])
group.append(gi[boost["id"]])
continue
if away is not None:
target.append(other)
must.append(None)
group.append(gi[away["id"]])
continue
hit = [(i, b) for i, b in enumerate(blocks) if in_block(b, wd, m)]
if not hit:
target.append(b0)
must.append(None)
group.append(-1)
continue
tc = pick(b["c"] for _, b in hit)
ms = [b["c"] for _, b in hit if b["how"] == "must"]
target.append(tc)
must.append(pick(ms) if ms else None)
# the group: the must that binds, else the block that sets the target
g = next((i for i, b in hit if b["how"] == "must" and b["c"] == must[-1]), None) if ms else None
group.append(g if g is not None else next(i for i, b in hit if b["c"] == tc))
return {"target": target, "must": must, "group": group, "base": base, "groups": groups}
def next_ready(value: dict | None, zone: ZoneInfo, now: float) -> tuple[float, dict] | None:
"""The next ready-by time after now and its entry."""
best = None
today = datetime.fromtimestamp(now, zone).date()
for r in (value or {}).get("ready") or []:
h, mi = (int(x) for x in r["at"].split(":"))
for k in range(8):
d = today + timedelta(days=k)
if d.weekday() not in r["days"]:
continue
t = _wall(d, h, mi, zone)
if t > now:
if best is None or t < best[0]:
best = (t, r)
break
return best
def ev_need(d: dict, value: dict | None, ovs: list[dict], state: dict, zone, now: float, col, c) -> dict | None:
"""The car's next ready-by (a one-off override wins while it lasts) as
the departure step and the energy needed: kWh as given, or % from the
car's state of charge: (target - soc) x capacity."""
car = next((o for o in ovs if o["kind"] == "car" and o.get("control") == d["id"]), None)
if car is not None:
when, r, src = car["at"], car, "override"
else:
nr = next_ready(value, zone, now)
if nr is None:
return None
(when, r), src = nr, "schedule"
start, step, T = state["start"], state["step_s"], state["steps"]
dep = int(max(0, min(T, (when - start) // step)))
st = d.get("state") or {}
soc = st.get("soc_pct")
cap = (value or {}).get("capacity_kwh")
note = ""
if r.get("kwh") is not None:
need = r["kwh"] * 1000.0
elif soc is not None and cap:
need = max(0.0, (r["pct"] - float(soc)) / 100.0 * cap * 1000.0)
else:
need = float(st["need_wh"]) if st.get("need_wh") is not None else float(d["params"]["need_wh"])
note = "no state of charge from the car" if soc is None else "the car's capacity is not set"
loc = datetime.fromtimestamp(when, zone)
out = {"group": f"{d['id']}:{'car' if src == 'override' else 'ready'}", "source": src, "dep_step": dep,
"at": int(when), "at_local": loc.strftime("%a %H:%M"), "need_wh": round(need, 1), "how": r["how"],
"max_ct": r.get("max_ct"), "min_wh": (r.get("min_kwh") or 0.0) * 1000.0 if r.get("min_kwh") else 0.0,
"pct": r.get("pct"), "kwh": r.get("kwh"), "soc_pct": soc, "capacity_wh": cap * 1000.0 if cap else None,
"note": note, "beyond": when > start + T * step}
if col is not None and c.entities.get("connected"):
st0 = col.states.get(c.entities["connected"]) or {}
if st0.get("s") == "on" and st0.get("t"):
out["plugged_at"] = int(float(st0["t"]))
out["text"] = f"{_target_text(r).replace(' must', '')} by {loc.strftime('%a %H:%M')}" + \
(" (one-off)" if src == "override" else "")
return out
def battery_series(d: dict, value: dict | None, ends) -> dict | None:
if not value:
return None
base = float(value.get("reserve_pct") or 0.0)
groups = [{"group": f"{d['id']}:reserve", "kind": "reserve", "how": "must", "pct": base,
"text": f"reserve {_fmt(base)} %"}] if base > 0 else []
for i, b in enumerate(value.get("blocks") or []):
groups.append({"group": f"{d['id']}:{i}", "kind": "block", "how": "must", "pct": b["pct"],
"text": f"reserve {_fmt(b['pct'])} % {days_text(b['days'])} {b['from']}-{b['to']}"})
floor, group = [], []
off = 1 if base > 0 else 0
for wd, m in ends:
best, g = base, 0 if base > 0 else -1
for i, b in enumerate(value.get("blocks") or []):
if in_block(b, wd, m) and b["pct"] > best:
best, g = b["pct"], i + off
floor.append(best)
group.append(g)
if not groups:
return None
return {"floor_pct": floor, "group": group, "groups": groups}
# -- words ----------------------------------------------------------------------------
def describe(key: str, value: dict, kind: str = "") -> str:
what = key.split(":", 1)[0]
if what == "room":
return room_text(value) or "no schedule"
if what == "ev":
s = ev_text(value) or "no ready-by times"
return s + (f"; capacity {_fmt(value['capacity_kwh'])} kWh" if value.get("capacity_kwh") else "")
if what == "battery":
return f"reserve {_fmt(value['reserve_pct'])} %" + (f"; {battery_text(value)}" if value.get("blocks") else "")
return json.dumps(value)[:200]
def text_of(key: str, value: dict) -> str:
what = key.split(":", 1)[0]
return {"room": room_text, "ev": ev_text, "battery": battery_text}.get(what, lambda v: "")(value)
def from_text(key: str, text: str, control, current: dict) -> dict:
what = key.split(":", 1)[0]
if control is None:
raise ValueError(f"no control for {key!r}")
if what == "room":
return parse_room_text(text, control.kind)
if what == "ev":
return parse_ev_text(text, current.get("capacity_kwh"))
if what == "battery":
return parse_battery_text(text, current.get("reserve_pct", 0.0))
raise ValueError(f"no preference {key!r}")
def car_text(o: dict, tz: str | None = None) -> str:
"""A car override in the text form: "2026-11-05 07:00 80% must"."""
when = datetime.fromtimestamp(o["at"], ZoneInfo(tz or "UTC")).strftime("%Y-%m-%d %H:%M")
t = _target_text(o)
return f"{when} {t}" if o.get("how") == "must" else f"{when} {t} prefer"
def override_words(o: dict, tz: str | None = None) -> str:
until = datetime.fromtimestamp(_end(o), ZoneInfo(tz or "UTC")).strftime("%a %d %b %H:%M")
if o["kind"] == "boost":
return f"boost to {_fmt(o['c'])} °C until {until}"
if o["kind"] == "away":
return f"away until {until}"
return f"car {_target_text(o).replace(' must', '')} at {until}" + (" (must)" if o.get("how") == "must" else "")
def feedback_words(f: dict) -> str:
"""One preference's result in words: "Mon-Fri 22:00-06:00 ≥ 21 °C: met,
+0.40 €/day"; "car 100% by Thu 08:00: short by 4.2 kWh, ready at full
power by 09:15"."""
head = f.get("label") or f.get("text") or f.get("group", "")
if f.get("met") is None:
bits = [f.get("detail") or "not in the plan's 48 h"]
elif f["met"]:
bits = ["met"] + ([f["detail"]] if f.get("detail") else [])
else:
bits = ["not met"] + ([f["detail"]] if f.get("detail") else [])
if f.get("cost_eur_day") is not None:
bits.append(f"{f['cost_eur_day']:+.2f} €/day")
return f"{head}: " + ", ".join(bits)
+136
View File
@@ -0,0 +1,136 @@
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""A room's thermal resistance R and capacity C from its history
(docs/methods.md §6): the planner's first-order model (edge optimizer.py
`room`),
θ_t = (1 − a) θ_{t−1} + a T_out,t + (Δt/C) P_t + γ G_t + c0,
a = Δt / (R C),
written as a regression on 15-minute steps (Δt = 0.25 h):
θ_t − θ_{t−1} = a (T_out,t − θ_{t−1}) + b P_t + γ G_t + c0, b = Δt / C,
with P the heating power (kW; thermal where the site meters the heat
output, else electric: the planner's COP turns that into heat), G the
global irradiance (kW/m², the sun through the windows) and c0 the internal
gains (people, appliances). Least squares on every step where all inputs
are there; then C = Δt / b, R = Δt / (a C).
The other two terms go to the planner with R and C (Finding B6-a: R and C
were fitted with them; without them the planner expects the room to cool
faster than it does), as heat in the same units as P:
gain_w = 1000 c0 / b the internal gain (W, constant),
solar_m2 = γ / b the sun's heat per irradiance (m²: W of heat
per W/m² of global irradiance, an effective
window area).
(electric fits: W and m² of electric equivalent; the box multiplies by
the COP like C.)
Sanity before a fit is used (else the planner keeps its defaults, R 10
K/kW and C 3 kWh/K): a > 0 and b > 0; R within 1..100 K/kW, C within
0.3..50 kWh/K, τ = RC within 3..1000 h; at least a day of steps, the
heating on and off in it, 2 K of spread in T_out − θ; and the fitted
model simulates the room better than the defaults do (4-hour open-loop
runs from every 4th hour: RMSE in K).
The site's heating power is used for every room (the sims: one zone per
home): the rooms of a home with one heat meter each get the whole
home's C. Per-room heating power, where metered per control, is the
next step."""
from __future__ import annotations
import math
import numpy as np
DT_H = 0.25
DEFAULT_R, DEFAULT_C = 10.0, 3.0
R_BOUNDS, C_BOUNDS, TAU_BOUNDS = (1.0, 100.0), (0.3, 50.0), (3.0, 1000.0)
MIN_STEPS = 96
SIM_STEPS = 16
SIM_EVERY = 16
def simulate(theta0: float, tout: np.ndarray, p_kw: np.ndarray, ghi_kw: np.ndarray, r: float, c: float,
gamma: float = 0.0, c0: float = 0.0) -> np.ndarray:
a, b = DT_H / (r * c), DT_H / c
out, th = [], theta0
for k in range(len(tout)):
th = (1 - a) * th + a * tout[k] + b * p_kw[k] + gamma * ghi_kw[k] + c0
out.append(th)
return np.array(out)
def sim_rmse(theta: np.ndarray, tout: np.ndarray, p_kw: np.ndarray, ghi_kw: np.ndarray, r: float, c: float,
gamma: float = 0.0, c0: float = 0.0) -> float | None:
"""4-hour open-loop runs from every 4th hour where all inputs exist."""
errs = []
for s in range(0, len(theta) - SIM_STEPS - 1, SIM_EVERY):
seg = slice(s + 1, s + 1 + SIM_STEPS)
if np.isnan(theta[s]) or np.isnan(theta[seg]).any() or np.isnan(tout[seg]).any() or np.isnan(p_kw[seg]).any():
continue
sim = simulate(theta[s], tout[seg], p_kw[seg], np.nan_to_num(ghi_kw[seg]), r, c, gamma, c0)
errs.append((sim - theta[seg]) ** 2)
if not errs:
return None
return float(math.sqrt(np.concatenate(errs).mean()))
def fit(theta: np.ndarray, tout: np.ndarray, p_w: np.ndarray, ghi: np.ndarray | None = None,
power: str = "thermal") -> dict:
"""theta, tout (°C), p_w (W), ghi (W/m²) per 15-minute slot, aligned.
Returns {"status": "fitted"|"default", r_k_per_kw, c_kwh_per_k, tau_h,
power, n, rmse_k (simulation, fitted), rmse_default_k, r2, note}, and
when fitted gain_w and solar_m2 (the module's docstring)."""
n_all = len(theta)
ghi = np.zeros(n_all) if ghi is None else np.where(np.isnan(ghi), 0.0, ghi)
p_kw = p_w / 1000.0
g_kw = ghi / 1000.0
prev = np.concatenate([[np.nan], theta[:-1]])
dy = theta - prev
x1 = tout - prev
ok = ~(np.isnan(dy) | np.isnan(x1) | np.isnan(p_kw))
out = {"status": "default", "r_k_per_kw": DEFAULT_R, "c_kwh_per_k": DEFAULT_C, "tau_h": DEFAULT_R * DEFAULT_C,
"power": power, "n": int(ok.sum()), "rmse_k": None, "rmse_default_k": None, "r2": None, "note": ""}
if ok.sum() < MIN_STEPS:
out["note"] = f"{int(ok.sum())} usable steps (at least {MIN_STEPS})"
return out
on = p_kw[ok] > 0.05
if on.all() or not on.any():
out["note"] = "the heating was never " + ("off" if on.all() else "on") + " in the data"
return out
if np.ptp(x1[ok]) < 2.0:
out["note"] = "less than 2 K between outside and inside"
return out
X = np.column_stack([x1[ok], p_kw[ok], g_kw[ok], np.ones(int(ok.sum()))])
coef, *_ = np.linalg.lstsq(X, dy[ok], rcond=None)
a, b, gamma, c0 = (float(v) for v in coef)
resid = dy[ok] - X @ coef
var = float(np.var(dy[ok]))
out["r2"] = round(1 - float(np.var(resid)) / var, 3) if var > 0 else None
if a <= 0 or b <= 0:
out["note"] = f"no physical fit (a {a:.4g}, b {b:.4g})"
return out
c = DT_H / b
r = DT_H / (a * c)
tau = r * c
out["fit"] = {"a": a, "b": b, "gamma": gamma, "c0": c0}
if not (R_BOUNDS[0] <= r <= R_BOUNDS[1] and C_BOUNDS[0] <= c <= C_BOUNDS[1] and TAU_BOUNDS[0] <= tau <= TAU_BOUNDS[1]):
out["note"] = f"outside the sanity bounds (R {r:.3g} K/kW, C {c:.3g} kWh/K, τ {tau:.3g} h)"
return out
fitted = sim_rmse(theta, tout, p_kw, g_kw, r, c, gamma, c0)
default = sim_rmse(theta, tout, p_kw, g_kw, DEFAULT_R, DEFAULT_C)
out["rmse_default_k"] = round(default, 3) if default is not None else None
out["rmse_k"] = round(fitted, 3) if fitted is not None else None
if fitted is None or default is None:
out["note"] = "too few complete 4-hour stretches to check the fit"
return out
if fitted >= default:
out["note"] = f"no better than the defaults ({fitted:.2f} K vs {default:.2f} K)"
return out
out.update(status="fitted", r_k_per_kw=round(r, 3), c_kwh_per_k=round(c, 3), tau_h=round(tau, 2),
gain_w=round(1000.0 * c0 / b, 1), solar_m2=round(gamma / b, 3),
note=f"simulates the room to {fitted:.2f} K (defaults {default:.2f} K)")
return out
+12 -4
View File
@@ -31,8 +31,11 @@ keeps the value in force, its why says so) unless all hold:
Not here, so never held back: the console's and the building's commands,
the safe defaults, the grid operator's order (the plan is made again at
once within its bounds; a kept value outside the bounds in force is never
kept), the owner's own changes (the guard's hands off) and the guard's
floors. The entry the plan made is kept in the entry as "planned".
kept), the owner's own changes (the guard's hands off; and the owner's
preferences, preferences.py: the first plan after a change of a control's
preferences or overrides, `free`, is taken as it is for that control) and
the guard's floors. The entry the plan made is kept in the entry as
"planned".
"""
import math
@@ -117,10 +120,13 @@ def _within(bounds, v) -> bool:
return (lo is None or v >= lo - 1e-9) and (hi is None or v <= hi + 1e-9)
def stabilise(plan: dict, state: dict, controls, col, last_set=None, now: float | None = None) -> dict:
def stabilise(plan: dict, state: dict, controls, col, last_set=None, now: float | None = None,
free: set | None = None) -> dict:
"""The plan with changes that do not matter kept out (module
docstring). last_set(control id, kind "mode"|"set") -> when the agent
last changed that (None: not known). In place; returns the plan."""
last changed that (None: not known). free: control ids (or "*": all)
whose preferences the owner just changed: their entries as planned.
In place; returns the plan."""
now = plan.get("made", 0) if now is None else now
step = plan.get("step_s", 900)
end = plan.get("start", now) + step * state.get("steps", 192)
@@ -133,6 +139,8 @@ def stabilise(plan: dict, state: dict, controls, col, last_set=None, now: float
entries = (plan.get("devices") or {}).get(c.id)
if not entries or c.kind not in ("battery", "heating", "cooling", "ev"):
continue
if free and (c.id in free or "*" in free):
continue # the owner's change: not held back
cur = current(c, col)
if cur is None:
continue
+272
View File
@@ -0,0 +1,272 @@
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""The box's own training history (docs/methods.md B10): what its nightly
training (boxtrain.py) learns from, kept in
the agent's SQLite (<state dir>/training.db), rolling.
slots 15-minute means per asset the trainer uses (load.power,
pv.power, the planned devices' power, room.<r>.temp, the
heating's power, the outdoor thermometer, the grid meter and
the battery), from the agent's own 1-minute windows: per slot
the mean of the windows' means, as the cloud's trainer reads
telemetry (time_bucket('15 minutes') of avg(mean)). A slot the
agent did not see can be filled once from Home Assistant's
statistics (fill_from_stats: 5-minute where HA still has them,
hourly before), never over the agent's own. 13 months (the
training weighs recent weeks more and last year's same season
still: fcmodels.recency_weights); compact: an asset's number,
the slot's number, the sum and count (WITHOUT ROWID; ~20 bytes
a row).
weather hourly records (pvforecast.Hour's fields) per kind: observed
(the cloud's DWD observations shipped in the model bundle,
else Home Assistant's recorded weather entity) and forecast
(the last forecast made before the hour was over: Home
Assistant's, the cloud's), each with its source. 13 months.
fc_slots per series, slot and source the last forecast made before the
slot began (the box's own model, the cloud's series, the cloud's
model on the box, the hour-of-week, the coordinator's week
mean): the live error per source against the meter. 14 days.
Nothing here leaves the site except the error figures the plan carries
(controller.public_plan "ml")."""
from __future__ import annotations
import math
import os
import re
import sqlite3
import threading
from datetime import datetime, timezone
SLOT_S = 900
KEEP_DAYS = 396 # 13 months
KEEP_FC_DAYS = 14
# the assets the trainer reads (building/trainlib.py: SITE_SERIES, ROOM_INPUTS,
# CONTROLLED; trainer.load_site), and the meter for the member's net draw
ASSETS = {"load.power", "pv.power", "weather.outside_temp", "heat.power", "heatpump.power", "heating.power",
"ac.power", "cooling.power", "ev.power", "grid.import_power", "grid.export_power", "battery.power"}
ASSET_RE = re.compile(r"^(room\.[^.]+\.temp|(heatpump|heating|ac|cooling|ev)\.[0-9]+\.power)$")
OBSERVED_PREF = ("cloud", "ha") # an observed hour: DWD's (with irradiation) before HA's entity
FORECAST_PREF = ("ha", "cloud") # a forecast hour: as the box forecasts (localml.merge_hours: HA's first)
def wanted(asset: str) -> bool:
return asset in ASSETS or bool(ASSET_RE.match(asset))
def _finite(v) -> bool:
return isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)
class Store:
"""The training history. Thread-safe (the nightly job reads it in a
worker thread while the agent's loop writes): one connection, a lock."""
def __init__(self, path: str):
self.path = path
self.lock = threading.Lock()
self.db = sqlite3.connect(path, isolation_level=None, check_same_thread=False)
self.db.execute("PRAGMA journal_mode=WAL")
self.db.execute("PRAGMA synchronous=NORMAL")
self.db.execute("CREATE TABLE IF NOT EXISTS assets (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE)")
# slot: the quarter hour's number (unix / 900); src: 0 the agent, 1 HA's 5-minute, 2 HA's hourly statistics
self.db.execute("CREATE TABLE IF NOT EXISTS slots (aid INTEGER NOT NULL, slot INTEGER NOT NULL, sum REAL NOT NULL, "
"n INTEGER NOT NULL, src INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (aid, slot)) WITHOUT ROWID")
self._ids = {name: i for i, name in self.db.execute("SELECT id, name FROM assets")}
self.db.execute("CREATE TABLE IF NOT EXISTS weather (end_ts INTEGER NOT NULL, kind TEXT NOT NULL, source TEXT NOT NULL, "
"solar REAL, cloud REAL, temp REAL, issued INTEGER, PRIMARY KEY (end_ts, kind, source))")
self.db.execute("CREATE TABLE IF NOT EXISTS fc_slots (series TEXT NOT NULL, slot INTEGER NOT NULL, source TEXT NOT NULL, "
"value REAL NOT NULL, issued INTEGER NOT NULL, PRIMARY KEY (series, slot, source))")
self.db.execute("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
def _aid(self, name: str) -> int:
i = self._ids.get(name)
if i is None:
self.db.execute("INSERT OR IGNORE INTO assets (name) VALUES (?)", (name,))
i = self._ids[name] = self.db.execute("SELECT id FROM assets WHERE name = ?", (name,)).fetchone()[0]
return i
def close(self):
with self.lock:
self.db.close()
# -- the agent's own samples -------------------------------------------------
def add_window(self, start: float, records: list[dict]) -> int:
"""A closed 1-minute window (agent.close_window's records): each
wanted gauge's mean into its 15-minute slot (by the window's start)."""
slot = int(start // SLOT_S)
recs = [(r["asset"], float(r["mean"])) for r in records
if r.get("kind") == "gauge" and _finite(r.get("mean")) and wanted(r.get("asset", ""))]
if not recs:
return 0
with self.lock:
rows = [(self._aid(a), slot, v) for a, v in recs]
self.db.executemany(
"INSERT INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, 0) ON CONFLICT (aid, slot) "
"DO UPDATE SET sum = CASE WHEN src = 0 THEN sum + excluded.sum ELSE excluded.sum END, "
"n = CASE WHEN src = 0 THEN n + 1 ELSE 1 END, src = 0", rows)
return len(rows)
def fill(self, asset: str, values: dict[int, float], src: str) -> int:
"""Slots the agent did not see (Home Assistant's statistics): never
over a slot that has a value."""
code = {"agent": 0, "ha-5minute": 1, "ha-hour": 2}.get(src, 2)
with self.lock:
aid = self._aid(asset)
rows = [(aid, int(k) // SLOT_S, float(v), code) for k, v in values.items() if _finite(v) and int(k) % SLOT_S == 0]
before = self.db.total_changes
self.db.executemany("INSERT OR IGNORE INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, ?)", rows)
return self.db.total_changes - before
def assets(self) -> list[str]:
with self.lock:
return [a for (a,) in self.db.execute("SELECT name FROM assets WHERE id IN (SELECT DISTINCT aid FROM slots)")]
def series(self, since: int, until: int, assets=None) -> dict[str, dict[int, float]]:
"""{asset: {slot: mean}} for slots in [since, until)."""
lo, hi = -(-int(since) // SLOT_S), -(-int(until) // SLOT_S)
with self.lock:
names = {i: n for n, i in self._ids.items()}
out: dict[str, dict[int, float]] = {}
for aid, name in names.items():
if assets is not None and name not in assets:
continue
rows = self.db.execute("SELECT slot, sum / n FROM slots WHERE aid = ? AND slot >= ? AND slot < ?",
(aid, lo, hi)).fetchall()
if rows:
out[name] = {int(s) * SLOT_S: float(v) for s, v in rows}
return out
def span(self) -> tuple[int | None, int | None, int]:
"""The first and last slot kept and how many (all assets)."""
with self.lock:
a, b, n = self.db.execute("SELECT min(slot), max(slot), count(*) FROM slots").fetchone()
return (a * SLOT_S if a is not None else None), (b * SLOT_S if b is not None else None), n
# -- the weather -------------------------------------------------------------
def put_weather(self, kind: str, source: str, hours: list, issued: float | None = None) -> int:
"""Hourly records (pvforecast.Hour: end, solar_kwh_m2, cloud_pct,
temp_c). A forecast only for hours not over when it was made, the
newest such kept; an observation replaces what was there."""
rows = []
for h in hours:
end = int(h.end.timestamp())
if end % 3600:
continue
if kind == "forecast" and (issued is None or issued >= end):
continue
if h.solar_kwh_m2 is None and h.cloud_pct is None and h.temp_c is None:
continue
rows.append((end, kind, source, h.solar_kwh_m2, h.cloud_pct, h.temp_c, int(issued) if issued else None))
with self.lock:
self.db.executemany("INSERT OR REPLACE INTO weather (end_ts, kind, source, solar, cloud, temp, issued) "
"VALUES (?, ?, ?, ?, ?, ?, ?)", rows)
return len(rows)
def weather_hours(self, since: int, until: int, kind: str) -> list:
"""One record per hour ending in (since, until] of this kind, by the
kind's preferred source (OBSERVED_PREF, FORECAST_PREF)."""
from .pvforecast import Hour
pref = OBSERVED_PREF if kind == "observed" else FORECAST_PREF
with self.lock:
rows = self.db.execute("SELECT end_ts, source, solar, cloud, temp FROM weather WHERE kind = ? AND end_ts > ? "
"AND end_ts <= ?", (kind, int(since), int(until))).fetchall()
by: dict[int, tuple] = {}
for end, src, sol, cl, tc in rows:
rank = pref.index(src) if src in pref else len(pref)
if end not in by or rank < by[end][0]:
by[end] = (rank, sol, cl, tc)
return [Hour(datetime.fromtimestamp(e, timezone.utc), *by[e][1:]) for e in sorted(by)]
def training_hours(self, since: int, until: int) -> list:
"""The weather the models learn from: observed, else (an hour never
observed) the last forecast for it (wxfetch.hours, prefer observed)."""
obs = {int(h.end.timestamp()): h for h in self.weather_hours(since, until, "observed")}
for h in self.weather_hours(since, until, "forecast"):
obs.setdefault(int(h.end.timestamp()), h)
return [obs[k] for k in sorted(obs)]
def missing_observed(self, since: int, until: int) -> list[tuple[int, int]]:
"""Spans of hours without an observation (for Home Assistant's history)."""
with self.lock:
have = {e for (e,) in self.db.execute("SELECT end_ts FROM weather WHERE kind = 'observed' AND end_ts > ? "
"AND end_ts <= ?", (int(since), int(until)))}
spans, cur = [], None
for e in range(int(since) // 3600 * 3600 + 3600, int(until) + 1, 3600):
if e in have:
if cur:
spans.append(tuple(cur))
cur = None
elif cur is None:
cur = [e - 3600, e]
else:
cur[1] = e
if cur:
spans.append(tuple(cur))
return spans
# -- the forecasts per source -------------------------------------------------
def put_forecasts(self, series: str, start: int, cands: dict[str, list], issued: float, first: int = 1,
steps: int = 96) -> int:
"""Per source its values for the slots start + k·900, k in
[first, steps): the slot not begun yet (the quarter under way is
not a forecast any more)."""
rows = []
for src, vals in cands.items():
for k in range(first, min(steps, len(vals or []))):
v = vals[k]
if _finite(v):
rows.append((series, int(start) + k * SLOT_S, src, float(v), int(issued)))
with self.lock:
self.db.executemany("INSERT OR REPLACE INTO fc_slots (series, slot, source, value, issued) VALUES (?, ?, ?, ?, ?)",
rows)
return len(rows)
def forecasts(self, series: str, since: int, until: int) -> dict[str, dict[int, float]]:
with self.lock:
rows = self.db.execute("SELECT source, slot, value FROM fc_slots WHERE series = ? AND slot >= ? AND slot < ?",
(series, int(since), int(until))).fetchall()
out: dict[str, dict[int, float]] = {}
for src, s, v in rows:
out.setdefault(src, {})[int(s)] = float(v)
return out
# -- meta, pruning ---------------------------------------------------------------
def get(self, key: str) -> str | None:
with self.lock:
row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
return row[0] if row else None
def set(self, key: str, value: str):
with self.lock:
self.db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, str(value)))
def prune(self, now: float, days: int = KEEP_DAYS):
cut = int(now) - days * 86400
with self.lock:
self.db.execute("DELETE FROM slots WHERE slot < ?", (cut // SLOT_S,))
self.db.execute("DELETE FROM weather WHERE end_ts < ?", (cut,))
self.db.execute("DELETE FROM fc_slots WHERE slot < ?", (int(now) - KEEP_FC_DAYS * 86400,))
def size_bytes(self) -> int:
try:
return sum(os.path.getsize(self.path + s) for s in ("", "-wal") if os.path.exists(self.path + s))
except OSError:
return 0
def stats_to_slots(rows: list[dict], period: str) -> dict[int, float]:
"""Home Assistant's statistics rows ({"start": unix, "mean"}) as
15-minute values: 5-minute means averaged per slot, an hourly mean held
for its four slots (a step, not a shape: noted where used)."""
out: dict[int, list[float]] = {}
for r in rows:
v = r.get("mean")
if not _finite(v):
continue
s = int(r["start"])
if period == "hour":
for q in range(4):
out.setdefault(s // 3600 * 3600 + q * SLOT_S, []).append(float(v))
else:
out.setdefault(s // SLOT_S * SLOT_S, []).append(float(v))
return {k: sum(v) / len(v) for k, v in out.items()}
+274
View File
@@ -0,0 +1,274 @@
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""Training one series and the rooms: the pure part of the trainer
(docs/methods.md B5, B6, B10), shared by the cloud's trainer (trainer.py)
and the Edge Agent's own nightly training (edge/cantabile_edge/boxtrain.py:
a byte-identical copy, building/test_boxmodel.py checks it). numpy,
LightGBM (through fcmodels.fit), no database, no network.
train_series per series the challengers (how, gbm, pvphys) fitted on
the window before the held-out one, scored on it with the
model in force (the champion), the best put in force only
if it beats the champion (fcmodels.choose)
fit_rooms each room's R and C (rcfit.py)
base_load load.power less the planned devices' power (methods.md A2-a)
held_out_weather, scored_on
the held-out window on the weather as it was forecast
where that is known (methods.md B5-a)
cut_before PV_METERED_SINCE: metered PV before it does not count"""
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from datetime import datetime
import numpy as np
try: # on the box: the Edge Agent's copies (edge/cantabile_edge; methods.md B10)
from . import fcmodels, rcfit
except ImportError:
import fcmodels
import rcfit
log = logging.getLogger("trainer")
SLOT_S = 900
SITE_SERIES = {"load": "load.power", "pv": "pv.power"}
# A site's `base_load` series (methods.md A2-a; the box's planner adds the
# devices it plans itself, and load.power includes them): load.power less
# the power assets of the kinds of control the site's inventory has, as the
# Edge Agent does (mapping.CONTROLLED_POWER). A system's total (ev.power),
# else the sum of its parts (ev.1.power, ...). The battery is not here:
# load.power does not include its charging (sim physics.py: house = load +
# heat pump + A/C + EV). Published as base_load_w next to load_w (as
# metered: the coordinator's B7 uses it).
BASE = "base_load"
CONTROLLED_POWER = {"heating": ("heatpump.power", "heating.power"), "cooling": ("ac.power", "cooling.power"),
"ev": ("ev.power",)}
CONTROLLED = tuple(a for names in CONTROLLED_POWER.values() for a in names)
ROOM_INPUTS = ["weather.outside_temp", "heat.power", "heatpump.power"]
MIN_SLOTS = 16 # a series with less metered is not trained
SINCE_SERIES = ("pv",) # the series PV_METERED_SINCE cuts
def cut_before(series: str, t: np.ndarray, y: np.ndarray, since: datetime | None) -> np.ndarray:
"""y without the metered slots before `since` (NaN) for the series it
applies to (SINCE_SERIES)."""
if since is None or series not in SINCE_SERIES:
return y
return np.where(t < int(since.timestamp()), np.nan, y)
def scored_on(w_test_hours: int, n_hours: int) -> str:
"""What the held-out scores were made on (methods.md B5-a): the weather
as it was forecast where it is stored, else the observed weather."""
if n_hours <= 0 or w_test_hours <= 0:
return "scored on observed weather (no stored forecast for the held-out window: optimistic)"
if w_test_hours >= n_hours:
return "scored on forecast weather"
return (f"scored on forecast weather for {100 * w_test_hours // n_hours} % of the held-out hours, "
"observed for the rest")
@dataclass
class Run:
series: str
kind: str
metrics: dict
params: dict | None = None
chosen: bool = False
note: str = ""
@dataclass
class SeriesResult:
series: str
runs: list[Run] = field(default_factory=list)
champion: dict | None = None # metrics of the model in force on the held-out window
baseline: dict | None = None
chosen: Run | None = None
window: tuple = () # (data_from, data_to, test_from, test_to) unix
n_train: int = 0
n_test: int = 0
note: str = ""
def wslice(w: fcmodels.Weather, sl: slice) -> fcmodels.Weather:
return fcmodels.Weather(w.temp[sl], w.ghi[sl], w.cloud[sl], w.elev[sl], w.az[sl], w.hours)
# ---- one series, champion against challengers -----------------------------------
def train_series(series: str, t: np.ndarray, y: np.ndarray, w: fcmodels.Weather, site: fcmodels.Site,
champion: dict | None, kinds=None, w_test: fcmodels.Weather | None = None) -> SeriesResult:
"""t, y: the series over the training weeks (and the week before, for
the lag), w: the weather on the same slots (observed: what the models
learn from). Fits each kind on the window before the held-out one,
scores it and the champion's params on the held-out window, picks
(fcmodels.choose). w_test: the weather the held-out window is scored
on, as it was forecast (methods.md B5-a: the published forecasts run on
forecasts, so the choice and the drift baseline should too); else w."""
wt = w_test if w_test is not None else w
res = SeriesResult(series)
have = np.flatnonzero(~np.isnan(y))
if len(have) < MIN_SLOTS:
res.note = f"{len(have)} metered slots (at least {MIN_SLOTS})"
return res
cut = fcmodels.holdout(t, y)
first = int(have[0])
tr, te = slice(first, cut), slice(cut, len(t))
res.window = (int(t[first]), int(t[cut - 1]) + SLOT_S if cut > first else int(t[first]), int(t[cut]) if cut < len(t) else None,
int(t[-1]) + SLOT_S)
res.n_train = int((~np.isnan(y[tr])).sum())
res.n_test = int((~np.isnan(y[te])).sum())
if res.n_train < MIN_SLOTS or res.n_test < 4:
res.note = f"too little data to hold a window out ({res.n_train} slots to train, {res.n_test} to test)"
return res
kinds = kinds or (["how", "gbm", "pvphys"] if series == "pv" else ["how", "gbm"])
hist_t, hist_y = t[:cut], y[:cut]
for kind in kinds:
try:
p = fcmodels.fit(kind, series, t[tr], y[tr], wslice(w, tr), site, t[:first], y[:first])
except Exception as e: # one kind's failure never stops the others
log.exception("fit %s %s", series, kind)
res.runs.append(Run(series, kind, {}, None, note=f"failed: {e}"))
continue
if p is None:
why = {"gbm": "under a day of data", "pvphys": "no location or no weather for it"}.get(kind, "no data")
res.runs.append(Run(series, kind, {}, None, note=f"not trained: {why}"))
continue
pred = fcmodels.predict(p, series, t[te], wslice(wt, te), site, hist_t, hist_y)
res.runs.append(Run(series, kind, fcmodels.metrics(y[te], pred), p))
base = next((r for r in res.runs if r.kind == "how" and r.params), None)
res.baseline = base.metrics if base else None
if champion is not None:
try:
pred = fcmodels.predict(champion, series, t[te], wslice(wt, te), site, hist_t, hist_y)
res.champion = fcmodels.metrics(y[te], pred)
except Exception as e: # a champion that can no longer run is replaced
res.champion = None
res.note = f"the model in force could not be scored ({e}): replaced"
# nMAE where the metered mean is not ~0, else MAE (a held-out night of PV)
key = score_key(res)
scores = {r.kind: r.metrics.get(key) for r in res.runs if r.params}
champ = res.champion.get(key) if res.champion else None
best = fcmodels.choose(scores, champ)
if best is None and champion is None and scores:
best = min((k for k in scores if scores[k] is not None), key=lambda k: scores[k], default=None)
if best:
res.chosen = next(r for r in res.runs if r.kind == best and r.params)
res.chosen.chosen = True
elif champ is not None:
res.note = (res.note + "; " if res.note else "") + f"the model in force stays ({key} {champ:.3g})"
return res
def score_key(res: SeriesResult) -> str:
"""The metric a choice is made on: nMAE where the metered mean of the
held-out window is not ~0, else MAE (a held-out night of PV)."""
return "nmae" if res.baseline and res.baseline.get("nmae") is not None else "mae"
def score_params(params: dict, series: str, t: np.ndarray, y: np.ndarray, w: fcmodels.Weather, site: fcmodels.Site,
w_test: fcmodels.Weather | None = None) -> dict | None:
"""Another model's metrics on the same held-out window as train_series
cuts it (a benchmark: the cloud's model on the box), or None where it
cannot run or there is no window."""
have = np.flatnonzero(~np.isnan(y))
if len(have) < MIN_SLOTS:
return None
cut = fcmodels.holdout(t, y)
wt = w_test if w_test is not None else w
te = slice(cut, len(t))
try:
pred = fcmodels.predict(params, series, t[te], wslice(wt, te), site, t[:cut], y[:cut])
except Exception as e:
log.warning("benchmark %s: %s", series, e)
return None
return fcmodels.metrics(y[te], pred)
# ---- the rooms ---------------------------------------------------------------------
def fit_rooms(t: np.ndarray, assets: dict[str, np.ndarray], w: fcmodels.Weather) -> dict[str, dict]:
"""Each room.<r>.temp against the outside (the site's thermometer, else
the weather) and the site's heating (thermal heat.power, else the heat
pump's electric power)."""
out = {}
tout = assets.get("weather.outside_temp")
if tout is None or np.isnan(tout).all():
tout = w.temp
else:
tout = np.where(np.isnan(tout), w.temp, tout)
if "heat.power" in assets:
p, power = assets["heat.power"], "thermal"
elif "heatpump.power" in assets:
p, power = assets["heatpump.power"], "electric"
else:
p, power = None, "thermal"
for a, theta in sorted(assets.items()):
if not (a.startswith("room.") and a.endswith(".temp")):
continue
room = a[len("room."):-len(".temp")]
if p is None:
out[room] = {**rcfit.fit(theta, tout, np.full(len(theta), np.nan)), "note": "no heating power metered"}
continue
out[room] = rcfit.fit(theta, tout, p, w.ghi, power)
return out
def room_inputs(assets: dict[str, np.ndarray], w: fcmodels.Weather) -> tuple:
"""The inputs fit_rooms fits every room on: (outside °C, heating power
W or None, irradiance W/m²), for scoring other R and C on the same data."""
tout = assets.get("weather.outside_temp")
tout = w.temp if tout is None or np.isnan(tout).all() else np.where(np.isnan(tout), w.temp, tout)
p = assets.get("heat.power", assets.get("heatpump.power"))
return tout, p, w.ghi
# ---- the base load -----------------------------------------------------------------
def _system(asset: str) -> str:
return asset.split(".", 1)[0]
def controlled_power(raw: dict[str, np.ndarray], kinds: set[str], n: int) -> np.ndarray:
"""The planned devices' metered power per slot (W): per kind its assets
(a system's total, else the sum of its parts <system>.<n>.power), each
once; a slot without a value counts 0."""
dev = np.zeros(n)
for kind in sorted(kinds):
for name in CONTROLLED_POWER.get(kind, ()):
sysname, q = name.split(".", 1)
parts = [a for a in raw if a.startswith(sysname + ".") and a.endswith("." + q)
and a[len(sysname) + 1:-len(q) - 1].isdigit()]
for a in ([name] if name in raw else sorted(parts)):
dev = dev + np.nan_to_num(np.maximum(raw[a], 0.0))
return dev
def base_load(raw: dict[str, np.ndarray], kinds: set[str]) -> np.ndarray | None:
"""load.power less the planned devices' metered power (controlled_power),
never below 0 (methods.md A2-a); NaN where load.power is."""
load = raw.get(SITE_SERIES["load"])
if load is None:
return None
return np.where(np.isnan(load), np.nan, np.maximum(load - controlled_power(raw, kinds, len(load)), 0.0))
# ---- the weather the held-out window is scored on -----------------------------------
def held_out_weather(t: np.ndarray, y: np.ndarray, w: fcmodels.Weather, fc_hours: list, lat: float | None,
lon: float | None) -> tuple[fcmodels.Weather | None, str]:
"""The weather the held-out window (fcmodels.holdout, as train_series
cuts it) is scored on: the observed hours with every hour that was
forecast put in their place, and what that was (scored_on). None where
nothing was forecast: the observed weather."""
if not fc_hours or lat is None or lon is None or (~np.isnan(y)).sum() < MIN_SLOTS:
return None, scored_on(0, 0)
cut = fcmodels.holdout(t, y)
test_ends = {(int(ts) // 3600 + 1) * 3600 for ts in t[cut:].tolist()}
fc = {int(h.end.timestamp()): h for h in fc_hours}
n_fc = len(test_ends & set(fc))
if n_fc == 0:
return None, scored_on(0, len(test_ends))
merged = {int(h.end.timestamp()): h for h in w.hours} | fc
return fcmodels.weather(t, [merged[k] for k in sorted(merged)], lat, lon), scored_on(n_fc, len(test_ends))
+14 -3
View File
@@ -10,7 +10,9 @@ package main
// a command). Fleet
// sends one as a signed command, `tariff` or `model`, with args
// {"bundle": base64 of the bundle's JSON bytes, "sig": base64 of their
// signature with the BUNDLE key}. The command itself is checked with the
// signature with the BUNDLE key}, or, since 2026-11-04 to a link that says
// so in its heartbeat (bundle_text), {"bundle_text": the JSON itself, "sig"}:
// one base64 layer less on the wire (fleet bundlecap.go). The command itself is checked with the
// command key (checkCommand); here the bundle's own signature, its kind, site
// and expiry, and that it is not older than the one the agent has. Then the
// JSON goes into the agent's state directory (tariff.json, model.json),
@@ -37,7 +39,10 @@ var devicesPolicy = func() {
}
}
const bundleMax = 256 << 10 // a tariff is a few kB; a model's data well under this
// bundleMax: the largest bundle taken, reported in the heartbeat
// (bundle_max; fleet sends an older link at most 136 KiB). 512 KiB as text is
// ~0.7 MiB on the wire, under NATS's 1 MiB and any 1 MiB WebSocket proxy.
const bundleMax = 512 << 10
// bundleCert: the bundle key's certificate (CANTABILE_BUNDLE_CERT). Read for
// each bundle: a box may get it after its link started.
@@ -60,7 +65,13 @@ type bundleHead struct {
// expired and not issued in the future.
func checkBundle(key crypto.PublicKey, kind, site string, args map[string]string, now time.Time) ([]byte, bundleHead, error) {
var h bundleHead
data, err1 := base64.StdEncoding.DecodeString(args["bundle"])
var data []byte
var err1 error
if t, ok := args["bundle_text"]; ok {
data = []byte(t)
} else {
data, err1 = base64.StdEncoding.DecodeString(args["bundle"])
}
sig, err2 := base64.StdEncoding.DecodeString(args["sig"])
switch {
case err1 != nil || err2 != nil || len(data) == 0 || len(sig) == 0:
+58
View File
@@ -0,0 +1,58 @@
// Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
package main
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"encoding/base64"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// A bundle as text (fleet bundlecap.go, for a link that reports bundle_text)
// is taken like the base64 one, the signature over the same bytes; up to
// bundleMax (reported in the heartbeat), larger refused; a tampered text refused.
func TestBundlesAsTextAndLarge(t *testing.T) {
k, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
site := strings.Repeat("ab", 16)
now := time.Now()
l := &link{id: site, cfg: config{BundleCert: certFile(t, k), AgentState: t.TempDir()}}
model := func(id string, issued int64, pad int) map[string]any {
return map[string]any{"v": 1, "kind": "model", "id": id, "site": site, "issued": issued,
"expires": now.Add(time.Hour).Unix(), "pad": strings.Repeat("x", pad)}
}
text := func(b map[string]any) map[string]string {
a := bundleArgs(t, k, b)
data, _ := base64.StdEncoding.DecodeString(a["bundle"])
return map[string]string{"bundle_text": string(data), "sig": a["sig"]}
}
cmd := func(args map[string]string) command { return command{ID: "c", Cmd: "model", Args: args} }
if ok, msg := l.bundleCmd(cmd(text(model("m1", now.Unix(), 400<<10))), now); !ok { // over the old 256 KiB
t.Fatalf("a 400 KiB model as text: %s", msg)
}
if b, _ := os.ReadFile(filepath.Join(l.cfg.AgentState, "model.json")); len(b) < 400<<10 {
t.Fatalf("model.json: %d bytes", len(b))
}
if ok, msg := l.bundleCmd(cmd(bundleArgs(t, k, model("m2", now.Unix()+1, 300<<10))), now); !ok {
t.Fatalf("a 300 KiB model as base64: %s", msg)
}
if ok, msg := l.bundleCmd(cmd(text(model("m3", now.Unix()+2, bundleMax))), now); ok || !strings.Contains(msg, "larger than") {
t.Errorf("over bundleMax: %v %q", ok, msg)
}
a := text(model("m4", now.Unix()+3, 10))
a["bundle_text"] = strings.Replace(a["bundle_text"], `"m4"`, `"m5"`, 1)
if ok, msg := l.bundleCmd(cmd(a), now); ok || !strings.HasSuffix(msg, "bad bundle signature") {
t.Errorf("tampered text: %v %q", ok, msg)
}
t.Setenv("CANTABILE_HUB", "0")
t.Setenv("CANTABILE_CONSENT_FILE", filepath.Join(t.TempDir(), "none.json"))
hb := (&link{started: now, cfg: config{AgentState: t.TempDir()}, loc: &local{}}).heartbeat()
if hb["bundle_max"] != bundleMax || hb["bundle_text"] != true || bundleMax < 512<<10 {
t.Errorf("heartbeat: %v %v", hb["bundle_max"], hb["bundle_text"])
}
}
+2
View File
@@ -756,6 +756,8 @@ func (l *link) heartbeat() map[string]any {
if p := l.pathInfo(); p != nil {
hb["path"] = p
}
// the bundles this link takes (bundles.go): as large, and as text
hb["bundle_max"], hb["bundle_text"] = bundleMax, true
c := consents(env("CANTABILE_CONSENT_FILE",
map[bool]string{true: "/data/cantabile/edge/consent.json", false: "/data/state/consent.json"}[isHub()]))
hb["consents"] = c
+4
View File
@@ -15,6 +15,10 @@ dependencies = [
"PyYAML==6.0.2",
"highspy==1.11.0", # the optimiser (HiGHS MILP), v1
"numpy==2.3.5", # the box's forecast models (localml.py, fcmodels.py): as building/requirements.txt
# the box trains its own models (boxtrain.py, methods.md B10; +152 MB in the image):
# LightGBM as building/requirements.txt, scipy (LightGBM requires it) as resolved with numpy 2.3.5
"lightgbm==4.5.0",
"scipy==1.18.1",
]
[project.optional-dependencies]
+260
View File
@@ -0,0 +1,260 @@
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""The box trains its own models (boxtrain.py, trainhist.py; methods.md
B10): the training history, the nightly job's caps (wall time, memory) and
that it never blocks the controller's loop, the box keeping the better
model (against its own and the cloud's), the planner taking the box's own
model first, the intraday correction, the plan's "ml". The parity with the
cloud's trainer on the same data: building/test_boxmodel.py."""
import asyncio
import json
import math
import sys
import time
from datetime import datetime, timezone
from types import SimpleNamespace
import numpy as np
import pytest
from cantabile_edge import boxtrain, fcmodels, pvforecast, trainhist
from cantabile_edge.bundles import Bundles
from cantabile_edge.inputs import Collector, History, model_state
from test_base_load import site_mapping, watts
from test_control_parts import SITE
from test_localml import START, bundle, gbm, put, weather
DAY = 86400
def cfg(tmp_path, **kw):
d = dict(state_dir=str(tmp_path), train=True, train_at="03:30", train_series_s=60.0, train_mem_mb=600,
train_min_free_mb=0, train_days=60, pv_metered_since="")
d.update(kw)
return SimpleNamespace(**d)
def store(tmp_path):
return trainhist.Store(str(tmp_path / "training.db"))
# -- the history -----------------------------------------------------------------
def test_windows_into_quarter_hour_means(tmp_path):
s = store(tmp_path)
t0 = START
for k in range(15): # 15 one-minute windows of the slot
s.add_window(t0 + 60 * k, [{"asset": "load.power", "kind": "gauge", "mean": 400.0 + 20 * k},
{"asset": "sensor.unrelated", "kind": "gauge", "mean": 1.0},
{"asset": "pv.power", "kind": "gauge"}, # unavailable: no mean
{"asset": "grid.energy", "kind": "counter", "delta": 3.0}])
got = s.series(t0, t0 + 900)
assert set(got) == {"load.power"} and got["load.power"][t0] == pytest.approx(540.0)
# HA's statistics fill only what the agent did not see; the agent's own replace them
assert s.fill("load.power", {t0: 1.0, t0 + 900: 300.0}, "ha-hour") == 1
assert s.series(t0, t0 + 1800)["load.power"] == {t0: pytest.approx(540.0), t0 + 900: 300.0}
s.add_window(t0 + 900, [{"asset": "load.power", "kind": "gauge", "mean": 700.0}])
assert s.series(t0 + 900, t0 + 1800)["load.power"][t0 + 900] == 700.0
assert trainhist.wanted("room.living.temp") and trainhist.wanted("ev.2.power") and not trainhist.wanted("ev.power.x")
s.prune(t0 + 400 * DAY)
assert s.span()[2] == 0
def test_weather_kinds_and_sources(tmp_path):
s = store(tmp_path)
H = lambda end, cloud: pvforecast.Hour(datetime.fromtimestamp(end, timezone.utc), None, cloud, 5.0) # noqa: E731
s.put_weather("forecast", "cloud", [H(START + 3600, 10.0), H(START + 7200, 10.0)], issued=START)
s.put_weather("forecast", "ha", [H(START + 3600, 20.0)], issued=START + 100)
s.put_weather("forecast", "ha", [H(START + 3600, 99.0)], issued=START + 3600) # made after the hour: no forecast
s.put_weather("observed", "ha", [H(START + 7200, 70.0)])
s.put_weather("observed", "cloud", [H(START + 7200, 60.0)])
fc = s.weather_hours(START, START + 7200, "forecast")
assert [h.cloud_pct for h in fc] == [20.0, 10.0] # HA's first, as the box forecasts
tr = s.training_hours(START, START + 7200)
assert [h.cloud_pct for h in tr] == [20.0, 60.0] # observed (DWD first), else the forecast
assert s.missing_observed(START, START + 4 * 3600) == [(START, START + 3600), (START + 7200, START + 4 * 3600)]
def test_ha_history_to_hours():
st = [{"a": {"temperature": 3.0, "cloud_coverage": 40}, "lu": START + 100},
{"a": {"temperature": 2.0, "cloud_coverage": 90}, "lu": START + 3700}]
hs = boxtrain.ha_history_hours(st, [(START, START + 3 * 3600)])
assert [(int(h.end.timestamp()) - START, h.temp_c, h.cloud_pct) for h in hs] == \
[(3600, 3.0, 40.0), (7200, 2.0, 90.0), (10800, 2.0, 90.0)]
# -- the caps: never in the controller's way -----------------------------------------
def slow_child(seconds, mb=0):
return [sys.executable, "-c", f"import time; x = bytearray({mb} << 20); [x.__setitem__(i, 1) for i in range(0, len(x), 4096)]; "
f"time.sleep({seconds})"]
async def ticking(job, period=0.05):
"""Run job while counting ticks of a loop like the controller's."""
ticks, done = [], asyncio.Event()
async def ticker():
while not done.is_set():
ticks.append(time.monotonic())
await asyncio.sleep(period)
tk = asyncio.create_task(ticker())
t0 = time.monotonic()
res = await job
done.set()
await tk
gaps = np.diff(ticks) if len(ticks) > 1 else np.array([0.0])
return res, time.monotonic() - t0, len(ticks), float(gaps.max())
async def test_a_slow_fit_is_stopped_at_its_wall_time_and_never_blocks(tmp_path):
b = boxtrain.BoxTraining(cfg(tmp_path, train_series_s=1.0), store(tmp_path), None, child_cmd=slow_child(30))
(res, how), took, n, gap = await ticking(b.child({"series": "load"}))
assert res is None and "wall time" in how["killed"]
assert 0.9 < took < 5.0
assert n >= 15 and gap < 0.5 # the loop ran all along
@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="/proc")
async def test_a_hungry_fit_is_stopped_at_its_memory_cap(tmp_path):
b = boxtrain.BoxTraining(cfg(tmp_path, train_mem_mb=80), store(tmp_path), None, child_cmd=slow_child(20, 200))
(res, how), took, n, gap = await ticking(b.child({"series": "load"}))
assert res is None and "memory cap" in how["killed"] and how["peak_mb"] > 80
assert took < 15 and gap < 0.5
async def test_low_memory_skips_the_night(tmp_path, monkeypatch):
b = boxtrain.BoxTraining(cfg(tmp_path, train_min_free_mb=10 ** 9), store(tmp_path), None)
monkeypatch.setattr(boxtrain, "mem_available_mb", lambda: 100.0)
monkeypatch.setattr(b, "prepare", lambda *a: ([{"series": "load"}], {}))
monkeypatch.setattr(b, "child", lambda job: pytest.fail("trained with too little memory"))
out = await b.train_once()
assert "100 MB available" in out["notes"][0]
def test_next_run_local_time():
now = datetime(2026, 11, 4, 12, 0, tzinfo=timezone.utc).timestamp()
nxt = datetime.fromtimestamp(boxtrain.next_run(now, "03:30", "Europe/Berlin"), timezone.utc)
assert (nxt.day, nxt.hour, nxt.minute) == (5, 2, 30) # 03:30 CET
# -- the box keeps the better model -----------------------------------------------------
def result(chosen, runs, champion=None, cloud=None):
return {"key": "nmae", "chosen": chosen, "params": {"kind": chosen, "values": [1.0] * 168} if chosen else None,
"runs": [{"kind": k, "metrics": {"nmae": v}} for k, v in runs.items()], "champion": champion and {"nmae": champion},
"cloud": cloud and {"nmae": cloud}, "window": [0, 1, 2, 3], "n_test": 672, "note": ""}
def test_the_box_keeps_the_better_model(tmp_path):
b = boxtrain.BoxTraining(cfg(tmp_path), store(tmp_path), None)
job = {"site": {"tz": "UTC"}, "scored_on": "scored on forecast weather"}
# the first model: in force only if it beats the cloud's
out = b.take("load", job, result("gbm", {"how": 0.4, "gbm": 0.30}, cloud=0.25), {"seconds": 1}, 1000)
assert not out["in_force"] and "not in force" in out["result"]
assert b.state["series"]["load"]["kind"] == "gbm" # kept: tomorrow's champion
out = b.take("load", job, result("gbm", {"how": 0.4, "gbm": 0.20}, champion=0.30, cloud=0.25), {}, 2000)
assert out["in_force"] and b.state["series"]["load"]["trained_at"] == 2000
# a new one that beats the box's own but not the cloud's: the old one stays (and is in force while it beats the cloud)
out = b.take("load", job, result("gbm", {"how": 0.4, "gbm": 0.19}, champion=0.21, cloud=0.18), {}, 3000)
e = b.state["series"]["load"]
assert e["trained_at"] == 2000 and not e["in_force"] and "does not beat" in out["result"]
# nothing beats the box's own, and it beats the cloud's: in force
out = b.take("load", job, result(None, {"how": 0.4, "gbm": 0.3}, champion=0.2, cloud=0.25), {}, 4000)
assert b.state["series"]["load"]["in_force"] and b.state["series"]["load"]["trained_at"] == 2000
# no cloud model on the box: anything but the hour-of-week that beat it
out = b.take("pv", job, result("how", {"how": 0.3}), {}, 5000)
assert not out["in_force"]
out = b.take("pv", job, result("pvphys", {"how": 0.3, "pvphys": 0.2}, champion=0.3), {}, 6000)
assert out["in_force"]
# a failed child changes nothing
out = b.take("pv", job, None, {"killed": "over its wall time (300 s)"}, 7000)
assert out["result"].startswith("failed") and b.state["series"]["pv"]["trained_at"] == 6000
def test_the_live_error_takes_a_model_out_of_force(tmp_path):
s = store(tmp_path)
now = START + 8 * DAY
for k in range(7 * 96):
ts = now - 7 * DAY + k * 900
s.add_window(ts, [{"asset": "load.power", "kind": "gauge", "mean": 500.0}])
s.put_forecasts("plan", ts - 900, {"own": [None, 800.0], "cloud": [None, 520.0]}, ts - 900)
b = boxtrain.BoxTraining(cfg(tmp_path), s, None)
le = b.live_errors("plan", now)
assert le["own"]["nmae"] == pytest.approx(0.6) and le["cloud"]["nmae"] == pytest.approx(0.04) and le["own"]["n"] == 672
out = b.take("load", {"site": {}}, result("gbm", {"how": 0.4, "gbm": 0.2}, cloud=0.25), {}, now)
assert not out["in_force"] and "live nMAE" in out["result"]
# -- the planner takes the box's own model first ------------------------------------------
class Own:
"""A stand-in for BoxTraining at the planner: a forecast for load_w."""
def __init__(self, vals):
self.vals = vals
def forecast(self, name, start, steps, ha, entity=""):
return (list(self.vals), {"kind": "gbm", "weather": "Home Assistant's forecast"}) if name == "base_load_w" else None
def correct(self, series, start, values):
return list(values), 0.0
def room_rc(self, *a):
return None
def state(tmp_path, b, ha=None):
m = site_mapping(tmp_path, [], [])
col = Collector()
watts(col, sensor__load=700)
return model_state(START, [], m.inputs, col, b, History(str(tmp_path / "h.json")), "UTC", ha)
def test_own_model_first_then_the_cloud(tmp_path):
b = put(tmp_path, bundle(models={"base_load_w": gbm()}, wx=weather(30)))
b.own = Own([1234.0] * 100 + [None] * 92)
s = state(tmp_path, b)
f = s["sources"]["forecast"]["load"]
assert s["load_w"][0] == 1234.0 and s["load_w"][99] == 1234.0
assert s["load_w"][100] != 1234.0 # past the box's own: the cloud's model on the box, the profile
assert f["own"] == 100 and f["own_model"] == "gbm" and f["source"] == "own"
assert "the box's own gbm model (trained on the box) for 100 of 192 steps" in s["sources"]["load"]
c = s["_cands"]["load"]
assert set(c) >= {"own", "cloud", "cloud_model", "hour_of_week", "used"}
assert c["cloud"][0] == 5000.0 and c["own"][0] == 1234.0 and c["hour_of_week"][0] == 111.0
# no own model in force: the cloud's series as before
b.own = Own([None] * 192)
s = state(tmp_path, b)
assert s["load_w"][0] == 5000.0 and "own" not in s["sources"]["forecast"]["load"]
def test_own_model_alone_without_a_cloud_model(tmp_path):
b = Bundles(str(tmp_path), SITE)
b.own = Own([800.0] * 192)
s = state(tmp_path, b)
assert s["load_w"] == [800.0] * 192 and s["sources"]["forecast"]["load"]["source"] == "own"
# -- the intraday correction on the box ------------------------------------------------------
def test_the_box_carries_the_last_hours_error(tmp_path):
s = store(tmp_path)
for k in range(12): # the last 3 hours: 300 W above what was planned on
ts = START - 3 * 3600 + k * 900
s.add_window(ts, [{"asset": "load.power", "kind": "gauge", "mean": 1300.0}])
s.put_forecasts("plan", ts - 900, {"used": [None, 1000.0]}, ts - 900)
b = boxtrain.BoxTraining(cfg(tmp_path), s, None)
out, e = b.correct("plan", START, [1000.0] * 192)
assert e == pytest.approx(300.0)
assert out[0] == pytest.approx(1300.0) and out[8] == pytest.approx(1150.0) and out[191] == pytest.approx(1000.0, abs=1e-3)
assert b.correct("pv", START, [0.0] * 192)[1] == 0.0 # nothing known: unchanged
# -- the plan's "ml" ----------------------------------------------------------------------
def test_status_for_the_plan(tmp_path):
b = boxtrain.BoxTraining(cfg(tmp_path), store(tmp_path), None)
b.take("load", {"site": {}, "scored_on": "x"}, result("gbm", {"how": 0.4, "gbm": 0.2}, cloud=0.25), {"seconds": 2.0}, 1000)
fc = {"load": {"start": START, "step_s": 900, "values": [1.0] * 192, "source": "own", "series": "load"}}
st = b.status(START, fc)
assert st["v"] == 1 and st["series"]["load"]["in_force"] and st["series"]["load"]["held_out"] == {"how": 0.4, "gbm": 0.2}
assert st["series"]["load"]["held_out_cloud"] == 0.25 and st["forecasts"] == fc
assert "forecasts" not in b.status(START + 900, fc) # at most hourly
json.dumps(st, allow_nan=False)
assert len(json.dumps(st)) < 8000
+72
View File
@@ -0,0 +1,72 @@
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""The box trains for itself without the share consent: its training
history fills from its own readings either way (they never leave the
box), and nothing of it goes upward without that consent (no "ml" in the
plan: no forecasts, no error figures). With the consent, as before: the
telemetry and the plan's "ml"."""
import pytest
from cantabile_edge import controller as controller_mod
from cantabile_edge.agent import Agent
from cantabile_edge.config import Config
from cantabile_edge.store import Queue
from test_base_load import gauge, heater, site_mapping
from test_control_parts import SITE
T0 = 1_800_000_000 // 900 * 900
class Sink:
def __init__(self):
self.published = []
async def publish(self, kind, data):
self.published.append((kind, data))
def agent(tmp_path, share: bool):
m = site_mapping(tmp_path, [gauge("sensor.pv", "pv.power")], [heater()])
cfg = Config(site_id=SITE, mapping=str(tmp_path / "mapping.yaml"), state_dir=str(tmp_path), dashboards=False)
a = Agent(cfg, m, Queue(str(tmp_path / "queue.db"), 100), Sink())
a.clock = lambda: T0 + 30
if share:
a.consent.set(True, "test")
a.agg.begin(T0)
a.train_agg.begin(T0)
a.on_entities({"a": {"sensor.load": {"s": "800", "a": {"unit_of_measurement": "W"}},
"sensor.pv": {"s": "300", "a": {"unit_of_measurement": "W"}}}})
a.close_window(T0 + 60)
return a
@pytest.mark.parametrize("share", [False, True])
def test_the_history_fills_with_or_without_the_consent(tmp_path, share):
a = agent(tmp_path, share)
got = a.control.boxml.store.series(T0, T0 + 900)
assert got["load.power"][T0] == pytest.approx(800.0) and got["pv.power"][T0] == pytest.approx(300.0)
# the cloud: only with the consent
assert a.queue.depth() == (1 if share else 0)
@pytest.mark.parametrize("share", [False, True])
async def test_nothing_goes_upward_without_the_consent(tmp_path, monkeypatch, share):
a = agent(tmp_path, share)
ctl = a.control
async def solve(state, timeout):
return {"v": 1, "made": T0 + 30, "start": state["start"], "step_s": 900, "status": "Optimal", "solve_s": 0.0,
"devices": {}, "flex": {"grid_w": [500] * 96, "shed_w": [0] * 96, "net_w": [500] * 96},
"cost_eur": 1.0, "baseline_eur": 1.2, "price_note": "", "sources": state["sources"]}
monkeypatch.setattr(controller_mod, "solve", solve)
ctl.client = object()
plan = await ctl.make_plan()
published = [d for k, d in ctl.sink.published if k == "plan"]
assert plan is not None and len(published) == 1
if share:
assert published[0]["ml"]["v"] == 1 and "forecasts" in published[0]["ml"]
else:
assert "ml" not in published[0] and "ml" not in plan
# the box keeps its forecasts per source for itself either way (the live error, the intraday correction)
assert ctl.boxml.store.forecasts("plan", 0, 2 ** 40)
+2 -1
View File
@@ -119,7 +119,8 @@ def test_a_bundle_with_the_new_keys_is_accepted_and_one_without_still_works(tmp_
s = state(tmp_path, b, history(tmp_path))
assert s["sources"]["load"] == "model (base_load_w)"
assert s["load_w"][0] == 5000.0 and s["load_w"][-1] == 111.0 # the series, then the profile
assert s["sources"]["forecast"]["load"] == {"series": "base_load_w", "cloud": 32, "box": 0, "hour_of_week": 160}
assert s["sources"]["forecast"]["load"] == {"series": "base_load_w", "cloud": 32, "box": 0, "hour_of_week": 160,
"source": "hour_of_week"}
def test_a_model_the_box_cannot_run_is_left_out(tmp_path):
+492
View File
@@ -0,0 +1,492 @@
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""The owner's preferences (preferences.py): the text forms, the target per
step (DST days too), must vs prefer in the plan, a car's ready-by with its
state of charge and when it cannot be met, overrides expiring, the levels
(the owner's win, an installer within the owner's limits), Home Assistant's
entities round trip, the panel's inbox, persistence across a restart."""
import asyncio
import json
import os
from datetime import datetime, timezone
from types import SimpleNamespace
from zoneinfo import ZoneInfo
import pytest
from cantabile_edge import preferences as pm
from cantabile_edge import steady
from cantabile_edge.controls import parse as parse_controls
BERLIN = ZoneInfo("Europe/Berlin")
START = 1793232000 # 2026-10-29 00:00 UTC, a Thursday (01:00 in Berlin)
class Clock:
def __init__(self, t):
self.t = t
def __call__(self):
return self.t
def controls():
raw = [{"id": "living", "kind": "heating", "entities": {"climate": "climate.living"},
"limits": {"temp_c": {"owner": [19, 24]}}},
{"id": "ac", "kind": "cooling", "entities": {"climate": "climate.ac"}},
{"id": "car", "kind": "ev", "entities": {"switch": "switch.ev", "connected": "binary_sensor.ev"}},
{"id": "battery", "kind": "battery", "entities": {"setpoint": "number.b"},
"limits": {"soc_pct": {"owner": [10, 100]}}}]
return {c.id: c for c in parse_controls(raw, set())}
def state(start=START, steps=192, tz="Europe/Berlin", devices=()):
return {"start": start, "step_s": 900, "steps": steps, "tz": tz, "occupied": [1] * steps,
"devices": list(devices)}
def room_dev(cid="living", kind="heating"):
return {"id": cid, "kind": kind, "params": {"comfort_c": 21.0, "setback_c": 17.0, "setup_c": 28.0},
"state": {"temp_c": 20.0}, "bounds": {"temp_c": [16, 26, "owner", "owner"]}}
def local(t):
return datetime.fromtimestamp(t, timezone.utc).astimezone(BERLIN)
# -- text forms ---------------------------------------------------------------
def test_room_text_round_trip():
v = pm.parse_room_text("Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19; Sa,So 7-23 20.5", "heating")
assert v["blocks"][0] == {"days": [0, 1, 2, 3, 4], "from": "22:00", "to": "06:00", "c": 21.0, "how": "must"}
assert v["blocks"][2]["days"] == [5, 6] and v["blocks"][2]["from"] == "07:00" and v["blocks"][2]["how"] == "prefer"
t = pm.room_text(v)
assert t == "Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19; Sa,Su 07:00-23:00 20.5"
assert pm.parse_room_text(t, "heating") == v
assert pm.parse_days("daily") == list(range(7)) and pm.parse_days("Di-Do") == [1, 2, 3]
assert pm.parse_days("Sa-Mo") == [0, 5, 6] # a range over the week's end
for bad in ("Mo-Fr 22:00 21", "Xx 22:00-06:00 21", "Mo 22:00-06:00 40", "Mo 25:00-06:00 21",
"Mo 22:00-06:00 21 maybe"):
with pytest.raises(ValueError):
pm.parse_room_text(bad, "heating")
assert len(t) <= 255
def test_ev_and_battery_text():
v = pm.parse_ev_text("Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct min 5kWh", 60)
assert v["capacity_kwh"] == 60
a, b = v["ready"]
assert a == {"days": [0, 1, 2, 3, 4], "at": "08:00", "pct": 100.0, "kwh": None, "how": "must",
"max_ct": None, "min_kwh": None}
assert b["kwh"] == 20 and b["how"] == "prefer" and b["max_ct"] == 25 and b["min_kwh"] == 5
assert pm.ev_text(v) == "Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct min 5kWh"
with pytest.raises(ValueError):
pm.parse_ev_text("Mo 08:00 80% 20kWh") # both
bt = pm.parse_battery_text("daily 17:00-21:00 50%", 30)
assert bt == {"reserve_pct": 30.0, "blocks": [{"days": list(range(7)), "from": "17:00", "to": "21:00", "pct": 50.0}]}
assert pm.battery_text(bt) == "daily 17:00-21:00 50%"
# -- the target per step ------------------------------------------------------------
def test_schedule_target_per_step():
v = pm.parse_room_text("Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19", "heating")
s = state(devices=[room_dev()])
ends = [pm._local(START + (t + 1) * 900, BERLIN) for t in range(192)]
p = pm.room_series(s["devices"][0], v, [], s, ends)
def at(y, mo, d, h, mi): # the step whose end is this local time
ts = datetime(y, mo, d, h, mi, tzinfo=BERLIN).timestamp()
return int((ts - START) // 900) - 1
t = at(2026, 10, 29, 3, 0) # Thu 03:00: Wed's night block
assert (p["target"][t], p["must"][t]) == (21, 21)
t = at(2026, 10, 29, 12, 0) # Thu noon: 19, prefer
assert (p["target"][t], p["must"][t]) == (19, None)
t = at(2026, 10, 30, 23, 0) # Fri 23:00: Fri's night block
assert p["must"][t] == 21
t = at(2026, 10, 31, 0, 30) # Sat 00:30: still Fri's night block
assert p["must"][t] == 21
# a day later: the weekend
s2 = state(start=START + 86400, devices=[room_dev()])
ends2 = [pm._local(START + 86400 + (t + 1) * 900, BERLIN) for t in range(192)]
p2 = pm.room_series(s2["devices"][0], v, [], s2, ends2)
t = at(2026, 10, 31, 5, 45) - 96 # Sat 05:45: Fri's night block
assert p2["must"][t] == 21
t = at(2026, 10, 31, 7, 0) - 96 # Sat 07:00: no block: the setback, nothing must
assert (p2["target"][t], p2["must"][t], p2["group"][t]) == (17, None, -1)
t = at(2026, 10, 31, 22, 30) - 96 # Sat 22:30: Saturday starts no night block
assert (p2["target"][t], p2["must"][t]) == (17, None)
assert p["groups"][0]["text"] == "Mo-Fr 22:00-06:00 ≥ 21 °C" and p["groups"][0]["how"] == "must"
# without a schedule (and no override): today's behaviour
assert pm.room_series(s["devices"][0], None, [], s, ends) is None
def test_schedule_across_the_change_to_winter_time():
"""2026-10-25 03:00 CEST -> 02:00 CET: a 25-hour Sunday. A block is on
the wall clock: 06:00 is 04:00 UTC on Saturday, 05:00 UTC on Sunday."""
start = int(datetime(2026, 10, 24, 0, 0, tzinfo=timezone.utc).timestamp())
v = pm.parse_room_text("daily 06:00-22:00 20", "heating")
s = state(start=start, devices=[room_dev()])
ends = [pm._local(start + (t + 1) * 900, BERLIN) for t in range(192)]
p = pm.room_series(s["devices"][0], v, [], s, ends)
step = lambda h, m=0, d=24: int((datetime(2026, 10, d, h, m, tzinfo=timezone.utc).timestamp() - start) // 900) - 1 # noqa: E731
assert p["target"][step(4, 0)] == 20 and p["target"][step(3, 45)] == 17 # Sat 06:00 CEST
assert p["target"][step(5, 0, 25)] == 20 and p["target"][step(4, 45, 25)] == 17 # Sun 06:00 CET
assert p["target"][step(21, 0, 25)] == 17 and p["target"][step(20, 45, 25)] == 20 # Sun 22:00 CET
# every step: the block's wall-clock rule, by local time
for t in range(192):
lt = local(start + (t + 1) * 900)
assert p["target"][t] == (20 if 6 <= lt.hour < 22 else 17), lt
def test_ready_by_on_the_spring_change():
"""2027-03-28 02:00 CET -> 03:00 CEST: a ready-by at 02:30 does not
exist that day; it is 03:30 CEST (as a clock moves on)."""
v = pm.parse_ev_text("daily 02:30 80%")
now = datetime(2027, 3, 28, 0, 0, tzinfo=BERLIN).timestamp()
t, r = pm.next_ready(v, BERLIN, now)
assert local(t).strftime("%H:%M %Z") == "03:30 CEST"
t2, _ = pm.next_ready(v, BERLIN, t + 60)
assert local(t2).strftime("%d %H:%M") == "29 02:30"
# -- the plan: must vs prefer, a car's ready-by ---------------------------------------------
optimizer = pytest.importorskip("cantabile_edge.optimizer")
from test_optimizer import house # noqa: E402
def heated(how):
"""A room whose 17:00-21:00 block costs 3 EUR/kWh to hold (a scarce
evening); the owner's limit 21.5 °C keeps it from storing heat before."""
s = house([{"id": "living", "kind": "heating", "channels": ["climate"],
"params": {"r_k_per_kw": 10, "c_kwh_per_k": 3, "max_power_w": 4000, "comfort_c": 21,
"setback_c": 17, "cop": 1.0},
"state": {"temp_c": 21.0}, "bounds": {"temp_c": [16, 21.5, "owner", "owner"]}}])
hour = lambda k: (k // 4 + 1) % 24 # noqa: E731
s["price_import"] = [300.0 if 17 <= hour(k) < 21 else p for k, p in enumerate(s["price_import"])]
ends = [pm._local(START + (t + 1) * 900, BERLIN) for t in range(192)]
v = pm.parse_room_text(f"daily 17:00-21:00 21 {how}", "heating")
s["devices"][0]["pref"] = pm.room_series(s["devices"][0], v, [], s, ends)
return s
def block_temps(s, p):
m = optimizer.Model(s)
m.solve(20)
val = m.h.getSolution().col_value
temps = [val[v.index] for v in m.devices["living"][1]["temp"]]
g = s["devices"][0]["pref"]["group"]
return [temps[t] for t in range(192) if g[t] == 0]
def test_must_is_hard_and_prefer_soft():
s = heated("must")
must = block_temps(s, None)
assert len(must) == 32 and min(must) >= 20.9, min(must)
pre = heated("prefer")
assert min(block_temps(pre, None)) < 20.5 # at 3 EUR/kWh comfort gives way
# the plan says so, with the must's cost
s["prefs_cost"] = True
out = optimizer.plan(s, 20)
f = out["prefs"][0]
assert f["how"] == "must" and f["met"] is True and f["label"] == "daily 17:00-21:00 ≥ 21 °C"
assert out["prefs_cost"]["living:0"] > 0.5 # holding 21 at 3 EUR/kWh costs
fp = optimizer.plan(heated("prefer"), 20)["prefs"][0]
assert fp["met"] is False and "°C at" in fp["detail"]
def test_must_gives_way_to_a_building_cap_and_the_owners_limit():
s = heated("must")
s["building"] = {"thermostat_cap": {"value": 19.0, "from_step": 0, "to_step": 192, "penalty": 2.0,
"level": "flexible"}}
out = optimizer.plan(s, 20)
f = out["prefs"][0]
assert f["met"] is True and f.get("held_by") == "building" and "held at 19 °C" in f["detail"]
s2 = heated("must")
v = pm.parse_room_text("daily 17:00-21:00 23 must", "heating")
ends = [pm._local(START + (t + 1) * 900, BERLIN) for t in range(192)]
s2["devices"][0]["pref"] = pm.room_series(s2["devices"][0], v, [], s2, ends)
f2 = optimizer.plan(s2, 20)["prefs"][0]
assert f2.get("held_by") == "owner" and "21.5" in f2["detail"]
def ev(soc, connected=True):
return {"id": "car", "kind": "ev", "channels": ["switch", "current"],
"params": {"need_wh": 8000, "departure": "07:00", "min_current_a": 6, "max_current_a": 16,
"phases": 1, "voltage": 230},
"state": {"connected": connected, "soc_pct": soc}, "bounds": {"current_a": [0, 16, "device", "device"]}}
def with_car(soc, text, capacity=50, connected=True, now=START):
s = house([ev(soc, connected)])
c = controls()["car"]
v = pm.parse_ev_text(text, capacity)
s["devices"][0]["pref"] = pm.ev_need(s["devices"][0], v, [], s, BERLIN, now, None, c)
return s
def test_ev_ready_by_from_the_state_of_charge():
s = with_car(40, "daily 08:00 80% must") # (80 - 40) % of 50 kWh = 20 kWh by Thu 08:00
p = s["devices"][0]["pref"]
assert p["need_wh"] == 20000 and p["dep_step"] == 28 and p["at_local"] == "Thu 08:00"
out = optimizer.plan(s, 20)
f = out["prefs"][0]
assert f["met"] is True and abs(f["planned_kwh"] - 20) < 0.3 and f["reached_pct"] == 80
assert f["detail"].startswith("80 % by Thu 08:00") and f["charge_eur"] > 0
def test_ev_must_infeasible_says_by_how_much_and_when():
s = with_car(10, "daily 08:00 100% must", capacity=60) # 54 kWh in 7 h at 3.68 kW: no
out = optimizer.plan(s, 20)
f = out["prefs"][0]
assert f["met"] is False and 27 < f["short_kwh"] < 29.5
ready = local(f["ready_at"])
assert ready.strftime("%a") == "Thu" and ready.hour >= 15 # 54 kWh / 3.68 kW: ~14.7 h from 01:00
assert "short by" in f["detail"] and "ready at full power by Thu" in f["detail"]
nc = optimizer.plan(with_car(10, "daily 08:00 100% must", 60, connected=False), 20)["prefs"][0]
assert nc["met"] is False and nc["detail"] == "not plugged in"
def test_ev_prefer_up_to_a_price_and_a_minimum():
# ready Thu 18:00: the night (12 ct, 01:00-06:00) gives 5 h x 3.68 kW = 18.4 kWh; the day costs 28 ct
s = with_car(None, "daily 18:00 30kWh max 20ct")
out = optimizer.plan(s, 20)
f = out["prefs"][0]
assert 17 < f["planned_kwh"] < 19 and f["met"] is False and "dearer" in f["detail"]
s = with_car(None, "daily 18:00 30kWh max 20ct min 22kWh")
f = optimizer.plan(s, 20)["prefs"][0]
assert 21.8 < f["planned_kwh"] < 23
s = with_car(None, "daily 18:00 30kWh max 50ct")
assert optimizer.plan(s, 20)["prefs"][0]["planned_kwh"] > 29.5
def test_battery_reserve_is_the_floor():
from test_optimizer import battery
s = house([battery(soc=60, floor=10)])
v = pm.check_battery({"reserve_pct": 30, "blocks": [{"days": "daily", "from": "17:00", "to": "21:00", "pct": 55}]})
ends = [pm._local(START + (t + 1) * 900, BERLIN) for t in range(192)]
s["devices"][0]["pref"] = pm.battery_series(s["devices"][0], v, ends)
m = optimizer.Model(s)
m.solve(20)
val = m.h.getSolution().col_value
soc = [val[x.index] / 100.0 for x in m.devices["battery"][1]["soc"]] # % of 10 kWh
fl = s["devices"][0]["pref"]["floor_pct"]
assert all(soc[t] >= fl[t] - 0.5 for t in range(192))
assert min(soc[t] for t in range(192) if fl[t] == 55) >= 54.5
assert min(soc) < 50 # outside the evening only 30 holds it
out = optimizer.plan(s, 20)
assert {f["group"] for f in out["prefs"]} == {"battery:reserve", "battery:0"} and all(f["met"] for f in out["prefs"])
# -- the store: levels, overrides, persistence ------------------------------------------------
def test_levels_owner_wins_installer_within_limits(tmp_path):
cs = controls()
p = pm.Preferences(str(tmp_path), Clock(START))
ok, why = p.set("room:living", {"blocks": [{"days": "daily", "from": "22:00", "to": "06:00", "c": 26}]},
"installer:bob", "installer", cs["living"])
assert not ok and "owner's limits" in why # 26 above the owner's 24
ok, _ = p.set("room:living", {"blocks": [{"days": "daily", "from": "22:00", "to": "06:00", "c": 22}]},
"installer:bob", "installer", cs["living"])
assert ok and p.effective("room:living")["level"] == "installer"
ok, why = p.set("room:living", {"blocks": [{"days": "daily", "from": "22:00", "to": "06:00", "c": 26, "how": "must"}]},
"Home Assistant", "owner", cs["living"])
assert ok and p.effective("room:living")["by"] == "Home Assistant" # the owner may (the plan keeps the bounds)
ok, why = p.set("room:living", {"blocks": []}, "installer:bob", "installer", cs["living"])
assert ok and "owner's own setting is in force" in why
p.set("room:living", None, "panel:alice", "owner", cs["living"])
assert p.effective("room:living")["level"] == "installer" # the owner cleared theirs
assert [e["by"] for e in p.log] == ["installer:bob", "Home Assistant", "installer:bob", "panel:alice"]
assert p.errors[-1]["by"] == "installer:bob"
ok, why = p.apply_msg({"override": {"kind": "away"}}, cs, "Europe/Berlin", "installer", "installer:bob")
assert not ok and "owner" in why
def test_overrides_win_and_expire(tmp_path):
cs = controls()
clock = Clock(START)
p = pm.Preferences(str(tmp_path), clock)
p.set("room:living", pm.parse_room_text("daily 00:00-24:00 19", "heating"), "ha", "owner", cs["living"])
ok, _ = p.add_override({"kind": "boost", "control": "living", "c": 23, "hours": 2}, "ha", cs)
assert ok
s = state(devices=[room_dev()])
p.apply(s, cs)
pr = s["devices"][0]["pref"]
assert pr["target"][0] == 23 and pr["must"][0] == 23 and pr["groups"][pr["group"][0]]["kind"] == "boost"
assert pr["target"][8] == 19 # after 2 h: the schedule again
assert p.free() == {"living"} and p.free() == set()
clock.t += 3 * 3600
assert p.prune() is True and p.overrides == [] and p.log[-1]["what"] == "expired"
s = state(start=START + 3 * 3600, devices=[room_dev()])
p.apply(s, cs)
assert s["devices"][0]["pref"]["target"][0] == 19
# away: every room at its setback, the schedule's musts suspended
p.set("room:living", pm.parse_room_text("daily 00:00-24:00 21 must", "heating"), "ha", "owner", cs["living"])
ok, _ = p.apply_msg({"override": {"kind": "away", "until_text": "2026-10-30"}}, cs, "Europe/Berlin", "owner", "ha")
assert ok
s = state(start=START + 3 * 3600, devices=[room_dev()])
p.apply(s, cs)
pr = s["devices"][0]["pref"]
back = int((datetime(2026, 10, 30, 0, 0, tzinfo=BERLIN).timestamp() - s["start"]) // 900)
assert pr["target"][0] == 17 and pr["must"][0] is None
assert pr["must"][back] == 21 and pr["target"][back - 2] == 17
# a car needed once wins over its schedule until then
p.set("ev:car", pm.parse_ev_text("daily 08:00 100% must", 50), "ha", "owner", cs["car"])
ok, msg = p.apply_msg({"override": {"kind": "car", "control": "car", "text": "2026-10-29 06:00 60%"}},
cs, "Europe/Berlin", "owner", "ha")
assert ok, msg
s = state(start=START + 3 * 3600, devices=[dict(ev(20), pref=None)])
s["devices"][0]["params"]["need_wh"] = 8000
p.apply(s, cs, now=START + 3 * 3600)
e = s["devices"][0]["pref"]
assert e["source"] == "override" and e["need_wh"] == 20000 and e["at_local"] == "Thu 06:00"
clock.t = datetime(2026, 10, 29, 6, 1, tzinfo=BERLIN).timestamp()
p.prune()
s = state(start=int(clock.t // 900 * 900), devices=[ev(20)])
p.apply(s, cs)
assert s["devices"][0]["pref"]["source"] == "schedule" and s["devices"][0]["pref"]["at_local"] == "Thu 08:00"
def test_persists_across_a_restart_and_the_panels_inbox(tmp_path):
cs = controls()
p = pm.Preferences(str(tmp_path), Clock(START))
os.makedirs(tmp_path / "prefs-inbox")
for i, m in enumerate([{"by": "alice", "set": {"item": "battery:battery", "value": {"reserve_pct": 30, "blocks": []}}},
{"by": "alice", "set": {"item": "ev:car", "value": "Mo-Fr 07:30 80% must"}},
{"by": "alice", "override": {"kind": "boost", "control": "living", "c": 22.5, "hours": 1}},
{"by": "alice", "set": {"item": "room:nope", "value": {"blocks": []}}}]):
(tmp_path / "prefs-inbox" / f"{i:03d}.json").write_text(json.dumps(m))
assert p.take_inbox(cs, "Europe/Berlin") == 4
assert os.listdir(tmp_path / "prefs-inbox") == []
assert p.errors[-1]["item"] == "room:nope"
p.write_view(list(cs.values()), "shadow")
view = json.loads((tmp_path / "preferences-view.json").read_text())
assert {c["id"] for c in view["controls"]} == {"living", "ac", "car", "battery"}
assert view["items"]["ev:car"]["levels"]["owner"]["text"] == "Mo-Fr 07:30 80% must"
assert view["items"]["ev:car"]["levels"]["owner"]["by"] == "panel:alice"
q = pm.Preferences(str(tmp_path), Clock(START + 60)) # the agent started again
assert q.items == p.items and q.overrides == p.overrides and q.log == p.log
assert q.value("battery:battery")["reserve_pct"] == 30 and q.active_overrides()[0]["c"] == 22.5
def test_costs_made_again_only_when_due(tmp_path):
cs = controls()
clock = Clock(START)
p = pm.Preferences(str(tmp_path), clock)
p.set("room:living", pm.parse_room_text("daily 22:00-06:00 21 must", "heating"), "ha", "owner", cs["living"])
s = p.apply(state(devices=[room_dev()]), cs)
assert s["prefs_cost"] is True
plan = {"prefs": [{"control": "living", "group": "living:0", "how": "must", "met": True, "label": "x"}],
"prefs_cost": {"living:0": 0.4}}
p.take_plan(plan)
assert plan["prefs"][0]["cost_eur_day"] == 0.4 and plan["prefs"][0]["text"] == "x: met, +0.40 €/day"
clock.t += 600
assert p.apply(state(devices=[room_dev()]), cs)["prefs_cost"] is False # not changed, not an hour yet
plan = {"prefs": [{"control": "living", "group": "living:0", "how": "must", "met": True, "label": "x"}]}
p.take_plan(plan)
assert plan["prefs"][0]["cost_eur_day"] == 0.4 # the last cost, as of then
clock.t += 3600
assert p.apply(state(devices=[room_dev()]), cs)["prefs_cost"] is True
def test_steady_does_not_hold_back_the_owners_change():
c = controls()["living"]
col = SimpleNamespace(state=lambda e: "heat", attr=lambda e, a: 20.0)
plan = {"made": START, "start": START, "step_s": 900,
"devices": {"living": [{"t": START, "set": 21.0, "why": ""}]}}
st = {"steps": 192, "devices": [{"id": "living", "params": {}, "bounds": {"temp_c": [16, 26]}}]}
held = steady.stabilise(json.loads(json.dumps(plan)), st, [c], col, lambda *_: START - 60, START)
assert any("kept" in e["why"] for e in held["devices"]["living"])
free = steady.stabilise(json.loads(json.dumps(plan)), st, [c], col, lambda *_: START - 60, START, free={"living"})
assert free["devices"]["living"] == plan["devices"]["living"]
# -- Home Assistant's entities ---------------------------------------------------------------
def test_ha_entities_round_trip(tmp_path):
from test_announce import make
a, ann = make(tmp_path)
ctl = a.control
ctl.tz = ctl.prefs.tz = "Europe/Berlin"
objs = ann.objects()
keys = {(c, o) for (_, c, o) in objs}
for k in [("text", "pref_living_schedule"), ("number", "pref_living_boost_c"), ("switch", "pref_living_boost"),
("switch", "pref_away"), ("text", "pref_away_until"), ("text", "pref_car_ready"),
("number", "pref_car_capacity"), ("text", "pref_car_needed"), ("number", "pref_battery_reserve"),
("text", "pref_battery_reserve_blocks"), ("text", "pref_boiler_schedule")]:
assert k in keys, k
sched = objs[(ann.uid, "text", "pref_living_schedule")]
assert sched["command_topic"] == f"cantabile/{ann.site}/prefs/pref_living_schedule/set" and sched["max"] == 255
assert sched["entity_category"] == "config" and sched["default_entity_id"] == "text.cantabile_pref_living_schedule"
assert ("sensor", "preferences") not in keys # shadow: no plan, no results in HA
assert ann.prefs_state()["pref_living_schedule"] == ""
def send(obj, text):
asyncio.run(ann.pref_message(obj, text.encode()))
send("pref_living_schedule", "mo-fr 22-6 21 must; mo-fr 6-22 19")
assert ctl.prefs.value("room:living")["blocks"][0]["how"] == "must"
assert ann.prefs_state()["pref_living_schedule"] == "Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19"
assert ctl.prefs.effective("room:living")["by"] == "Home Assistant"
send("pref_living_schedule", "Mo-Fr 22:00-06:00 99") # refused: as it was
assert ann.prefs_state()["pref_living_schedule"].startswith("Mo-Fr 22:00-06:00 21 must")
assert ann.prefs_result()["last_error"].startswith("room:living")
send("pref_living_boost_c", "23.5")
send("pref_living_boost", "ON")
st = ann.prefs_state()
assert st["pref_living_boost"] == "ON" and st["pref_living_boost_c"] == 23.5
send("pref_living_boost", "OFF")
assert ann.prefs_state()["pref_living_boost"] == "OFF"
send("pref_car_capacity", "60")
send("pref_car_ready", "Mo-Fr 08:00 100% must")
assert ctl.prefs.value("ev:car") == {"ready": [{"days": [0, 1, 2, 3, 4], "at": "08:00", "pct": 100.0, "kwh": None,
"how": "must", "max_ct": None, "min_kwh": None}], "capacity_kwh": 60.0}
send("pref_car_needed", "2030-01-01 07:00 80%") # too far ahead: refused
assert ann.prefs_state()["pref_car_needed"] == ""
when = datetime.fromtimestamp(ctl.prefs.clock() + 86400, BERLIN).strftime("%Y-%m-%d")
send("pref_car_needed", f"{when} 07:00 80%")
assert ann.prefs_state()["pref_car_needed"] == f"{when} 07:00 80% must"
send("pref_car_needed", "")
assert ann.prefs_state()["pref_car_needed"] == ""
send("pref_battery_reserve", "25")
send("pref_battery_reserve_blocks", "daily 17:00-21:00 50%")
assert ctl.prefs.value("battery:battery") == {"reserve_pct": 25.0, "blocks": [
{"days": list(range(7)), "from": "17:00", "to": "21:00", "pct": 50.0}]}
send("pref_away_until", "2030-01-01") # stored; away is off
assert ann.prefs_state()["pref_away"] == "OFF" and ann.prefs_state()["pref_away_until"] == "2030-01-01"
send("pref_away", "ON") # beyond 60 days: refused
assert ann.prefs_state()["pref_away"] == "OFF"
send("pref_away_until", "")
send("pref_away", "ON") # no date: until switched off
assert ann.prefs_state()["pref_away"] == "ON"
send("pref_away", "OFF")
assert ann.prefs_state()["pref_away"] == "OFF"
# with the plan shown (advisory): the results
ctl.mode.current = "advisory"
ctl.prefs.feedback = [{"control": "living", "group": "living:0", "how": "must", "met": False,
"text": "Mo-Fr 22:00-06:00 ≥ 21 °C: not met", "cost_eur_day": 0.4}]
keys = {(c, o) for (_, c, o) in ann.objects()}
assert ("sensor", "preferences") in keys and ("sensor", "pref_living_result") in keys
r = ann.prefs_result()
assert r["summary"] == "1 not met" and r["controls"]["living"]["state"] == "not met"
assert r["controls"]["car"]["state"] == "none"
# who did what, kept across a restart
q = pm.Preferences(str(tmp_path))
assert q.value("battery:battery")["reserve_pct"] == 25 and all(e["by"] == "Home Assistant" for e in q.log)
def test_console_command_sets_an_installers_level(tmp_path):
from test_announce import make
a, _ = make(tmp_path)
ctl = a.control
msg = json.dumps({"set": {"item": "room:living", "value": "daily 22:00-06:00 25"}})
ok, why = asyncio.run(ctl.command("preferences", {"msg": msg, "level": "installer", "by": "bob"}))
assert not ok and "owner's limits" in why # the owner's 19..24
msg = json.dumps({"set": {"item": "room:living", "value": "daily 22:00-06:00 22"}})
ok, why = asyncio.run(ctl.command("preferences", {"msg": msg, "level": "installer", "by": "bob"}))
assert ok and ctl.prefs.effective("room:living")["by"] == "installer:bob"
ok, why = asyncio.run(ctl.command("preferences", {"msg": msg, "level": "owner", "by": "bob"}))
assert not ok # the console is never the owner
ok, why = asyncio.run(ctl.command("preferences", {"msg": json.dumps({"override": {"kind": "away"}}),
"level": "installer"}))
assert not ok
pub = ctl.public_plan({"id": "x", "made": START, "start": START, "step_s": 900, "mode": "shadow", "devices": {},
"cost_eur": 0, "baseline_eur": 0, "status": "Optimal"})
pr = pub["preferences"]
assert pr["items"]["room:living"]["in_force"] == "installer"
assert pr["items"]["room:living"]["levels"]["installer"]["text"] == "daily 22:00-06:00 22"
assert pr["log"][-1]["by"] == "installer:bob"