cantabile_edge, cantabile_modbus, cantabile_eebus 0.1.676: full source
This commit is contained in:
1 parent
3edd015039
commit
d48b9aff87
32 files changed
+4633
-133
No files matched your search
@@ -15,7 +15,7 @@ import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from . import areas, ha, propose, sinks
|
||||
from . import areas, ha, propose, sinks, trainhist
|
||||
from .announce import Announcer
|
||||
from .consent import Consent
|
||||
from .controller import Controller
|
||||
@@ -62,6 +62,15 @@ class Agent:
|
||||
self.control = Controller(cfg, mapping, self.consent, sink) if mapping.controls else None
|
||||
if self.control is not None and hasattr(sink, "on_command"):
|
||||
sink.on_command = self.control.command
|
||||
# the box's own training history (trainhist.py): its own 1-minute means,
|
||||
# with or without the share consent: they never leave the box (nothing
|
||||
# of it goes upward without that consent); only the consented values
|
||||
# go to the cloud (self.agg)
|
||||
self.train_agg = None
|
||||
self.tparts: dict[str, float | None] = {}
|
||||
if self.control is not None and getattr(self.control, "boxml", None) is not None:
|
||||
self.control.boxml.mapping = mapping # HA's statistics into its history (boxtrain.fill_from_ha)
|
||||
self.train_agg = Aggregator(cfg.interval_s)
|
||||
# the grid operator's orders (EEBUS LPC, gridorder.py): the controller's,
|
||||
# or, on a site without controls, checked and told to the cloud only
|
||||
self.gridorders = self.control.gridorders if self.control is not None else GridOrders(cfg.state_dir)
|
||||
@@ -169,9 +178,15 @@ class Agent:
|
||||
return self.last_sent > 0
|
||||
|
||||
def sample(self, entity_id: str, state: str | None, ts: float):
|
||||
if not self.consent.share:
|
||||
share = self.consent.share # without it nothing is collected to send
|
||||
local = self.train_agg is not None
|
||||
if not share and not local:
|
||||
return # the owner has not (or no longer) agreed
|
||||
for a in self.mapping.consented_all(entity_id): # none: not mapped or no consent, never leaves
|
||||
for a in self.mapping.of_entity.get(entity_id, []):
|
||||
up = share and a.consent # to the cloud: the site's and the asset's consent
|
||||
mine = local and trainhist.wanted(a.asset) # the box's own training: never leaves the box
|
||||
if not (up or mine):
|
||||
continue
|
||||
raw = value_of(a, state, self.attrs.get(entity_id))
|
||||
v = number(raw)
|
||||
if v is not None and not a.equals:
|
||||
@@ -180,24 +195,33 @@ class Agent:
|
||||
v = scaled(convert(v, unit, a.unit), a)
|
||||
if v is None:
|
||||
log.warning("%s: unit %s cannot become %s; value dropped", a.asset, unit, a.unit)
|
||||
self.agg.sample(a.asset, a.kind, v, ts)
|
||||
total = self.mapping.part_total.get(a.asset)
|
||||
if total:
|
||||
self.sample_total(self.mapping.totals[total], a.asset, v, ts)
|
||||
if up:
|
||||
self.agg.sample(a.asset, a.kind, v, ts)
|
||||
if total:
|
||||
self.sample_total(self.mapping.totals[total], a.asset, v, ts)
|
||||
if local and a.kind == "gauge" and (mine or (total and trainhist.wanted(total))):
|
||||
if mine:
|
||||
self.train_agg.sample(a.asset, a.kind, v, ts)
|
||||
if total and trainhist.wanted(total):
|
||||
self.sample_total(self.mapping.totals[total], a.asset, v, ts, self.train_agg, self.tparts, False)
|
||||
|
||||
def sample_total(self, t, part: str, v: float | None, ts: float):
|
||||
def sample_total(self, t, part: str, v: float | None, ts: float, agg=None, store=None, consented: bool = True):
|
||||
"""A total of parts (pv.power from pv.1.power, pv.2.power: mapping.py)
|
||||
from its consented parts' last values: their sum (the mean for % and
|
||||
°C); unknown while a part is (a counter then waits: a missing part
|
||||
must not look like a meter reset)."""
|
||||
self.parts[part] = v
|
||||
parts = [p for p in t.parts if getattr(self.mapping.by_asset.get(p), "consent", False)]
|
||||
vals = [self.parts.get(p) for p in parts]
|
||||
must not look like a meter reset). agg, store, consented: the box's
|
||||
own training history's (every part, consented or not)."""
|
||||
agg = self.agg if agg is None else agg
|
||||
store = self.parts if store is None else store
|
||||
store[part] = v
|
||||
parts = [p for p in t.parts if not consented or getattr(self.mapping.by_asset.get(p), "consent", False)]
|
||||
vals = [store.get(p) for p in parts]
|
||||
if not vals or None in vals:
|
||||
if t.kind == "gauge":
|
||||
self.agg.sample(t.asset, t.kind, None, ts)
|
||||
agg.sample(t.asset, t.kind, None, ts)
|
||||
return
|
||||
self.agg.sample(t.asset, t.kind, sum(vals) / (len(vals) if t.mean else 1), ts)
|
||||
agg.sample(t.asset, t.kind, sum(vals) / (len(vals) if t.mean else 1), ts)
|
||||
|
||||
def on_entities(self, ev: dict):
|
||||
now = self.clock()
|
||||
@@ -229,6 +253,9 @@ class Agent:
|
||||
except Exception as e: # never keeps the agent from reading
|
||||
log.warning("mapping: %s", e)
|
||||
ids = list(dict.fromkeys(a.entity for a in self.mapping.assets if a.consent)) # one entity may feed several
|
||||
if self.train_agg is not None: # and the box's own training's (never sent: sample())
|
||||
ids = list(dict.fromkeys(ids + [a.entity for a in self.mapping.assets if trainhist.wanted(a.asset)
|
||||
or trainhist.wanted(self.mapping.part_total.get(a.asset, ""))]))
|
||||
await client.subscribe_entities(ids, self.on_entities)
|
||||
log.info("subscribed to %d consented entities (mapping v%d)", len(ids), self.mapping.version)
|
||||
if self.control is not None:
|
||||
@@ -258,6 +285,13 @@ class Agent:
|
||||
def close_window(self, end: float):
|
||||
start = self.agg.start
|
||||
records = self.agg.flush(end)
|
||||
if self.train_agg is not None:
|
||||
tstart, trecs = self.train_agg.start, self.train_agg.flush(end)
|
||||
try: # the box's own training history (trainhist.py): 15-minute means, kept 13 months
|
||||
if trecs:
|
||||
self.control.boxml.store.add_window(tstart, trecs)
|
||||
except Exception as e:
|
||||
log.warning("training history: %s", e)
|
||||
if not records:
|
||||
return
|
||||
seq = self.queue.put("tel", {
|
||||
@@ -327,6 +361,8 @@ class Agent:
|
||||
async def run(self):
|
||||
self.started = time.time()
|
||||
self.agg.begin(self.started)
|
||||
if self.train_agg is not None:
|
||||
self.train_agg.begin(self.started)
|
||||
if self.cfg.mode == "off":
|
||||
log.info("mode off: not reading Home Assistant")
|
||||
await self.stop.wait()
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
"Cantabile" (one per site) under HA's MQTT integration, in the area
|
||||
"Cantabile" (suggested_area, in HA's language: names.py), with
|
||||
availability (offline when the agent stops: MQTT's last will, or said
|
||||
before a clean stop). Read only: nothing in HA changes the agent; the
|
||||
owner's consent is given in the box's setup wizard and panel only (sensitive).
|
||||
before a clean stop). Nothing in HA changes the agent but the owner's
|
||||
preferences (below); the owner's consent is given
|
||||
in the box's setup wizard and panel only (sensitive),
|
||||
and the mode only from the console.
|
||||
|
||||
What it shows (what goes between the cloud or the
|
||||
building and the home, and what the agent does to each device):
|
||||
@@ -36,6 +38,19 @@ building and the home, and what the agent does to each device):
|
||||
actuator its contract target vs its state now;
|
||||
sensors only, unavailable when the bundle is
|
||||
stale (cantabile/<site>/market/availability)
|
||||
the owner's preferences (preferences.py; every mode but off; entity
|
||||
category config, usable in automations; HA's MQTT integration has no
|
||||
time or date entity: times are text in the preferences' text forms):
|
||||
text pref_<room>_schedule "Mo-Fr 22:00-06:00 21 must; ..." (empty: none)
|
||||
number pref_<room>_boost_c, pref_<room>_boost_h, switch pref_<room>_boost
|
||||
switch pref_away, text pref_away_until ("2026-11-10"; empty: until off)
|
||||
text pref_<car>_ready "Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct"
|
||||
number pref_<car>_capacity (kWh, for %), text pref_<car>_needed (one-off)
|
||||
number pref_<battery>_reserve (%), text pref_<battery>_reserve_blocks
|
||||
sensor pref_<control>_result, preferences (advisory and auto, with the
|
||||
plan): per preference met or not and each must's cost
|
||||
Commands on cantabile/<site>/prefs/<object>/set (the owner's, "Home
|
||||
Assistant"); states on cantabile/<site>/prefs and /prefs/result.
|
||||
On a building's own box (flats.json, a signed `flats` bundle from the
|
||||
cloud: bundles.py) each flat is a device of its own ("Flat", via the
|
||||
building's Cantabile device, area "Flats"): its grid power, its
|
||||
@@ -64,7 +79,8 @@ log = logging.getLogger(__name__)
|
||||
INTERVAL = 30 # state at least this often (seconds)
|
||||
RETIRED = (("switch", "share"), ("sensor", "queued"), ("sensor", "last_sent"), ("sensor", "mode"))
|
||||
RECEIVED = 10 # commands listed in HA (sensor received)
|
||||
COMMANDS = ("mode", "control", "release", "limits", "tariff", "model", "building")
|
||||
COMMANDS = ("mode", "control", "release", "limits", "tariff", "model", "building", "preferences")
|
||||
PREF_TOPIC = re.compile(r"^cantabile/[^/]+/prefs/([^/]+)/set$")
|
||||
MAX_FLATS = 200
|
||||
|
||||
|
||||
@@ -86,9 +102,15 @@ class Announcer:
|
||||
self.sent: dict[tuple, str] = {} # (node, component, object) -> config sent (JSON)
|
||||
ctl = getattr(agent, "control", None)
|
||||
self.bundles = ctl.bundles if ctl is not None else Bundles(agent.cfg.state_dir, site)
|
||||
self.pref_cmds: dict[str, tuple[str, str]] = {} # object -> (control id, what): the commands HA may send
|
||||
if ctl is not None:
|
||||
ctl.on_plan.append(self._planned)
|
||||
ctl.on_received.append(self._received)
|
||||
if hasattr(ctl, "on_prefs"):
|
||||
ctl.on_prefs.append(self._prefs_changed)
|
||||
|
||||
async def _prefs_changed(self):
|
||||
self.changed.set()
|
||||
|
||||
@property
|
||||
def lang(self) -> str:
|
||||
@@ -394,6 +416,193 @@ class Announcer:
|
||||
"last": a.get("last"), "shadow": True}
|
||||
return out
|
||||
|
||||
# -- the owner's preferences (preferences.py) -------------------------------
|
||||
def _pref_controls(self):
|
||||
ctl = getattr(self.agent, "control", None)
|
||||
if ctl is None or not hasattr(ctl, "prefs") or ctl.mode() == "off":
|
||||
return None, []
|
||||
return ctl, [c for c in ctl.controls if c.kind in ("heating", "cooling", "ev", "battery")]
|
||||
|
||||
def _prefs_objects(self) -> dict:
|
||||
"""The preferences' entities: settings in every mode but off, their
|
||||
results with the plan (advisory, auto)."""
|
||||
ctl, cs = self._pref_controls()
|
||||
if not cs:
|
||||
self.pref_cmds = {}
|
||||
return {}
|
||||
lang, dev, top = self.lang, self._device(), f"{self.base}/prefs"
|
||||
common = {"state_topic": top, "entity_category": "config", **self._avail(), "device": dev}
|
||||
out, cmds = {}, {}
|
||||
|
||||
def add(comp, obj, cid, what, name, **conf):
|
||||
cmds[obj] = (cid, what)
|
||||
out[(comp, obj)] = {"name": name, "unique_id": f"{self.uid}_{obj}", "command_topic": f"{top}/{obj}/set",
|
||||
"value_template": "{{ value_json['%s'] }}" % obj, **common, **conf}
|
||||
rooms = [c for c in cs if c.kind in ("heating", "cooling")]
|
||||
for c in rooms:
|
||||
lo, hi = (16.0, 32.0) if c.kind == "cooling" else (5.0, 30.0)
|
||||
k, n = _slug(c.id), c.label()
|
||||
add("text", f"pref_{k}_schedule", c.id, "schedule", names.name("pref_schedule_cool" if c.kind == "cooling"
|
||||
else "pref_schedule", lang, n),
|
||||
max=255, mode="text", icon="mdi:calendar-clock")
|
||||
add("number", f"pref_{k}_boost_c", c.id, "boost_c", names.name("pref_boost_c", lang, n), min=lo, max=hi,
|
||||
step=0.5, mode="box", unit_of_measurement="°C", icon="mdi:thermometer-chevron-up")
|
||||
add("number", f"pref_{k}_boost_h", c.id, "boost_h", names.name("pref_boost_h", lang, n), min=0.5, max=24,
|
||||
step=0.5, mode="box", unit_of_measurement="h", icon="mdi:timer-outline")
|
||||
add("switch", f"pref_{k}_boost", c.id, "boost", names.name("pref_boost", lang, n),
|
||||
payload_on="ON", payload_off="OFF", state_on="ON", state_off="OFF", icon="mdi:fire")
|
||||
if rooms:
|
||||
add("switch", "pref_away", "", "away", names.name("pref_away", lang),
|
||||
payload_on="ON", payload_off="OFF", state_on="ON", state_off="OFF", icon="mdi:home-export-outline")
|
||||
add("text", "pref_away_until", "", "away_until", names.name("pref_away_until", lang), max=40, mode="text",
|
||||
icon="mdi:calendar-arrow-right")
|
||||
for c in cs:
|
||||
k, n = _slug(c.id), c.label()
|
||||
if c.kind == "ev":
|
||||
add("text", f"pref_{k}_ready", c.id, "ready", names.name("pref_ready", lang, n), max=255, mode="text",
|
||||
icon="mdi:car-clock")
|
||||
add("number", f"pref_{k}_capacity", c.id, "capacity", names.name("pref_capacity", lang, n), min=1,
|
||||
max=250, step=0.5, mode="box", unit_of_measurement="kWh", icon="mdi:car-battery")
|
||||
add("text", f"pref_{k}_needed", c.id, "needed", names.name("pref_needed", lang, n), max=80, mode="text",
|
||||
icon="mdi:car-arrow-right")
|
||||
elif c.kind == "battery":
|
||||
add("number", f"pref_{k}_reserve", c.id, "reserve", names.name("pref_reserve", lang, n), min=0, max=100,
|
||||
step=1, mode="box", unit_of_measurement="%", icon="mdi:battery-lock")
|
||||
add("text", f"pref_{k}_reserve_blocks", c.id, "reserve_blocks", names.name("pref_reserve_blocks", lang, n),
|
||||
max=255, mode="text", icon="mdi:battery-clock")
|
||||
self.pref_cmds = cmds
|
||||
if ctl.mode() in ("advisory", "auto"):
|
||||
res = {"state_topic": f"{top}/result", "json_attributes_topic": f"{top}/result", **self._avail(), "device": dev}
|
||||
out[("sensor", "preferences")] = {
|
||||
"name": names.name("pref_result", lang), "unique_id": f"{self.uid}_preferences", "icon": "mdi:clipboard-check",
|
||||
"value_template": "{{ value_json.summary }}",
|
||||
"json_attributes_template": "{{ {'results': value_json.lines, 'last_error': value_json.last_error} | tojson }}",
|
||||
**res}
|
||||
for c in cs:
|
||||
k = _slug(c.id)
|
||||
out[("sensor", f"pref_{k}_result")] = {
|
||||
"name": names.name("pref_result_of", lang, c.label()), "unique_id": f"{self.uid}_pref_{k}_result",
|
||||
"icon": "mdi:clipboard-check-outline",
|
||||
"value_template": "{{ value_json.controls['%s'].state }}" % c.id,
|
||||
"json_attributes_template": "{{ value_json.controls['%s'] | tojson }}" % c.id, **res}
|
||||
return out
|
||||
|
||||
def prefs_state(self) -> dict:
|
||||
"""The settings' values as HA shows them (cantabile/<site>/prefs)."""
|
||||
ctl, cs = self._pref_controls()
|
||||
if not cs:
|
||||
return {}
|
||||
from . import preferences as pm
|
||||
p, out = ctl.prefs, {}
|
||||
act = p.active_overrides()
|
||||
for c in cs:
|
||||
k = _slug(c.id)
|
||||
if c.kind in ("heating", "cooling"):
|
||||
out[f"pref_{k}_schedule"] = pm.room_text(p.value(f"room:{c.id}"))
|
||||
b = next((o for o in act if o["kind"] == "boost" and o.get("control") == c.id), None)
|
||||
out[f"pref_{k}_boost_c"] = b["c"] if b else self._boost(p, c.id)["c"]
|
||||
out[f"pref_{k}_boost_h"] = p.ha.get(f"boost_h:{c.id}", 2.0)
|
||||
out[f"pref_{k}_boost"] = "ON" if b else "OFF"
|
||||
elif c.kind == "ev":
|
||||
v = p.value(f"ev:{c.id}") or {}
|
||||
out[f"pref_{k}_ready"] = pm.ev_text(v)
|
||||
out[f"pref_{k}_capacity"] = v.get("capacity_kwh")
|
||||
car = next((o for o in act if o["kind"] == "car" and o.get("control") == c.id), None)
|
||||
out[f"pref_{k}_needed"] = pm.car_text(car, p.tz) if car else ""
|
||||
elif c.kind == "battery":
|
||||
v = p.value(f"battery:{c.id}") or {}
|
||||
out[f"pref_{k}_reserve"] = v.get("reserve_pct", 0.0)
|
||||
out[f"pref_{k}_reserve_blocks"] = pm.battery_text(v)
|
||||
if any(c.kind in ("heating", "cooling") for c in cs):
|
||||
away = next((o for o in act if o["kind"] == "away"), None)
|
||||
out["pref_away"] = "ON" if away else "OFF"
|
||||
out["pref_away_until"] = p.ha.get("away_until", "")
|
||||
return out
|
||||
|
||||
def prefs_result(self) -> dict:
|
||||
"""The plan's feedback per control (cantabile/<site>/prefs/result)."""
|
||||
ctl, cs = self._pref_controls()
|
||||
p = ctl.prefs if ctl is not None else None
|
||||
fb = p.feedback if p is not None else []
|
||||
per = {}
|
||||
for c in cs:
|
||||
mine = [f for f in fb if f.get("control") == c.id]
|
||||
musts = [f for f in mine if f.get("how") == "must" and f.get("met") is not None]
|
||||
state = "none" if not mine else "not met" if any(f.get("met") is False for f in mine) else \
|
||||
"met" if any(f.get("met") for f in mine) else "open"
|
||||
per[c.id] = {"state": state, "lines": [f["text"] for f in mine],
|
||||
"cost_eur_day": round(sum(f.get("cost_eur_day") or 0.0 for f in musts), 2) if musts else None,
|
||||
"results": mine}
|
||||
bad = [f for f in fb if f.get("met") is False]
|
||||
last = (p.errors[-1] if p is not None and p.errors else None)
|
||||
return {"summary": "none" if not fb else f"{len(bad)} not met" if bad else "met",
|
||||
"lines": [f["text"] for f in fb], "controls": per,
|
||||
"last_error": f"{last['item']}: {last['why']}" if last else ""}
|
||||
|
||||
async def pref_message(self, obj: str, payload: bytes):
|
||||
"""A preference set in HA (its entity's command): as the owner's."""
|
||||
ctl = getattr(self.agent, "control", None)
|
||||
if ctl is None or obj not in self.pref_cmds:
|
||||
return
|
||||
cid, what = self.pref_cmds[obj]
|
||||
text = payload.decode(errors="replace").strip()
|
||||
p, by = ctl.prefs, "Home Assistant"
|
||||
msg = None
|
||||
if what == "schedule":
|
||||
msg = {"set": {"item": f"room:{cid}", "value": text or None}}
|
||||
elif what in ("boost_c", "boost_h"):
|
||||
try:
|
||||
p.ha[f"{what}:{cid}"] = float(text)
|
||||
except ValueError:
|
||||
return
|
||||
p.save()
|
||||
b = next((o for o in p.active_overrides() if o["kind"] == "boost" and o.get("control") == cid), None)
|
||||
if b is None:
|
||||
self.changed.set()
|
||||
return
|
||||
msg = {"override": self._boost(p, cid)}
|
||||
elif what == "boost":
|
||||
msg = {"override": self._boost(p, cid)} if text.upper() == "ON" else {"cancel": {"kind": "boost", "control": cid}}
|
||||
elif what == "away":
|
||||
msg = {"override": {"kind": "away", "until_text": p.ha.get("away_until", "")}} if text.upper() == "ON" \
|
||||
else {"cancel": {"kind": "away", "control": ""}}
|
||||
elif what == "away_until":
|
||||
p.ha["away_until"] = text[:40]
|
||||
p.save()
|
||||
if not any(o["kind"] == "away" for o in p.active_overrides()):
|
||||
self.changed.set()
|
||||
return
|
||||
msg = {"override": {"kind": "away", "until_text": text}}
|
||||
elif what in ("ready", "capacity"):
|
||||
v = dict(p.value(f"ev:{cid}") or {"ready": [], "capacity_kwh": None})
|
||||
if what == "capacity":
|
||||
v["capacity_kwh"] = text if text not in ("", "None") else None
|
||||
msg = {"set": {"item": f"ev:{cid}", "value": v}}
|
||||
else:
|
||||
msg = {"set": {"item": f"ev:{cid}", "value": text}} if text else \
|
||||
{"set": {"item": f"ev:{cid}", "value": dict(v, ready=[]) if v.get("capacity_kwh") else None}}
|
||||
elif what == "needed":
|
||||
msg = {"override": {"kind": "car", "control": cid, "text": text}}
|
||||
elif what in ("reserve", "reserve_blocks"):
|
||||
v = dict(p.value(f"battery:{cid}") or {"reserve_pct": 0.0, "blocks": []})
|
||||
if what == "reserve":
|
||||
v["reserve_pct"] = text
|
||||
msg = {"set": {"item": f"battery:{cid}", "value": v}}
|
||||
else:
|
||||
msg = {"set": {"item": f"battery:{cid}", "value": text}} if text else \
|
||||
{"set": {"item": f"battery:{cid}", "value": dict(v, blocks=[])}}
|
||||
if msg is not None:
|
||||
await ctl.prefs_change(msg, "owner", by)
|
||||
self.changed.set()
|
||||
|
||||
def _boost(self, p, cid) -> dict:
|
||||
"""A boost with HA's numbers (their defaults: comfort + 1 °C, 2 h)."""
|
||||
ctl = self.agent.control
|
||||
c = next((c for c in ctl.controls if c.id == cid), None)
|
||||
default = float(c.p("comfort_c")) + (-1.0 if c is not None and c.kind == "cooling" else 1.0) if c else 21.0
|
||||
return {"kind": "boost", "control": cid, "c": p.ha.get(f"boost_c:{cid}", default),
|
||||
"hours": p.ha.get(f"boost_h:{cid}", 2.0)}
|
||||
|
||||
def objects(self) -> dict:
|
||||
"""Every discovery config HA should have now: (node, component,
|
||||
object) -> config; node is the site's or a flat's."""
|
||||
@@ -401,7 +610,7 @@ class Announcer:
|
||||
tariff, model = self.bundles.tariff_now(now), self.bundles.model_info()
|
||||
out = {}
|
||||
for (comp, obj), conf in {**self._configs(), **self._plan_objects(),
|
||||
**self._exchange_objects(tariff, model),
|
||||
**self._exchange_objects(tariff, model), **self._prefs_objects(),
|
||||
**self._market_objects(self.bundles.market_now(now))}.items():
|
||||
out[(self.uid, comp, obj)] = conf
|
||||
out.update(self._flat_objects(self.bundles.flat_list()))
|
||||
@@ -495,6 +704,11 @@ class Announcer:
|
||||
await c.publish(f"{self.base}/model", json.dumps(model), retain=True)
|
||||
if ctl is not None and ctl.mode() != "off":
|
||||
await c.publish(f"{self.base}/actuators", json.dumps(ctl.actuators(), default=str), retain=True)
|
||||
prefs = self.prefs_state()
|
||||
if prefs:
|
||||
await c.publish(f"{self.base}/prefs", json.dumps(prefs, default=str), retain=True)
|
||||
if ctl.mode() in ("advisory", "auto"):
|
||||
await c.publish(f"{self.base}/prefs/result", json.dumps(self.prefs_result(), default=str), retain=True)
|
||||
if ctl is not None and ctl.received:
|
||||
# kept (retained) across restarts until new ones come
|
||||
await c.publish(f"{self.base}/received", json.dumps({"commands": self.received_list(ctl)}), retain=True)
|
||||
@@ -540,19 +754,35 @@ class Announcer:
|
||||
|
||||
async def _grid_orders(self, c):
|
||||
"""The EEBUS bridge's grid order (gridorder.py), to the
|
||||
controller: the grid operator's limit as a higher-rank decision."""
|
||||
controller: the grid operator's limit as a higher-rank decision; and
|
||||
the preferences HA sets (pref_message)."""
|
||||
take = getattr(self.agent, "grid_order", None)
|
||||
if take is None:
|
||||
return
|
||||
prefs = getattr(getattr(self.agent, "control", None), "prefs", None) is not None
|
||||
|
||||
def on(topic, payload):
|
||||
if topic == gridorder.TOPIC and payload:
|
||||
if topic == gridorder.TOPIC and payload and take is not None:
|
||||
asyncio.ensure_future(take(payload))
|
||||
return
|
||||
m = PREF_TOPIC.match(topic)
|
||||
if m and topic.startswith(f"{self.base}/prefs/") and prefs:
|
||||
asyncio.ensure_future(self._pref_safe(m.group(1), payload))
|
||||
c.on_message = on
|
||||
if take is not None:
|
||||
try:
|
||||
await c.subscribe(gridorder.TOPIC)
|
||||
except Exception as e:
|
||||
log.info("mqtt: grid orders not subscribed: %s", e)
|
||||
if prefs:
|
||||
try:
|
||||
await c.subscribe(f"{self.base}/prefs/+/set")
|
||||
except Exception as e:
|
||||
log.info("mqtt: preferences not subscribed: %s", e)
|
||||
|
||||
async def _pref_safe(self, obj, payload):
|
||||
try:
|
||||
await c.subscribe(gridorder.TOPIC)
|
||||
await self.pref_message(obj, payload)
|
||||
except Exception as e:
|
||||
log.info("mqtt: grid orders not subscribed: %s", e)
|
||||
log.warning("preferences from HA (%s): %s", obj, e)
|
||||
|
||||
def _password(self) -> str:
|
||||
if self.password_file:
|
||||
|
||||
@@ -56,6 +56,40 @@ async def backfill(client: HAClient, mapping, queue, since: datetime, until: dat
|
||||
return batches
|
||||
|
||||
|
||||
async def collect(client: HAClient, mapping, since: datetime, until: datetime, period: str = "hour",
|
||||
want=None, consented: bool = True) -> dict[str, list[dict]]:
|
||||
"""HA's statistics of the gauges (want(asset): which; consented: only
|
||||
those with consent) as {asset: [{"start": unix, "mean"}]} in the assets'
|
||||
units: for the box's own training history (trainhist.py, every mapped
|
||||
gauge: nothing of it is queued or sent). A total of parts (pv.power from
|
||||
pv.1.power, ...) has no entity of its own: left out."""
|
||||
assets = {a.entity: a for a in mapping.assets
|
||||
if (a.consent or not consented) and a.kind == "gauge" and not a.attribute and not a.equals
|
||||
and (want is None or want(a.asset))}
|
||||
if not assets:
|
||||
return {}
|
||||
meta = await client.send("recorder/list_statistic_ids", statistic_type="mean")
|
||||
units = {m["statistic_id"]: m.get("statistics_unit_of_measurement") for m in meta}
|
||||
ids = [e for e in assets if e in units]
|
||||
out: dict[str, list[dict]] = {}
|
||||
start = since
|
||||
while start < until and ids:
|
||||
end = min(start + CHUNK, until)
|
||||
res = await client.send("recorder/statistics_during_period", timeout=300, start_time=start.isoformat(),
|
||||
end_time=end.isoformat(), statistic_ids=ids, period=period, types=["mean"])
|
||||
for eid, rows in (res or {}).items():
|
||||
a, unit = assets.get(eid), units.get(eid)
|
||||
if a is None:
|
||||
continue
|
||||
for r in rows:
|
||||
v = number(r.get("mean"))
|
||||
v = None if v is None else scaled(convert(v, unit, a.unit), a)
|
||||
if v is not None:
|
||||
out.setdefault(a.asset, []).append({"start": _ts(r["start"]), "mean": v})
|
||||
start = end
|
||||
return out
|
||||
|
||||
|
||||
def parse_since(s: str) -> datetime:
|
||||
"""An ISO date/time (UTC unless it says otherwise) or days back, e.g. 30d."""
|
||||
if s.endswith("d") and s[:-1].isdigit():
|
||||
|
||||
@@ -0,0 +1,809 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The box trains its own models (docs/methods.md B10; step 2 of the
|
||||
distributed models).
|
||||
|
||||
Nightly (cfg.train_at, default 03:30 in Home Assistant's time zone) from
|
||||
the agent's own training history (trainhist.py, 13 months, recent weeks weighing more): per series
|
||||
(load, pv, and base_load where the box plans devices) the cloud's own code
|
||||
(trainlib.train_series: how, gbm, pvphys; byte-identical copies of
|
||||
building/'s trainlib.py, fcmodels.py, pvcalib.py, rcfit.py) fitted on the
|
||||
window before the held-out one and scored on it, on the weather as it was
|
||||
forecast where the box kept it (else observed, and the run says so); the
|
||||
rooms' R and C (trainlib.fit_rooms).
|
||||
|
||||
Never in the controller's way: each series is a child process (python -m
|
||||
cantabile_edge.boxtrain <job> <result>) at the lowest priority (nice 19,
|
||||
one LightGBM thread), one at a time, killed past its wall time
|
||||
(train_series_s) or its resident memory (train_mem_mb), not started while
|
||||
less than train_min_free_mb is available; the agent's loop only awaits
|
||||
it. The data are read in a worker thread.
|
||||
|
||||
Which model the box plans on (inputs._model_series), per step: the box's
|
||||
own model where it is in force, else the cloud's series, else the cloud's
|
||||
model on the box (localml.py), else the hour-of-week. In force means: on
|
||||
the night's held-out window it beat the model the box had (the champion;
|
||||
trainlib's rule) AND the cloud's model in force (the bundle's "models",
|
||||
scored on the same window): the cloud's model is the standing benchmark
|
||||
until the cloud's teacher model exists. Without a cloud model to compare,
|
||||
a box model other than the hour-of-week that beats the hour-of-week. A
|
||||
model in force whose live error over 7 days (the meter against what was
|
||||
planned on, 2 days at least) is worse than the cloud's series' leaves
|
||||
force until it wins again.
|
||||
|
||||
Upward (the plan's "ml", controller.public_plan): the forecasts the box
|
||||
planned on (hourly), the held-out errors per kind, the live error per
|
||||
source, and the training's own figures (seconds, peak memory)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import numpy as np
|
||||
|
||||
from . import fcmodels, localml, pvforecast, trainhist
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SLOT_S = 900
|
||||
NAMES = {"load": "load_w", "pv": "pv_w", "base_load": "base_load_w"} # the bundle's names (boxmodel.NAMES)
|
||||
SERIES = {v: k for k, v in NAMES.items()}
|
||||
CTL_KINDS = ("heating", "cooling", "ev") # trainlib.CONTROLLED_POWER's keys
|
||||
ROOM_DAYS = 42 # the rooms on the cloud's window (trainer: 6 weeks)
|
||||
LIVE_DAYS = 7
|
||||
LIVE_MIN = 2 * 96 # live errors count from 2 days of slots
|
||||
RUNS_KEPT = 30
|
||||
PUBLISH_EVERY_S = 3600 # the forecasts upward at most hourly (the plan comes every 15 min)
|
||||
|
||||
|
||||
def _f(v):
|
||||
return None if v is None or (isinstance(v, float) and not math.isfinite(v)) else float(v)
|
||||
|
||||
|
||||
def _clean(v):
|
||||
"""JSON without NaN, numpy scalars or private keys (lgbtrees' cache)."""
|
||||
if isinstance(v, float):
|
||||
return v if math.isfinite(v) else None
|
||||
if isinstance(v, dict):
|
||||
return {k: _clean(x) for k, x in v.items() if not str(k).startswith("_")}
|
||||
if isinstance(v, (list, tuple)):
|
||||
return [_clean(x) for x in v]
|
||||
if isinstance(v, np.generic):
|
||||
return _clean(v.item())
|
||||
if isinstance(v, np.ndarray):
|
||||
return _clean(v.tolist())
|
||||
return v
|
||||
|
||||
|
||||
def parse_hhmm(s: str) -> tuple[int, int]:
|
||||
try:
|
||||
h, m = (int(x) for x in str(s).split(":")[:2])
|
||||
if 0 <= h < 24 and 0 <= m < 60:
|
||||
return h, m
|
||||
except ValueError:
|
||||
pass
|
||||
return 3, 30
|
||||
|
||||
|
||||
def next_run(now: float, at: str, tz: str) -> float:
|
||||
"""The next train_at (local) after now."""
|
||||
zone = ZoneInfo(tz or "UTC")
|
||||
h, m = parse_hhmm(at)
|
||||
d = datetime.fromtimestamp(now, timezone.utc).astimezone(zone)
|
||||
t = d.replace(hour=h, minute=m, second=0, microsecond=0)
|
||||
if t.timestamp() <= now:
|
||||
t = (d + timedelta(days=1)).replace(hour=h, minute=m, second=0, microsecond=0)
|
||||
return t.timestamp()
|
||||
|
||||
|
||||
def mem_available_mb() -> float | None:
|
||||
try:
|
||||
with open("/proc/meminfo") as f:
|
||||
for line in f:
|
||||
if line.startswith("MemAvailable:"):
|
||||
return int(line.split()[1]) / 1024.0
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def rss_mb(pid: int) -> float | None:
|
||||
try:
|
||||
with open(f"/proc/{pid}/status") as f:
|
||||
for line in f:
|
||||
if line.startswith("VmRSS:"):
|
||||
return int(line.split()[1]) / 1024.0
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def hours_json(hours: list) -> list:
|
||||
return [[int(h.end.timestamp()), h.solar_kwh_m2, h.cloud_pct, h.temp_c] for h in hours]
|
||||
|
||||
|
||||
def hours_from(rows) -> list:
|
||||
return [pvforecast.Hour(datetime.fromtimestamp(int(r[0]), timezone.utc), r[1], r[2], r[3]) for r in rows or []]
|
||||
|
||||
|
||||
def arrays(series: dict[int, float], t: np.ndarray) -> np.ndarray:
|
||||
return np.array([series.get(int(k), np.nan) for k in t.tolist()], dtype=float)
|
||||
|
||||
|
||||
def net_series(raw: dict[str, np.ndarray], n: int) -> np.ndarray | None:
|
||||
"""A member's draw per slot (the coordinator's core.member_use_w): grid
|
||||
import less export where metered, else load less PV and battery."""
|
||||
if "grid.import_power" in raw:
|
||||
exp = raw.get("grid.export_power")
|
||||
return raw["grid.import_power"] - (np.nan_to_num(exp) if exp is not None else 0.0)
|
||||
if "load.power" in raw:
|
||||
out = raw["load.power"].copy()
|
||||
for a in ("pv.power", "battery.power"):
|
||||
if a in raw:
|
||||
out = out - np.nan_to_num(raw[a])
|
||||
return out
|
||||
return None
|
||||
|
||||
|
||||
def score(m: dict | None, key: str) -> float | None:
|
||||
v = (m or {}).get(key)
|
||||
return float(v) if isinstance(v, (int, float)) and math.isfinite(v) else None
|
||||
|
||||
|
||||
# ---- the child process: one job -------------------------------------------------
|
||||
|
||||
def run_job(job: dict) -> dict:
|
||||
"""One series (or the rooms), as the cloud trains it (trainlib). Pure:
|
||||
the parent wrote everything it needs."""
|
||||
from . import trainlib
|
||||
t0 = time.monotonic()
|
||||
t = np.asarray(job["t"], dtype=np.int64)
|
||||
s = job["site"]
|
||||
site = fcmodels.Site(s.get("tz") or "UTC", s.get("lat"), s.get("lon"), s.get("kwp"), s.get("tilt", 30.0),
|
||||
s.get("azimuth", 180.0), s.get("pvgis_monthly"), s.get("state"))
|
||||
w = fcmodels.weather(t, hours_from(job.get("hours")), site.lat, site.lon)
|
||||
out: dict = {"series": job["series"]}
|
||||
if job["series"] == "rooms":
|
||||
assets = {a: np.array([np.nan if v is None else v for v in vals], dtype=float) for a, vals in job["assets"].items()}
|
||||
fits = trainlib.fit_rooms(t, assets, w)
|
||||
tout, p, ghi = trainlib.room_inputs(assets, w)
|
||||
for room, f in fits.items():
|
||||
c = (job.get("cloud_rooms") or {}).get(room)
|
||||
theta = assets.get(f"room.{room}.temp")
|
||||
if c and p is not None and theta is not None and c.get("power", "thermal") == f.get("power"):
|
||||
from . import rcfit
|
||||
b = rcfit.DT_H / c["c_kwh_per_k"]
|
||||
gain, solar = c.get("gain_w") or 0.0, c.get("solar_m2") or 0.0
|
||||
gain = sum(gain) / len(gain) if isinstance(gain, list) else gain
|
||||
f["cloud_rmse_k"] = rcfit.sim_rmse(theta, tout, p / 1000.0, np.nan_to_num(ghi) / 1000.0, c["r_k_per_kw"],
|
||||
c["c_kwh_per_k"], solar * b, gain * b / 1000.0)
|
||||
out["rooms"] = _clean(fits)
|
||||
else:
|
||||
series = job["series"]
|
||||
y = np.array([np.nan if v is None else v for v in job["y"]], dtype=float)
|
||||
wt = fcmodels.weather(t, hours_from(job["test_hours"]), site.lat, site.lon) if job.get("test_hours") else None
|
||||
res = trainlib.train_series(series, t, y, w, site, job.get("champion"), w_test=wt)
|
||||
key = trainlib.score_key(res)
|
||||
cloud = trainlib.score_params(job["cloud"], series, t, y, w, site, wt) if job.get("cloud") else None
|
||||
out.update(key=key, window=list(res.window), n_train=res.n_train, n_test=res.n_test, note=res.note,
|
||||
champion=res.champion, baseline=res.baseline, cloud=cloud,
|
||||
runs=[{"kind": r.kind, "metrics": r.metrics, "note": r.note, "chosen": r.chosen} for r in res.runs],
|
||||
chosen=res.chosen.kind if res.chosen else None,
|
||||
params=fcmodels.box_params(res.chosen.params) if res.chosen else None)
|
||||
if job.get("want_predictions") and res.chosen: # the parity test
|
||||
out["predictions"] = fcmodels.predict(res.chosen.params, series, t, w, site).tolist()
|
||||
out["seconds"] = round(time.monotonic() - t0, 2)
|
||||
try: # this process's own peak (VmHWM: getrusage's maxrss keeps the parent's across exec)
|
||||
with open("/proc/self/status") as f:
|
||||
out["peak_mb"] = round(int(f.read().split("VmHWM:")[1].split()[0]) / 1024.0, 1)
|
||||
except (OSError, IndexError, ValueError):
|
||||
pass
|
||||
return _clean(out)
|
||||
|
||||
|
||||
def child_main(argv: list[str]) -> int:
|
||||
try:
|
||||
os.nice(19)
|
||||
except (OSError, AttributeError):
|
||||
pass
|
||||
with open(argv[0]) as f:
|
||||
job = json.load(f)
|
||||
out = run_job(job)
|
||||
tmp = argv[1] + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(out, f, separators=(",", ":"))
|
||||
os.replace(tmp, argv[1])
|
||||
return 0
|
||||
|
||||
|
||||
# ---- the box's own models and the nightly job --------------------------------------
|
||||
|
||||
class StoreHistory:
|
||||
"""inputs.History's interface over one series of the training history
|
||||
(the lag of the box's own models: the right series, not the agent's
|
||||
base load for all)."""
|
||||
|
||||
def __init__(self, name: str, values: dict[int, float]):
|
||||
self.slots = {name: {k: [v, 1] for k, v in values.items()}}
|
||||
|
||||
def mean(self, name: str, slot: int):
|
||||
s = self.slots.get(name, {}).get(slot)
|
||||
return s[0] if s else None
|
||||
|
||||
|
||||
class BoxTraining:
|
||||
def __init__(self, cfg, store: trainhist.Store, bundles, kinds=None, clock=time.time, child_cmd=None):
|
||||
self.cfg, self.store, self.bundles, self.clock = cfg, store, bundles, clock
|
||||
self.kinds = kinds or (lambda: set()) # the planned devices' kinds (CTL_KINDS) now
|
||||
self.path = os.path.join(cfg.state_dir, "boxmodels.json")
|
||||
self.child_cmd = child_cmd or [sys.executable, "-m", "cantabile_edge.boxtrain"]
|
||||
self.state = self._load()
|
||||
self.running = False
|
||||
self.progress = ""
|
||||
self._models: dict = {} # (name, trained_at) -> localml.Model
|
||||
self._fc_cache: dict = {} # (name, start) -> forecast
|
||||
self._hist_cache: dict = {}
|
||||
self.last_published = 0.0
|
||||
self.enabled = bool(getattr(cfg, "train", True))
|
||||
# LightGBM is imported in the child only (the agent's own memory stays
|
||||
# as it was); here only whether it is there at all
|
||||
import importlib.util
|
||||
self.lightgbm = importlib.util.find_spec("lightgbm") is not None
|
||||
if not self.lightgbm:
|
||||
log.warning("LightGBM not in this image: the box's gbm fits fail, the other kinds train")
|
||||
|
||||
# -- the state ---------------------------------------------------------------
|
||||
def _load(self) -> dict:
|
||||
try:
|
||||
with open(self.path) as f:
|
||||
d = json.load(f)
|
||||
if isinstance(d, dict) and d.get("v") == 1:
|
||||
d.setdefault("series", {})
|
||||
d.setdefault("rooms", {})
|
||||
d.setdefault("runs", [])
|
||||
return d
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return {"v": 1, "series": {}, "rooms": {}, "runs": [], "last": None}
|
||||
|
||||
def save(self):
|
||||
tmp = self.path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(_clean(self.state), f, separators=(",", ":"))
|
||||
os.replace(tmp, self.path)
|
||||
|
||||
def in_force(self, series: str) -> dict | None:
|
||||
e = self.state["series"].get(series)
|
||||
return e if e and e.get("in_force") and e.get("params") else None
|
||||
|
||||
# -- what the planner asks -------------------------------------------------------
|
||||
def model(self, name: str):
|
||||
"""The box's own model for a bundle series name (load_w, pv_w,
|
||||
base_load_w) as localml runs it, or None (none in force)."""
|
||||
e = self.in_force(SERIES.get(name, ""))
|
||||
if e is None:
|
||||
return None
|
||||
key = (name, e.get("trained_at"))
|
||||
m = self._models.get(key)
|
||||
if m is None:
|
||||
try:
|
||||
m = localml.Model(name, {"params": e["params"], "site": e.get("site") or {}})
|
||||
except (ValueError, TypeError, KeyError) as ex:
|
||||
log.warning("the box's own %s model cannot run: %s", name, ex)
|
||||
e["in_force"], e["why"] = False, f"cannot run: {ex}"[:200]
|
||||
return None
|
||||
self._models = {k: v for k, v in self._models.items() if k[0] != name}
|
||||
self._models[key] = m
|
||||
return m
|
||||
|
||||
def history_for(self, series: str, start: int) -> StoreHistory:
|
||||
"""The series' 8 days before start from the training history."""
|
||||
key = (series, start)
|
||||
h = self._hist_cache.get(key)
|
||||
if h is None:
|
||||
since = start - 8 * 86400
|
||||
raw = {a: v for a, v in self.store.series(since, start).items()}
|
||||
t = np.arange(since, start, SLOT_S, dtype=np.int64)
|
||||
arr = self.actual(series, t, {a: arrays(v, t) for a, v in raw.items()})
|
||||
vals = {} if arr is None else {int(k): float(v) for k, v in zip(t.tolist(), arr.tolist()) if not math.isnan(v)}
|
||||
h = StoreHistory(localml.HISTORY[NAMES.get(series, "load_w")], vals)
|
||||
self._hist_cache = {key: h}
|
||||
return h
|
||||
|
||||
def forecast(self, name: str, start: int, steps: int, ha_forecast, weather_entity: str = ""):
|
||||
"""The box's own forecast of a series (localml.predict on its own
|
||||
model, HA's weather, else the cloud's bundle weather): (values with
|
||||
None, info) or None without a model in force. Once per quarter hour."""
|
||||
m = self.model(name)
|
||||
if m is None:
|
||||
return None
|
||||
e = self.in_force(SERIES[name])
|
||||
key = (name, start, e.get("trained_at"))
|
||||
hit = self._fc_cache.get(key)
|
||||
if hit is None:
|
||||
b = self.bundles.model.get() if self.bundles is not None else None
|
||||
b = b if isinstance(b, dict) and b.get("site") == getattr(self.bundles, "site", None) else {}
|
||||
try:
|
||||
hit = localml.predict(m, [start + k * SLOT_S for k in range(steps)], self.history_for(SERIES[name], start),
|
||||
ha_forecast, b.get("weather"), weather_entity, b.get("issued"))
|
||||
except Exception as ex: # never stops a plan: the next source then
|
||||
log.warning("the box's own %s model: %s", name, ex)
|
||||
hit = ([None] * steps, {"kind": m.kind, "weather": "", "steps": 0, "error": str(ex)[:200]})
|
||||
self._fc_cache = {k: v for k, v in self._fc_cache.items() if k[1] == start}
|
||||
self._fc_cache[key] = hit
|
||||
return hit
|
||||
|
||||
def correct(self, series: str, start: int, values: list) -> tuple[list, float]:
|
||||
"""The intraday correction (fcmodels.intraday) of what the box plans
|
||||
on: the last 3 hours' forecast as planned on ("used", before any
|
||||
correction) against the meter. series: plan (the load) or pv."""
|
||||
since = start - int(fcmodels.INTRADAY_WINDOW_H * 3600)
|
||||
fc = self.store.forecasts(series, since, start).get("used")
|
||||
if not fc:
|
||||
return list(values), 0.0
|
||||
t = np.arange(since, start, SLOT_S, dtype=np.int64)
|
||||
raw = {a: arrays(v, t) for a, v in self.store.series(since, start).items()}
|
||||
y = self.actual(series, t, raw)
|
||||
if y is None:
|
||||
return list(values), 0.0
|
||||
out, e = fcmodels.intraday("pv" if series == "pv" else "load", np.asarray(values, dtype=float), arrays(fc, t), y)
|
||||
return [float(v) for v in out], e
|
||||
|
||||
def room_rc(self, control_id: str, area: str = "", only_one: bool = False) -> dict | None:
|
||||
"""A room the box fitted itself and put in force (it simulated the
|
||||
room better than the cloud's fit on the same data, or the cloud has
|
||||
none): as bundles.room_rc gives the cloud's."""
|
||||
rooms = {k: v for k, v in (self.state.get("rooms") or {}).items() if v.get("in_force")}
|
||||
import re
|
||||
key = next((k for k in (control_id, re.sub(r"_(heating|cooling|heat|cool)$", "", control_id or ""), area or "")
|
||||
if k and k in rooms), None)
|
||||
if key is None and only_one and len(rooms) == 1:
|
||||
key = next(iter(rooms))
|
||||
if key is None:
|
||||
return None
|
||||
r = rooms[key]
|
||||
return {"r_k_per_kw": float(r["r_k_per_kw"]), "c_kwh_per_k": float(r["c_kwh_per_k"]),
|
||||
"power": r.get("power") or "thermal", "gain_w": float(r.get("gain_w") or 0.0),
|
||||
"solar_m2": min(max(float(r.get("solar_m2") or 0.0), 0.0), 50.0)}
|
||||
|
||||
# -- the series and their actual values -------------------------------------------
|
||||
def train_series_list(self) -> list[str]:
|
||||
out = ["load", "pv"]
|
||||
if self.kinds():
|
||||
out.insert(1, "base_load")
|
||||
return out
|
||||
|
||||
def actual(self, series: str, t: np.ndarray, raw: dict[str, np.ndarray]) -> np.ndarray | None:
|
||||
"""What was metered for a series: load.power, pv.power, the base
|
||||
load (trainlib.base_load with the planned devices' kinds), the
|
||||
member's net draw; for "plan" the series the box plans on."""
|
||||
from . import trainlib
|
||||
if series == "plan":
|
||||
series = "base_load" if self.kinds() else "load"
|
||||
if series == "base_load":
|
||||
return trainlib.base_load(raw, set(self.kinds()))
|
||||
if series == "net":
|
||||
return net_series(raw, len(t))
|
||||
a = trainlib.SITE_SERIES.get(series)
|
||||
return raw.get(a) if a else None
|
||||
|
||||
# -- live errors ----------------------------------------------------------------
|
||||
def record(self, series: str, start: int, cands: dict[str, list], now: float):
|
||||
"""The forecasts per source the box had for the next 24 h (the
|
||||
live error against the meter, trainhist fc_slots). series: plan
|
||||
(the load the box plans on), pv or net."""
|
||||
try:
|
||||
self.store.put_forecasts(series, start, {k: v for k, v in cands.items() if v}, now)
|
||||
except Exception as e: # never stops a plan
|
||||
log.warning("forecasts per source: %s", e)
|
||||
|
||||
def week_mean(self, series: str, start: int, steps: int = 96) -> list[float | None]:
|
||||
"""The coordinator's member forecast (methods.md B7) as the box
|
||||
sees it: the same quarter hour's mean over the last 7 days."""
|
||||
since = start - 7 * 86400
|
||||
t = np.arange(since, start, SLOT_S, dtype=np.int64)
|
||||
raw = {a: arrays(v, t) for a, v in self.store.series(since, start).items()}
|
||||
y = self.actual(series, t, raw)
|
||||
if y is None:
|
||||
return [None] * steps
|
||||
by: dict[int, list[float]] = {}
|
||||
for ts, v in zip(t.tolist(), y.tolist()):
|
||||
if not math.isnan(v):
|
||||
by.setdefault((ts % 86400) // SLOT_S, []).append(v)
|
||||
return [(sum(by[q]) / len(by[q])) if (q := ((start + k * SLOT_S) % 86400) // SLOT_S) in by else None
|
||||
for k in range(steps)]
|
||||
|
||||
def live_errors(self, series: str, now: float, days: int = LIVE_DAYS) -> dict[str, dict]:
|
||||
"""Per source the error of what was forecast against the meter
|
||||
over the last days: {"nmae", "mae", "n"}."""
|
||||
until = int(now // SLOT_S * SLOT_S)
|
||||
since = until - days * 86400
|
||||
fc = self.store.forecasts(series, since, until)
|
||||
if not fc:
|
||||
return {}
|
||||
t = np.arange(since, until, SLOT_S, dtype=np.int64)
|
||||
raw = {a: arrays(v, t) for a, v in self.store.series(since, until).items()}
|
||||
y = self.actual(series, t, raw)
|
||||
if y is None:
|
||||
return {}
|
||||
out = {}
|
||||
for src, vals in fc.items():
|
||||
p = arrays(vals, t)
|
||||
m = fcmodels.metrics(y, p)
|
||||
if m["n"]:
|
||||
out[src] = {"nmae": _f(m["nmae"]), "mae": _f(m["mae"]), "n": m["n"]}
|
||||
return out
|
||||
|
||||
# -- the nightly job ----------------------------------------------------------------
|
||||
async def run(self, stop: asyncio.Event, tz=lambda: "UTC", location=lambda: {}, client=lambda: None,
|
||||
weather_entity=lambda: ""):
|
||||
"""Each night at train_at: train (never more than once at a time)."""
|
||||
if not self.enabled:
|
||||
log.info("the box's own training is off (train: false)")
|
||||
await stop.wait()
|
||||
return
|
||||
while not stop.is_set():
|
||||
wait = max(1.0, next_run(self.clock(), self.cfg.train_at, tz()) - self.clock())
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), wait)
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
try:
|
||||
await self.train_once(tz(), location(), client(), weather_entity())
|
||||
except Exception as e: # the next night again
|
||||
log.exception("the box's training: %s", e)
|
||||
|
||||
def site(self, tz: str, location: dict) -> fcmodels.Site:
|
||||
"""Where and what: the cloud's model's site where the bundle has one
|
||||
(its kWp, orientation and PVGIS), else Home Assistant's location."""
|
||||
for name in ("pv_w", "load_w", "base_load_w"):
|
||||
m = self.bundles.box_model(name) if self.bundles is not None and hasattr(self.bundles, "box_model") else None
|
||||
if m is not None and m.site.lat is not None:
|
||||
return fcmodels.Site(tz or m.site.tz, m.site.lat, m.site.lon, m.site.kwp, m.site.tilt, m.site.azimuth,
|
||||
m.site.pvgis_monthly)
|
||||
return fcmodels.Site(tz or "UTC", (location or {}).get("latitude"), (location or {}).get("longitude"))
|
||||
|
||||
def metered_since(self) -> datetime | None:
|
||||
v = (getattr(self.cfg, "pv_metered_since", "") or "").strip()
|
||||
if not v:
|
||||
return None
|
||||
try:
|
||||
t = datetime.fromisoformat(v.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
log.warning("pv_metered_since %r: not ISO 8601", v)
|
||||
return None
|
||||
return t if t.tzinfo else t.replace(tzinfo=timezone.utc)
|
||||
|
||||
def prepare(self, now: float, tz: str, location: dict) -> tuple[list[dict], dict]:
|
||||
"""The jobs (in a worker thread: SQLite and numpy, no await)."""
|
||||
from . import trainlib
|
||||
until = int(now // SLOT_S * SLOT_S)
|
||||
since = until - int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS)) * 86400
|
||||
t = np.arange(since, until, SLOT_S, dtype=np.int64)
|
||||
stored = self.store.series(since, until)
|
||||
raw = {a: arrays(v, t) for a, v in stored.items()}
|
||||
site = self.site(tz, location)
|
||||
obs = self.store.training_hours(since - 3600, until + 3600)
|
||||
fc = self.store.weather_hours(since - 3600, until + 3600, "forecast")
|
||||
w = fcmodels.weather(t, obs, site.lat, site.lon)
|
||||
pv_since = self.metered_since()
|
||||
jobs, info = [], {"slots": int(sum((~np.isnan(v)).sum() for v in raw.values())), "weather_hours": len(obs),
|
||||
"forecast_hours": len(fc), "site": asdict(site)}
|
||||
for series in self.train_series_list():
|
||||
y = self.actual(series, t, raw)
|
||||
if y is None or not (~np.isnan(y)).any():
|
||||
continue
|
||||
y = trainlib.cut_before(series, t, y, pv_since)
|
||||
own = self.state["series"].get(series) or {}
|
||||
champ = own.get("params")
|
||||
if champ is not None and pv_since is not None and series in trainlib.SINCE_SERIES \
|
||||
and (own.get("window") or [0])[0] and own["window"][0] < pv_since.timestamp():
|
||||
champ = None # learned from PV before the cut: any new model replaces it
|
||||
cloud = self.bundles.box_model(NAMES[series]) if self.bundles is not None and hasattr(self.bundles, "box_model") else None
|
||||
w_test, on = trainlib.held_out_weather(t, y, w, fc, site.lat, site.lon)
|
||||
jobs.append({"series": series, "t": t.tolist(), "y": [None if math.isnan(v) else float(v) for v in y.tolist()],
|
||||
"hours": hours_json(obs), "test_hours": hours_json(w_test.hours) if w_test is not None else None,
|
||||
"site": asdict(site), "champion": champ, "scored_on": on,
|
||||
"cloud": _clean(cloud.params) if cloud is not None else None,
|
||||
"cloud_model": ((self.bundles.model.get() or {}).get("models") or {}).get(NAMES[series], {}).get("model")
|
||||
if cloud is not None else None})
|
||||
r0 = int(np.searchsorted(t, until - ROOM_DAYS * 86400))
|
||||
rooms_raw = {a: v[r0:] for a, v in raw.items() if a.startswith("room.") or a in trainlib.ROOM_INPUTS}
|
||||
if any(a.startswith("room.") for a in rooms_raw):
|
||||
m = self.bundles.model.get() if self.bundles is not None else None
|
||||
cloud_rooms = (m or {}).get("rooms") if isinstance(m, dict) and m.get("site") == getattr(self.bundles, "site", None) else {}
|
||||
jobs.append({"series": "rooms", "t": t[r0:].tolist(), "site": asdict(site),
|
||||
"hours": hours_json(obs), "cloud_rooms": cloud_rooms or {},
|
||||
"assets": {a: [None if math.isnan(x) else float(x) for x in v.tolist()] for a, v in rooms_raw.items()}})
|
||||
return jobs, info
|
||||
|
||||
async def child(self, job: dict) -> tuple[dict | None, dict]:
|
||||
"""One job in a child process within the caps: (result or None, how
|
||||
it went: seconds, peak_mb, killed)."""
|
||||
d = self.cfg.state_dir
|
||||
jp, op = os.path.join(d, "train-job.json"), os.path.join(d, "train-result.json")
|
||||
await asyncio.to_thread(_write_json, jp, job)
|
||||
try:
|
||||
os.unlink(op)
|
||||
except OSError:
|
||||
pass
|
||||
env = dict(os.environ, OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1", MKL_NUM_THREADS="1")
|
||||
cap_s = float(getattr(self.cfg, "train_series_s", 300.0))
|
||||
cap_mb = float(getattr(self.cfg, "train_mem_mb", 600))
|
||||
t0 = time.monotonic()
|
||||
proc = await asyncio.create_subprocess_exec(*self.child_cmd, jp, op, env=env, stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE)
|
||||
peak, killed = 0.0, ""
|
||||
err_task = asyncio.ensure_future(proc.stderr.read(64 << 10))
|
||||
while True:
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), 0.25)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
r = rss_mb(proc.pid)
|
||||
if r is not None:
|
||||
peak = max(peak, r)
|
||||
if r is not None and r > cap_mb:
|
||||
killed = f"over its memory cap ({r:.0f} MB > {cap_mb:.0f} MB)"
|
||||
elif time.monotonic() - t0 > cap_s:
|
||||
killed = f"over its wall time ({cap_s:.0f} s)"
|
||||
if killed:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
break
|
||||
err = b""
|
||||
try:
|
||||
err = await asyncio.wait_for(err_task, 2)
|
||||
except (asyncio.TimeoutError, Exception):
|
||||
pass
|
||||
how = {"seconds": round(time.monotonic() - t0, 2), "peak_mb": round(peak, 1), "killed": killed}
|
||||
res = None
|
||||
if not killed:
|
||||
try:
|
||||
with open(op) as f:
|
||||
res = json.load(f)
|
||||
how["peak_mb"] = max(how["peak_mb"], float(res.get("peak_mb") or 0.0))
|
||||
except (OSError, ValueError):
|
||||
how["error"] = (err.decode(errors="replace").strip().splitlines() or [f"exit {proc.returncode}"])[-1][:300]
|
||||
for p in (jp, op):
|
||||
try:
|
||||
os.unlink(p)
|
||||
except OSError:
|
||||
pass
|
||||
return res, how
|
||||
|
||||
async def train_once(self, tz: str = "UTC", location: dict | None = None, client=None, weather_entity: str = "") -> dict:
|
||||
"""One night's training; the summary (also in state["last"])."""
|
||||
if self.running:
|
||||
return {"skipped": "already running"}
|
||||
self.running = True
|
||||
t_start = self.clock()
|
||||
summary: dict = {"at": int(t_start), "series": {}, "rooms": None, "notes": []}
|
||||
try:
|
||||
if client is not None:
|
||||
for what, fn in (("statistics", self.fill_from_ha), ("weather", self.weather_from_ha)):
|
||||
try:
|
||||
n = await fn(client, weather_entity) if what == "weather" else await fn(client)
|
||||
if n:
|
||||
summary["notes"].append(f"{n} {what} slots/hours from Home Assistant")
|
||||
except Exception as e: # HA's history is a help, not a need
|
||||
log.info("training: %s from Home Assistant: %s", what, e)
|
||||
jobs, info = await asyncio.to_thread(self.prepare, t_start, tz, location or {})
|
||||
summary["data"] = info
|
||||
for job in jobs:
|
||||
free = mem_available_mb()
|
||||
need = float(getattr(self.cfg, "train_min_free_mb", 300))
|
||||
name = job["series"]
|
||||
if free is not None and free < need:
|
||||
summary["notes"].append(f"{name}: not trained, {free:.0f} MB available (at least {need:.0f})")
|
||||
continue
|
||||
self.progress = name
|
||||
res, how = await self.child(job)
|
||||
if name == "rooms":
|
||||
summary["rooms"] = {"how": how, **self.take_rooms(res, how)}
|
||||
else:
|
||||
summary["series"][name] = self.take(name, job, res, how, t_start)
|
||||
self.state["last"] = summary
|
||||
self.state["runs"] = (self.state.get("runs", []) + [_runlog(summary)])[-RUNS_KEPT:]
|
||||
await asyncio.to_thread(self.save)
|
||||
await asyncio.to_thread(self.store.prune, self.clock(), int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS)))
|
||||
log.info("the box's training: %s in %.0f s", ", ".join(f"{k} {v.get('result')}" for k, v in summary["series"].items())
|
||||
or "nothing to train", self.clock() - t_start)
|
||||
finally:
|
||||
self.running, self.progress = False, ""
|
||||
self._models, self._fc_cache = {}, {}
|
||||
return summary
|
||||
|
||||
def take(self, series: str, job: dict, res: dict | None, how: dict, now: float) -> dict:
|
||||
"""A series' result: the new model in force only if it beats the
|
||||
box's model (trainlib) and the cloud's (score on the same window)."""
|
||||
out = {"how": how, "scored_on": job.get("scored_on", "")}
|
||||
if res is None:
|
||||
out["result"] = "failed: " + (how.get("killed") or how.get("error") or "no result")
|
||||
return out
|
||||
key = res.get("key") or "nmae"
|
||||
runs = {r["kind"]: score(r.get("metrics"), key) for r in res.get("runs") or [] if r.get("metrics")}
|
||||
cloud = score(res.get("cloud"), key)
|
||||
own_now = score(res.get("champion"), key)
|
||||
out.update(key=key, runs=runs, cloud=cloud, champion=own_now, n_test=res.get("n_test"), note=res.get("note", ""))
|
||||
cur = self.state["series"].get(series)
|
||||
chosen = res.get("chosen")
|
||||
|
||||
def beats(kind, v) -> bool:
|
||||
"""In force: better than the cloud's model on the window; without
|
||||
one, any kind but the hour-of-week (it beat the hour-of-week)."""
|
||||
return (cloud is None and kind != "how") or (cloud is not None and v is not None and v < cloud)
|
||||
|
||||
def vs() -> str:
|
||||
return f"the cloud's model {cloud:.3g}" if cloud is not None else "no cloud model on the box to compare"
|
||||
|
||||
new = runs.get(chosen) if chosen else None
|
||||
if chosen and res.get("params") and (beats(chosen, new) or cur is None):
|
||||
# a new model: it beat the box's own (trainlib) and, to be in force, the cloud's
|
||||
cur = {"kind": chosen, "params": res["params"], "site": job["site"], "trained_at": int(now),
|
||||
"window": res.get("window"), "score": new, "key": key, "in_force": beats(chosen, new)}
|
||||
self.state["series"][series] = cur
|
||||
out["result"] = (("new model in force: " if cur["in_force"] else "trained, not in force: ")
|
||||
+ f"{chosen} {key} {new:.3g}, {vs()}"
|
||||
+ (f", the box's own {own_now:.3g}" if own_now is not None else ""))
|
||||
elif cur is not None:
|
||||
# the box's model stays (nothing beat it, or the new one did not beat
|
||||
# the cloud's): in force while it beats the cloud's on this window
|
||||
stays = own_now if own_now is not None else cur.get("score")
|
||||
cur["in_force"] = beats(cur.get("kind"), stays)
|
||||
if own_now is not None:
|
||||
cur["score"] = own_now
|
||||
out["result"] = (f"the box's {cur.get('kind')} stays" + (f" ({key} {stays:.3g}, {vs()})" if stays is not None else "")
|
||||
+ ("" if cur["in_force"] else ": not in force"))
|
||||
if chosen and new is not None:
|
||||
out["result"] += f"; the new {chosen} ({new:.3g}) does not beat {vs()}"
|
||||
else:
|
||||
out["result"] = "nothing trained: " + (res.get("note") or "no model")
|
||||
if cur is not None:
|
||||
cur.update(cloud=cloud, runs=runs, scored_on=job.get("scored_on", ""), n_test=res.get("n_test"),
|
||||
checked_at=int(now), cloud_model=job.get("cloud_model"))
|
||||
# the live error decides too: worse than the cloud's series over a week, out of force
|
||||
live = self.live_errors("plan" if series == ("base_load" if self.kinds() else "load") else series, now)
|
||||
mine, theirs = live.get("own"), live.get("cloud")
|
||||
if cur.get("in_force") and mine and theirs and mine["n"] >= LIVE_MIN and theirs["n"] >= LIVE_MIN \
|
||||
and mine["nmae"] is not None and theirs["nmae"] is not None and mine["nmae"] > theirs["nmae"]:
|
||||
cur["in_force"] = False
|
||||
out["result"] += (f"; out of force: live nMAE {mine['nmae']:.3g} vs the cloud's series "
|
||||
f"{theirs['nmae']:.3g} over {LIVE_DAYS} days")
|
||||
cur["why"] = out["result"][:300]
|
||||
out["in_force"] = cur["in_force"]
|
||||
return out
|
||||
|
||||
def take_rooms(self, res: dict | None, how: dict) -> dict:
|
||||
if res is None:
|
||||
return {"result": "failed: " + (how.get("killed") or how.get("error") or "no result")}
|
||||
out = {}
|
||||
rooms = self.state.setdefault("rooms", {})
|
||||
for room, f in (res.get("rooms") or {}).items():
|
||||
fitted = f.get("status") == "fitted"
|
||||
cloud = f.get("cloud_rmse_k")
|
||||
better = fitted and (cloud is None or (f.get("rmse_k") is not None and f["rmse_k"] < cloud))
|
||||
f["in_force"] = bool(better)
|
||||
f["why"] = ("fitted, simulates the room better than " + (f"the cloud's fit ({f['rmse_k']} K vs {cloud:.3f} K)"
|
||||
if cloud is not None else "the defaults (no cloud fit to compare)")) if better else \
|
||||
(f"fitted, but the cloud's fit simulates it better ({cloud:.3f} K vs {f.get('rmse_k')} K)" if fitted
|
||||
else f.get("note", "default"))
|
||||
rooms[room] = f
|
||||
out[room] = {"in_force": f["in_force"], "why": f["why"]}
|
||||
return {"result": out}
|
||||
|
||||
# -- Home Assistant's history ------------------------------------------------------
|
||||
async def fill_from_ha(self, client) -> int:
|
||||
"""Slots the agent never saw, from HA's statistics (backfill.collect),
|
||||
once a day: 5-minute means for HA's last 10 days, hourly before."""
|
||||
mapping = getattr(self, "mapping", None)
|
||||
if mapping is None:
|
||||
return 0
|
||||
from . import backfill
|
||||
now = datetime.fromtimestamp(self.clock(), timezone.utc)
|
||||
days = int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS))
|
||||
n = 0
|
||||
for period, a, b in (("hour", now - timedelta(days=days), now - timedelta(days=9)),
|
||||
("5minute", now - timedelta(days=9), now)):
|
||||
got = await backfill.collect(client, mapping, a, b, period, trainhist.wanted, consented=False)
|
||||
for asset, rows in got.items():
|
||||
n += await asyncio.to_thread(self.store.fill, asset, trainhist.stats_to_slots(rows, period),
|
||||
"ha-" + period)
|
||||
return n
|
||||
|
||||
async def weather_from_ha(self, client, entity: str) -> int:
|
||||
"""Observed hours the box lacks, from HA's recorded weather entity
|
||||
(temperature and cloud cover at each hour's end; no irradiation)."""
|
||||
if not entity:
|
||||
return 0
|
||||
now = self.clock()
|
||||
days = int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS))
|
||||
spans = await asyncio.to_thread(self.store.missing_observed, now - days * 86400, now)
|
||||
if not spans:
|
||||
return 0
|
||||
a = datetime.fromtimestamp(spans[0][0], timezone.utc)
|
||||
res = await client.send("history/history_during_period", timeout=120, start_time=a.isoformat(),
|
||||
end_time=datetime.fromtimestamp(now, timezone.utc).isoformat(), entity_ids=[entity],
|
||||
minimal_response=False, no_attributes=False, significant_changes_only=False)
|
||||
hours = ha_history_hours((res or {}).get(entity) or [], spans)
|
||||
return await asyncio.to_thread(self.store.put_weather, "observed", "ha", hours, now)
|
||||
|
||||
# -- upward ------------------------------------------------------------------------
|
||||
def status(self, now: float, forecasts: dict | None = None) -> dict:
|
||||
"""The plan's "ml": the training's figures and the error per source
|
||||
(and, at most hourly, the forecasts the box planned on)."""
|
||||
series = {}
|
||||
for s, e in (self.state.get("series") or {}).items():
|
||||
series[s] = {"kind": e.get("kind"), "in_force": bool(e.get("in_force")), "trained_at": e.get("trained_at"),
|
||||
"checked_at": e.get("checked_at"), "key": e.get("key"), "held_out": e.get("runs") or {},
|
||||
"held_out_box": e.get("score"), "held_out_cloud": e.get("cloud"), "scored_on": e.get("scored_on", ""),
|
||||
"n_test": e.get("n_test"), "why": (e.get("why") or "")[:300]}
|
||||
live = {}
|
||||
for s in ("plan", "pv", "net"):
|
||||
le = self.live_errors(s, now)
|
||||
if le:
|
||||
live[s] = le
|
||||
last = self.state.get("last") or {}
|
||||
out = {"v": 1, "series": series, "live": live, "live_days": LIVE_DAYS,
|
||||
"planned": "base_load" if self.kinds() else "load",
|
||||
"rooms": {k: {"in_force": bool(v.get("in_force")), "r_k_per_kw": v.get("r_k_per_kw"),
|
||||
"c_kwh_per_k": v.get("c_kwh_per_k"), "rmse_k": v.get("rmse_k"),
|
||||
"cloud_rmse_k": v.get("cloud_rmse_k")} for k, v in (self.state.get("rooms") or {}).items()},
|
||||
"train": {"enabled": self.enabled, "lightgbm": self.lightgbm, "at": getattr(self.cfg, "train_at", ""),
|
||||
"last": last.get("at"), "running": self.running,
|
||||
"per_series": {k: v.get("how") for k, v in (last.get("series") or {}).items()},
|
||||
"notes": (last.get("notes") or [])[:5]}}
|
||||
if forecasts and now - self.last_published >= PUBLISH_EVERY_S:
|
||||
out["forecasts"] = forecasts
|
||||
self.last_published = now
|
||||
return _clean(out)
|
||||
|
||||
|
||||
def _write_json(path: str, d: dict):
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(d, f, separators=(",", ":"))
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def _runlog(summary: dict) -> dict:
|
||||
return {"at": summary.get("at"), "series": {k: {"result": v.get("result", "")[:200], "how": v.get("how")}
|
||||
for k, v in (summary.get("series") or {}).items()}}
|
||||
|
||||
|
||||
def ha_history_hours(states: list[dict], spans: list[tuple[int, int]]) -> list:
|
||||
"""A weather entity's recorded states → per hour end in the spans the
|
||||
temperature and cloud cover in force then (the last state before)."""
|
||||
pts = []
|
||||
for s in states:
|
||||
a = s.get("a") or s.get("attributes") or {}
|
||||
ts = s.get("lu") or s.get("lc") or s.get("last_updated") or s.get("last_changed")
|
||||
try:
|
||||
ts = float(ts) if isinstance(ts, (int, float)) else datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp()
|
||||
except ValueError:
|
||||
continue
|
||||
temp, cloud = a.get("temperature"), a.get("cloud_coverage")
|
||||
pts.append((ts, float(temp) if isinstance(temp, (int, float)) else None,
|
||||
float(cloud) if isinstance(cloud, (int, float)) else None))
|
||||
pts.sort()
|
||||
out, j = [], -1
|
||||
for a, b in spans:
|
||||
for end in range(a + 3600, b + 1, 3600):
|
||||
while j + 1 < len(pts) and pts[j + 1][0] <= end:
|
||||
j += 1
|
||||
if j >= 0 and (pts[j][1] is not None or pts[j][2] is not None) and end - pts[j][0] <= 3 * 3600:
|
||||
out.append(pvforecast.Hour(datetime.fromtimestamp(end, timezone.utc), None, pts[j][2], pts[j][1]))
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(child_main(sys.argv[1:]))
|
||||
@@ -440,6 +440,19 @@ class Bundles:
|
||||
out.append((float(f["values"][d.weekday() * 24 + d.hour]), "profile"))
|
||||
return out
|
||||
|
||||
def profile(self, name: str, start: int, step_s: int, steps: int, tz: str) -> list[float] | None:
|
||||
"""The model's hour-of-week profile of a series for every step (the
|
||||
live error per source, boxtrain.py), or None."""
|
||||
m = self.model.get()
|
||||
if m is None or m.get("site") not in (self.site, "*"):
|
||||
return None
|
||||
f = (m.get("forecasts") or {}).get(name)
|
||||
if f is None:
|
||||
return None
|
||||
zone = ZoneInfo(tz or "UTC")
|
||||
return [float(f["values"][d.weekday() * 24 + d.hour]) for d in
|
||||
(datetime.fromtimestamp(start + k * step_s, timezone.utc).astimezone(zone) for k in range(steps))]
|
||||
|
||||
def box_model(self, name: str):
|
||||
"""The bundle's own model for a series (localml.Model), or None: no
|
||||
model bundle for this site, none for the series, or one the box
|
||||
|
||||
@@ -62,6 +62,19 @@ class Config:
|
||||
override_hold_s: int = 7200 # hands off a control the owner changed by hand
|
||||
min_switch_s: int = 600 # anti-cycling: a switch stays on/off at least this long
|
||||
max_actions_per_hour: int = 12 # per control
|
||||
# The box trains its own models (boxtrain.py, methods.md B10): nightly at
|
||||
# train_at (local time, HA's time zone), one series at a time in a child
|
||||
# process at the lowest priority, each at most train_series_s and
|
||||
# train_mem_mb (resident) and only with train_min_free_mb available; the
|
||||
# history kept and trained on: train_days. pv_metered_since (ISO 8601, UTC; as the
|
||||
# cloud's PV_METERED_SINCE): metered PV before it does not count.
|
||||
train: bool = True
|
||||
train_at: str = "03:30"
|
||||
train_series_s: float = 300.0
|
||||
train_mem_mb: int = 600
|
||||
train_min_free_mb: int = 300
|
||||
train_days: int = 396 # 13 months: recent weeks weigh more (fcmodels.recency_weights)
|
||||
pv_metered_since: str = ""
|
||||
|
||||
|
||||
def _read_secret(path: str) -> str:
|
||||
@@ -94,6 +107,10 @@ def load(path: str | None, require_site: bool = True) -> Config:
|
||||
cfg.dashboards = False
|
||||
if env.get("CANTABILE_PROPOSE") in ("1", "true", "on"):
|
||||
cfg.propose = True
|
||||
if env.get("CANTABILE_TRAIN") in ("0", "false", "off"):
|
||||
cfg.train = False
|
||||
cfg.train_at = env.get("CANTABILE_TRAIN_AT", cfg.train_at)
|
||||
cfg.pv_metered_since = env.get("CANTABILE_PV_METERED_SINCE", cfg.pv_metered_since)
|
||||
if token_file:
|
||||
cfg.ha_token = _read_secret(token_file)
|
||||
# Inside a Supervisor add-on the Supervisor proxies Core's API and hands
|
||||
|
||||
@@ -24,6 +24,15 @@ Commands (from the console through the link, which checked the signature):
|
||||
"floor"?, "limit"? (empty: none), "until"?}
|
||||
building {"v", "id", "building", "level", "control", "value", "from",
|
||||
"until", "why"}: the building's requests (building.py)
|
||||
preferences {"msg": JSON of preferences.apply_msg's {"set": {"item",
|
||||
"value"}}, "level": "installer|operator", "by"}: an installer's or
|
||||
Cantabile's setting within the owner's limits (preferences.py);
|
||||
the owner's own come from Home Assistant and the panel
|
||||
|
||||
The owner's preferences (preferences.py): into every model state (a room's
|
||||
schedule, a car's ready-by, a battery's reserve, the overrides), a plan at
|
||||
once after a change (not held back by steady.py), their feedback and the
|
||||
settings with the plan to the cloud (public_plan "preferences").
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
@@ -41,8 +50,10 @@ from .executor import Executor, building_msg
|
||||
from .gridorder import GridOrders
|
||||
from .guard import Guard
|
||||
from .policy import Policy
|
||||
from .preferences import Preferences
|
||||
from .switchover import Switchover
|
||||
from .inputs import Collector, History
|
||||
from . import localml
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -73,6 +84,7 @@ class Mode:
|
||||
class Controller:
|
||||
def __init__(self, cfg, mapping, consent, sink, clock=time.time):
|
||||
self.cfg, self.mapping, self.sink, self.clock = cfg, mapping, sink, clock
|
||||
self.consent = consent
|
||||
self.controls = mapping.controls
|
||||
self.inputs = mapping.inputs
|
||||
self.mode = Mode(cfg.state_dir, cfg.mode)
|
||||
@@ -108,6 +120,25 @@ class Controller:
|
||||
self.on_plan = [] # callbacks (the announcer) when a plan is made
|
||||
self.received: list[dict] = [] # the last commands and bundles (shown in HA, announce.py)
|
||||
self.on_received = [] # callbacks (the announcer) for each
|
||||
self.prefs = Preferences(cfg.state_dir, clock) # the owner's preferences (preferences.py)
|
||||
self.on_prefs = [] # callbacks (the announcer) when they change
|
||||
try:
|
||||
self.prefs.write_view(self.controls, self.mode())
|
||||
except Exception as e:
|
||||
log.warning("preferences view: %s", e)
|
||||
# the box trains its own models (boxtrain.py, methods.md B10): its
|
||||
# history, the nightly job, and its models for the planner (bundles.own)
|
||||
self.boxml = None
|
||||
if getattr(cfg, "train", True):
|
||||
try:
|
||||
from .boxtrain import BoxTraining
|
||||
from .trainhist import Store
|
||||
self.boxml = BoxTraining(cfg, Store(os.path.join(cfg.state_dir, "training.db")), self.bundles,
|
||||
kinds=self.planned_kinds, clock=clock)
|
||||
self.bundles.own = self.boxml
|
||||
except Exception as e: # never stops the controller
|
||||
log.warning("the box's own training is off: %s", e)
|
||||
self.boxml = None
|
||||
|
||||
# -- Home Assistant ----------------------------------------------------
|
||||
def outside_entity(self) -> str:
|
||||
@@ -181,6 +212,7 @@ class Controller:
|
||||
try:
|
||||
cfg = await client.send("get_config")
|
||||
self.tz = (cfg or {}).get("time_zone") or "UTC"
|
||||
self.prefs.tz = self.tz
|
||||
lat, lon = (cfg or {}).get("latitude"), (cfg or {}).get("longitude")
|
||||
if isinstance(lat, (int, float)) and isinstance(lon, (int, float)):
|
||||
self.location = {"latitude": float(lat), "longitude": float(lon)}
|
||||
@@ -215,6 +247,14 @@ class Controller:
|
||||
value = st.get("s")
|
||||
self.guard.observe(c.id, eid, value, now)
|
||||
|
||||
def planned_kinds(self) -> set[str]:
|
||||
"""The kinds of the devices the box plans whose power is inside the
|
||||
load (heating, cooling, EV): its base load is trained where any is
|
||||
metered (inputs.controlled_power), as the planner uses it."""
|
||||
if not (getattr(self.inputs, "controlled_power", None) or []):
|
||||
return set()
|
||||
return {c.kind for c in self.controls if c.kind in ("heating", "cooling", "ev")}
|
||||
|
||||
# -- plans ---------------------------------------------------------------
|
||||
def model_state(self) -> dict:
|
||||
now = self.clock()
|
||||
@@ -223,13 +263,15 @@ class Controller:
|
||||
# and of a thermostat that heats and cools only its active side
|
||||
self.switch_sides()
|
||||
controls = [c for c in self.controls if self.policy.plannable(c) and self.switchover.active(c)]
|
||||
return inputs_mod.model_state(now, controls, self.inputs, self.col, self.bundles,
|
||||
self.history, self.tz, self.forecast, self.guard.extra_limits,
|
||||
building=self.building.for_model(start, inputs_mod.STEP_S, inputs_mod.STEPS),
|
||||
location=self.location, shade_ctx=self.shade_ctx(),
|
||||
grid_order=self.gridorders.for_model(start, inputs_mod.STEP_S, inputs_mod.STEPS),
|
||||
grid_max_w=getattr(self.cfg, "grid_max_w", None),
|
||||
ev_phases=self.ev_phases())
|
||||
state = inputs_mod.model_state(now, controls, self.inputs, self.col, self.bundles,
|
||||
self.history, self.tz, self.forecast, self.guard.extra_limits,
|
||||
building=self.building.for_model(start, inputs_mod.STEP_S, inputs_mod.STEPS),
|
||||
location=self.location, shade_ctx=self.shade_ctx(),
|
||||
grid_order=self.gridorders.for_model(start, inputs_mod.STEP_S, inputs_mod.STEPS),
|
||||
grid_max_w=getattr(self.cfg, "grid_max_w", None),
|
||||
ev_phases=self.ev_phases())
|
||||
# the owner's preferences per step (preferences.py)
|
||||
return self.prefs.apply(state, {c.id: c for c in controls}, self.col, now)
|
||||
|
||||
async def make_plan(self) -> dict | None:
|
||||
self.forecast = []
|
||||
@@ -239,6 +281,7 @@ class Controller:
|
||||
except Exception as e:
|
||||
log.info("weather forecast: %s", e)
|
||||
state = self.model_state()
|
||||
cands, net_load = state.pop("_cands", None) or {}, state.pop("_net_load", None)
|
||||
plan = await solve(state, self.cfg.solve_timeout_s)
|
||||
if "error" in plan:
|
||||
self.plan_error = plan["error"]
|
||||
@@ -247,12 +290,25 @@ class Controller:
|
||||
calm, self.calm = self.calm, False
|
||||
if calm:
|
||||
steady.carry_quarter(plan, self.plan, self.clock())
|
||||
# changes that do not matter stay out of it (deadband, hold, gain: steady.py)
|
||||
# what the plan does for each preference, and what each must costs
|
||||
self.prefs.take_plan(plan, self.clock())
|
||||
# changes that do not matter stay out of it (deadband, hold, gain: steady.py);
|
||||
# the owner's preference changes are not held back
|
||||
steady.stabilise(plan, state, [c for c in self.controls if c.id in plan.get("devices", {})], self.col,
|
||||
self.guard.last_change, self.clock())
|
||||
self.guard.last_change, self.clock(), free=self.prefs.free())
|
||||
self.prefs.write_view(self.controls, self.mode())
|
||||
plan["mode"] = self.mode()
|
||||
plan["id"] = f"{self.cfg.site_id[:8]}-{plan['made']}"
|
||||
plan["model"] = self.bundles.model_id()
|
||||
if self.boxml is not None:
|
||||
try: # SQLite and numpy: in a worker thread, the loop goes on
|
||||
ml = await asyncio.to_thread(self.ml_after_plan, state, plan, cands, net_load)
|
||||
# upward only with the owner's share consent: without it the box
|
||||
# trains for itself and nothing of it leaves (no forecasts, no errors)
|
||||
if getattr(self.consent, "share", False):
|
||||
plan["ml"] = ml
|
||||
except Exception as e:
|
||||
log.warning("the box's model figures: %s", e)
|
||||
self.plan, self.plan_error = plan, ""
|
||||
self.executor.plan = plan
|
||||
self.executor.inputs_ok = self.inputs_fresh()
|
||||
@@ -276,7 +332,82 @@ class Controller:
|
||||
# for the building (shed_w/grid_w per step, 24 h): absent from older agents
|
||||
"flex": plan.get("flex", {}),
|
||||
# the market's contracts in the plan (shadow): planned kWh and saving vs the grid
|
||||
"market": plan.get("market", [])}
|
||||
"market": plan.get("market", []),
|
||||
# the owner's preferences: per level who set what and when, the
|
||||
# overrides, and per preference met or not and each must's cost
|
||||
"preferences": self.prefs.public(),
|
||||
# the box's own models (boxtrain.py): held-out and live errors per source,
|
||||
# hourly the forecasts it planned on; absent from older agents
|
||||
**({"ml": plan["ml"]} if plan.get("ml") else {})}
|
||||
|
||||
# -- the owner's preferences (preferences.py) -------------------------------
|
||||
async def prefs_change(self, msg: dict, level: str, by: str) -> tuple[bool, str]:
|
||||
"""A change of the preferences (Home Assistant's entities and the
|
||||
panel: the owner's; the console: an installer's or operator's): a
|
||||
plan with it at once (not held back: steady.py), shown in HA."""
|
||||
ok, why = self.prefs.apply_msg(msg, {c.id: c for c in self.controls}, self.tz, level, by)
|
||||
log.info("preferences (%s, %s): %s %s", by, level, "applied" if ok else "refused", why)
|
||||
await self._prefs_changed()
|
||||
return ok, why
|
||||
|
||||
async def _prefs_changed(self):
|
||||
self.prefs.write_view(self.controls, self.mode())
|
||||
self.wake.set()
|
||||
for cb in self.on_prefs:
|
||||
try:
|
||||
await cb()
|
||||
except Exception as e:
|
||||
log.warning("preferences: %s", e)
|
||||
|
||||
async def _prefs_watch(self, stop):
|
||||
"""The panel's changes (prefs-inbox/), and overrides that expire."""
|
||||
while not stop.is_set():
|
||||
try:
|
||||
n = self.prefs.take_inbox({c.id: c for c in self.controls}, self.tz)
|
||||
if n or self.prefs.prune(self.clock()):
|
||||
await self._prefs_changed()
|
||||
except Exception as e:
|
||||
log.warning("preferences: %s", e)
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), PREFS_POLL_S)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
def ml_after_plan(self, state: dict, plan: dict, cands: dict, net_load) -> dict:
|
||||
"""After a plan (in a worker thread): the weather the box forecast
|
||||
with and each source's forecast kept for the training and the live
|
||||
error (trainhist.py), and the figures for the plan's "ml"
|
||||
(boxtrain.status), with, at most hourly, the forecasts it planned on:
|
||||
the load (as planned: the base load where it plans devices), PV and
|
||||
the member's net draw (the plan's import less export where it acts,
|
||||
auto; else the metered load's forecast less PV) for the building."""
|
||||
b, now, start = self.boxml, self.clock(), int(state["start"])
|
||||
st = b.store
|
||||
if self.forecast:
|
||||
st.put_weather("forecast", "ha", localml.ha_hours(self.forecast), now)
|
||||
m = self.bundles.model.get()
|
||||
if isinstance(m, dict) and m.get("site") == self.cfg.site_id:
|
||||
if m.get("weather"):
|
||||
st.put_weather("forecast", "cloud", localml.cloud_hours(m["weather"]), m.get("issued"))
|
||||
if m.get("weather_past"):
|
||||
st.put_weather("observed", "cloud", localml.cloud_hours(m["weather_past"]), now)
|
||||
for key, series in (("load", "plan"), ("pv", "pv")):
|
||||
if cands.get(key):
|
||||
b.record(series, start, cands[key], now)
|
||||
auto = plan.get("mode") == "auto"
|
||||
net = (plan.get("flex") or {}).get("net_w") if auto else \
|
||||
([round(lo - pv) for lo, pv in zip(net_load, state["pv_w"])] if net_load else None)
|
||||
if net:
|
||||
b.record("net", start, {"own": net, "week_mean": b.week_mean("net", start, len(net))}, now)
|
||||
fsrc = (state.get("sources") or {}).get("forecast") or {}
|
||||
ups = {k: {"start": start, "step_s": state["step_s"], "values": [round(float(v), 1) for v in state[f"{k}_w"]],
|
||||
"source": (fsrc.get(k) or {}).get("source") or (state.get("sources") or {}).get(k, "")[:80]}
|
||||
for k in ("load", "pv")}
|
||||
ups["load"]["series"] = "base_load" if b.kinds() else "load"
|
||||
if net:
|
||||
ups["net"] = {"start": start, "step_s": state["step_s"], "values": [float(v) for v in net],
|
||||
"source": "plan" if auto else "forecast"}
|
||||
return b.status(now, ups)
|
||||
|
||||
def inputs_fresh(self) -> bool:
|
||||
"""The plan's room temperatures and the battery's state have a value
|
||||
@@ -341,7 +472,12 @@ class Controller:
|
||||
await stop.wait()
|
||||
return
|
||||
tasks = [asyncio.create_task(self._planner(stop)), asyncio.create_task(self._actor(stop)),
|
||||
asyncio.create_task(self._bundle_watch(stop))]
|
||||
asyncio.create_task(self._bundle_watch(stop)), asyncio.create_task(self._prefs_watch(stop))]
|
||||
if self.boxml is not None:
|
||||
# the nightly training (boxtrain.py): child processes, the loop only awaits them
|
||||
tasks.append(asyncio.create_task(self.boxml.run(
|
||||
stop, tz=lambda: self.tz, location=lambda: self.location, client=lambda: self.client,
|
||||
weather_entity=lambda: self.inputs.weather or "")))
|
||||
await stop.wait()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
@@ -472,6 +608,18 @@ class Controller:
|
||||
return ok, msg
|
||||
if cmd == "building":
|
||||
return await self.building_command(args, now)
|
||||
if cmd == "preferences":
|
||||
# the console's: an installer's or Cantabile's, within the owner's limits
|
||||
level = args.get("level", "")
|
||||
if level not in ("installer", "operator"):
|
||||
return False, "level: installer or operator (the owner's own come from Home Assistant or the panel)"
|
||||
try:
|
||||
msg = json.loads(args.get("msg") or "{}")
|
||||
except ValueError:
|
||||
return False, "msg: JSON"
|
||||
if not isinstance(msg, dict) or "set" not in msg:
|
||||
return False, "msg: {\"set\": {\"item\", \"value\"}} (overrides are the owner's)"
|
||||
return await self.prefs_change(msg, level, f"{level}:{str(args.get('by') or 'console')[:60]}")
|
||||
return False, f"unknown command {cmd!r}"
|
||||
|
||||
async def building_command(self, args: dict, now: float) -> tuple[bool, str]:
|
||||
@@ -668,6 +816,7 @@ async def solve(state: dict, timeout: float) -> dict:
|
||||
|
||||
|
||||
PLAN_AT_S = 5.0 # a plan this long after a quarter hour: its first step is the quarter just begun
|
||||
PREFS_POLL_S = 2.0 # how often the panel's preference changes are looked for
|
||||
|
||||
|
||||
def plan_wait(now: float, interval: float, at: float = PLAN_AT_S) -> float:
|
||||
|
||||
@@ -10,15 +10,21 @@ metered). The kinds:
|
||||
week (168 values; the box's fallback format, ml/model_bundle.py).
|
||||
The baseline every other kind has to beat.
|
||||
gbm LightGBM on calendar (local hour, weekday, a day off: weekend or
|
||||
a German public holiday), the hour-of-week value, the same slot a
|
||||
a German public holiday, the special day's type as a categorical
|
||||
feature, the yearly rhythm: day of year as sine and cosine, the
|
||||
day's daylight hours), the hour-of-week value, the same slot a
|
||||
week before, the weather (temperature, irradiance, cloud cover)
|
||||
and, for PV, the sun and the physical prior. Small (150 trees, 15
|
||||
leaves, one thread): hundreds of sites a night.
|
||||
leaves, one thread): hundreds of sites a night. Recent weeks weigh
|
||||
more (recency_weights); a model's own feature list is kept with
|
||||
it, so a model trained before a feature was added still runs.
|
||||
pvphys PV only: the physical irradiance model (pvforecast.py, shared
|
||||
with the Mieterstrom pricing) times the site's calibration
|
||||
(pvcalib.py).
|
||||
|
||||
A model is a JSON-able dict ("params"); fit_* makes it, predict() uses it.
|
||||
intraday() carries the last hours' error against the meter into the next
|
||||
hours (the box every 15 minutes; methods.md B10).
|
||||
The weather per slot comes from hourly records (pvforecast.Hour: the hour
|
||||
ENDING at `end`, solar in kWh/m² over that hour, cloud %, temp °C)."""
|
||||
from __future__ import annotations
|
||||
@@ -45,8 +51,23 @@ GBM_PARAMS = dict(objective="regression", learning_rate=0.05, num_leaves=15, min
|
||||
bagging_freq=1, feature_fraction=0.9, lambda_l2=1.0, num_threads=1, verbose=-1, seed=1)
|
||||
GBM_ROUNDS = 150
|
||||
MIN_GBM_SLOTS = 96 # a day of data at least
|
||||
LOAD_FEATURES = ["hod", "dow", "off", "how", "lag7", "temp", "ghi", "cloud", "hdd"]
|
||||
PV_FEATURES = ["hod", "how", "lag7", "temp", "ghi", "cloud", "elev", "az", "phys"]
|
||||
LOAD_FEATURES = ["hod", "dow", "off", "how", "lag7", "temp", "ghi", "cloud", "hdd", "special", "doy_sin", "doy_cos",
|
||||
"daylight"]
|
||||
PV_FEATURES = ["hod", "how", "lag7", "temp", "ghi", "cloud", "elev", "az", "phys", "doy_sin", "doy_cos", "daylight"]
|
||||
FEATURES_V1 = {"load": ["hod", "dow", "off", "how", "lag7", "temp", "ghi", "cloud", "hdd"], # before 2026-11-04
|
||||
"pv": ["hod", "how", "lag7", "temp", "ghi", "cloud", "elev", "az", "phys"]}
|
||||
CATEGORICAL = ("special",)
|
||||
# linear leaves (LightGBM linear_tree: each leaf a linear model on its
|
||||
# branch's features, so the forecast keeps rising with heating degrees past
|
||||
# the coldest day trained on): for these series only where it measured better
|
||||
# (measured 2026-11-04, building's synthetic site: a cold snap 12 K below the
|
||||
# coldest day trained on, held-out nMAE 29.5 % -> 6.4 %; an ordinary winter
|
||||
# week 3.8 % -> 3.3 %; methods.md B10)
|
||||
LINEAR_SERIES: tuple = ("load", "base_load")
|
||||
# recency (methods.md B10): a slot's weight max(0.5^(age/30 d), 0.25 x the
|
||||
# season's likeness (cos of the age over a year, last year's same weeks
|
||||
# count), 0.05)
|
||||
HALF_LIFE_D, SEASON_W, FLOOR_W = 30.0, 0.25, 0.05
|
||||
|
||||
|
||||
# ---- the calendar -------------------------------------------------------------
|
||||
@@ -74,12 +95,91 @@ def holidays_de(year: int) -> frozenset:
|
||||
return frozenset(fixed + moving)
|
||||
|
||||
|
||||
def calendar(t: np.ndarray, tz: str) -> dict[str, np.ndarray]:
|
||||
# The special days' types (one categorical feature, "special"; the trees
|
||||
# combine it with the hour of day). A type the site has under 2 days of in
|
||||
# its training data counts as normal (fit_gbm: params["special_kept"]): the
|
||||
# cloud's teacher model will give such a type its shape later.
|
||||
SPECIAL = ("normal", "christmas_eve", "christmas_day", "boxing_day", "new_years_eve", "new_years_day", "easter",
|
||||
"carnival", "bridge_day", "school_holiday", "dst_change", "holiday")
|
||||
SPECIAL_CODE = {k: i for i, k in enumerate(SPECIAL)}
|
||||
# carnival Monday and Tuesday where it is kept (a site's state unknown: everywhere)
|
||||
CARNIVAL_STATES = frozenset({"NW", "RP", "HE", "BW", "BY", "SL"})
|
||||
# School holidays per German state: {state: [(first day, last day), ...]}.
|
||||
# A hook: no compact source in the repository yet (the KMK's yearly tables);
|
||||
# empty, so no day is typed school_holiday until it is filled.
|
||||
SCHOOL_HOLIDAYS: dict[str, list[tuple[date, date]]] = {}
|
||||
|
||||
|
||||
def _last_sunday(year: int, month: int) -> date:
|
||||
d = date(year, month + 1, 1) - timedelta(days=1) if month < 12 else date(year, 12, 31)
|
||||
return d - timedelta(days=(d.weekday() + 1) % 7)
|
||||
|
||||
|
||||
@lru_cache(maxsize=64)
|
||||
def special_days(year: int, state: str | None = None) -> dict:
|
||||
"""{date: type code} for the year's special days (SPECIAL); the rest
|
||||
normal."""
|
||||
e = _easter(year)
|
||||
hol = holidays_de(year) | holidays_de(year - 1) | holidays_de(year + 1)
|
||||
out: dict[date, int] = {}
|
||||
if state in SCHOOL_HOLIDAYS:
|
||||
for a, b in SCHOOL_HOLIDAYS[state]:
|
||||
d = a
|
||||
while d <= b:
|
||||
if d.year == year:
|
||||
out[d] = SPECIAL_CODE["school_holiday"]
|
||||
d += timedelta(days=1)
|
||||
for m in (3, 10):
|
||||
out[_last_sunday(year, m)] = SPECIAL_CODE["dst_change"]
|
||||
d = date(year, 1, 1)
|
||||
while d.year == year: # bridge days: a working day between a holiday and a weekend
|
||||
if d.weekday() < 5 and d not in hol:
|
||||
prev, nxt = d - timedelta(days=1), d + timedelta(days=1)
|
||||
off_prev, off_next = prev.weekday() >= 5 or prev in hol, nxt.weekday() >= 5 or nxt in hol
|
||||
if (prev in hol and off_next) or (nxt in hol and off_prev):
|
||||
out[d] = SPECIAL_CODE["bridge_day"]
|
||||
d += timedelta(days=1)
|
||||
for x in holidays_de(year):
|
||||
out[x] = SPECIAL_CODE["holiday"]
|
||||
if state is None or state in CARNIVAL_STATES:
|
||||
out[e - timedelta(days=48)] = out[e - timedelta(days=47)] = SPECIAL_CODE["carnival"]
|
||||
for x in (e - timedelta(days=2), e, e + timedelta(days=1)):
|
||||
out[x] = SPECIAL_CODE["easter"]
|
||||
out.update({date(year, 12, 24): SPECIAL_CODE["christmas_eve"], date(year, 12, 25): SPECIAL_CODE["christmas_day"],
|
||||
date(year, 12, 26): SPECIAL_CODE["boxing_day"], date(year, 12, 31): SPECIAL_CODE["new_years_eve"],
|
||||
date(year, 1, 1): SPECIAL_CODE["new_years_day"]})
|
||||
return out
|
||||
|
||||
|
||||
def daylight_h(doy: np.ndarray, lat: float | None) -> np.ndarray:
|
||||
"""The day's hours of daylight at a latitude (the sun's declination by
|
||||
day of year; polar day and night clipped), NaN without a location."""
|
||||
if lat is None:
|
||||
return np.full(len(doy), np.nan)
|
||||
decl = np.radians(23.44) * np.sin(2 * np.pi * (284 + doy) / 365.0)
|
||||
x = np.clip(-np.tan(np.radians(lat)) * np.tan(decl), -1.0, 1.0)
|
||||
return 24.0 * np.degrees(np.arccos(x)) / 180.0
|
||||
|
||||
|
||||
def recency_weights(t: np.ndarray, end: int | None = None) -> np.ndarray:
|
||||
"""A slot's weight by its age before end (the last slot): recent weeks
|
||||
weigh most, last year's same season still counts (HALF_LIFE_D,
|
||||
SEASON_W, FLOOR_W)."""
|
||||
if not len(t):
|
||||
return np.zeros(0)
|
||||
end = int(t[-1]) if end is None else int(end)
|
||||
age = (end - t.astype(float)) / 86400.0
|
||||
return np.maximum.reduce([0.5 ** (age / HALF_LIFE_D), SEASON_W * np.maximum(np.cos(2 * np.pi * age / 365.25), 0.0),
|
||||
np.full(len(t), FLOOR_W)])
|
||||
|
||||
|
||||
def calendar(t: np.ndarray, tz: str, state: str | None = None) -> dict[str, np.ndarray]:
|
||||
"""Per slot: local hour of day (fractional), weekday (Monday 0), hour of
|
||||
the week, day of year, and whether it is a day off."""
|
||||
the week, day of year, whether it is a day off, and the special day's
|
||||
type (SPECIAL's code)."""
|
||||
zone = ZoneInfo(tz or "UTC")
|
||||
n = len(t)
|
||||
hod, dow, how, doy, off = (np.zeros(n) for _ in range(5))
|
||||
hod, dow, how, doy, off, special = (np.zeros(n) for _ in range(6))
|
||||
cache: dict[int, tuple] = {}
|
||||
for i, ts in enumerate(t.tolist()):
|
||||
hour_key = int(ts) // 3600
|
||||
@@ -87,12 +187,13 @@ def calendar(t: np.ndarray, tz: str) -> dict[str, np.ndarray]:
|
||||
if c is None:
|
||||
d = datetime.fromtimestamp(hour_key * 3600, timezone.utc).astimezone(zone)
|
||||
c = (d.hour, d.weekday(), d.timetuple().tm_yday,
|
||||
1.0 if d.weekday() >= 5 or d.date() in holidays_de(d.year) else 0.0)
|
||||
1.0 if d.weekday() >= 5 or d.date() in holidays_de(d.year) else 0.0,
|
||||
special_days(d.year, state).get(d.date(), 0))
|
||||
cache[hour_key] = c
|
||||
h, wd, dy, of = c
|
||||
h, wd, dy, of, sp = c
|
||||
hod[i] = h + (int(ts) % 3600) / 3600.0
|
||||
dow[i], how[i], doy[i], off[i] = wd, wd * 24 + h, dy, of
|
||||
return {"hod": hod, "dow": dow, "how": how, "doy": doy, "off": off}
|
||||
dow[i], how[i], doy[i], off[i], special[i] = wd, wd * 24 + h, dy, of, sp
|
||||
return {"hod": hod, "dow": dow, "how": how, "doy": doy, "off": off, "special": special}
|
||||
|
||||
|
||||
# ---- the weather per slot -------------------------------------------------------
|
||||
@@ -183,14 +284,16 @@ def metrics(y: np.ndarray, p: np.ndarray) -> dict:
|
||||
|
||||
# ---- how: the hour-of-week profile -----------------------------------------------
|
||||
|
||||
def fit_how(t: np.ndarray, y: np.ndarray, tz: str) -> dict:
|
||||
def fit_how(t: np.ndarray, y: np.ndarray, tz: str, weights: np.ndarray | None = None) -> dict:
|
||||
"""weights: per slot (recency_weights); None: all alike."""
|
||||
cal = calendar(t, tz)
|
||||
vals: list[float | None] = []
|
||||
have = ~np.isnan(y)
|
||||
overall = float(y[have].mean()) if have.any() else 0.0
|
||||
wt = np.ones(len(y)) if weights is None else np.asarray(weights, dtype=float)
|
||||
overall = float(np.average(y[have], weights=wt[have])) if have.any() and wt[have].sum() > 0 else 0.0
|
||||
for h in range(168):
|
||||
m = have & (cal["how"] == h)
|
||||
vals.append(float(y[m].mean()) if m.any() else None)
|
||||
vals.append(float(np.average(y[m], weights=wt[m])) if m.any() and wt[m].sum() > 0 else None)
|
||||
out = []
|
||||
for h, v in enumerate(vals):
|
||||
if v is None: # the same hour on the other weekdays, else the mean
|
||||
@@ -217,27 +320,65 @@ def _lag7(t: np.ndarray, y: np.ndarray, hist_t: np.ndarray | None, hist_y: np.nd
|
||||
|
||||
|
||||
def features(series: str, t: np.ndarray, w: Weather, how_p: dict, tz: str, lag7: np.ndarray,
|
||||
phys: np.ndarray | None) -> tuple[np.ndarray, list[str]]:
|
||||
cal = calendar(t, tz)
|
||||
phys: np.ndarray | None, names: list[str] | None = None, lat: float | None = None,
|
||||
state: str | None = None, kept: list | None = None) -> tuple[np.ndarray, list[str]]:
|
||||
"""The feature matrix; names: a model's own list (default: the
|
||||
series' current one). kept: the special days' types the model learnt
|
||||
(the others count as normal)."""
|
||||
cal = calendar(t, tz, state)
|
||||
special = cal["special"]
|
||||
if kept is not None:
|
||||
special = np.where(np.isin(special, np.asarray(list(kept) + [0], dtype=float)), special, 0.0)
|
||||
ang = 2 * np.pi * (cal["doy"] - 1) / 365.25
|
||||
cols = {"hod": cal["hod"], "dow": cal["dow"], "off": cal["off"], "doy": cal["doy"],
|
||||
"how": how_values(how_p, t, tz), "lag7": lag7, "temp": w.temp, "ghi": w.ghi, "cloud": w.cloud,
|
||||
"hdd": np.maximum(15.0 - w.temp, 0.0), "elev": w.elev, "az": w.az,
|
||||
"phys": phys if phys is not None else np.full(len(t), np.nan)}
|
||||
names = PV_FEATURES if series == "pv" else LOAD_FEATURES
|
||||
"phys": phys if phys is not None else np.full(len(t), np.nan), "special": special,
|
||||
"doy_sin": np.sin(ang), "doy_cos": np.cos(ang), "daylight": daylight_h(cal["doy"], lat)}
|
||||
names = list(names) if names is not None else (PV_FEATURES if series == "pv" else LOAD_FEATURES)
|
||||
return np.column_stack([cols[c] for c in names]), names
|
||||
|
||||
|
||||
def special_kept(t: np.ndarray, y: np.ndarray, tz: str, state: str | None = None, at_least: int = 2) -> list[int]:
|
||||
"""The special days' types with at least 2 metered days in (t, y)."""
|
||||
cal = calendar(t, tz, state)
|
||||
have = ~np.isnan(y)
|
||||
days: dict[int, set] = {}
|
||||
# a local day by its local midnight (a day spans two UTC dates)
|
||||
for sp, ts, h in zip(cal["special"][have].tolist(), t[have].tolist(), cal["hod"][have].tolist()):
|
||||
if sp:
|
||||
days.setdefault(int(sp), set()).add(round((int(ts) - h * 3600) / 86400))
|
||||
return sorted(k for k, v in days.items() if len(v) >= at_least)
|
||||
|
||||
|
||||
def fit_gbm(series: str, t: np.ndarray, y: np.ndarray, w: Weather, tz: str, lag7: np.ndarray,
|
||||
phys: np.ndarray | None) -> dict | None:
|
||||
phys: np.ndarray | None, lat: float | None = None, state: str | None = None,
|
||||
weights: np.ndarray | None = None, linear: bool | None = None,
|
||||
names: list[str] | None = None) -> dict | None:
|
||||
"""weights: per slot (recency_weights; None: all alike); linear: leaves
|
||||
with linear models (default: series in LINEAR_SERIES); names: the
|
||||
features (default: the series' current list)."""
|
||||
import lightgbm as lgb
|
||||
have = ~np.isnan(y)
|
||||
if have.sum() < MIN_GBM_SLOTS:
|
||||
return None
|
||||
how_p = fit_how(t, y, tz)
|
||||
X, names = features(series, t, w, how_p, tz, lag7, phys)
|
||||
m = lgb.train(GBM_PARAMS, lgb.Dataset(X[have], y[have], feature_name=names), num_boost_round=GBM_ROUNDS)
|
||||
return {"kind": "gbm", "model": m.model_to_string(), "features": names, "how": how_p["values"], "tz": tz,
|
||||
"cap_w": float(np.nanpercentile(y[have], 99.9)) * 1.2 if have.any() else None}
|
||||
how_p = fit_how(t, y, tz, weights)
|
||||
kept = special_kept(t, y, tz, state)
|
||||
X, names = features(series, t, w, how_p, tz, lag7, phys, names, lat, state, kept)
|
||||
params = dict(GBM_PARAMS)
|
||||
if linear if linear is not None else series in LINEAR_SERIES:
|
||||
params["linear_tree"] = True
|
||||
cat = [c for c in CATEGORICAL if c in names]
|
||||
ds = lgb.Dataset(X[have], y[have], feature_name=names, categorical_feature=cat or "auto",
|
||||
weight=None if weights is None else np.asarray(weights, dtype=float)[have])
|
||||
m = lgb.train(params, ds, num_boost_round=GBM_ROUNDS)
|
||||
out = {"kind": "gbm", "model": m.model_to_string(), "features": names, "how": how_p["values"], "tz": tz,
|
||||
"cap_w": float(np.nanpercentile(y[have], 99.9)) * 1.2 if have.any() else None}
|
||||
if "special" in names:
|
||||
out["special_kept"] = kept
|
||||
if params.get("linear_tree"):
|
||||
out["linear"] = True
|
||||
return out
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
@@ -269,11 +410,13 @@ def box_params(params: dict) -> dict:
|
||||
|
||||
|
||||
def predict_gbm(params: dict, series: str, t: np.ndarray, w: Weather, lag7: np.ndarray,
|
||||
phys: np.ndarray | None) -> np.ndarray:
|
||||
phys: np.ndarray | None, lat: float | None = None, state: str | None = None) -> np.ndarray:
|
||||
tz = params.get("tz", "UTC")
|
||||
X, names = features(series, t, w, {"values": params["how"]}, tz, lag7, phys)
|
||||
if names != params["features"]:
|
||||
raise ValueError("gbm: the features changed since it was trained")
|
||||
names = params["features"]
|
||||
if not isinstance(names, list) or any(n not in FEATURE_NAMES for n in names):
|
||||
raise ValueError("gbm: features this code does not know")
|
||||
X, names = features(series, t, w, {"values": params["how"]}, tz, lag7, phys, names, lat, state,
|
||||
params.get("special_kept"))
|
||||
p = np.asarray(_predictor(params).predict(X), dtype=float)
|
||||
p = np.maximum(p, 0.0)
|
||||
if params.get("cap_w"):
|
||||
@@ -283,6 +426,9 @@ def predict_gbm(params: dict, series: str, t: np.ndarray, w: Weather, lag7: np.n
|
||||
return p
|
||||
|
||||
|
||||
FEATURE_NAMES = frozenset(LOAD_FEATURES) | frozenset(PV_FEATURES) | {"doy"}
|
||||
|
||||
|
||||
# ---- one entry point -----------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
@@ -296,6 +442,7 @@ class Site:
|
||||
tilt: float = 30.0
|
||||
azimuth: float = 180.0
|
||||
pvgis_monthly: list | None = None # PVGIS's kWh per kWp per month (pvcalib.pvgis_monthly)
|
||||
state: str | None = None # the German state (NW, BY, ...): carnival, school holidays; None: unknown
|
||||
|
||||
|
||||
def phys_prior(t: np.ndarray, w: Weather, s: Site) -> np.ndarray | None:
|
||||
@@ -309,12 +456,13 @@ def phys_prior(t: np.ndarray, w: Weather, s: Site) -> np.ndarray | None:
|
||||
def fit(kind: str, series: str, t: np.ndarray, y: np.ndarray, w: Weather, s: Site,
|
||||
hist_t: np.ndarray | None = None, hist_y: np.ndarray | None = None) -> dict | None:
|
||||
"""A model of this kind on (t, y), or None where the kind cannot be made
|
||||
(too little data, PV physics without a location or weather)."""
|
||||
(too little data, PV physics without a location or weather). Recent
|
||||
slots weigh more (recency_weights; pvcalib has its own)."""
|
||||
if kind == "how":
|
||||
return fit_how(t, y, s.tz) if (~np.isnan(y)).any() else None
|
||||
return fit_how(t, y, s.tz, recency_weights(t)) if (~np.isnan(y)).any() else None
|
||||
if kind == "gbm":
|
||||
prior = phys_prior(t, w, s) if series == "pv" else None
|
||||
return fit_gbm(series, t, y, w, s.tz, _lag7(t, y, hist_t, hist_y), prior)
|
||||
return fit_gbm(series, t, y, w, s.tz, _lag7(t, y, hist_t, hist_y), prior, s.lat, s.state, recency_weights(t))
|
||||
if kind == "pvphys":
|
||||
if series != "pv":
|
||||
return None
|
||||
@@ -339,9 +487,9 @@ def predict(params: dict, series: str, t: np.ndarray, w: Weather, s: Site,
|
||||
if kind == "gbm":
|
||||
prior = phys_prior(t, w, s) if series == "pv" else None
|
||||
nan = np.full(len(t), np.nan)
|
||||
return predict_gbm(params, series, t, w, _lag7(t, nan, hist_t, hist_y), prior)
|
||||
return predict_gbm(params, series, t, w, _lag7(t, nan, hist_t, hist_y), prior, s.lat, s.state)
|
||||
if kind == "pvphys":
|
||||
ss = Site(s.tz, s.lat, s.lon, s.kwp, params.get("tilt", s.tilt), params.get("azimuth", s.azimuth))
|
||||
ss = Site(s.tz, s.lat, s.lon, s.kwp, params.get("tilt", s.tilt), params.get("azimuth", s.azimuth), state=s.state)
|
||||
prior = phys_prior(t, w, ss)
|
||||
if prior is None:
|
||||
return np.full(len(t), np.nan)
|
||||
@@ -374,3 +522,41 @@ def choose(challengers: dict[str, float | None], champion: float | None) -> str
|
||||
if champion is not None and math.isfinite(champion) and scored[best] >= champion:
|
||||
return None
|
||||
return best
|
||||
|
||||
|
||||
# ---- the intraday correction ------------------------------------------------------
|
||||
|
||||
INTRADAY_WINDOW_H = 3.0 # the recent error over the last 3 hours,
|
||||
INTRADAY_RECENT_H = 1.0 # the latest weighing most (half-life 1 h: an evening that starts at 18:00 shows by 19:00)
|
||||
INTRADAY_HALF_LIFE_H = 2.0 # fading with the step ahead
|
||||
INTRADAY_PV_RATIO = (0.5, 1.5) # PV: the metered over the forecast energy, capped
|
||||
|
||||
|
||||
def intraday(series: str, pred: np.ndarray, past_pred: np.ndarray, past_y: np.ndarray,
|
||||
half_life_h: float = INTRADAY_HALF_LIFE_H) -> tuple[np.ndarray, float]:
|
||||
"""The forecast for the next steps (pred, the first the slot under way
|
||||
or the next) moved by the last hours' error against the meter
|
||||
(past_pred, past_y: the same slots, NaN where unknown; at least an hour
|
||||
of both; the latest weighing most, half-life 1 h), fading with the step
|
||||
ahead (half-life 2 h): load additive (W),
|
||||
PV multiplicative (the ratio of energies, capped 0.5..1.5; none at
|
||||
night). Returns the corrected forecast and the error carried (W, or the
|
||||
PV ratio less 1). It catches any unusual day: a special day, guests,
|
||||
illness."""
|
||||
pred = np.asarray(pred, dtype=float)
|
||||
past_pred, past_y = np.asarray(past_pred, dtype=float), np.asarray(past_y, dtype=float)
|
||||
m = ~(np.isnan(past_pred) | np.isnan(past_y))
|
||||
if m.sum() < 4:
|
||||
return pred, 0.0
|
||||
decay = 0.5 ** (np.arange(len(pred)) * (SLOT_S / 3600.0) / half_life_h)
|
||||
# the past slots' weights: the latest most (the last slot ends where pred starts)
|
||||
age_h = (len(past_y) - np.arange(len(past_y)) - 0.5) * (SLOT_S / 3600.0)
|
||||
wt = np.where(m, 0.5 ** (age_h / INTRADAY_RECENT_H), 0.0)
|
||||
if series == "pv":
|
||||
e_pred, e_y = float(np.sum(wt * np.nan_to_num(past_pred))), float(np.sum(wt * np.nan_to_num(past_y)))
|
||||
if e_pred < 50.0 * float(wt.sum()): # under 50 W on average: night or dusk, nothing to scale
|
||||
return pred, 0.0
|
||||
r = min(max(e_y / e_pred, INTRADAY_PV_RATIO[0]), INTRADAY_PV_RATIO[1]) - 1.0
|
||||
return np.maximum(pred * (1.0 + r * decay), 0.0), r
|
||||
e = float(np.sum(wt * np.nan_to_num(past_y - past_pred)) / wt.sum())
|
||||
return np.maximum(pred + e * decay, 0.0), e
|
||||
@@ -37,6 +37,7 @@ READ_ONLY = frozenset({
|
||||
"config/area_registry/list",
|
||||
"recorder/statistics_during_period",
|
||||
"recorder/list_statistic_ids",
|
||||
"history/history_during_period", # the weather entity's past, for the box's own training (boxtrain.py)
|
||||
"ping",
|
||||
})
|
||||
|
||||
|
||||
@@ -163,37 +163,74 @@ def base_load(col: Collector, inputs) -> float | None:
|
||||
return max(0.0, sum(vals) - ctl)
|
||||
|
||||
|
||||
FC_SOURCES = {"own": "the box's own model", "cloud": "the cloud's series", "box": "the cloud's model on the box",
|
||||
"hour_of_week": "the hour-of-week profile"}
|
||||
|
||||
|
||||
def _model_series(m, start, bundles, history, tz, box) -> tuple[list[float], dict] | None:
|
||||
"""The model bundle's series m for the horizon: the cloud's forecast
|
||||
where its series covers a step, else the box's own model (localml.py;
|
||||
"""A model series m for the horizon, per step the best fresh source
|
||||
(methods.md B10): the box's own model where it is in force and has a
|
||||
value (boxtrain.py: bundles.own), else the cloud's forecast where its
|
||||
series covers the step, else the cloud's model on the box (localml.py;
|
||||
box: HA's weather forecast and its entity) where it has a value, else
|
||||
the hour-of-week profile; and how many steps each gave."""
|
||||
the hour-of-week profile; how many steps each gave, the source that
|
||||
gave most ("source"), and every source's values ("_cands": the live
|
||||
error per source, taken out before the plan)."""
|
||||
trainer = getattr(bundles, "own", None) if bundles is not None else None
|
||||
mine = None
|
||||
if trainer is not None and box is not None:
|
||||
try:
|
||||
mine = trainer.forecast(m, start, STEPS, box[0], box[1])
|
||||
except Exception as e: # never stops a plan
|
||||
log.warning("the box's own %s model: %s", m, e)
|
||||
if not hasattr(bundles, "forecast_parts"):
|
||||
s = bundles.forecast(m, start, STEP_S, STEPS, tz)
|
||||
return None if s is None else (s, {})
|
||||
parts = bundles.forecast_parts(m, start, STEP_S, STEPS, tz)
|
||||
if parts is None:
|
||||
return None
|
||||
# no cloud model for it: the box's own alone, where it covers the horizon
|
||||
if mine is None or any(v is None for v in mine[0]):
|
||||
return None
|
||||
vals = [float(v) for v in mine[0]]
|
||||
return vals, {"series": m, "own": STEPS, "cloud": 0, "box": 0, "hour_of_week": 0, "own_model": mine[1].get("kind"),
|
||||
"source": "own", "_cands": {"own": vals}}
|
||||
vals = [v for v, _ in parts]
|
||||
info = {"series": m, "cloud": sum(src == "series" for _, src in parts), "box": 0, "hour_of_week": 0}
|
||||
info = {"series": m, "own": 0, "cloud": 0, "box": 0, "hour_of_week": 0}
|
||||
cloud_series = [v if src == "series" else None for v, src in parts]
|
||||
own = None
|
||||
if box is not None and info["cloud"] < STEPS:
|
||||
if box is not None and (trainer is not None or not all(v is not None for v in cloud_series)):
|
||||
own = bundles.box_forecast(m, start, STEPS, history, box[0], box[1])
|
||||
for k, (v, src) in enumerate(parts):
|
||||
if src == "series":
|
||||
continue
|
||||
o = mine[0][k] if mine is not None else None
|
||||
b = own[0][k] if own is not None else None
|
||||
if b is not None:
|
||||
if o is not None:
|
||||
vals[k] = o
|
||||
info["own"] += 1
|
||||
elif src == "series":
|
||||
info["cloud"] += 1
|
||||
elif b is not None:
|
||||
vals[k] = b
|
||||
info["box"] += 1
|
||||
else:
|
||||
info["hour_of_week"] += 1
|
||||
if mine is not None:
|
||||
info["own_model"] = mine[1].get("kind")
|
||||
if mine[1].get("weather"):
|
||||
info["own_weather"] = mine[1]["weather"]
|
||||
if own is not None:
|
||||
info["box_model"] = own[1].get("kind")
|
||||
if own[1].get("weather"):
|
||||
info["box_weather"] = own[1]["weather"]
|
||||
if own[1].get("error"):
|
||||
info["box_error"] = own[1]["error"]
|
||||
info["source"] = max(("own", "cloud", "box", "hour_of_week"), key=lambda k: info[k])
|
||||
if trainer is not None:
|
||||
info["_cands"] = {"own": list(mine[0]) if mine is not None else None, "cloud": cloud_series,
|
||||
"cloud_model": list(own[0]) if own is not None else None,
|
||||
"hour_of_week": (bundles.profile(m, start, STEP_S, STEPS, tz) if hasattr(bundles, "profile")
|
||||
else [v if src == "profile" else None for v, src in parts])}
|
||||
if not info["own"]:
|
||||
info.pop("own")
|
||||
return vals, info
|
||||
|
||||
|
||||
@@ -207,6 +244,9 @@ def _series(name, start, bundles, history, tz, now_value, default, model_names=N
|
||||
if got is not None:
|
||||
s, n = got
|
||||
src = "model" if m == name else f"model ({m})"
|
||||
if n.get("own"):
|
||||
src += (f"; the box's own {n.get('own_model')} model (trained on the box) for {n['own']} of {STEPS} steps"
|
||||
f" (weather: {n.get('own_weather') or 'none'})")
|
||||
if n.get("box"):
|
||||
src += (f"; the box's own {n.get('box_model')} model for {n['box']} of {STEPS} steps"
|
||||
f" (weather: {n.get('box_weather') or 'none'})")
|
||||
@@ -326,6 +366,39 @@ def model_state(now: float, controls, inputs, col: Collector, bundles, history,
|
||||
bundles.forecast("load_w", start, STEP_S, 1, tz) is not None:
|
||||
load_src += " (the model's load_w includes the planned devices: not used)"
|
||||
pv, pv_src = _series("pv_w", start, bundles, history, tz, None, 0.0, None, box, fc_src["pv"])
|
||||
# the forecasts per source for the live error (boxtrain.py), not for the plan
|
||||
cands = {k: v.pop("_cands") for k, v in fc_src.items() if "_cands" in v}
|
||||
net_load = None
|
||||
intraday = {}
|
||||
trainer = getattr(bundles, "own", None)
|
||||
if trainer is not None:
|
||||
# the intraday correction (fcmodels.intraday, methods.md B10): the last
|
||||
# 3 hours' error of what was planned on against the meter, carried into
|
||||
# the next hours and fading (half-life 2 h); load in W, PV as a ratio
|
||||
for key, series in (("load", "plan"), ("pv", "pv")):
|
||||
base = load if key == "load" else pv
|
||||
c = cands.setdefault(key, {})
|
||||
c["used"] = list(base)
|
||||
try:
|
||||
new, e = trainer.correct(series, start, base)
|
||||
except Exception as ex: # never stops a plan
|
||||
log.warning("intraday correction %s: %s", key, ex)
|
||||
continue
|
||||
c["corrected"] = list(new)
|
||||
intraday[key] = round(float(e), 3)
|
||||
if key == "load":
|
||||
load = new
|
||||
else:
|
||||
pv = new
|
||||
if trainer is not None:
|
||||
# the house as metered (devices in), for the member's net draw the box
|
||||
# publishes upward where it does not act itself (controller.forecasts_up)
|
||||
if ctl:
|
||||
nl: dict = {}
|
||||
got = _series("load_w", start, bundles, history, tz, None, 500.0, ("load_w",), box, nl)
|
||||
net_load = got[0] if got[1].startswith("model") else None
|
||||
else:
|
||||
net_load = load
|
||||
t_out = _hourly(weather_forecast or [], "temperature", start, STEPS)
|
||||
t_src = "weather forecast"
|
||||
measured = col.value(inputs.outside_temp) if inputs.outside_temp else None
|
||||
@@ -390,6 +463,7 @@ def model_state(now: float, controls, inputs, col: Collector, bundles, history,
|
||||
"sources": {"load": load_src, "pv": pv_src, "t_out": t_src, "model": bundles.model_id(),
|
||||
"sky": "weather forecast" if clouds else "clear", "rooms": rooms_src, "ghi": ghi_src,
|
||||
"ev_phases": phases_src,
|
||||
**({"intraday": intraday} if intraday else {}),
|
||||
# per series the steps from the cloud's forecast, the box's own model
|
||||
# (its kind and weather) and the hour-of-week profile (methods.md B9)
|
||||
"forecast": {k: v for k, v in fc_src.items() if v}},
|
||||
@@ -398,6 +472,9 @@ def model_state(now: float, controls, inputs, col: Collector, bundles, history,
|
||||
"market": bundles.market_for_model(start, STEP_S, STEPS, now) if hasattr(bundles, "market_for_model") else []}
|
||||
if grid_max_w:
|
||||
out["grid_max_w"] = float(grid_max_w)
|
||||
if cands or net_load is not None:
|
||||
out["_cands"] = cands # taken out by the controller before the plan
|
||||
out["_net_load"] = net_load
|
||||
# a shade's moves by rules on the sun and its room (shades.py); the
|
||||
# optimiser takes them as they are
|
||||
from . import shades
|
||||
@@ -437,8 +514,13 @@ def _room_rc(c, params: dict, bundles, only_one: bool, now: float) -> str:
|
||||
heat is cop * P). Into params; where they came from."""
|
||||
if "r_k_per_kw" in c.params or "c_kwh_per_k" in c.params:
|
||||
return "mapping"
|
||||
get = getattr(bundles, "room_rc", None)
|
||||
rc = get(c.id, str(c.params.get("area") or ""), only_one, now) if get else None
|
||||
# the box's own fit where it is in force (boxtrain.py), else the cloud's
|
||||
own = getattr(bundles, "own", None)
|
||||
rc = own.room_rc(c.id, str(c.params.get("area") or ""), only_one) if own is not None else None
|
||||
src = "box model" if rc is not None else "model"
|
||||
if rc is None:
|
||||
get = getattr(bundles, "room_rc", None)
|
||||
rc = get(c.id, str(c.params.get("area") or ""), only_one, now) if get else None
|
||||
if rc is None:
|
||||
return "defaults"
|
||||
r, cap = rc["r_k_per_kw"], rc["c_kwh_per_k"]
|
||||
@@ -456,7 +538,7 @@ def _room_rc(c, params: dict, bundles, only_one: bool, now: float) -> str:
|
||||
params["internal_gain_w"] = gain
|
||||
if "solar_m2" not in c.params:
|
||||
params["solar_m2"] = solar
|
||||
return "model"
|
||||
return src
|
||||
|
||||
|
||||
def _quantities(c) -> list[str]:
|
||||
|
||||
@@ -17,7 +17,19 @@ are.
|
||||
Trees(enc, n_features).predict(rows) sums the leaves in LightGBM's order
|
||||
with its decision rule (tree.h NumericalDecision), so it gives LightGBM's
|
||||
raw score to the last bit (measured 2026-11-04: max
|
||||
relative difference 0.0 over 192 slots)."""
|
||||
relative difference 0.0 over 192 slots).
|
||||
|
||||
lgbm-trees-v2 (2026-11-04, methods.md B10), written only for a booster
|
||||
that needs it (a categorical split or linear leaves; else v1, which older
|
||||
agents read): per tree uint16 L, uint8 flags (bit 0: linear leaves); the
|
||||
L-1 nodes as v1, decision bit 0 set for a categorical split, whose
|
||||
threshold is then the index of its category set; uint16 the number of
|
||||
sets, each uint8 n and n uint16 categories (those going left); the L
|
||||
leaf values; with linear leaves per leaf float64 constant, uint8 n and n
|
||||
(uint8 feature, float64 coefficient). Categorical rule (tree.h
|
||||
CategoricalDecision): NaN or a negative value right, else left if the
|
||||
integer is in the set. A linear leaf: constant plus the coefficients
|
||||
times the values in LightGBM's order; a NaN among them: the leaf's value."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
@@ -26,6 +38,7 @@ import struct
|
||||
import zlib
|
||||
|
||||
ENCODING = "lgbm-trees-v1"
|
||||
ENCODING_V2 = "lgbm-trees-v2"
|
||||
NODE = struct.Struct("<BBhhd")
|
||||
MISSING = {"None": 0, "Zero": 1, "NaN": 2}
|
||||
ZERO = 1.0000000180025095e-35 # LightGBM's kZeroThreshold (1e-35f)
|
||||
@@ -36,44 +49,86 @@ def _ref(n: dict) -> int:
|
||||
return int(n["split_index"]) if "split_index" in n else ~int(n.get("leaf_index", 0))
|
||||
|
||||
|
||||
def pack_tree(root: dict) -> bytes:
|
||||
"""One tree (dump_model()'s tree_info[i]["tree_structure"])."""
|
||||
if "split_index" not in root: # no split: one leaf
|
||||
return struct.pack("<Hd", 1, float(root["leaf_value"]))
|
||||
def _walk(root: dict) -> tuple[dict[int, dict], dict[int, dict]]:
|
||||
nodes: dict[int, dict] = {}
|
||||
leaves: dict[int, float] = {}
|
||||
leaves: dict[int, dict] = {}
|
||||
stack = [root]
|
||||
while stack:
|
||||
n = stack.pop()
|
||||
if "split_index" in n:
|
||||
if n.get("decision_type", "<=") != "<=":
|
||||
raise ValueError("a categorical split: not supported")
|
||||
if n.get("decision_type", "<=") not in ("<=", "=="):
|
||||
raise ValueError(f"a split {n.get('decision_type')!r}: not supported")
|
||||
nodes[int(n["split_index"])] = n
|
||||
stack += [n["left_child"], n["right_child"]]
|
||||
else:
|
||||
leaves[int(n["leaf_index"])] = float(n["leaf_value"])
|
||||
leaves[int(n.get("leaf_index", 0))] = n
|
||||
k = len(leaves)
|
||||
if sorted(nodes) != list(range(k - 1)) or sorted(leaves) != list(range(k)):
|
||||
raise ValueError("a tree's node or leaf indices are not contiguous")
|
||||
return nodes, leaves
|
||||
|
||||
|
||||
def _needs_v2(dump: dict) -> bool:
|
||||
for t in dump["tree_info"]:
|
||||
nodes, leaves = _walk(t["tree_structure"])
|
||||
if any(n.get("decision_type") == "==" for n in nodes.values()) or any("leaf_coeff" in x for x in leaves.values()):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def pack_tree(root: dict, v2: bool = False) -> bytes:
|
||||
"""One tree (dump_model()'s tree_info[i]["tree_structure"])."""
|
||||
if "split_index" not in root and not v2: # no split: one leaf
|
||||
return struct.pack("<Hd", 1, float(root["leaf_value"]))
|
||||
nodes, leaves = _walk(root)
|
||||
k = len(leaves)
|
||||
linear = v2 and any("leaf_coeff" in x for x in leaves.values())
|
||||
out = [struct.pack("<H", k)]
|
||||
if v2:
|
||||
out.append(struct.pack("<B", 1 if linear else 0))
|
||||
cats: list[list[int]] = []
|
||||
for i in range(k - 1):
|
||||
n = nodes[i]
|
||||
dec = (2 if n.get("default_left") else 0) | (MISSING[n.get("missing_type", "None")] << 2)
|
||||
out.append(NODE.pack(int(n["split_feature"]), dec, _ref(n["left_child"]), _ref(n["right_child"]),
|
||||
float(n["threshold"])))
|
||||
out.append(struct.pack(f"<{k}d", *(leaves[i] for i in range(k))))
|
||||
threshold = n["threshold"]
|
||||
if n.get("decision_type") == "==":
|
||||
if not v2:
|
||||
raise ValueError("a categorical split: lgbm-trees-v2")
|
||||
dec |= 1
|
||||
cats.append([int(c) for c in str(threshold).split("||") if c != ""])
|
||||
threshold = len(cats) - 1
|
||||
out.append(NODE.pack(int(n["split_feature"]), dec, _ref(n["left_child"]), _ref(n["right_child"]), float(threshold)))
|
||||
if v2:
|
||||
out.append(struct.pack("<H", len(cats)))
|
||||
for c in cats:
|
||||
if len(c) > 255 or any(not 0 <= x < 65536 for x in c):
|
||||
raise ValueError("a categorical split: at most 255 categories below 65536")
|
||||
out.append(struct.pack(f"<B{len(c)}H", len(c), *c))
|
||||
out.append(struct.pack(f"<{k}d", *(float(leaves[i]["leaf_value"]) for i in range(k))))
|
||||
if linear:
|
||||
for i in range(k):
|
||||
x = leaves[i]
|
||||
feats, coeff = [int(f) for f in x.get("leaf_features") or []], [float(c) for c in x.get("leaf_coeff") or []]
|
||||
if len(feats) != len(coeff) or len(feats) > 255:
|
||||
raise ValueError("a linear leaf")
|
||||
out.append(struct.pack("<dB", float(x.get("leaf_const", 0.0)), len(feats)))
|
||||
for f, c in zip(feats, coeff):
|
||||
out.append(struct.pack("<Bd", f, c))
|
||||
return b"".join(out)
|
||||
|
||||
|
||||
def pack(dump: dict) -> dict:
|
||||
"""A booster's dump_model() as lgbm-trees-v1."""
|
||||
"""A booster's dump_model() as lgbm-trees-v1, or v2 where it has
|
||||
categorical splits or linear leaves."""
|
||||
if dump.get("num_class", 1) != 1 or dump.get("num_tree_per_iteration", 1) != 1:
|
||||
raise ValueError("one tree per round only")
|
||||
obj = str(dump.get("objective") or "").split(" ")[0]
|
||||
if obj != "regression":
|
||||
raise ValueError(f"objective {obj!r}: regression only (no output transform)")
|
||||
raw = b"".join(pack_tree(t["tree_structure"]) for t in dump["tree_info"])
|
||||
return {"enc": ENCODING, "n": len(dump["tree_info"]), "data": base64.b64encode(zlib.compress(raw, 9)).decode()}
|
||||
v2 = _needs_v2(dump)
|
||||
raw = b"".join(pack_tree(t["tree_structure"], v2) for t in dump["tree_info"])
|
||||
return {"enc": ENCODING_V2 if v2 else ENCODING, "n": len(dump["tree_info"]),
|
||||
"data": base64.b64encode(zlib.compress(raw, 9)).decode()}
|
||||
|
||||
|
||||
class Trees:
|
||||
@@ -81,8 +136,9 @@ class Trees:
|
||||
leaf (no loops), features in range, finite leaves."""
|
||||
|
||||
def __init__(self, enc: dict, n_features: int):
|
||||
if not isinstance(enc, dict) or enc.get("enc") != ENCODING:
|
||||
raise ValueError(f"trees: {ENCODING} only")
|
||||
if not isinstance(enc, dict) or enc.get("enc") not in (ENCODING, ENCODING_V2):
|
||||
raise ValueError(f"trees: {ENCODING} or {ENCODING_V2} only")
|
||||
v2 = enc["enc"] == ENCODING_V2
|
||||
z = zlib.decompressobj()
|
||||
try:
|
||||
raw = z.decompress(base64.b64decode(str(enc.get("data")), validate=True), MAX_RAW)
|
||||
@@ -100,6 +156,13 @@ class Trees:
|
||||
off += 2
|
||||
if not 1 <= k <= MAX_LEAVES:
|
||||
raise ValueError("trees: leaves")
|
||||
linear = False
|
||||
if v2:
|
||||
(flags,) = struct.unpack_from("<B", raw, off)
|
||||
off += 1
|
||||
if flags & ~1:
|
||||
raise ValueError("trees: flags")
|
||||
linear = bool(flags & 1)
|
||||
nodes = []
|
||||
for i in range(k - 1):
|
||||
f, dec, left, right, threshold = NODE.unpack_from(raw, off)
|
||||
@@ -107,14 +170,44 @@ class Trees:
|
||||
for c in (left, right):
|
||||
if not (i < c < k - 1 or -k <= c < 0):
|
||||
raise ValueError("trees: a child out of range")
|
||||
if f >= n_features or dec & 1 or (dec >> 2) & 3 > 2 or math.isnan(threshold):
|
||||
if f >= n_features or (dec & 1 and not v2) or (dec >> 2) & 3 > 2 or math.isnan(threshold):
|
||||
raise ValueError("trees: a split")
|
||||
nodes.append((f, threshold, bool(dec & 2), (dec >> 2) & 3, left, right))
|
||||
nodes.append([f, threshold, bool(dec & 2), (dec >> 2) & 3, left, right, bool(dec & 1)])
|
||||
if v2:
|
||||
(ncat,) = struct.unpack_from("<H", raw, off)
|
||||
off += 2
|
||||
sets = []
|
||||
for _ in range(ncat):
|
||||
(m,) = struct.unpack_from("<B", raw, off)
|
||||
sets.append(frozenset(struct.unpack_from(f"<{m}H", raw, off + 1)))
|
||||
off += 1 + 2 * m
|
||||
for nd in nodes:
|
||||
if nd[6]:
|
||||
if not 0 <= nd[1] < len(sets) or nd[1] != int(nd[1]):
|
||||
raise ValueError("trees: a category set")
|
||||
nd[1] = sets[int(nd[1])]
|
||||
nodes = [tuple(nd) for nd in nodes]
|
||||
leaves = struct.unpack_from(f"<{k}d", raw, off)
|
||||
off += 8 * k
|
||||
if not all(math.isfinite(v) for v in leaves):
|
||||
raise ValueError("trees: a leaf")
|
||||
self.trees.append((nodes, leaves))
|
||||
lin = None
|
||||
if linear:
|
||||
lin = []
|
||||
for _ in range(k):
|
||||
const, m = struct.unpack_from("<dB", raw, off)
|
||||
off += 9
|
||||
terms = []
|
||||
for _ in range(m):
|
||||
f, c = struct.unpack_from("<Bd", raw, off)
|
||||
off += 9
|
||||
if f >= n_features or not math.isfinite(c):
|
||||
raise ValueError("trees: a linear leaf")
|
||||
terms.append((f, c))
|
||||
if not math.isfinite(const):
|
||||
raise ValueError("trees: a linear leaf")
|
||||
lin.append((const, terms))
|
||||
self.trees.append((nodes, leaves, lin))
|
||||
if len(self.trees) > MAX_TREES:
|
||||
raise ValueError("trees: too many")
|
||||
except struct.error as e:
|
||||
@@ -128,21 +221,31 @@ class Trees:
|
||||
split knows NaN; the split's missing values (zero or NaN, by its
|
||||
type) go its default way; the leaves summed in order."""
|
||||
total = 0.0
|
||||
for nodes, leaves in self.trees:
|
||||
if not nodes:
|
||||
total += leaves[0]
|
||||
continue
|
||||
i = 0
|
||||
for nodes, leaves, lin in self.trees:
|
||||
i = 0 if nodes else ~0
|
||||
while i >= 0:
|
||||
f, threshold, default_left, missing, left, right = nodes[i]
|
||||
f, threshold, default_left, missing, left, right, cat = nodes[i]
|
||||
v = x[f]
|
||||
if cat:
|
||||
i = right if v != v or v < 0 or int(v) not in threshold else left
|
||||
continue
|
||||
if v != v and missing != 2:
|
||||
v = 0.0
|
||||
if (missing == 1 and -ZERO <= v <= ZERO) or (missing == 2 and v != v):
|
||||
i = left if default_left else right
|
||||
else:
|
||||
i = left if v <= threshold else right
|
||||
total += leaves[~i]
|
||||
if lin is None:
|
||||
total += leaves[~i]
|
||||
continue
|
||||
out = lin[~i][0]
|
||||
for f, c in lin[~i][1]:
|
||||
v = x[f]
|
||||
if v != v:
|
||||
out = leaves[~i]
|
||||
break
|
||||
out += c * v
|
||||
total += out
|
||||
return total
|
||||
|
||||
def predict(self, rows) -> list[float]:
|
||||
|
||||
@@ -45,7 +45,7 @@ SERIES = {"load_w": "load", "pv_w": "pv", "base_load_w": "base_load"} # the bu
|
||||
# is the base load already (the house without the planned devices), which
|
||||
# is the load where the box plans none (the only case its load_w is used)
|
||||
HISTORY = {"load_w": "load_w", "base_load_w": "load_w", "pv_w": "pv_w"}
|
||||
SITE_FIELDS = {"tz": str, "lat": float, "lon": float, "kwp": float, "tilt": float, "azimuth": float}
|
||||
SITE_FIELDS = {"tz": str, "lat": float, "lon": float, "kwp": float, "tilt": float, "azimuth": float, "state": str}
|
||||
|
||||
|
||||
def _num(v) -> bool:
|
||||
@@ -86,8 +86,12 @@ class Model:
|
||||
if not _numbers(p.get("values"), 168):
|
||||
raise ValueError("how: 168 numbers")
|
||||
elif self.kind == "gbm":
|
||||
want = fcmodels.PV_FEATURES if self.series == "pv" else fcmodels.LOAD_FEATURES
|
||||
if p.get("features") != want or not _numbers(p.get("how"), 168) or "model" in p:
|
||||
# the features it was trained with: today's, or those before the
|
||||
# special days and the yearly rhythm (fcmodels.FEATURES_V1)
|
||||
key = "pv" if self.series == "pv" else "load"
|
||||
want = fcmodels.PV_FEATURES if key == "pv" else fcmodels.LOAD_FEATURES
|
||||
if p.get("features") not in (want, fcmodels.FEATURES_V1[key]) or not _numbers(p.get("how"), 168) \
|
||||
or "model" in p or not all(isinstance(k, int) for k in p.get("special_kept") or []):
|
||||
raise ValueError(f"gbm: the features {', '.join(want)}, 168 hour-of-week values, trees")
|
||||
if p.get("cap_w") is not None and not _num(p["cap_w"]):
|
||||
raise ValueError("gbm: cap_w")
|
||||
|
||||
@@ -171,6 +171,21 @@ NAMES = {
|
||||
"control": {"en": "Control", "de": "Gerät"},
|
||||
"value": {"en": "Value", "de": "Wert"},
|
||||
"why": {"en": "Why", "de": "Warum"},
|
||||
# the owner's preferences (preferences.py; announce.py)
|
||||
"pref_schedule": {"en": "{0}: minimum temperatures", "de": "{0}: Mindesttemperaturen"},
|
||||
"pref_schedule_cool": {"en": "{0}: maximum temperatures", "de": "{0}: Höchsttemperaturen"},
|
||||
"pref_boost_c": {"en": "{0}: boost temperature", "de": "{0}: Boost-Temperatur"},
|
||||
"pref_boost_h": {"en": "{0}: boost hours", "de": "{0}: Boost-Dauer"},
|
||||
"pref_boost": {"en": "{0}: boost now", "de": "{0}: Boost jetzt"},
|
||||
"pref_away": {"en": "Away", "de": "Abwesend"},
|
||||
"pref_away_until": {"en": "Away until", "de": "Abwesend bis"},
|
||||
"pref_ready": {"en": "{0}: ready by", "de": "{0}: fertig bis"},
|
||||
"pref_capacity": {"en": "{0}: battery capacity", "de": "{0}: Akkukapazität"},
|
||||
"pref_needed": {"en": "{0}: needed once at", "de": "{0}: einmalig gebraucht um"},
|
||||
"pref_reserve": {"en": "{0}: reserve", "de": "{0}: Reserve"},
|
||||
"pref_reserve_blocks": {"en": "{0}: reserve by time", "de": "{0}: Reserve nach Uhrzeit"},
|
||||
"pref_result": {"en": "Preferences", "de": "Präferenzen"},
|
||||
"pref_result_of": {"en": "{0}: preferences", "de": "{0}: Präferenzen"},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -48,7 +48,18 @@ The building's requests (building.py, state["building"]): a load limit on
|
||||
the grid import and a thermostat cap are constraints with a penalty (binding
|
||||
far above any price, flexible with a price), a shift request a soft target
|
||||
(the window's import lowered by its kWh against the house's own net load).
|
||||
|
||||
The owner's preferences (preferences.py, a device's "pref"): a room's
|
||||
target per step from its schedule (in place of the occupied hours) at the
|
||||
comfort cost, and a "must" as a floor (heating) or limit (cooling) with the
|
||||
penalty MUST_EUR_K, inside the bounds and under a building's thermostat cap; a
|
||||
car's ready-by time and energy (a must at EV_MUST_EUR_KWH, a prefer valued
|
||||
at its max_ct per kWh with min_kwh at EV_SHORT_EUR_KWH); a battery's
|
||||
reserve as its floor per step. The plan's "prefs" says per preference
|
||||
whether it is met; with state["prefs_cost"] its "prefs_cost" the cost of
|
||||
each must (pref_costs: the plan made again without it).
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
@@ -67,6 +78,13 @@ COMFORT_EUR_KH = 2.0 # EUR per K and hour below (heating) or above (cooling
|
||||
# while someone is home: above any price (0.30 let a room cool by 1.3 K
|
||||
# at a 40 ct peak); pre-heating in cheap hours still pays
|
||||
EV_SHORT_EUR_KWH = 1.0 # EUR per kWh the car lacks at departure
|
||||
EV_MUST_EUR_KWH = 5.0 # EUR per kWh a car lacks at a "must" ready-by (preferences.py): above any price,
|
||||
# below a building's binding load limit (50) and the grid operator's (500)
|
||||
MUST_EUR_K = 20.0 # EUR per K and slot across a room's "must" (preferences.py): 80 EUR per K·h, so the
|
||||
# heat a room holds (C kWh/K) is not worth letting it fall even at 3 EUR/kWh; it stays
|
||||
# inside the bounds and under a building's cap, so those still win
|
||||
COST_SOLVE_S = 8.0 # s at most per extra plan for a must's cost (pref_costs)
|
||||
COST_MAX = 6 # musts costed per plan at most
|
||||
LIGHT_EUR = 0.002 # EUR per step for a closed cover while someone is home in daylight
|
||||
CHANGE_EUR_KW = 0.002 # EUR per kW a device's power changes from one step to the next: smooth
|
||||
# plans (equal prices would otherwise give any order, and a charger or
|
||||
@@ -245,7 +263,13 @@ class Model:
|
||||
soc0 = cap * min(max(float(st["soc_pct"]), 0.0), 100.0) / 100.0
|
||||
mean_price = sum(self.s["price_import"]) / self.T
|
||||
soc, prev, ch, dis = [], soc0, [], []
|
||||
pref = d.get("pref") # the owner's reserve per step (preferences.py)
|
||||
floors = []
|
||||
for t in range(self.T):
|
||||
lo_t = lo_pct
|
||||
if pref and not (pref.get("off") is not None and pref["group"][t] == pref["off"]):
|
||||
lo_t = min(max(lo_pct, pref["floor_pct"][t]), hi_pct)
|
||||
floors.append(lo_t)
|
||||
c = self.var(0.0, max_ch)
|
||||
x = self.var(0.0, max_dis)
|
||||
z = self.var(0.0, 1.0, integer=True)
|
||||
@@ -255,7 +279,7 @@ class Model:
|
||||
under = self.var(cost=PENALTY_LIMIT / cap * 100) # Wh below the floor, per % of the battery
|
||||
over = self.var(cost=PENALTY_LIMIT / cap * 100)
|
||||
self.h.addConstr(e - prev - self.dt * eff * c + self.dt / eff * x == 0)
|
||||
self.h.addConstr(e + under >= lo_pct / 100.0 * cap)
|
||||
self.h.addConstr(e + under >= lo_t / 100.0 * cap)
|
||||
self.h.addConstr(e - over <= hi_pct / 100.0 * cap)
|
||||
self.flows[t] += [(-1, c), (1, x)]
|
||||
soc.append(e)
|
||||
@@ -267,7 +291,7 @@ class Model:
|
||||
self.h.addConstr(prev + short >= min(soc0, hi_pct / 100.0 * cap))
|
||||
self.smooth([c - x for c, x in zip(ch, dis)])
|
||||
self.devices[d["id"]] = ("battery", {"ch": ch, "dis": dis, "soc": soc, "cap": cap, "max_dis": max_dis,
|
||||
"floor_wh": lo_pct / 100.0 * cap})
|
||||
"floor_wh": lo_pct / 100.0 * cap, "floors": floors})
|
||||
|
||||
def room(self, d, cooling: bool, gains):
|
||||
p, st, b = d["params"], d["state"], d["bounds"]
|
||||
@@ -284,6 +308,8 @@ class Model:
|
||||
internal = self.hourly(p.get("internal_gain_w") or 0.0)
|
||||
solar = float(p.get("solar_m2") or 0.0)
|
||||
ghi = self.s.get("ghi_w_m2") or [0.0] * self.T
|
||||
pref = d.get("pref") # the owner's schedule and overrides (preferences.py)
|
||||
cap = (self.s.get("building") or {}).get("thermostat_cap")
|
||||
for t in range(self.T):
|
||||
if switched:
|
||||
u = self.var(0.0, 1.0, integer=True)
|
||||
@@ -302,7 +328,23 @@ class Model:
|
||||
self.h.addConstr(tt - (1 - a) * prev - sign * self.dt / c * cop / 1000.0 * pw
|
||||
- self.dt / c / 1000.0 * gain == a * self.s["t_out"][t] + self.dt / c / 1000.0 * fixed)
|
||||
occ = self.s["occupied"][t + 1] if t + 1 < self.T else self.s["occupied"][t]
|
||||
target = min(max(comfort if occ else other, lo), hi)
|
||||
want, must = (comfort if occ else other), None
|
||||
if pref:
|
||||
want, must = pref["target"][t], pref["must"][t]
|
||||
if pref.get("off") is not None and pref["group"][t] == pref["off"]:
|
||||
want, must = pref["base"][t], None # pref_costs: the plan without this group
|
||||
target = min(max(want, lo), hi)
|
||||
capped = bool(cap) and cap["from_step"] <= t < cap["to_step"]
|
||||
if must is not None:
|
||||
# a bound inside the bounds in force, and under a building's cap
|
||||
mv = min(max(must, lo), hi)
|
||||
if capped:
|
||||
mv = max(mv, cap["value"]) if cooling else min(mv, cap["value"])
|
||||
cross = self.var(cost=MUST_EUR_K)
|
||||
if cooling:
|
||||
self.h.addConstr(tt - cross <= mv)
|
||||
else:
|
||||
self.h.addConstr(tt + cross >= mv)
|
||||
miss = self.var(cost=COMFORT_EUR_KH * self.dt)
|
||||
under = self.var(cost=PENALTY_LIMIT)
|
||||
over = self.var(cost=PENALTY_LIMIT)
|
||||
@@ -312,8 +354,7 @@ class Model:
|
||||
self.h.addConstr(tt + miss >= target)
|
||||
self.h.addConstr(tt + under >= lo)
|
||||
self.h.addConstr(tt - over <= hi)
|
||||
cap = (self.s.get("building") or {}).get("thermostat_cap")
|
||||
if cap and cap["from_step"] <= t < cap["to_step"]:
|
||||
if capped:
|
||||
miss_cap = self.var(cost=cap["penalty"] * self.dt)
|
||||
if cooling:
|
||||
self.h.addConstr(tt + miss_cap >= cap["value"]) # cooling: not below the cap
|
||||
@@ -345,6 +386,11 @@ class Model:
|
||||
imin = max(float(p["min_current_a"]), b["current_a"][0])
|
||||
dep = departure_step(self.s, str(st.get("departure") or p["departure"]))
|
||||
need = float(st["need_wh"]) if st.get("need_wh") is not None else float(p["need_wh"])
|
||||
pref = d.get("pref") # the owner's ready-by (preferences.py)
|
||||
if pref:
|
||||
dep, need = int(pref["dep_step"]), float(pref["need_wh"])
|
||||
if pref.get("off"):
|
||||
need = 0.0
|
||||
powers, total = [], Lin()
|
||||
for t in range(self.T):
|
||||
if t >= dep or imax < imin:
|
||||
@@ -357,10 +403,21 @@ class Model:
|
||||
self.flows[t].append((-1, pw))
|
||||
powers.append(pw)
|
||||
total += self.dt * pw
|
||||
short = self.var(cost=EV_SHORT_EUR_KWH / 1000.0)
|
||||
self.h.addConstr(total + short >= need)
|
||||
if pref and pref["how"] == "prefer":
|
||||
# a kWh is worth max_ct to the owner (without one: today's), min_kwh at any price
|
||||
worth = EV_SHORT_EUR_KWH if pref.get("max_ct") is None else max(0.0, pref["max_ct"]) / 100.0
|
||||
short = self.var(cost=worth / 1000.0)
|
||||
self.h.addConstr(total + short >= need)
|
||||
mwh = min(float(pref.get("min_wh") or 0.0), need)
|
||||
if mwh > 0:
|
||||
short_min = self.var(cost=EV_SHORT_EUR_KWH / 1000.0)
|
||||
self.h.addConstr(total + short_min >= mwh)
|
||||
else:
|
||||
short = self.var(cost=(EV_MUST_EUR_KWH if pref else EV_SHORT_EUR_KWH) / 1000.0)
|
||||
self.h.addConstr(total + short >= need)
|
||||
self.smooth([p for p in powers if p is not None])
|
||||
self.devices[d["id"]] = ("ev", {"power": powers, "volts": volts, "dep": dep})
|
||||
self.devices[d["id"]] = ("ev", {"power": powers, "volts": volts, "dep": dep,
|
||||
"pmax": imax * volts if imax >= imin else 0.0})
|
||||
|
||||
def cover(self, d, room_ids):
|
||||
p, b = d["params"], d["bounds"]
|
||||
@@ -463,16 +520,198 @@ def plan(s: dict, time_limit: float = 30.0) -> dict:
|
||||
if e is not None and (not entries or _key(entries[-1]) != _key(e)):
|
||||
entries.append({"t": start + t * step, **e})
|
||||
devices[d["id"]] = entries
|
||||
energy = sum(m.eur_wh(t, pi[t]) * x(m.gin[t]) - m.eur_wh(t, s["price_export"][t]) * x(m.gout[t])
|
||||
for t in range(T))
|
||||
market = market_plan(s, m, x)
|
||||
energy += sum(c["cost_eur"] for c in market)
|
||||
energy = energy_of(s, m, x, market)
|
||||
out = {"v": 1, "made": int(time.time()), "start": start, "step_s": step, "status": status,
|
||||
"solve_s": round(time.time() - t0, 2), "devices": devices, "flex": flexibility(m, x),
|
||||
"cost_eur": round(energy, 2), "baseline_eur": round(baseline(s, m, x), 2),
|
||||
"price_note": s.get("price_note", ""), "sources": s.get("sources", {})}
|
||||
if market:
|
||||
out["market"] = market
|
||||
if any(d.get("pref") for d in s["devices"]):
|
||||
out["prefs"] = pref_feedback(s, m, x)
|
||||
if s.get("prefs_cost"):
|
||||
out["prefs_cost"] = pref_costs(s, energy, t0 + time_limit)
|
||||
return out
|
||||
|
||||
|
||||
def energy_of(s, m, x, market=None) -> float:
|
||||
"""The plan's cost for power (EUR): the grid's import less its feed-in,
|
||||
and the market's contracts."""
|
||||
pi, pe = s["price_import"], s["price_export"]
|
||||
e = sum(m.eur_wh(t, pi[t]) * x(m.gin[t]) - m.eur_wh(t, pe[t]) * x(m.gout[t]) for t in range(m.T))
|
||||
return e + sum(c["cost_eur"] for c in (market if market is not None else market_plan(s, m, x)))
|
||||
|
||||
|
||||
def _when(s, ts: float) -> str:
|
||||
return datetime.fromtimestamp(ts, timezone.utc).astimezone(ZoneInfo(s.get("tz") or "UTC")).strftime("%a %H:%M")
|
||||
|
||||
|
||||
TOL_K = 0.3 # a room's target counts as met within this
|
||||
TOL_PCT = 0.5 # a battery's reserve
|
||||
TOL_WH = 50.0 # a car's energy
|
||||
|
||||
|
||||
def pref_feedback(s, m, x) -> list[dict]:
|
||||
"""Per preference (a room's block, boost or away; a car's next ready-by;
|
||||
a battery's reserve): met or not in the plan, and in a few words why.
|
||||
Fields: control, group, kind, how, label, met (None: not in the
|
||||
plan's 48 h, or not plugged in), detail, and per kind the numbers."""
|
||||
out = []
|
||||
start, step, T = s["start"], s["step_s"], m.T
|
||||
for d in s["devices"]:
|
||||
pref = d.get("pref")
|
||||
if not pref:
|
||||
continue
|
||||
kind, v = m.devices.get(d["id"], (d["kind"], None))
|
||||
if kind in ("heating", "cooling") and v:
|
||||
out += _room_feedback(s, d, pref, v, x, kind == "cooling")
|
||||
elif kind == "battery" and v:
|
||||
for gi, g in enumerate(pref["groups"]):
|
||||
ts = [t for t in range(T) if pref["group"][t] == gi]
|
||||
f = {"control": d["id"], "group": g["group"], "kind": "battery", "how": "must", "label": g["text"]}
|
||||
if not ts:
|
||||
out.append(dict(f, met=None, detail="not in the next 48 h"))
|
||||
continue
|
||||
soc = [100.0 * x(v["soc"][t]) / v["cap"] for t in ts]
|
||||
worst = min(range(len(ts)), key=lambda k: soc[k] - v["floors"][ts[k]])
|
||||
miss = v["floors"][ts[worst]] - soc[worst]
|
||||
met = miss <= TOL_PCT
|
||||
f.update(met=met, min_pct=round(min(soc), 1), floor_pct=round(v["floors"][ts[worst]], 1))
|
||||
if not met:
|
||||
f["detail"] = f"{soc[worst]:.0f} % at {_when(s, start + (ts[worst] + 1) * step)}"
|
||||
if d["state"].get("soc_pct") is not None and float(d["state"]["soc_pct"]) < v["floors"][ts[0]]:
|
||||
f["detail"] += f" (starts at {float(d['state']['soc_pct']):.0f} %)"
|
||||
out.append(f)
|
||||
elif kind == "ev":
|
||||
out.append(_ev_feedback(s, d, pref, v, x, m))
|
||||
return out
|
||||
|
||||
|
||||
def _room_feedback(s, d, pref, v, x, cooling) -> list[dict]:
|
||||
out = []
|
||||
start, step, T = s["start"], s["step_s"], len(pref["target"])
|
||||
lo, hi, lo_by, hi_by = (d["bounds"]["temp_c"] + ["device", "device"])[:4]
|
||||
cap = (s.get("building") or {}).get("thermostat_cap")
|
||||
sign = -1.0 if cooling else 1.0
|
||||
for gi, g in enumerate(pref["groups"]):
|
||||
ts = [t for t in range(T) if pref["group"][t] == gi]
|
||||
f = {"control": d["id"], "group": g["group"], "kind": g["kind"], "how": g["how"], "label": g["text"]}
|
||||
if not ts:
|
||||
out.append(dict(f, met=None, detail="not in the next 48 h"))
|
||||
continue
|
||||
notes = []
|
||||
want = []
|
||||
for t in ts:
|
||||
w0 = pref["must"][t] if g["how"] == "must" and pref["must"][t] is not None else pref["target"][t]
|
||||
w = min(max(w0, lo), hi)
|
||||
by = (hi_by if not cooling else lo_by) if sign * (w0 - w) > 1e-9 else None
|
||||
if cap and cap["from_step"] <= t < cap["to_step"]:
|
||||
wc = max(w, cap["value"]) if cooling else min(w, cap["value"])
|
||||
if wc != w:
|
||||
by, w = "building", wc
|
||||
if by and by not in notes:
|
||||
notes.append(by)
|
||||
want.append(w)
|
||||
temps = [x(v["temp"][t]) for t in ts]
|
||||
gap = [sign * (w - tt) for w, tt in zip(want, temps)]
|
||||
k = max(range(len(ts)), key=lambda i: gap[i])
|
||||
met = gap[k] <= TOL_K
|
||||
f.update(met=met, first=start + ts[0] * step, worst_c=round(temps[k], 1), want_c=round(want[k], 1))
|
||||
bits = []
|
||||
if notes:
|
||||
held = min(want) if not cooling else max(want)
|
||||
bits.append(f"held at {held:g} °C by the {notes[0]}'s limit")
|
||||
f["held_by"] = notes[0]
|
||||
if not met:
|
||||
bits.append(f"{temps[k]:.1f} °C at {_when(s, start + (ts[k] + 1) * step)}")
|
||||
if bits:
|
||||
f["detail"] = "; ".join(bits)
|
||||
out.append(f)
|
||||
return out
|
||||
|
||||
|
||||
def _ev_feedback(s, d, pref, v, x, m) -> dict:
|
||||
start, step = s["start"], s["step_s"]
|
||||
f = {"control": d["id"], "group": pref["group"], "kind": "ev", "how": pref["how"],
|
||||
"label": f"car {pref['text']}", "at": pref["at"], "need_kwh": round(pref["need_wh"] / 1000.0, 2)}
|
||||
plug = f"; plugged in at {_when(s, pref['plugged_at'])}" if pref.get("plugged_at") else ""
|
||||
if pref.get("note"):
|
||||
f["note"] = pref["note"]
|
||||
if v is None:
|
||||
return dict(f, met=None if pref["how"] == "prefer" else False, detail="not plugged in")
|
||||
pw = [x(p) if p is not None else 0.0 for p in v["power"]]
|
||||
dep = min(v["dep"], len(pw))
|
||||
wh = sum(m.dt * p for p in pw[:dep])
|
||||
short = max(0.0, pref["need_wh"] - wh)
|
||||
pi = s["price_import"]
|
||||
f["planned_kwh"] = round(wh / 1000.0, 2)
|
||||
f["charge_eur"] = round(sum(m.eur_wh(t, pi[t]) * pw[t] for t in range(dep)), 2)
|
||||
reached = ""
|
||||
if pref.get("soc_pct") is not None and pref.get("capacity_wh"):
|
||||
pct = min(100.0, float(pref["soc_pct"]) + 100.0 * wh / pref["capacity_wh"])
|
||||
f["reached_pct"] = round(pct, 0)
|
||||
reached = f"{pct:.0f} %"
|
||||
else:
|
||||
reached = f"{wh / 1000.0:.1f} kWh"
|
||||
when = _when(s, pref["at"])
|
||||
if pref.get("beyond"):
|
||||
f.update(met=None, detail=f"{reached} by the end of the plan (the ready-by is later){plug}")
|
||||
return f
|
||||
if pref["how"] == "prefer":
|
||||
f["met"] = short <= max(TOL_WH, 0.01 * pref["need_wh"])
|
||||
f["detail"] = f"{reached} by {when}" + ("" if f["met"] else
|
||||
f" ({short / 1000.0:.1f} kWh less: dearer than the price wanted)") + plug
|
||||
return f
|
||||
f["met"] = short <= max(TOL_WH, 0.01 * pref["need_wh"])
|
||||
if f["met"]:
|
||||
f["detail"] = f"{reached} by {when}{plug}"
|
||||
return f
|
||||
# a must it cannot meet: by how much, and when it could be ready at full power from now
|
||||
f["short_kwh"] = round(short / 1000.0, 2)
|
||||
rate = v.get("pmax") or 0.0
|
||||
if rate > 0:
|
||||
n = math.ceil(pref["need_wh"] / (rate * m.dt) - 1e-9)
|
||||
f["ready_at"] = int(start + n * step)
|
||||
can = f"; ready at full power by {_when(s, f['ready_at'])}"
|
||||
else:
|
||||
can = "; it may not charge now (a limit holds it at 0 A)"
|
||||
f["detail"] = f"{reached} by {when}, short by {short / 1000.0:.1f} kWh{can}{plug}"
|
||||
return f
|
||||
|
||||
|
||||
def pref_costs(s, energy: float, deadline: float) -> dict:
|
||||
"""Each must's cost (EUR a day): the plan's cost for power less that of
|
||||
the plan made again without it (a room's block at its setback, a
|
||||
battery's reserve at its floor before); a car's must: what its charging
|
||||
costs (no second plan). At most COST_MAX, each within COST_SOLVE_S and
|
||||
the time left."""
|
||||
out = {}
|
||||
days = s["steps"] * s["step_s"] / 86400.0
|
||||
n = 0
|
||||
for i, d in enumerate(s["devices"]):
|
||||
pref = d.get("pref")
|
||||
if not pref or d["kind"] not in ("heating", "cooling", "battery"):
|
||||
continue
|
||||
for gi, g in enumerate(pref["groups"]):
|
||||
if g["how"] != "must" or gi not in pref["group"]:
|
||||
continue
|
||||
left = deadline - time.time() - 1.0
|
||||
if n >= COST_MAX or left < 1.0:
|
||||
return out
|
||||
n += 1
|
||||
s2 = copy.deepcopy(s)
|
||||
s2["prefs_cost"] = False
|
||||
s2["devices"][i]["pref"]["off"] = gi
|
||||
try:
|
||||
m2 = Model(s2)
|
||||
m2.solve(min(COST_SOLVE_S, left))
|
||||
val = m2.h.getSolution().col_value
|
||||
e2 = energy_of(s2, m2, lambda v: val[v.index])
|
||||
except Exception: # noqa: BLE001 - a cost is not worth failing the plan
|
||||
continue
|
||||
c = (energy - e2) / days
|
||||
out[g["group"]] = round(c if abs(c) >= 0.005 else 0.0, 2)
|
||||
return out
|
||||
|
||||
|
||||
@@ -546,7 +785,9 @@ def flexibility(m, x) -> dict:
|
||||
for t in range(min(n, len(cols or []))):
|
||||
if cols[t] is not None and hasattr(cols[t], "index"): # an EV's steps after departure are empty
|
||||
shed[t] += max(x(cols[t]), 0.0)
|
||||
return {"grid_w": grid, "shed_w": [round(min(sh, g)) for sh, g in zip(shed, grid)]}
|
||||
# net_w: import less export, the member's draw as planned (the box's forecast of it upward, boxtrain.py)
|
||||
return {"grid_w": grid, "shed_w": [round(min(sh, g)) for sh, g in zip(shed, grid)],
|
||||
"net_w": [round(x(m.gin[t]) - x(m.gout[t])) for t in range(n)]}
|
||||
|
||||
|
||||
def _charge_why(pi, t) -> str:
|
||||
@@ -607,6 +848,14 @@ def _entry(d, kind, v, t, x, s, pi, grid_w=(1e9, 1e9)):
|
||||
temp = x(v["temp"][t])
|
||||
pw = x(v["power"][t])
|
||||
ahead = s["occupied"][min(t + 4, s["steps"] - 1)]
|
||||
pref = d.get("pref")
|
||||
if pref:
|
||||
# a schedule: "someone home" is a target above the setback ahead
|
||||
# (below the setup, cooling); its target in place of comfort/setback
|
||||
sign = -1 if kind == "cooling" else 1
|
||||
tg, k0 = pref["target"], max(t - 1, 0)
|
||||
occ = sign * (tg[k0] - pref["base"][k0]) > 0
|
||||
ahead = any(sign * (tg[k] - tg[k0]) > 0 for k in range(t, min(t + 4, len(tg))))
|
||||
if kind == "heating" and v["switched"]:
|
||||
on = pw > 0.5 * v["pmax"]
|
||||
why = ("pre-heat before someone is home" if on and not occ and ahead else
|
||||
@@ -630,6 +879,10 @@ def _entry(d, kind, v, t, x, s, pi, grid_w=(1e9, 1e9)):
|
||||
occ1 = s["occupied"][min(t + 1, s["steps"] - 1)]
|
||||
p = d["params"]
|
||||
want = float(p["comfort_c"]) if occ1 else float(p["setback_c"])
|
||||
if pref:
|
||||
want = float(pref["target"][t])
|
||||
if pref["must"][t] is not None:
|
||||
want = max(want, float(pref["must"][t]))
|
||||
floor_t = min(max(math.floor(want * 2) / 2, math.ceil(lo * 2) / 2), math.floor(hi * 2) / 2)
|
||||
return {"set": min(target, floor_t), "why": "setback: away or night" if not occ else "warm enough"}
|
||||
return {"set": target, "why": f"pre-heat while power is cheap ({pi[t]:.1f} ct)" if not occ and ahead
|
||||
|
||||
@@ -0,0 +1,975 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The owner's preferences for the plan ("I may want
|
||||
my car to be fully charged in any case at 8 AM ... I want my home to be min
|
||||
21 degree in night, 19 degree in day").
|
||||
|
||||
Per control, kept in <state dir>/preferences.json:
|
||||
room:<id> a heating room's minimum / a cooling room's maximum per
|
||||
weekday and time: blocks {days, from, to, c, how}
|
||||
ev:<id> ready-by times per weekday {days, at, pct | kwh, how,
|
||||
max_ct, min_kwh}, and the car's capacity_kwh (for %)
|
||||
battery:<id> a reserve (min state of charge, %) and optional blocks
|
||||
{days, from, to, pct}: the owner's floor
|
||||
and overrides that win over them until they expire:
|
||||
boost {control, c, from, until} a room to c (a must) for some hours
|
||||
away {from, until} every room at its setback/setup
|
||||
car {control, at, pct | kwh, how} the car needed at a one-off time
|
||||
|
||||
"must": a bound (a floor or limit in the plan: MUST_EUR_K, EV_MUST_EUR_KWH,
|
||||
inside the bounds in force: the grid operator's, the building's and the
|
||||
owner's limits still win). "prefer": today's comfort cost (rooms), or a
|
||||
value per kWh (a car: charge where a kWh costs at most max_ct, and at least
|
||||
min_kwh at any price). A room without a schedule keeps today's occupied
|
||||
hours (inputs.py) and comfort/setback targets.
|
||||
|
||||
Who sets them, and who wins (limits.py's spirit): the owner (Home
|
||||
Assistant, the Hub's panel: level "owner"), an installer or Cantabile's
|
||||
operators (the console, level "installer" / "operator": only within the
|
||||
owner's limits, never above them). Each level's setting is kept; the
|
||||
highest level's is in force, so the owner's own changes win and an
|
||||
installer's comes back when the owner clears theirs. Overrides are the
|
||||
owner's only. Every change is logged with who, when and what.
|
||||
|
||||
Where they come in: Home Assistant's entities (announce.py, the text
|
||||
formats below), the panel (an inbox directory, prefs-inbox/, the panel
|
||||
writes one JSON file per change; the agent writes preferences-view.json
|
||||
for it), the console (a signed command "preferences"). They go to the
|
||||
cloud with the plan (controller.public_plan), with the feedback per
|
||||
preference: met or not, and the cost of each must (the plan made again
|
||||
without it: optimizer.pref_costs).
|
||||
|
||||
Text formats (an HA text entity holds 255 characters), blocks separated by
|
||||
";": days are Mo Tu We Th Fr Sa Su (or Mo Di Mi Do Fr Sa So), ranges
|
||||
"Mo-Fr", lists "Sa,Su", or daily / weekdays / weekend.
|
||||
room Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19; Sa,Su 07:00-23:00 20
|
||||
ev Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct min 5kWh
|
||||
battery daily 17:00-21:00 50%
|
||||
car 2026-11-05 07:00 80% must (or "07:00 80%": the next 07:00)
|
||||
away 2026-11-10 (back that day; or with a time)
|
||||
A block from 22:00 to 06:00 belongs to the day it starts. Times are the
|
||||
home's wall clock (HA's time zone), so a schedule holds across the change
|
||||
to and from summer time.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DAYS = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
|
||||
DAY_NAMES = {"mo": 0, "mon": 0, "monday": 0, "montag": 0,
|
||||
"tu": 1, "tue": 1, "tuesday": 1, "di": 1, "dienstag": 1,
|
||||
"we": 2, "wed": 2, "wednesday": 2, "mi": 2, "mittwoch": 2,
|
||||
"th": 3, "thu": 3, "thursday": 3, "do": 3, "donnerstag": 3,
|
||||
"fr": 4, "fri": 4, "friday": 4, "freitag": 4,
|
||||
"sa": 5, "sat": 5, "saturday": 5, "samstag": 5,
|
||||
"su": 6, "sun": 6, "sunday": 6, "so": 6, "sonntag": 6}
|
||||
DAY_SETS = {"daily": list(range(7)), "all": list(range(7)), "täglich": list(range(7)), "taeglich": list(range(7)),
|
||||
"weekdays": list(range(5)), "werktags": list(range(5)), "weekend": [5, 6], "wochenende": [5, 6]}
|
||||
HOW = ("must", "prefer")
|
||||
HOW_WORDS = {"must": "must", "muss": "must", "prefer": "prefer", "bevorzugt": "prefer", "gern": "prefer"}
|
||||
LEVELS = ("operator", "installer", "owner") # low to high: the highest set is in force
|
||||
RANK = {lvl: i + 1 for i, lvl in enumerate(LEVELS)}
|
||||
KINDS = {"heating": "room", "cooling": "room", "ev": "ev", "battery": "battery"}
|
||||
RANGE_C = {"heating": (5.0, 30.0), "cooling": (16.0, 32.0)} # controls.RANGE
|
||||
MAX_BLOCKS = 16
|
||||
MAX_OVERRIDES = 20
|
||||
MAX_LOG = 200
|
||||
AWAY_MAX_S = 60 * 86400 # away without a date: at most this long
|
||||
BOOST_MAX_H = 24
|
||||
COST_EVERY_S = 3600 # the musts' costs are made again at most this often (unless something changed)
|
||||
HHMM = re.compile(r"^([01]?\d|2[0-4])(?::([0-5]\d))?$")
|
||||
|
||||
|
||||
# -- small parsers -------------------------------------------------------------
|
||||
def hhmm(v, end=False) -> str:
|
||||
""""7", "07:00", "7:30" -> "07:00"/"07:30"; "24:00" only as an end."""
|
||||
m = HHMM.match(str(v).strip())
|
||||
if not m:
|
||||
raise ValueError(f"{v!r} is not a time (HH:MM)")
|
||||
h, mi = int(m.group(1)), int(m.group(2) or 0)
|
||||
if h == 24 and (mi or not end):
|
||||
raise ValueError(f"{v!r}: 24:00 only as the end of a block")
|
||||
return f"{h:02d}:{mi:02d}"
|
||||
|
||||
|
||||
def minutes(s: str) -> int:
|
||||
h, m = s.split(":")
|
||||
return int(h) * 60 + int(m)
|
||||
|
||||
|
||||
def parse_days(v) -> list[int]:
|
||||
""""Mo-Fr", "Sa,Su", "daily", [0, 1], "mon-fri" -> sorted weekday numbers (Mon 0)."""
|
||||
if isinstance(v, (list, tuple)):
|
||||
out = sorted({int(x) for x in v})
|
||||
if not out or out[0] < 0 or out[-1] > 6:
|
||||
raise ValueError("days: weekday numbers 0 (Monday) to 6 (Sunday)")
|
||||
return out
|
||||
s = str(v).strip().lower()
|
||||
if s in DAY_SETS:
|
||||
return list(DAY_SETS[s])
|
||||
out = set()
|
||||
for part in re.split(r"[,/ ]+", s):
|
||||
if not part:
|
||||
continue
|
||||
if part in DAY_SETS:
|
||||
out.update(DAY_SETS[part])
|
||||
continue
|
||||
a, _, b = part.partition("-")
|
||||
if a not in DAY_NAMES or (b and b not in DAY_NAMES):
|
||||
raise ValueError(f"{part!r} is not a day (Mo Tu We Th Fr Sa Su, a range Mo-Fr, or daily)")
|
||||
i, j = DAY_NAMES[a], DAY_NAMES[b] if b else DAY_NAMES[a]
|
||||
k = i
|
||||
while True:
|
||||
out.add(k)
|
||||
if k == j:
|
||||
break
|
||||
k = (k + 1) % 7
|
||||
if not out:
|
||||
raise ValueError("days: none given")
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def days_text(days: list[int]) -> str:
|
||||
d = sorted(set(days))
|
||||
if d == list(range(7)):
|
||||
return "daily"
|
||||
runs, cur = [], [d[0]]
|
||||
for x in d[1:]:
|
||||
if x == cur[-1] + 1:
|
||||
cur.append(x)
|
||||
else:
|
||||
runs.append(cur)
|
||||
cur = [x]
|
||||
runs.append(cur)
|
||||
return ",".join(DAYS[r[0]][:2] if len(r) == 1 else f"{DAYS[r[0]][:2]}-{DAYS[r[-1]][:2]}" if len(r) > 2
|
||||
else f"{DAYS[r[0]][:2]},{DAYS[r[1]][:2]}" for r in runs)
|
||||
|
||||
|
||||
def _num(v, what, lo, hi) -> float:
|
||||
try:
|
||||
x = float(str(v).replace(",", ".").strip())
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError(f"{what}: {v!r} is not a number") from None
|
||||
if not math.isfinite(x) or not lo <= x <= hi:
|
||||
raise ValueError(f"{what}: {x:g} is outside {lo:g}..{hi:g}")
|
||||
return x
|
||||
|
||||
|
||||
def _how(v, default="prefer") -> str:
|
||||
h = HOW_WORDS.get(str(v or default).strip().lower())
|
||||
if h is None:
|
||||
raise ValueError(f"how: must or prefer, not {v!r}")
|
||||
return h
|
||||
|
||||
|
||||
def _fmt(x: float) -> str:
|
||||
return f"{x:g}"
|
||||
|
||||
|
||||
# -- checked values (every source comes through these) --------------------------
|
||||
def check_room(v: dict, kind: str) -> dict:
|
||||
lo, hi = RANGE_C["cooling" if kind == "cooling" else "heating"]
|
||||
blocks = (v or {}).get("blocks")
|
||||
if not isinstance(blocks, list) or len(blocks) > MAX_BLOCKS:
|
||||
raise ValueError(f"blocks: a list of at most {MAX_BLOCKS}")
|
||||
out = []
|
||||
for b in blocks:
|
||||
out.append({"days": parse_days(b.get("days", "daily")), "from": hhmm(b.get("from", "00:00")),
|
||||
"to": hhmm(b.get("to", "24:00"), end=True),
|
||||
"c": _num(b.get("c"), "temperature", lo, hi), "how": _how(b.get("how"))})
|
||||
return {"blocks": out}
|
||||
|
||||
|
||||
def check_ev(v: dict) -> dict:
|
||||
v = v or {}
|
||||
ready = v.get("ready") or []
|
||||
if not isinstance(ready, list) or len(ready) > MAX_BLOCKS:
|
||||
raise ValueError(f"ready: a list of at most {MAX_BLOCKS}")
|
||||
out = {"ready": [], "capacity_kwh": None}
|
||||
if v.get("capacity_kwh") not in (None, ""):
|
||||
out["capacity_kwh"] = _num(v["capacity_kwh"], "capacity", 1, 250)
|
||||
for r in ready:
|
||||
out["ready"].append(_ev_target(r, {"days": parse_days(r.get("days", "daily")), "at": hhmm(r.get("at", ""))}))
|
||||
return out
|
||||
|
||||
|
||||
def _ev_target(r: dict, into: dict) -> dict:
|
||||
pct, kwh = r.get("pct"), r.get("kwh")
|
||||
if (pct in (None, "")) == (kwh in (None, "")):
|
||||
raise ValueError("a car's target: % or kWh, one of them")
|
||||
into["pct"] = _num(pct, "target %", 1, 100) if pct not in (None, "") else None
|
||||
into["kwh"] = _num(kwh, "target kWh", 0.1, 250) if kwh not in (None, "") else None
|
||||
into["how"] = _how(r.get("how"))
|
||||
into["max_ct"] = _num(r["max_ct"], "max price", -100, 500) if r.get("max_ct") not in (None, "") else None
|
||||
into["min_kwh"] = _num(r["min_kwh"], "min kWh", 0, 250) if r.get("min_kwh") not in (None, "") else None
|
||||
if into["how"] == "must":
|
||||
into["max_ct"] = into["min_kwh"] = None # a must charges at any price
|
||||
return into
|
||||
|
||||
|
||||
def check_battery(v: dict) -> dict:
|
||||
v = v or {}
|
||||
blocks = v.get("blocks") or []
|
||||
if not isinstance(blocks, list) or len(blocks) > MAX_BLOCKS:
|
||||
raise ValueError(f"blocks: a list of at most {MAX_BLOCKS}")
|
||||
out = {"reserve_pct": _num(v.get("reserve_pct", 0) or 0, "reserve", 0, 100), "blocks": []}
|
||||
for b in blocks:
|
||||
out["blocks"].append({"days": parse_days(b.get("days", "daily")), "from": hhmm(b.get("from", "00:00")),
|
||||
"to": hhmm(b.get("to", "24:00"), end=True), "pct": _num(b.get("pct"), "reserve", 0, 100)})
|
||||
return out
|
||||
|
||||
|
||||
def check(key: str, value: dict, kind: str) -> dict:
|
||||
what = key.split(":", 1)[0]
|
||||
if what == "room":
|
||||
return check_room(value, kind)
|
||||
if what == "ev":
|
||||
return check_ev(value)
|
||||
if what == "battery":
|
||||
return check_battery(value)
|
||||
raise ValueError(f"no preference {key!r}")
|
||||
|
||||
|
||||
# -- text forms (Home Assistant's text entities) -----------------------------------
|
||||
def _split(text: str) -> list[list[str]]:
|
||||
return [p.split() for p in re.split(r"[;\n]+", str(text or "")) if p.strip()]
|
||||
|
||||
|
||||
def _days_and_rest(words: list[str]) -> tuple[list[int], list[str]]:
|
||||
"""A block's leading days, else daily."""
|
||||
try:
|
||||
return parse_days(words[0]), words[1:]
|
||||
except ValueError:
|
||||
return list(range(7)), words
|
||||
|
||||
|
||||
def _range(w: str) -> tuple[str, str]:
|
||||
a, sep, b = w.partition("-")
|
||||
if not sep:
|
||||
raise ValueError(f"{w!r}: a time range, e.g. 22:00-06:00")
|
||||
return hhmm(a), hhmm(b, end=True)
|
||||
|
||||
|
||||
def parse_room_text(text: str, kind: str) -> dict:
|
||||
blocks = []
|
||||
for words in _split(text):
|
||||
days, rest = _days_and_rest(words)
|
||||
if len(rest) < 2:
|
||||
raise ValueError(f"{' '.join(words)!r}: days, a time range and a temperature (e.g. Mo-Fr 22:00-06:00 21 must)")
|
||||
a, b = _range(rest[0])
|
||||
c = rest[1].strip("≥≤>=<°Cc")
|
||||
how = rest[2] if len(rest) > 2 else "prefer"
|
||||
blocks.append({"days": days, "from": a, "to": b, "c": c, "how": how})
|
||||
return check_room({"blocks": blocks}, kind)
|
||||
|
||||
|
||||
def room_text(v: dict | None) -> str:
|
||||
return "; ".join(f"{days_text(b['days'])} {b['from']}-{b['to']} {_fmt(b['c'])}" + (" must" if b["how"] == "must" else "")
|
||||
for b in (v or {}).get("blocks") or [])
|
||||
|
||||
|
||||
def _target_words(words: list[str]) -> dict:
|
||||
""""80%", "20kWh", "20 kWh", "must", "max 25ct", "<25ct", "min 5kWh"."""
|
||||
out, i = {}, 0
|
||||
ws = [w.lower() for w in words]
|
||||
while i < len(ws):
|
||||
w = ws[i]
|
||||
nxt = ws[i + 1] if i + 1 < len(ws) else ""
|
||||
if w in HOW_WORDS:
|
||||
out["how"] = w
|
||||
elif w.endswith("%"):
|
||||
out["pct"] = w[:-1]
|
||||
elif w.endswith("kwh") and w[:-3]:
|
||||
out["kwh"] = w[:-3]
|
||||
elif nxt == "kwh" and "pct" not in out and "kwh" not in out:
|
||||
out["kwh"] = w
|
||||
i += 1
|
||||
elif w in ("max", "<", "bis") or (w.startswith("<") and w[1:]):
|
||||
v = w[1:] if w.startswith("<") and w[1:] else nxt
|
||||
if not (w.startswith("<") and w[1:]):
|
||||
i += 1
|
||||
out["max_ct"] = v.removesuffix("ct")
|
||||
elif w == "min":
|
||||
v = nxt
|
||||
i += 1
|
||||
if v.endswith("kwh"):
|
||||
out["min_kwh"] = v[:-3]
|
||||
elif i + 1 < len(ws) and ws[i + 1] == "kwh":
|
||||
out["min_kwh"] = v
|
||||
i += 1
|
||||
else:
|
||||
out["min_kwh"] = v
|
||||
else:
|
||||
raise ValueError(f"{w!r}: not understood (a target like 80% or 20kWh, must/prefer, max 25ct, min 5kWh)")
|
||||
i += 1
|
||||
return out
|
||||
|
||||
|
||||
def parse_ev_text(text: str, capacity_kwh=None) -> dict:
|
||||
ready = []
|
||||
for words in _split(text):
|
||||
days, rest = _days_and_rest(words)
|
||||
if not rest:
|
||||
raise ValueError(f"{' '.join(words)!r}: a time and a target (e.g. Mo-Fr 08:00 100% must)")
|
||||
ready.append({"days": days, "at": hhmm(rest[0]), **_target_words(rest[1:])})
|
||||
return check_ev({"ready": ready, "capacity_kwh": capacity_kwh})
|
||||
|
||||
|
||||
def _target_text(r: dict) -> str:
|
||||
s = f"{_fmt(r['pct'])}%" if r.get("pct") is not None else f"{_fmt(r['kwh'])}kWh"
|
||||
s += " must" if r["how"] == "must" else ""
|
||||
if r.get("max_ct") is not None:
|
||||
s += f" max {_fmt(r['max_ct'])}ct"
|
||||
if r.get("min_kwh") is not None:
|
||||
s += f" min {_fmt(r['min_kwh'])}kWh"
|
||||
return s
|
||||
|
||||
|
||||
def ev_text(v: dict | None) -> str:
|
||||
return "; ".join(f"{days_text(r['days'])} {r['at']} {_target_text(r)}" for r in (v or {}).get("ready") or [])
|
||||
|
||||
|
||||
def parse_battery_text(text: str, reserve_pct=0.0) -> dict:
|
||||
blocks = []
|
||||
for words in _split(text):
|
||||
days, rest = _days_and_rest(words)
|
||||
if len(rest) < 2:
|
||||
raise ValueError(f"{' '.join(words)!r}: days, a time range and a reserve (e.g. daily 17:00-21:00 50%)")
|
||||
a, b = _range(rest[0])
|
||||
blocks.append({"days": days, "from": a, "to": b, "pct": rest[1].rstrip("%")})
|
||||
return check_battery({"reserve_pct": reserve_pct, "blocks": blocks})
|
||||
|
||||
|
||||
def battery_text(v: dict | None) -> str:
|
||||
return "; ".join(f"{days_text(b['days'])} {b['from']}-{b['to']} {_fmt(b['pct'])}%" for b in (v or {}).get("blocks") or [])
|
||||
|
||||
|
||||
def local_when(text: str, zone: ZoneInfo, now: float, default_time: str | None = None) -> float:
|
||||
""""2026-11-05 07:00", "2026-11-05" (default_time, else 00:00), "07:00"
|
||||
(the next one) -> unix time, in the home's time zone."""
|
||||
words = str(text).split()
|
||||
if not words:
|
||||
raise ValueError("a date or time is needed")
|
||||
d, t = None, None
|
||||
for w in words:
|
||||
if re.match(r"^\d{4}-\d{2}-\d{2}$", w):
|
||||
d = date.fromisoformat(w)
|
||||
else:
|
||||
t = hhmm(w)
|
||||
t = t or default_time or "00:00"
|
||||
h, mi = (int(x) for x in t.split(":"))
|
||||
if d is None:
|
||||
n = datetime.fromtimestamp(now, zone)
|
||||
cand = _wall(n.date(), h, mi, zone)
|
||||
if cand <= now:
|
||||
cand = _wall(n.date() + timedelta(days=1), h, mi, zone)
|
||||
return cand
|
||||
return _wall(d, h, mi, zone)
|
||||
|
||||
|
||||
def _wall(d: date, h: int, mi: int, zone: ZoneInfo) -> float:
|
||||
"""A wall-clock time's unix time; one that does not exist (the hour
|
||||
skipped in spring) moves on by the gap, as a clock does (02:30 is 03:30
|
||||
summer time: PEP 495's fold 0 takes the offset before the change); one
|
||||
that comes twice (autumn) is the first."""
|
||||
naive = datetime(d.year, d.month, d.day, h % 24, mi) + timedelta(days=h // 24)
|
||||
return naive.replace(tzinfo=zone).timestamp()
|
||||
|
||||
|
||||
def parse_car_text(text: str, zone: ZoneInfo, now: float) -> dict | None:
|
||||
"""The one-off "car needed" text: None when empty (cancel)."""
|
||||
words = str(text or "").split()
|
||||
if not words:
|
||||
return None
|
||||
when = [w for w in words if re.match(r"^\d{4}-\d{2}-\d{2}$", w) or HHMM.match(w)]
|
||||
rest = [w for w in words if w not in when]
|
||||
if not when:
|
||||
raise ValueError("a time (and a date): e.g. 2026-11-05 07:00 80% must")
|
||||
at = local_when(" ".join(when), zone, now, default_time="07:00")
|
||||
tgt = _target_words(rest)
|
||||
tgt.setdefault("how", "must")
|
||||
if "pct" not in tgt and "kwh" not in tgt:
|
||||
tgt["pct"] = 100
|
||||
return _ev_target(tgt, {"kind": "car", "at": at})
|
||||
|
||||
|
||||
# -- the store --------------------------------------------------------------------
|
||||
class Preferences:
|
||||
def __init__(self, state_dir: str, clock=time.time):
|
||||
self.dir, self.clock = state_dir, clock
|
||||
self.tz = "UTC" # the home's (HA's), for times in words
|
||||
self.path = os.path.join(state_dir, "preferences.json")
|
||||
self.inbox = os.path.join(state_dir, "prefs-inbox")
|
||||
self.view_path = os.path.join(state_dir, "preferences-view.json")
|
||||
self.items: dict[str, dict[str, dict]] = {} # key -> level -> {"value", "by", "at"}
|
||||
self.overrides: list[dict] = []
|
||||
self.ha: dict = {} # HA's helper values: boost_c/boost_h per room, away_until text
|
||||
self.log: list[dict] = []
|
||||
self.changed_at = 0.0 # last change (the plan's costs, steady's free controls)
|
||||
self.dirty: set[str] = set() # controls changed since the last plan (steady.py: not held back)
|
||||
self.costs: dict = {"at": 0, "of": 0.0, "eur_day": {}} # the musts' costs, as of a plan
|
||||
self.feedback: list[dict] = []
|
||||
self.errors: list[dict] = [] # the last refused changes (the panel shows them)
|
||||
try:
|
||||
with open(self.path) as f:
|
||||
raw = json.load(f)
|
||||
self.items = {k: {lvl: e for lvl, e in v.items() if lvl in RANK} for k, v in (raw.get("items") or {}).items()}
|
||||
self.overrides = list(raw.get("overrides") or [])
|
||||
self.ha = dict(raw.get("ha") or {})
|
||||
self.log = list(raw.get("log") or [])[-MAX_LOG:]
|
||||
self.costs = raw.get("costs") or self.costs
|
||||
self.changed_at = float(raw.get("changed_at") or 0)
|
||||
except (OSError, ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
def save(self):
|
||||
os.makedirs(self.dir, exist_ok=True)
|
||||
tmp = self.path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump({"v": 1, "items": self.items, "overrides": self.overrides, "ha": self.ha, "log": self.log,
|
||||
"costs": self.costs, "changed_at": self.changed_at}, f)
|
||||
os.replace(tmp, self.path)
|
||||
|
||||
# -- reading -------------------------------------------------------------
|
||||
def effective(self, key: str) -> dict | None:
|
||||
"""The setting in force: the highest level's ({"value", "by", "at", "level"})."""
|
||||
per = self.items.get(key) or {}
|
||||
for lvl in reversed(LEVELS):
|
||||
if lvl in per:
|
||||
return dict(per[lvl], level=lvl)
|
||||
return None
|
||||
|
||||
def value(self, key: str) -> dict | None:
|
||||
e = self.effective(key)
|
||||
return e["value"] if e else None
|
||||
|
||||
def active_overrides(self, now: float | None = None) -> list[dict]:
|
||||
now = self.clock() if now is None else now
|
||||
return [o for o in self.overrides if _end(o) > now]
|
||||
|
||||
def prune(self, now: float | None = None) -> bool:
|
||||
"""Expired overrides out; True when one went (a plan without it)."""
|
||||
now = self.clock() if now is None else now
|
||||
keep = [o for o in self.overrides if _end(o) > now]
|
||||
gone = [o for o in self.overrides if _end(o) <= now]
|
||||
if not gone:
|
||||
return False
|
||||
self.overrides = keep
|
||||
for o in gone:
|
||||
self._log(now, o.get("by", ""), "owner", f"override:{o['kind']}", "expired", o.get("control", ""))
|
||||
if o.get("control"):
|
||||
self.dirty.add(o["control"])
|
||||
self.save()
|
||||
return True
|
||||
|
||||
# -- changing --------------------------------------------------------------
|
||||
def _log(self, now, by, level, key, what, control=""):
|
||||
self.log = (self.log + [{"at": int(now), "by": by, "level": level, "item": key, "what": str(what)[:200],
|
||||
**({"control": control} if control else {})}])[-MAX_LOG:]
|
||||
|
||||
def _refused(self, now, by, key, why):
|
||||
self.errors = (self.errors + [{"at": int(now), "by": by, "item": key, "why": why}])[-10:]
|
||||
return False, why
|
||||
|
||||
def set(self, key: str, value: dict | None, by: str, level: str, control, now: float | None = None) -> tuple[bool, str]:
|
||||
"""A setting at a level (None: that level's cleared). control: the
|
||||
control it is for (its kind; its owner limits for lower levels)."""
|
||||
now = self.clock() if now is None else now
|
||||
if level not in RANK:
|
||||
return self._refused(now, by, key, f"level: one of {', '.join(LEVELS)}")
|
||||
what, _, cid = key.partition(":")
|
||||
if control is None or control.id != cid or KINDS.get(control.kind) != what:
|
||||
return self._refused(now, by, key, f"no {what} control {cid!r}")
|
||||
if value is None:
|
||||
if level in (self.items.get(key) or {}):
|
||||
del self.items[key][level]
|
||||
if not self.items[key]:
|
||||
del self.items[key]
|
||||
self._log(now, by, level, key, "cleared", cid)
|
||||
else:
|
||||
try:
|
||||
value = check(key, value, control.kind)
|
||||
if level != "owner":
|
||||
within_owner(key, value, control)
|
||||
except ValueError as e:
|
||||
return self._refused(now, by, key, str(e))
|
||||
self.items.setdefault(key, {})[level] = {"value": value, "by": by, "at": int(now)}
|
||||
self._log(now, by, level, key, describe(key, value, control.kind), cid)
|
||||
self.changed_at = now
|
||||
self.dirty.add(cid)
|
||||
self.save()
|
||||
top = self.effective(key)
|
||||
note = "" if top is None or top["level"] == level else f" (the {top['level']}'s own setting is in force)"
|
||||
return True, f"{key}: {'cleared' if value is None else describe(key, value, control.kind)}{note}"
|
||||
|
||||
def add_override(self, o: dict, by: str, controls: dict, now: float | None = None) -> tuple[bool, str]:
|
||||
"""An override (the owner's): boost, away or car; one of each kind
|
||||
per control (a new one replaces it)."""
|
||||
now = self.clock() if now is None else now
|
||||
kind = o.get("kind")
|
||||
try:
|
||||
if kind == "boost":
|
||||
c = controls.get(o.get("control", ""))
|
||||
if c is None or c.kind not in ("heating", "cooling"):
|
||||
raise ValueError(f"no room {o.get('control')!r}")
|
||||
lo, hi = RANGE_C[c.kind]
|
||||
hours = _num(o.get("hours", 2), "hours", 0.25, BOOST_MAX_H)
|
||||
o = {"kind": "boost", "control": c.id, "c": _num(o.get("c"), "temperature", lo, hi),
|
||||
"from": int(now), "until": int(o.get("until") or now + hours * 3600)}
|
||||
elif kind == "away":
|
||||
until = int(o.get("until") or now + AWAY_MAX_S)
|
||||
if until <= now or until > now + AWAY_MAX_S:
|
||||
raise ValueError("away: until a time within the next 60 days")
|
||||
o = {"kind": "away", "from": int(o.get("from") or now), "until": until}
|
||||
elif kind == "car":
|
||||
c = controls.get(o.get("control", ""))
|
||||
if c is None or c.kind != "ev":
|
||||
raise ValueError(f"no car charger {o.get('control')!r}")
|
||||
at = int(o.get("at") or 0)
|
||||
if not now < at <= now + 7 * 86400:
|
||||
raise ValueError("car needed: a time within the next 7 days")
|
||||
o = _ev_target(dict(o, how=o.get("how") or "must"), {"kind": "car", "control": c.id, "at": at})
|
||||
else:
|
||||
raise ValueError("override: boost, away or car")
|
||||
except (ValueError, TypeError) as e:
|
||||
return self._refused(now, by, f"override:{kind}", str(e))
|
||||
o.update(id=uuid.uuid4().hex[:8], by=by, set_at=int(now))
|
||||
self.overrides = [x for x in self.overrides if not (x["kind"] == o["kind"] and x.get("control") == o.get("control"))]
|
||||
if len(self.overrides) >= MAX_OVERRIDES:
|
||||
return self._refused(now, by, f"override:{kind}", f"at most {MAX_OVERRIDES} overrides")
|
||||
self.overrides.append(o)
|
||||
self._log(now, by, "owner", f"override:{kind}", override_words(o, self.tz), o.get("control", ""))
|
||||
self._touch(now, o)
|
||||
return True, override_words(o, self.tz)
|
||||
|
||||
def cancel_override(self, kind: str, control: str, by: str, now: float | None = None) -> tuple[bool, str]:
|
||||
now = self.clock() if now is None else now
|
||||
gone = [x for x in self.overrides if x["kind"] == kind and x.get("control", "") == (control or "")]
|
||||
if not gone:
|
||||
return True, "no such override"
|
||||
self.overrides = [x for x in self.overrides if x not in gone]
|
||||
self._log(now, by, "owner", f"override:{kind}", "cancelled", control)
|
||||
for o in gone:
|
||||
self._touch(now, o)
|
||||
return True, f"{kind} cancelled"
|
||||
|
||||
def _touch(self, now, o):
|
||||
self.changed_at = now
|
||||
if o.get("control"):
|
||||
self.dirty.add(o["control"])
|
||||
else:
|
||||
self.dirty.add("*")
|
||||
self.save()
|
||||
|
||||
def free(self) -> set[str]:
|
||||
"""Controls whose next plan is taken as it is (steady.py), once."""
|
||||
d, self.dirty = self.dirty, set()
|
||||
return d
|
||||
|
||||
# -- the panel's inbox ---------------------------------------------------------
|
||||
def take_inbox(self, controls: dict, tz: str) -> int:
|
||||
"""The panel's changes (prefs-inbox/*.json, one per change, in name
|
||||
order): {"by", "set": {"item", "value" | null}} | {"override": {...}}
|
||||
| {"cancel": {"kind", "control"}}; each applied as the owner's."""
|
||||
try:
|
||||
names = sorted(n for n in os.listdir(self.inbox) if n.endswith(".json"))
|
||||
except OSError:
|
||||
return 0
|
||||
n = 0
|
||||
for name in names:
|
||||
p = os.path.join(self.inbox, name)
|
||||
try:
|
||||
with open(p) as f:
|
||||
msg = json.load(f)
|
||||
except (OSError, ValueError):
|
||||
msg = None
|
||||
try:
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
if isinstance(msg, dict):
|
||||
self.apply_msg(msg, controls, tz, "owner", "panel:" + str(msg.get("by") or "owner")[:40])
|
||||
n += 1
|
||||
return n
|
||||
|
||||
def apply_msg(self, msg: dict, controls: dict, tz: str, level: str, by: str) -> tuple[bool, str]:
|
||||
"""One change in the inbox's form (the panel's, the console's)."""
|
||||
now = self.clock()
|
||||
zone = ZoneInfo(tz or "UTC")
|
||||
if "set" in msg:
|
||||
s = msg["set"] or {}
|
||||
key = str(s.get("item", ""))
|
||||
val = s.get("value")
|
||||
if isinstance(val, str): # a text form
|
||||
try:
|
||||
val = from_text(key, val, controls.get(key.partition(":")[2]),
|
||||
(self.value(key) or {}) if key.startswith(("ev:", "battery:")) else {})
|
||||
except ValueError as e:
|
||||
return self._refused(now, by, key, str(e))
|
||||
return self.set(key, val, by, level, controls.get(key.partition(":")[2]), now)
|
||||
if level != "owner":
|
||||
return self._refused(now, by, "override", "overrides are the owner's (Home Assistant, the panel)")
|
||||
if "override" in msg:
|
||||
o = dict(msg["override"] or {})
|
||||
if o.get("kind") == "car" and isinstance(o.get("text"), str):
|
||||
try:
|
||||
car = parse_car_text(o["text"], zone, now)
|
||||
except ValueError as e:
|
||||
return self._refused(now, by, "override:car", str(e))
|
||||
if car is None:
|
||||
return self.cancel_override("car", o.get("control", ""), by, now)
|
||||
o = dict(car, control=o.get("control", ""))
|
||||
if o.get("kind") == "away" and isinstance(o.get("until_text"), str) and o["until_text"].strip():
|
||||
try:
|
||||
o["until"] = local_when(o["until_text"], zone, now)
|
||||
except ValueError as e:
|
||||
return self._refused(now, by, "override:away", str(e))
|
||||
return self.add_override(o, by, controls, now)
|
||||
if "cancel" in msg:
|
||||
c = msg["cancel"] or {}
|
||||
return self.cancel_override(str(c.get("kind", "")), str(c.get("control", "")), by, now)
|
||||
return self._refused(now, by, "", "set, override or cancel")
|
||||
|
||||
# -- into the plan ---------------------------------------------------------------
|
||||
def apply(self, state: dict, controls: dict, col=None, now: float | None = None) -> dict:
|
||||
"""The model state with each control's preferences per step
|
||||
(d["pref"]: optimizer.py). Controls without any keep today's."""
|
||||
now = self.clock() if now is None else now
|
||||
self.prune(now)
|
||||
zone = ZoneInfo(state.get("tz") or "UTC")
|
||||
T, start, step = state["steps"], state["start"], state["step_s"]
|
||||
ends = [_local(start + (t + 1) * step, zone) for t in range(T)] # a room's or battery's value at a step's end
|
||||
ovs = self.active_overrides(now)
|
||||
for d in state["devices"]:
|
||||
c = controls.get(d["id"])
|
||||
if c is None:
|
||||
continue
|
||||
if d["kind"] in ("heating", "cooling"):
|
||||
p = room_series(d, self.value(f"room:{d['id']}"), ovs, state, ends)
|
||||
elif d["kind"] == "ev":
|
||||
p = ev_need(d, self.value(f"ev:{d['id']}"), ovs, state, zone, now, col, c)
|
||||
elif d["kind"] == "battery":
|
||||
p = battery_series(d, self.value(f"battery:{d['id']}"), ends)
|
||||
else:
|
||||
p = None
|
||||
if p is not None:
|
||||
d["pref"] = p
|
||||
state["prefs_cost"] = bool(any(d.get("pref") for d in state["devices"])) and (
|
||||
self.changed_at > self.costs.get("of", 0) or now - self.costs.get("at", 0) >= COST_EVERY_S)
|
||||
return state
|
||||
|
||||
def take_plan(self, plan: dict, now: float | None = None):
|
||||
"""The plan's feedback (optimizer.pref_feedback), with the musts'
|
||||
costs: the plan's own when it made them, else the last ones."""
|
||||
now = self.clock() if now is None else now
|
||||
made = plan.get("prefs_cost")
|
||||
if made is not None:
|
||||
self.costs = {"at": int(now), "of": max(self.changed_at, 0.0), "eur_day": made}
|
||||
try:
|
||||
self.save()
|
||||
except OSError as e:
|
||||
log.warning("preferences: %s", e)
|
||||
eur = self.costs.get("eur_day") or {}
|
||||
fb = []
|
||||
for f in plan.get("prefs") or []:
|
||||
f = dict(f)
|
||||
if f.get("how") == "must" and f.get("group") in eur:
|
||||
f["cost_eur_day"] = eur[f["group"]]
|
||||
f["cost_as_of"] = self.costs.get("at")
|
||||
f["text"] = feedback_words(f)
|
||||
fb.append(f)
|
||||
self.feedback = fb
|
||||
plan["prefs"] = fb
|
||||
plan.pop("prefs_cost", None)
|
||||
return fb
|
||||
|
||||
# -- out ---------------------------------------------------------------------------
|
||||
def public(self) -> dict:
|
||||
"""For the cloud (with the plan) and the panel: the settings per
|
||||
level with who and when, the overrides in force, the feedback, the
|
||||
last changes."""
|
||||
items = {}
|
||||
for key, per in self.items.items():
|
||||
top = self.effective(key)
|
||||
items[key] = {"in_force": top["level"] if top else None,
|
||||
"levels": {lvl: {"value": e["value"], "by": e["by"], "at": e["at"],
|
||||
"text": text_of(key, e["value"])} for lvl, e in per.items()}}
|
||||
return {"v": 1, "items": items, "overrides": self.active_overrides(), "feedback": self.feedback,
|
||||
"log": self.log[-20:], "changed_at": int(self.changed_at)}
|
||||
|
||||
def write_view(self, controls, mode: str = ""):
|
||||
"""preferences-view.json for the Hub's panel: the controls (id,
|
||||
kind, name, the owner's limits), and public()."""
|
||||
cl = []
|
||||
for c in controls:
|
||||
if c.kind not in KINDS:
|
||||
continue
|
||||
q = {"heating": "temp_c", "cooling": "temp_c", "battery": "soc_pct", "ev": "current_a"}[c.kind]
|
||||
b = c.bounds(q)
|
||||
cl.append({"id": c.id, "kind": c.kind, "name": c.label(), "bounds": [b.lo, b.hi],
|
||||
"setback_c": _p(c, "setback_c"), "setup_c": _p(c, "setup_c"), "comfort_c": _p(c, "comfort_c")})
|
||||
v = dict(self.public(), controls=cl, mode=mode, errors=self.errors, ha=self.ha, made=int(self.clock()))
|
||||
try:
|
||||
tmp = self.view_path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(v, f)
|
||||
os.replace(tmp, self.view_path)
|
||||
except OSError as e:
|
||||
log.warning("preferences view: %s", e)
|
||||
|
||||
|
||||
def _p(c, k):
|
||||
try:
|
||||
return float(c.p(k))
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _end(o: dict) -> float:
|
||||
return float(o.get("until") or o.get("at") or 0)
|
||||
|
||||
|
||||
def _local(t: float, zone: ZoneInfo) -> tuple[int, int]:
|
||||
d = datetime.fromtimestamp(t, timezone.utc).astimezone(zone)
|
||||
return d.weekday(), d.hour * 60 + d.minute
|
||||
|
||||
|
||||
def in_block(b: dict, wd: int, m: int) -> bool:
|
||||
"""A block {days, from, to} covers the wall-clock (weekday, minute)? A
|
||||
block past midnight belongs to the day it starts; from == to: all day."""
|
||||
a, z = minutes(b["from"]), minutes(b["to"])
|
||||
if a < z:
|
||||
return wd in b["days"] and a <= m < z
|
||||
if a == z:
|
||||
return wd in b["days"]
|
||||
return (wd in b["days"] and m >= a) or ((wd - 1) % 7 in b["days"] and m < z)
|
||||
|
||||
|
||||
def within_owner(key: str, value: dict, c):
|
||||
"""An installer's or operator's setting within the owner's limits (the
|
||||
mapping's owner level), never above them."""
|
||||
what = key.split(":", 1)[0]
|
||||
owner = lambda q: (c.limits.get(q) or {}).get("owner") or (None, None) # noqa: E731
|
||||
if what == "room":
|
||||
lo, hi = owner("temp_c")
|
||||
for b in value["blocks"]:
|
||||
if (lo is not None and b["c"] < lo) or (hi is not None and b["c"] > hi):
|
||||
raise ValueError(f"{_fmt(b['c'])} °C is outside the owner's limits "
|
||||
f"{_fmt(lo) if lo is not None else '-'}..{_fmt(hi) if hi is not None else '-'} °C")
|
||||
elif what == "battery":
|
||||
lo, hi = owner("soc_pct")
|
||||
for p in [value["reserve_pct"]] + [b["pct"] for b in value["blocks"]]:
|
||||
if (lo is not None and p < lo) or (hi is not None and p > hi):
|
||||
raise ValueError(f"a reserve of {_fmt(p)} % is outside the owner's limits")
|
||||
|
||||
|
||||
# -- per step ----------------------------------------------------------------------
|
||||
def room_series(d: dict, value: dict | None, ovs: list[dict], state: dict, ends) -> dict | None:
|
||||
"""A room's target per step (its temperature at the step's end), the
|
||||
must's bound (None: none), and the group (a block, a boost, away) it
|
||||
comes from. base: what it would be without the group (setback/setup;
|
||||
today's occupied hours without a schedule)."""
|
||||
cooling = d["kind"] == "cooling"
|
||||
p = d["params"]
|
||||
other = float(p["setup_c"] if cooling else p["setback_c"])
|
||||
comfort = float(p["comfort_c"])
|
||||
blocks = (value or {}).get("blocks") or []
|
||||
mine = [o for o in ovs if (o["kind"] == "boost" and o.get("control") == d["id"]) or o["kind"] == "away"]
|
||||
if not blocks and not mine:
|
||||
return None
|
||||
T, start, step = state["steps"], state["start"], state["step_s"]
|
||||
occ = state.get("occupied") or [1] * T
|
||||
groups = [{"group": f"{d['id']}:{i}", "kind": "block", "how": b["how"], "c": b["c"],
|
||||
"text": f"{days_text(b['days'])} {b['from']}-{b['to']} {'≤' if cooling else '≥'} {_fmt(b['c'])} °C"}
|
||||
for i, b in enumerate(blocks)]
|
||||
gi = {}
|
||||
for o in mine:
|
||||
gi[o["id"]] = len(groups)
|
||||
groups.append({"group": f"{d['id']}:{o['kind']}", "kind": o["kind"], "how": "must" if o["kind"] == "boost" else "prefer",
|
||||
"c": o.get("c", other), "text": override_words(o, state.get("tz")), "until": o.get("until")})
|
||||
target, must, group, base = [], [], [], []
|
||||
pick = min if cooling else max
|
||||
for t in range(T):
|
||||
wd, m = ends[t]
|
||||
tend = start + (t + 1) * step
|
||||
b0 = other if blocks else (comfort if occ[min(t + 1, T - 1)] else other)
|
||||
base.append(b0)
|
||||
boost = next((o for o in mine if o["kind"] == "boost" and o["from"] <= tend < o["until"]), None)
|
||||
away = next((o for o in mine if o["kind"] == "away" and o.get("from", 0) <= tend < o["until"]), None)
|
||||
if boost is not None:
|
||||
target.append(boost["c"])
|
||||
must.append(boost["c"])
|
||||
group.append(gi[boost["id"]])
|
||||
continue
|
||||
if away is not None:
|
||||
target.append(other)
|
||||
must.append(None)
|
||||
group.append(gi[away["id"]])
|
||||
continue
|
||||
hit = [(i, b) for i, b in enumerate(blocks) if in_block(b, wd, m)]
|
||||
if not hit:
|
||||
target.append(b0)
|
||||
must.append(None)
|
||||
group.append(-1)
|
||||
continue
|
||||
tc = pick(b["c"] for _, b in hit)
|
||||
ms = [b["c"] for _, b in hit if b["how"] == "must"]
|
||||
target.append(tc)
|
||||
must.append(pick(ms) if ms else None)
|
||||
# the group: the must that binds, else the block that sets the target
|
||||
g = next((i for i, b in hit if b["how"] == "must" and b["c"] == must[-1]), None) if ms else None
|
||||
group.append(g if g is not None else next(i for i, b in hit if b["c"] == tc))
|
||||
return {"target": target, "must": must, "group": group, "base": base, "groups": groups}
|
||||
|
||||
|
||||
def next_ready(value: dict | None, zone: ZoneInfo, now: float) -> tuple[float, dict] | None:
|
||||
"""The next ready-by time after now and its entry."""
|
||||
best = None
|
||||
today = datetime.fromtimestamp(now, zone).date()
|
||||
for r in (value or {}).get("ready") or []:
|
||||
h, mi = (int(x) for x in r["at"].split(":"))
|
||||
for k in range(8):
|
||||
d = today + timedelta(days=k)
|
||||
if d.weekday() not in r["days"]:
|
||||
continue
|
||||
t = _wall(d, h, mi, zone)
|
||||
if t > now:
|
||||
if best is None or t < best[0]:
|
||||
best = (t, r)
|
||||
break
|
||||
return best
|
||||
|
||||
|
||||
def ev_need(d: dict, value: dict | None, ovs: list[dict], state: dict, zone, now: float, col, c) -> dict | None:
|
||||
"""The car's next ready-by (a one-off override wins while it lasts) as
|
||||
the departure step and the energy needed: kWh as given, or % from the
|
||||
car's state of charge: (target - soc) x capacity."""
|
||||
car = next((o for o in ovs if o["kind"] == "car" and o.get("control") == d["id"]), None)
|
||||
if car is not None:
|
||||
when, r, src = car["at"], car, "override"
|
||||
else:
|
||||
nr = next_ready(value, zone, now)
|
||||
if nr is None:
|
||||
return None
|
||||
(when, r), src = nr, "schedule"
|
||||
start, step, T = state["start"], state["step_s"], state["steps"]
|
||||
dep = int(max(0, min(T, (when - start) // step)))
|
||||
st = d.get("state") or {}
|
||||
soc = st.get("soc_pct")
|
||||
cap = (value or {}).get("capacity_kwh")
|
||||
note = ""
|
||||
if r.get("kwh") is not None:
|
||||
need = r["kwh"] * 1000.0
|
||||
elif soc is not None and cap:
|
||||
need = max(0.0, (r["pct"] - float(soc)) / 100.0 * cap * 1000.0)
|
||||
else:
|
||||
need = float(st["need_wh"]) if st.get("need_wh") is not None else float(d["params"]["need_wh"])
|
||||
note = "no state of charge from the car" if soc is None else "the car's capacity is not set"
|
||||
loc = datetime.fromtimestamp(when, zone)
|
||||
out = {"group": f"{d['id']}:{'car' if src == 'override' else 'ready'}", "source": src, "dep_step": dep,
|
||||
"at": int(when), "at_local": loc.strftime("%a %H:%M"), "need_wh": round(need, 1), "how": r["how"],
|
||||
"max_ct": r.get("max_ct"), "min_wh": (r.get("min_kwh") or 0.0) * 1000.0 if r.get("min_kwh") else 0.0,
|
||||
"pct": r.get("pct"), "kwh": r.get("kwh"), "soc_pct": soc, "capacity_wh": cap * 1000.0 if cap else None,
|
||||
"note": note, "beyond": when > start + T * step}
|
||||
if col is not None and c.entities.get("connected"):
|
||||
st0 = col.states.get(c.entities["connected"]) or {}
|
||||
if st0.get("s") == "on" and st0.get("t"):
|
||||
out["plugged_at"] = int(float(st0["t"]))
|
||||
out["text"] = f"{_target_text(r).replace(' must', '')} by {loc.strftime('%a %H:%M')}" + \
|
||||
(" (one-off)" if src == "override" else "")
|
||||
return out
|
||||
|
||||
|
||||
def battery_series(d: dict, value: dict | None, ends) -> dict | None:
|
||||
if not value:
|
||||
return None
|
||||
base = float(value.get("reserve_pct") or 0.0)
|
||||
groups = [{"group": f"{d['id']}:reserve", "kind": "reserve", "how": "must", "pct": base,
|
||||
"text": f"reserve {_fmt(base)} %"}] if base > 0 else []
|
||||
for i, b in enumerate(value.get("blocks") or []):
|
||||
groups.append({"group": f"{d['id']}:{i}", "kind": "block", "how": "must", "pct": b["pct"],
|
||||
"text": f"reserve {_fmt(b['pct'])} % {days_text(b['days'])} {b['from']}-{b['to']}"})
|
||||
floor, group = [], []
|
||||
off = 1 if base > 0 else 0
|
||||
for wd, m in ends:
|
||||
best, g = base, 0 if base > 0 else -1
|
||||
for i, b in enumerate(value.get("blocks") or []):
|
||||
if in_block(b, wd, m) and b["pct"] > best:
|
||||
best, g = b["pct"], i + off
|
||||
floor.append(best)
|
||||
group.append(g)
|
||||
if not groups:
|
||||
return None
|
||||
return {"floor_pct": floor, "group": group, "groups": groups}
|
||||
|
||||
|
||||
# -- words ----------------------------------------------------------------------------
|
||||
def describe(key: str, value: dict, kind: str = "") -> str:
|
||||
what = key.split(":", 1)[0]
|
||||
if what == "room":
|
||||
return room_text(value) or "no schedule"
|
||||
if what == "ev":
|
||||
s = ev_text(value) or "no ready-by times"
|
||||
return s + (f"; capacity {_fmt(value['capacity_kwh'])} kWh" if value.get("capacity_kwh") else "")
|
||||
if what == "battery":
|
||||
return f"reserve {_fmt(value['reserve_pct'])} %" + (f"; {battery_text(value)}" if value.get("blocks") else "")
|
||||
return json.dumps(value)[:200]
|
||||
|
||||
|
||||
def text_of(key: str, value: dict) -> str:
|
||||
what = key.split(":", 1)[0]
|
||||
return {"room": room_text, "ev": ev_text, "battery": battery_text}.get(what, lambda v: "")(value)
|
||||
|
||||
|
||||
def from_text(key: str, text: str, control, current: dict) -> dict:
|
||||
what = key.split(":", 1)[0]
|
||||
if control is None:
|
||||
raise ValueError(f"no control for {key!r}")
|
||||
if what == "room":
|
||||
return parse_room_text(text, control.kind)
|
||||
if what == "ev":
|
||||
return parse_ev_text(text, current.get("capacity_kwh"))
|
||||
if what == "battery":
|
||||
return parse_battery_text(text, current.get("reserve_pct", 0.0))
|
||||
raise ValueError(f"no preference {key!r}")
|
||||
|
||||
|
||||
def car_text(o: dict, tz: str | None = None) -> str:
|
||||
"""A car override in the text form: "2026-11-05 07:00 80% must"."""
|
||||
when = datetime.fromtimestamp(o["at"], ZoneInfo(tz or "UTC")).strftime("%Y-%m-%d %H:%M")
|
||||
t = _target_text(o)
|
||||
return f"{when} {t}" if o.get("how") == "must" else f"{when} {t} prefer"
|
||||
|
||||
|
||||
def override_words(o: dict, tz: str | None = None) -> str:
|
||||
until = datetime.fromtimestamp(_end(o), ZoneInfo(tz or "UTC")).strftime("%a %d %b %H:%M")
|
||||
if o["kind"] == "boost":
|
||||
return f"boost to {_fmt(o['c'])} °C until {until}"
|
||||
if o["kind"] == "away":
|
||||
return f"away until {until}"
|
||||
return f"car {_target_text(o).replace(' must', '')} at {until}" + (" (must)" if o.get("how") == "must" else "")
|
||||
|
||||
|
||||
def feedback_words(f: dict) -> str:
|
||||
"""One preference's result in words: "Mon-Fri 22:00-06:00 ≥ 21 °C: met,
|
||||
+0.40 €/day"; "car 100% by Thu 08:00: short by 4.2 kWh, ready at full
|
||||
power by 09:15"."""
|
||||
head = f.get("label") or f.get("text") or f.get("group", "")
|
||||
if f.get("met") is None:
|
||||
bits = [f.get("detail") or "not in the plan's 48 h"]
|
||||
elif f["met"]:
|
||||
bits = ["met"] + ([f["detail"]] if f.get("detail") else [])
|
||||
else:
|
||||
bits = ["not met"] + ([f["detail"]] if f.get("detail") else [])
|
||||
if f.get("cost_eur_day") is not None:
|
||||
bits.append(f"{f['cost_eur_day']:+.2f} €/day")
|
||||
return f"{head}: " + ", ".join(bits)
|
||||
@@ -0,0 +1,136 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""A room's thermal resistance R and capacity C from its history
|
||||
(docs/methods.md §6): the planner's first-order model (edge optimizer.py
|
||||
`room`),
|
||||
|
||||
θ_t = (1 − a) θ_{t−1} + a T_out,t + (Δt/C) P_t + γ G_t + c0,
|
||||
a = Δt / (R C),
|
||||
|
||||
written as a regression on 15-minute steps (Δt = 0.25 h):
|
||||
|
||||
θ_t − θ_{t−1} = a (T_out,t − θ_{t−1}) + b P_t + γ G_t + c0, b = Δt / C,
|
||||
|
||||
with P the heating power (kW; thermal where the site meters the heat
|
||||
output, else electric: the planner's COP turns that into heat), G the
|
||||
global irradiance (kW/m², the sun through the windows) and c0 the internal
|
||||
gains (people, appliances). Least squares on every step where all inputs
|
||||
are there; then C = Δt / b, R = Δt / (a C).
|
||||
|
||||
The other two terms go to the planner with R and C (Finding B6-a: R and C
|
||||
were fitted with them; without them the planner expects the room to cool
|
||||
faster than it does), as heat in the same units as P:
|
||||
|
||||
gain_w = 1000 c0 / b the internal gain (W, constant),
|
||||
solar_m2 = γ / b the sun's heat per irradiance (m²: W of heat
|
||||
per W/m² of global irradiance, an effective
|
||||
window area).
|
||||
|
||||
(electric fits: W and m² of electric equivalent; the box multiplies by
|
||||
the COP like C.)
|
||||
|
||||
Sanity before a fit is used (else the planner keeps its defaults, R 10
|
||||
K/kW and C 3 kWh/K): a > 0 and b > 0; R within 1..100 K/kW, C within
|
||||
0.3..50 kWh/K, τ = RC within 3..1000 h; at least a day of steps, the
|
||||
heating on and off in it, 2 K of spread in T_out − θ; and the fitted
|
||||
model simulates the room better than the defaults do (4-hour open-loop
|
||||
runs from every 4th hour: RMSE in K).
|
||||
|
||||
The site's heating power is used for every room (the sims: one zone per
|
||||
home): the rooms of a home with one heat meter each get the whole
|
||||
home's C. Per-room heating power, where metered per control, is the
|
||||
next step."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
|
||||
DT_H = 0.25
|
||||
DEFAULT_R, DEFAULT_C = 10.0, 3.0
|
||||
R_BOUNDS, C_BOUNDS, TAU_BOUNDS = (1.0, 100.0), (0.3, 50.0), (3.0, 1000.0)
|
||||
MIN_STEPS = 96
|
||||
SIM_STEPS = 16
|
||||
SIM_EVERY = 16
|
||||
|
||||
|
||||
def simulate(theta0: float, tout: np.ndarray, p_kw: np.ndarray, ghi_kw: np.ndarray, r: float, c: float,
|
||||
gamma: float = 0.0, c0: float = 0.0) -> np.ndarray:
|
||||
a, b = DT_H / (r * c), DT_H / c
|
||||
out, th = [], theta0
|
||||
for k in range(len(tout)):
|
||||
th = (1 - a) * th + a * tout[k] + b * p_kw[k] + gamma * ghi_kw[k] + c0
|
||||
out.append(th)
|
||||
return np.array(out)
|
||||
|
||||
|
||||
def sim_rmse(theta: np.ndarray, tout: np.ndarray, p_kw: np.ndarray, ghi_kw: np.ndarray, r: float, c: float,
|
||||
gamma: float = 0.0, c0: float = 0.0) -> float | None:
|
||||
"""4-hour open-loop runs from every 4th hour where all inputs exist."""
|
||||
errs = []
|
||||
for s in range(0, len(theta) - SIM_STEPS - 1, SIM_EVERY):
|
||||
seg = slice(s + 1, s + 1 + SIM_STEPS)
|
||||
if np.isnan(theta[s]) or np.isnan(theta[seg]).any() or np.isnan(tout[seg]).any() or np.isnan(p_kw[seg]).any():
|
||||
continue
|
||||
sim = simulate(theta[s], tout[seg], p_kw[seg], np.nan_to_num(ghi_kw[seg]), r, c, gamma, c0)
|
||||
errs.append((sim - theta[seg]) ** 2)
|
||||
if not errs:
|
||||
return None
|
||||
return float(math.sqrt(np.concatenate(errs).mean()))
|
||||
|
||||
|
||||
def fit(theta: np.ndarray, tout: np.ndarray, p_w: np.ndarray, ghi: np.ndarray | None = None,
|
||||
power: str = "thermal") -> dict:
|
||||
"""theta, tout (°C), p_w (W), ghi (W/m²) per 15-minute slot, aligned.
|
||||
Returns {"status": "fitted"|"default", r_k_per_kw, c_kwh_per_k, tau_h,
|
||||
power, n, rmse_k (simulation, fitted), rmse_default_k, r2, note}, and
|
||||
when fitted gain_w and solar_m2 (the module's docstring)."""
|
||||
n_all = len(theta)
|
||||
ghi = np.zeros(n_all) if ghi is None else np.where(np.isnan(ghi), 0.0, ghi)
|
||||
p_kw = p_w / 1000.0
|
||||
g_kw = ghi / 1000.0
|
||||
prev = np.concatenate([[np.nan], theta[:-1]])
|
||||
dy = theta - prev
|
||||
x1 = tout - prev
|
||||
ok = ~(np.isnan(dy) | np.isnan(x1) | np.isnan(p_kw))
|
||||
out = {"status": "default", "r_k_per_kw": DEFAULT_R, "c_kwh_per_k": DEFAULT_C, "tau_h": DEFAULT_R * DEFAULT_C,
|
||||
"power": power, "n": int(ok.sum()), "rmse_k": None, "rmse_default_k": None, "r2": None, "note": ""}
|
||||
if ok.sum() < MIN_STEPS:
|
||||
out["note"] = f"{int(ok.sum())} usable steps (at least {MIN_STEPS})"
|
||||
return out
|
||||
on = p_kw[ok] > 0.05
|
||||
if on.all() or not on.any():
|
||||
out["note"] = "the heating was never " + ("off" if on.all() else "on") + " in the data"
|
||||
return out
|
||||
if np.ptp(x1[ok]) < 2.0:
|
||||
out["note"] = "less than 2 K between outside and inside"
|
||||
return out
|
||||
X = np.column_stack([x1[ok], p_kw[ok], g_kw[ok], np.ones(int(ok.sum()))])
|
||||
coef, *_ = np.linalg.lstsq(X, dy[ok], rcond=None)
|
||||
a, b, gamma, c0 = (float(v) for v in coef)
|
||||
resid = dy[ok] - X @ coef
|
||||
var = float(np.var(dy[ok]))
|
||||
out["r2"] = round(1 - float(np.var(resid)) / var, 3) if var > 0 else None
|
||||
if a <= 0 or b <= 0:
|
||||
out["note"] = f"no physical fit (a {a:.4g}, b {b:.4g})"
|
||||
return out
|
||||
c = DT_H / b
|
||||
r = DT_H / (a * c)
|
||||
tau = r * c
|
||||
out["fit"] = {"a": a, "b": b, "gamma": gamma, "c0": c0}
|
||||
if not (R_BOUNDS[0] <= r <= R_BOUNDS[1] and C_BOUNDS[0] <= c <= C_BOUNDS[1] and TAU_BOUNDS[0] <= tau <= TAU_BOUNDS[1]):
|
||||
out["note"] = f"outside the sanity bounds (R {r:.3g} K/kW, C {c:.3g} kWh/K, τ {tau:.3g} h)"
|
||||
return out
|
||||
fitted = sim_rmse(theta, tout, p_kw, g_kw, r, c, gamma, c0)
|
||||
default = sim_rmse(theta, tout, p_kw, g_kw, DEFAULT_R, DEFAULT_C)
|
||||
out["rmse_default_k"] = round(default, 3) if default is not None else None
|
||||
out["rmse_k"] = round(fitted, 3) if fitted is not None else None
|
||||
if fitted is None or default is None:
|
||||
out["note"] = "too few complete 4-hour stretches to check the fit"
|
||||
return out
|
||||
if fitted >= default:
|
||||
out["note"] = f"no better than the defaults ({fitted:.2f} K vs {default:.2f} K)"
|
||||
return out
|
||||
out.update(status="fitted", r_k_per_kw=round(r, 3), c_kwh_per_k=round(c, 3), tau_h=round(tau, 2),
|
||||
gain_w=round(1000.0 * c0 / b, 1), solar_m2=round(gamma / b, 3),
|
||||
note=f"simulates the room to {fitted:.2f} K (defaults {default:.2f} K)")
|
||||
return out
|
||||
@@ -31,8 +31,11 @@ keeps the value in force, its why says so) unless all hold:
|
||||
Not here, so never held back: the console's and the building's commands,
|
||||
the safe defaults, the grid operator's order (the plan is made again at
|
||||
once within its bounds; a kept value outside the bounds in force is never
|
||||
kept), the owner's own changes (the guard's hands off) and the guard's
|
||||
floors. The entry the plan made is kept in the entry as "planned".
|
||||
kept), the owner's own changes (the guard's hands off; and the owner's
|
||||
preferences, preferences.py: the first plan after a change of a control's
|
||||
preferences or overrides, `free`, is taken as it is for that control) and
|
||||
the guard's floors. The entry the plan made is kept in the entry as
|
||||
"planned".
|
||||
"""
|
||||
import math
|
||||
|
||||
@@ -117,10 +120,13 @@ def _within(bounds, v) -> bool:
|
||||
return (lo is None or v >= lo - 1e-9) and (hi is None or v <= hi + 1e-9)
|
||||
|
||||
|
||||
def stabilise(plan: dict, state: dict, controls, col, last_set=None, now: float | None = None) -> dict:
|
||||
def stabilise(plan: dict, state: dict, controls, col, last_set=None, now: float | None = None,
|
||||
free: set | None = None) -> dict:
|
||||
"""The plan with changes that do not matter kept out (module
|
||||
docstring). last_set(control id, kind "mode"|"set") -> when the agent
|
||||
last changed that (None: not known). In place; returns the plan."""
|
||||
last changed that (None: not known). free: control ids (or "*": all)
|
||||
whose preferences the owner just changed: their entries as planned.
|
||||
In place; returns the plan."""
|
||||
now = plan.get("made", 0) if now is None else now
|
||||
step = plan.get("step_s", 900)
|
||||
end = plan.get("start", now) + step * state.get("steps", 192)
|
||||
@@ -133,6 +139,8 @@ def stabilise(plan: dict, state: dict, controls, col, last_set=None, now: float
|
||||
entries = (plan.get("devices") or {}).get(c.id)
|
||||
if not entries or c.kind not in ("battery", "heating", "cooling", "ev"):
|
||||
continue
|
||||
if free and (c.id in free or "*" in free):
|
||||
continue # the owner's change: not held back
|
||||
cur = current(c, col)
|
||||
if cur is None:
|
||||
continue
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The box's own training history (docs/methods.md B10): what its nightly
|
||||
training (boxtrain.py) learns from, kept in
|
||||
the agent's SQLite (<state dir>/training.db), rolling.
|
||||
|
||||
slots 15-minute means per asset the trainer uses (load.power,
|
||||
pv.power, the planned devices' power, room.<r>.temp, the
|
||||
heating's power, the outdoor thermometer, the grid meter and
|
||||
the battery), from the agent's own 1-minute windows: per slot
|
||||
the mean of the windows' means, as the cloud's trainer reads
|
||||
telemetry (time_bucket('15 minutes') of avg(mean)). A slot the
|
||||
agent did not see can be filled once from Home Assistant's
|
||||
statistics (fill_from_stats: 5-minute where HA still has them,
|
||||
hourly before), never over the agent's own. 13 months (the
|
||||
training weighs recent weeks more and last year's same season
|
||||
still: fcmodels.recency_weights); compact: an asset's number,
|
||||
the slot's number, the sum and count (WITHOUT ROWID; ~20 bytes
|
||||
a row).
|
||||
weather hourly records (pvforecast.Hour's fields) per kind: observed
|
||||
(the cloud's DWD observations shipped in the model bundle,
|
||||
else Home Assistant's recorded weather entity) and forecast
|
||||
(the last forecast made before the hour was over: Home
|
||||
Assistant's, the cloud's), each with its source. 13 months.
|
||||
fc_slots per series, slot and source the last forecast made before the
|
||||
slot began (the box's own model, the cloud's series, the cloud's
|
||||
model on the box, the hour-of-week, the coordinator's week
|
||||
mean): the live error per source against the meter. 14 days.
|
||||
|
||||
Nothing here leaves the site except the error figures the plan carries
|
||||
(controller.public_plan "ml")."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
|
||||
SLOT_S = 900
|
||||
KEEP_DAYS = 396 # 13 months
|
||||
KEEP_FC_DAYS = 14
|
||||
# the assets the trainer reads (building/trainlib.py: SITE_SERIES, ROOM_INPUTS,
|
||||
# CONTROLLED; trainer.load_site), and the meter for the member's net draw
|
||||
ASSETS = {"load.power", "pv.power", "weather.outside_temp", "heat.power", "heatpump.power", "heating.power",
|
||||
"ac.power", "cooling.power", "ev.power", "grid.import_power", "grid.export_power", "battery.power"}
|
||||
ASSET_RE = re.compile(r"^(room\.[^.]+\.temp|(heatpump|heating|ac|cooling|ev)\.[0-9]+\.power)$")
|
||||
OBSERVED_PREF = ("cloud", "ha") # an observed hour: DWD's (with irradiation) before HA's entity
|
||||
FORECAST_PREF = ("ha", "cloud") # a forecast hour: as the box forecasts (localml.merge_hours: HA's first)
|
||||
|
||||
|
||||
def wanted(asset: str) -> bool:
|
||||
return asset in ASSETS or bool(ASSET_RE.match(asset))
|
||||
|
||||
|
||||
def _finite(v) -> bool:
|
||||
return isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)
|
||||
|
||||
|
||||
class Store:
|
||||
"""The training history. Thread-safe (the nightly job reads it in a
|
||||
worker thread while the agent's loop writes): one connection, a lock."""
|
||||
|
||||
def __init__(self, path: str):
|
||||
self.path = path
|
||||
self.lock = threading.Lock()
|
||||
self.db = sqlite3.connect(path, isolation_level=None, check_same_thread=False)
|
||||
self.db.execute("PRAGMA journal_mode=WAL")
|
||||
self.db.execute("PRAGMA synchronous=NORMAL")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS assets (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE)")
|
||||
# slot: the quarter hour's number (unix / 900); src: 0 the agent, 1 HA's 5-minute, 2 HA's hourly statistics
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS slots (aid INTEGER NOT NULL, slot INTEGER NOT NULL, sum REAL NOT NULL, "
|
||||
"n INTEGER NOT NULL, src INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (aid, slot)) WITHOUT ROWID")
|
||||
self._ids = {name: i for i, name in self.db.execute("SELECT id, name FROM assets")}
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS weather (end_ts INTEGER NOT NULL, kind TEXT NOT NULL, source TEXT NOT NULL, "
|
||||
"solar REAL, cloud REAL, temp REAL, issued INTEGER, PRIMARY KEY (end_ts, kind, source))")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS fc_slots (series TEXT NOT NULL, slot INTEGER NOT NULL, source TEXT NOT NULL, "
|
||||
"value REAL NOT NULL, issued INTEGER NOT NULL, PRIMARY KEY (series, slot, source))")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
||||
|
||||
def _aid(self, name: str) -> int:
|
||||
i = self._ids.get(name)
|
||||
if i is None:
|
||||
self.db.execute("INSERT OR IGNORE INTO assets (name) VALUES (?)", (name,))
|
||||
i = self._ids[name] = self.db.execute("SELECT id FROM assets WHERE name = ?", (name,)).fetchone()[0]
|
||||
return i
|
||||
|
||||
def close(self):
|
||||
with self.lock:
|
||||
self.db.close()
|
||||
|
||||
# -- the agent's own samples -------------------------------------------------
|
||||
def add_window(self, start: float, records: list[dict]) -> int:
|
||||
"""A closed 1-minute window (agent.close_window's records): each
|
||||
wanted gauge's mean into its 15-minute slot (by the window's start)."""
|
||||
slot = int(start // SLOT_S)
|
||||
recs = [(r["asset"], float(r["mean"])) for r in records
|
||||
if r.get("kind") == "gauge" and _finite(r.get("mean")) and wanted(r.get("asset", ""))]
|
||||
if not recs:
|
||||
return 0
|
||||
with self.lock:
|
||||
rows = [(self._aid(a), slot, v) for a, v in recs]
|
||||
self.db.executemany(
|
||||
"INSERT INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, 0) ON CONFLICT (aid, slot) "
|
||||
"DO UPDATE SET sum = CASE WHEN src = 0 THEN sum + excluded.sum ELSE excluded.sum END, "
|
||||
"n = CASE WHEN src = 0 THEN n + 1 ELSE 1 END, src = 0", rows)
|
||||
return len(rows)
|
||||
|
||||
def fill(self, asset: str, values: dict[int, float], src: str) -> int:
|
||||
"""Slots the agent did not see (Home Assistant's statistics): never
|
||||
over a slot that has a value."""
|
||||
code = {"agent": 0, "ha-5minute": 1, "ha-hour": 2}.get(src, 2)
|
||||
with self.lock:
|
||||
aid = self._aid(asset)
|
||||
rows = [(aid, int(k) // SLOT_S, float(v), code) for k, v in values.items() if _finite(v) and int(k) % SLOT_S == 0]
|
||||
before = self.db.total_changes
|
||||
self.db.executemany("INSERT OR IGNORE INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, ?)", rows)
|
||||
return self.db.total_changes - before
|
||||
|
||||
def assets(self) -> list[str]:
|
||||
with self.lock:
|
||||
return [a for (a,) in self.db.execute("SELECT name FROM assets WHERE id IN (SELECT DISTINCT aid FROM slots)")]
|
||||
|
||||
def series(self, since: int, until: int, assets=None) -> dict[str, dict[int, float]]:
|
||||
"""{asset: {slot: mean}} for slots in [since, until)."""
|
||||
lo, hi = -(-int(since) // SLOT_S), -(-int(until) // SLOT_S)
|
||||
with self.lock:
|
||||
names = {i: n for n, i in self._ids.items()}
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for aid, name in names.items():
|
||||
if assets is not None and name not in assets:
|
||||
continue
|
||||
rows = self.db.execute("SELECT slot, sum / n FROM slots WHERE aid = ? AND slot >= ? AND slot < ?",
|
||||
(aid, lo, hi)).fetchall()
|
||||
if rows:
|
||||
out[name] = {int(s) * SLOT_S: float(v) for s, v in rows}
|
||||
return out
|
||||
|
||||
def span(self) -> tuple[int | None, int | None, int]:
|
||||
"""The first and last slot kept and how many (all assets)."""
|
||||
with self.lock:
|
||||
a, b, n = self.db.execute("SELECT min(slot), max(slot), count(*) FROM slots").fetchone()
|
||||
return (a * SLOT_S if a is not None else None), (b * SLOT_S if b is not None else None), n
|
||||
|
||||
# -- the weather -------------------------------------------------------------
|
||||
def put_weather(self, kind: str, source: str, hours: list, issued: float | None = None) -> int:
|
||||
"""Hourly records (pvforecast.Hour: end, solar_kwh_m2, cloud_pct,
|
||||
temp_c). A forecast only for hours not over when it was made, the
|
||||
newest such kept; an observation replaces what was there."""
|
||||
rows = []
|
||||
for h in hours:
|
||||
end = int(h.end.timestamp())
|
||||
if end % 3600:
|
||||
continue
|
||||
if kind == "forecast" and (issued is None or issued >= end):
|
||||
continue
|
||||
if h.solar_kwh_m2 is None and h.cloud_pct is None and h.temp_c is None:
|
||||
continue
|
||||
rows.append((end, kind, source, h.solar_kwh_m2, h.cloud_pct, h.temp_c, int(issued) if issued else None))
|
||||
with self.lock:
|
||||
self.db.executemany("INSERT OR REPLACE INTO weather (end_ts, kind, source, solar, cloud, temp, issued) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?)", rows)
|
||||
return len(rows)
|
||||
|
||||
def weather_hours(self, since: int, until: int, kind: str) -> list:
|
||||
"""One record per hour ending in (since, until] of this kind, by the
|
||||
kind's preferred source (OBSERVED_PREF, FORECAST_PREF)."""
|
||||
from .pvforecast import Hour
|
||||
pref = OBSERVED_PREF if kind == "observed" else FORECAST_PREF
|
||||
with self.lock:
|
||||
rows = self.db.execute("SELECT end_ts, source, solar, cloud, temp FROM weather WHERE kind = ? AND end_ts > ? "
|
||||
"AND end_ts <= ?", (kind, int(since), int(until))).fetchall()
|
||||
by: dict[int, tuple] = {}
|
||||
for end, src, sol, cl, tc in rows:
|
||||
rank = pref.index(src) if src in pref else len(pref)
|
||||
if end not in by or rank < by[end][0]:
|
||||
by[end] = (rank, sol, cl, tc)
|
||||
return [Hour(datetime.fromtimestamp(e, timezone.utc), *by[e][1:]) for e in sorted(by)]
|
||||
|
||||
def training_hours(self, since: int, until: int) -> list:
|
||||
"""The weather the models learn from: observed, else (an hour never
|
||||
observed) the last forecast for it (wxfetch.hours, prefer observed)."""
|
||||
obs = {int(h.end.timestamp()): h for h in self.weather_hours(since, until, "observed")}
|
||||
for h in self.weather_hours(since, until, "forecast"):
|
||||
obs.setdefault(int(h.end.timestamp()), h)
|
||||
return [obs[k] for k in sorted(obs)]
|
||||
|
||||
def missing_observed(self, since: int, until: int) -> list[tuple[int, int]]:
|
||||
"""Spans of hours without an observation (for Home Assistant's history)."""
|
||||
with self.lock:
|
||||
have = {e for (e,) in self.db.execute("SELECT end_ts FROM weather WHERE kind = 'observed' AND end_ts > ? "
|
||||
"AND end_ts <= ?", (int(since), int(until)))}
|
||||
spans, cur = [], None
|
||||
for e in range(int(since) // 3600 * 3600 + 3600, int(until) + 1, 3600):
|
||||
if e in have:
|
||||
if cur:
|
||||
spans.append(tuple(cur))
|
||||
cur = None
|
||||
elif cur is None:
|
||||
cur = [e - 3600, e]
|
||||
else:
|
||||
cur[1] = e
|
||||
if cur:
|
||||
spans.append(tuple(cur))
|
||||
return spans
|
||||
|
||||
# -- the forecasts per source -------------------------------------------------
|
||||
def put_forecasts(self, series: str, start: int, cands: dict[str, list], issued: float, first: int = 1,
|
||||
steps: int = 96) -> int:
|
||||
"""Per source its values for the slots start + k·900, k in
|
||||
[first, steps): the slot not begun yet (the quarter under way is
|
||||
not a forecast any more)."""
|
||||
rows = []
|
||||
for src, vals in cands.items():
|
||||
for k in range(first, min(steps, len(vals or []))):
|
||||
v = vals[k]
|
||||
if _finite(v):
|
||||
rows.append((series, int(start) + k * SLOT_S, src, float(v), int(issued)))
|
||||
with self.lock:
|
||||
self.db.executemany("INSERT OR REPLACE INTO fc_slots (series, slot, source, value, issued) VALUES (?, ?, ?, ?, ?)",
|
||||
rows)
|
||||
return len(rows)
|
||||
|
||||
def forecasts(self, series: str, since: int, until: int) -> dict[str, dict[int, float]]:
|
||||
with self.lock:
|
||||
rows = self.db.execute("SELECT source, slot, value FROM fc_slots WHERE series = ? AND slot >= ? AND slot < ?",
|
||||
(series, int(since), int(until))).fetchall()
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for src, s, v in rows:
|
||||
out.setdefault(src, {})[int(s)] = float(v)
|
||||
return out
|
||||
|
||||
# -- meta, pruning ---------------------------------------------------------------
|
||||
def get(self, key: str) -> str | None:
|
||||
with self.lock:
|
||||
row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def set(self, key: str, value: str):
|
||||
with self.lock:
|
||||
self.db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, str(value)))
|
||||
|
||||
def prune(self, now: float, days: int = KEEP_DAYS):
|
||||
cut = int(now) - days * 86400
|
||||
with self.lock:
|
||||
self.db.execute("DELETE FROM slots WHERE slot < ?", (cut // SLOT_S,))
|
||||
self.db.execute("DELETE FROM weather WHERE end_ts < ?", (cut,))
|
||||
self.db.execute("DELETE FROM fc_slots WHERE slot < ?", (int(now) - KEEP_FC_DAYS * 86400,))
|
||||
|
||||
def size_bytes(self) -> int:
|
||||
try:
|
||||
return sum(os.path.getsize(self.path + s) for s in ("", "-wal") if os.path.exists(self.path + s))
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def stats_to_slots(rows: list[dict], period: str) -> dict[int, float]:
|
||||
"""Home Assistant's statistics rows ({"start": unix, "mean"}) as
|
||||
15-minute values: 5-minute means averaged per slot, an hourly mean held
|
||||
for its four slots (a step, not a shape: noted where used)."""
|
||||
out: dict[int, list[float]] = {}
|
||||
for r in rows:
|
||||
v = r.get("mean")
|
||||
if not _finite(v):
|
||||
continue
|
||||
s = int(r["start"])
|
||||
if period == "hour":
|
||||
for q in range(4):
|
||||
out.setdefault(s // 3600 * 3600 + q * SLOT_S, []).append(float(v))
|
||||
else:
|
||||
out.setdefault(s // SLOT_S * SLOT_S, []).append(float(v))
|
||||
return {k: sum(v) / len(v) for k, v in out.items()}
|
||||
@@ -0,0 +1,274 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""Training one series and the rooms: the pure part of the trainer
|
||||
(docs/methods.md B5, B6, B10), shared by the cloud's trainer (trainer.py)
|
||||
and the Edge Agent's own nightly training (edge/cantabile_edge/boxtrain.py:
|
||||
a byte-identical copy, building/test_boxmodel.py checks it). numpy,
|
||||
LightGBM (through fcmodels.fit), no database, no network.
|
||||
|
||||
train_series per series the challengers (how, gbm, pvphys) fitted on
|
||||
the window before the held-out one, scored on it with the
|
||||
model in force (the champion), the best put in force only
|
||||
if it beats the champion (fcmodels.choose)
|
||||
fit_rooms each room's R and C (rcfit.py)
|
||||
base_load load.power less the planned devices' power (methods.md A2-a)
|
||||
held_out_weather, scored_on
|
||||
the held-out window on the weather as it was forecast
|
||||
where that is known (methods.md B5-a)
|
||||
cut_before PV_METERED_SINCE: metered PV before it does not count"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
import numpy as np
|
||||
|
||||
try: # on the box: the Edge Agent's copies (edge/cantabile_edge; methods.md B10)
|
||||
from . import fcmodels, rcfit
|
||||
except ImportError:
|
||||
import fcmodels
|
||||
import rcfit
|
||||
|
||||
log = logging.getLogger("trainer")
|
||||
SLOT_S = 900
|
||||
SITE_SERIES = {"load": "load.power", "pv": "pv.power"}
|
||||
# A site's `base_load` series (methods.md A2-a; the box's planner adds the
|
||||
# devices it plans itself, and load.power includes them): load.power less
|
||||
# the power assets of the kinds of control the site's inventory has, as the
|
||||
# Edge Agent does (mapping.CONTROLLED_POWER). A system's total (ev.power),
|
||||
# else the sum of its parts (ev.1.power, ...). The battery is not here:
|
||||
# load.power does not include its charging (sim physics.py: house = load +
|
||||
# heat pump + A/C + EV). Published as base_load_w next to load_w (as
|
||||
# metered: the coordinator's B7 uses it).
|
||||
BASE = "base_load"
|
||||
CONTROLLED_POWER = {"heating": ("heatpump.power", "heating.power"), "cooling": ("ac.power", "cooling.power"),
|
||||
"ev": ("ev.power",)}
|
||||
CONTROLLED = tuple(a for names in CONTROLLED_POWER.values() for a in names)
|
||||
ROOM_INPUTS = ["weather.outside_temp", "heat.power", "heatpump.power"]
|
||||
MIN_SLOTS = 16 # a series with less metered is not trained
|
||||
SINCE_SERIES = ("pv",) # the series PV_METERED_SINCE cuts
|
||||
|
||||
|
||||
def cut_before(series: str, t: np.ndarray, y: np.ndarray, since: datetime | None) -> np.ndarray:
|
||||
"""y without the metered slots before `since` (NaN) for the series it
|
||||
applies to (SINCE_SERIES)."""
|
||||
if since is None or series not in SINCE_SERIES:
|
||||
return y
|
||||
return np.where(t < int(since.timestamp()), np.nan, y)
|
||||
|
||||
|
||||
def scored_on(w_test_hours: int, n_hours: int) -> str:
|
||||
"""What the held-out scores were made on (methods.md B5-a): the weather
|
||||
as it was forecast where it is stored, else the observed weather."""
|
||||
if n_hours <= 0 or w_test_hours <= 0:
|
||||
return "scored on observed weather (no stored forecast for the held-out window: optimistic)"
|
||||
if w_test_hours >= n_hours:
|
||||
return "scored on forecast weather"
|
||||
return (f"scored on forecast weather for {100 * w_test_hours // n_hours} % of the held-out hours, "
|
||||
"observed for the rest")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Run:
|
||||
series: str
|
||||
kind: str
|
||||
metrics: dict
|
||||
params: dict | None = None
|
||||
chosen: bool = False
|
||||
note: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SeriesResult:
|
||||
series: str
|
||||
runs: list[Run] = field(default_factory=list)
|
||||
champion: dict | None = None # metrics of the model in force on the held-out window
|
||||
baseline: dict | None = None
|
||||
chosen: Run | None = None
|
||||
window: tuple = () # (data_from, data_to, test_from, test_to) unix
|
||||
n_train: int = 0
|
||||
n_test: int = 0
|
||||
note: str = ""
|
||||
|
||||
|
||||
def wslice(w: fcmodels.Weather, sl: slice) -> fcmodels.Weather:
|
||||
return fcmodels.Weather(w.temp[sl], w.ghi[sl], w.cloud[sl], w.elev[sl], w.az[sl], w.hours)
|
||||
|
||||
|
||||
# ---- one series, champion against challengers -----------------------------------
|
||||
|
||||
def train_series(series: str, t: np.ndarray, y: np.ndarray, w: fcmodels.Weather, site: fcmodels.Site,
|
||||
champion: dict | None, kinds=None, w_test: fcmodels.Weather | None = None) -> SeriesResult:
|
||||
"""t, y: the series over the training weeks (and the week before, for
|
||||
the lag), w: the weather on the same slots (observed: what the models
|
||||
learn from). Fits each kind on the window before the held-out one,
|
||||
scores it and the champion's params on the held-out window, picks
|
||||
(fcmodels.choose). w_test: the weather the held-out window is scored
|
||||
on, as it was forecast (methods.md B5-a: the published forecasts run on
|
||||
forecasts, so the choice and the drift baseline should too); else w."""
|
||||
wt = w_test if w_test is not None else w
|
||||
res = SeriesResult(series)
|
||||
have = np.flatnonzero(~np.isnan(y))
|
||||
if len(have) < MIN_SLOTS:
|
||||
res.note = f"{len(have)} metered slots (at least {MIN_SLOTS})"
|
||||
return res
|
||||
cut = fcmodels.holdout(t, y)
|
||||
first = int(have[0])
|
||||
tr, te = slice(first, cut), slice(cut, len(t))
|
||||
res.window = (int(t[first]), int(t[cut - 1]) + SLOT_S if cut > first else int(t[first]), int(t[cut]) if cut < len(t) else None,
|
||||
int(t[-1]) + SLOT_S)
|
||||
res.n_train = int((~np.isnan(y[tr])).sum())
|
||||
res.n_test = int((~np.isnan(y[te])).sum())
|
||||
if res.n_train < MIN_SLOTS or res.n_test < 4:
|
||||
res.note = f"too little data to hold a window out ({res.n_train} slots to train, {res.n_test} to test)"
|
||||
return res
|
||||
kinds = kinds or (["how", "gbm", "pvphys"] if series == "pv" else ["how", "gbm"])
|
||||
hist_t, hist_y = t[:cut], y[:cut]
|
||||
for kind in kinds:
|
||||
try:
|
||||
p = fcmodels.fit(kind, series, t[tr], y[tr], wslice(w, tr), site, t[:first], y[:first])
|
||||
except Exception as e: # one kind's failure never stops the others
|
||||
log.exception("fit %s %s", series, kind)
|
||||
res.runs.append(Run(series, kind, {}, None, note=f"failed: {e}"))
|
||||
continue
|
||||
if p is None:
|
||||
why = {"gbm": "under a day of data", "pvphys": "no location or no weather for it"}.get(kind, "no data")
|
||||
res.runs.append(Run(series, kind, {}, None, note=f"not trained: {why}"))
|
||||
continue
|
||||
pred = fcmodels.predict(p, series, t[te], wslice(wt, te), site, hist_t, hist_y)
|
||||
res.runs.append(Run(series, kind, fcmodels.metrics(y[te], pred), p))
|
||||
base = next((r for r in res.runs if r.kind == "how" and r.params), None)
|
||||
res.baseline = base.metrics if base else None
|
||||
if champion is not None:
|
||||
try:
|
||||
pred = fcmodels.predict(champion, series, t[te], wslice(wt, te), site, hist_t, hist_y)
|
||||
res.champion = fcmodels.metrics(y[te], pred)
|
||||
except Exception as e: # a champion that can no longer run is replaced
|
||||
res.champion = None
|
||||
res.note = f"the model in force could not be scored ({e}): replaced"
|
||||
# nMAE where the metered mean is not ~0, else MAE (a held-out night of PV)
|
||||
key = score_key(res)
|
||||
scores = {r.kind: r.metrics.get(key) for r in res.runs if r.params}
|
||||
champ = res.champion.get(key) if res.champion else None
|
||||
best = fcmodels.choose(scores, champ)
|
||||
if best is None and champion is None and scores:
|
||||
best = min((k for k in scores if scores[k] is not None), key=lambda k: scores[k], default=None)
|
||||
if best:
|
||||
res.chosen = next(r for r in res.runs if r.kind == best and r.params)
|
||||
res.chosen.chosen = True
|
||||
elif champ is not None:
|
||||
res.note = (res.note + "; " if res.note else "") + f"the model in force stays ({key} {champ:.3g})"
|
||||
return res
|
||||
|
||||
|
||||
def score_key(res: SeriesResult) -> str:
|
||||
"""The metric a choice is made on: nMAE where the metered mean of the
|
||||
held-out window is not ~0, else MAE (a held-out night of PV)."""
|
||||
return "nmae" if res.baseline and res.baseline.get("nmae") is not None else "mae"
|
||||
|
||||
|
||||
def score_params(params: dict, series: str, t: np.ndarray, y: np.ndarray, w: fcmodels.Weather, site: fcmodels.Site,
|
||||
w_test: fcmodels.Weather | None = None) -> dict | None:
|
||||
"""Another model's metrics on the same held-out window as train_series
|
||||
cuts it (a benchmark: the cloud's model on the box), or None where it
|
||||
cannot run or there is no window."""
|
||||
have = np.flatnonzero(~np.isnan(y))
|
||||
if len(have) < MIN_SLOTS:
|
||||
return None
|
||||
cut = fcmodels.holdout(t, y)
|
||||
wt = w_test if w_test is not None else w
|
||||
te = slice(cut, len(t))
|
||||
try:
|
||||
pred = fcmodels.predict(params, series, t[te], wslice(wt, te), site, t[:cut], y[:cut])
|
||||
except Exception as e:
|
||||
log.warning("benchmark %s: %s", series, e)
|
||||
return None
|
||||
return fcmodels.metrics(y[te], pred)
|
||||
|
||||
|
||||
# ---- the rooms ---------------------------------------------------------------------
|
||||
|
||||
def fit_rooms(t: np.ndarray, assets: dict[str, np.ndarray], w: fcmodels.Weather) -> dict[str, dict]:
|
||||
"""Each room.<r>.temp against the outside (the site's thermometer, else
|
||||
the weather) and the site's heating (thermal heat.power, else the heat
|
||||
pump's electric power)."""
|
||||
out = {}
|
||||
tout = assets.get("weather.outside_temp")
|
||||
if tout is None or np.isnan(tout).all():
|
||||
tout = w.temp
|
||||
else:
|
||||
tout = np.where(np.isnan(tout), w.temp, tout)
|
||||
if "heat.power" in assets:
|
||||
p, power = assets["heat.power"], "thermal"
|
||||
elif "heatpump.power" in assets:
|
||||
p, power = assets["heatpump.power"], "electric"
|
||||
else:
|
||||
p, power = None, "thermal"
|
||||
for a, theta in sorted(assets.items()):
|
||||
if not (a.startswith("room.") and a.endswith(".temp")):
|
||||
continue
|
||||
room = a[len("room."):-len(".temp")]
|
||||
if p is None:
|
||||
out[room] = {**rcfit.fit(theta, tout, np.full(len(theta), np.nan)), "note": "no heating power metered"}
|
||||
continue
|
||||
out[room] = rcfit.fit(theta, tout, p, w.ghi, power)
|
||||
return out
|
||||
|
||||
|
||||
def room_inputs(assets: dict[str, np.ndarray], w: fcmodels.Weather) -> tuple:
|
||||
"""The inputs fit_rooms fits every room on: (outside °C, heating power
|
||||
W or None, irradiance W/m²), for scoring other R and C on the same data."""
|
||||
tout = assets.get("weather.outside_temp")
|
||||
tout = w.temp if tout is None or np.isnan(tout).all() else np.where(np.isnan(tout), w.temp, tout)
|
||||
p = assets.get("heat.power", assets.get("heatpump.power"))
|
||||
return tout, p, w.ghi
|
||||
|
||||
|
||||
# ---- the base load -----------------------------------------------------------------
|
||||
|
||||
def _system(asset: str) -> str:
|
||||
return asset.split(".", 1)[0]
|
||||
|
||||
|
||||
def controlled_power(raw: dict[str, np.ndarray], kinds: set[str], n: int) -> np.ndarray:
|
||||
"""The planned devices' metered power per slot (W): per kind its assets
|
||||
(a system's total, else the sum of its parts <system>.<n>.power), each
|
||||
once; a slot without a value counts 0."""
|
||||
dev = np.zeros(n)
|
||||
for kind in sorted(kinds):
|
||||
for name in CONTROLLED_POWER.get(kind, ()):
|
||||
sysname, q = name.split(".", 1)
|
||||
parts = [a for a in raw if a.startswith(sysname + ".") and a.endswith("." + q)
|
||||
and a[len(sysname) + 1:-len(q) - 1].isdigit()]
|
||||
for a in ([name] if name in raw else sorted(parts)):
|
||||
dev = dev + np.nan_to_num(np.maximum(raw[a], 0.0))
|
||||
return dev
|
||||
|
||||
|
||||
def base_load(raw: dict[str, np.ndarray], kinds: set[str]) -> np.ndarray | None:
|
||||
"""load.power less the planned devices' metered power (controlled_power),
|
||||
never below 0 (methods.md A2-a); NaN where load.power is."""
|
||||
load = raw.get(SITE_SERIES["load"])
|
||||
if load is None:
|
||||
return None
|
||||
return np.where(np.isnan(load), np.nan, np.maximum(load - controlled_power(raw, kinds, len(load)), 0.0))
|
||||
|
||||
|
||||
# ---- the weather the held-out window is scored on -----------------------------------
|
||||
|
||||
def held_out_weather(t: np.ndarray, y: np.ndarray, w: fcmodels.Weather, fc_hours: list, lat: float | None,
|
||||
lon: float | None) -> tuple[fcmodels.Weather | None, str]:
|
||||
"""The weather the held-out window (fcmodels.holdout, as train_series
|
||||
cuts it) is scored on: the observed hours with every hour that was
|
||||
forecast put in their place, and what that was (scored_on). None where
|
||||
nothing was forecast: the observed weather."""
|
||||
if not fc_hours or lat is None or lon is None or (~np.isnan(y)).sum() < MIN_SLOTS:
|
||||
return None, scored_on(0, 0)
|
||||
cut = fcmodels.holdout(t, y)
|
||||
test_ends = {(int(ts) // 3600 + 1) * 3600 for ts in t[cut:].tolist()}
|
||||
fc = {int(h.end.timestamp()): h for h in fc_hours}
|
||||
n_fc = len(test_ends & set(fc))
|
||||
if n_fc == 0:
|
||||
return None, scored_on(0, len(test_ends))
|
||||
merged = {int(h.end.timestamp()): h for h in w.hours} | fc
|
||||
return fcmodels.weather(t, [merged[k] for k in sorted(merged)], lat, lon), scored_on(n_fc, len(test_ends))
|
||||
Reference in new issue
Block a user