cantabile_edge, cantabile_modbus, cantabile_eebus 0.1.676: full source
This commit is contained in:
1 parent
3edd015039
commit
d48b9aff87
32 files changed
+4617
-117
No files matched your search
@@ -22,7 +22,7 @@ Settings → Add-ons → Add-on store → ⋮ → Repositories, add
|
||||
`https://git.setti.solutions/Daniel/cantabile-ha-addons`, then install the
|
||||
add-on. The Supervisor pulls the prebuilt images
|
||||
(`git.setti.solutions/daniel/cantabile-ha-<edge|modbus|eebus>-<arch>:<version>`,
|
||||
named by `image:` in each `config.yaml`); this version: `0.1.650`, arch amd64.
|
||||
named by `image:` in each `config.yaml`); this version: `0.1.676`, arch amd64.
|
||||
|
||||
## Layout
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# the Hub's, wrapped by addon-run (Dockerfile). The image is prebuilt,
|
||||
# per arch (README.md: building it from src/).
|
||||
name: Cantabile Edge Agent
|
||||
version: "0.1.650"
|
||||
version: "0.1.676"
|
||||
slug: cantabile_edge
|
||||
description: Plans your home's energy with Cantabile and, with your consent, runs the devices you choose.
|
||||
url: https://cantabile.cloud
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# Assistant that isn't a Cantabile Hub. Devices are paired on its page
|
||||
# (ingress), by their SKI.
|
||||
name: Cantabile EEBUS
|
||||
version: "0.1.650"
|
||||
version: "0.1.676"
|
||||
slug: cantabile_eebus
|
||||
description: Pairs with EEBUS devices (the grid operator's control box, heat pumps, wallboxes, inverters), shows them in Home Assistant under Cantabile's names and hands the grid operator's §14a limit to the Cantabile agent; every control with its protection.
|
||||
url: https://cantabile.cloud
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# Home Assistant that isn't a Cantabile Hub. Devices come from the options;
|
||||
# the page (ingress) shows them and each control's protection.
|
||||
name: Cantabile Modbus bridge
|
||||
version: "0.1.650"
|
||||
version: "0.1.676"
|
||||
slug: cantabile_modbus
|
||||
description: Reads inverters, meters, relays and circuits by Cantabile's device profiles (Modbus, or Home Assistant's own entities) and shows them in Home Assistant, each control with its protection.
|
||||
url: https://cantabile.cloud
|
||||
|
||||
@@ -17,6 +17,9 @@ RUN CGO_ENABLED=0 GOARCH=$TARGETARCH GOAMD64=v1 go build -trimpath -ldflags="-s
|
||||
|
||||
FROM python:3.12.12-slim-bookworm
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
# LightGBM's OpenMP runtime: the box trains its own models (boxtrain.py,
|
||||
# methods.md B10), as the cloud's trainer image does (building/Dockerfile)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 && rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /src
|
||||
COPY pyproject.toml ./
|
||||
COPY cantabile_edge ./cantabile_edge
|
||||
|
||||
@@ -15,7 +15,7 @@ import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from . import areas, ha, propose, sinks
|
||||
from . import areas, ha, propose, sinks, trainhist
|
||||
from .announce import Announcer
|
||||
from .consent import Consent
|
||||
from .controller import Controller
|
||||
@@ -62,6 +62,15 @@ class Agent:
|
||||
self.control = Controller(cfg, mapping, self.consent, sink) if mapping.controls else None
|
||||
if self.control is not None and hasattr(sink, "on_command"):
|
||||
sink.on_command = self.control.command
|
||||
# the box's own training history (trainhist.py): its own 1-minute means,
|
||||
# with or without the share consent: they never leave the box (nothing
|
||||
# of it goes upward without that consent); only the consented values
|
||||
# go to the cloud (self.agg)
|
||||
self.train_agg = None
|
||||
self.tparts: dict[str, float | None] = {}
|
||||
if self.control is not None and getattr(self.control, "boxml", None) is not None:
|
||||
self.control.boxml.mapping = mapping # HA's statistics into its history (boxtrain.fill_from_ha)
|
||||
self.train_agg = Aggregator(cfg.interval_s)
|
||||
# the grid operator's orders (EEBUS LPC, gridorder.py): the controller's,
|
||||
# or, on a site without controls, checked and told to the cloud only
|
||||
self.gridorders = self.control.gridorders if self.control is not None else GridOrders(cfg.state_dir)
|
||||
@@ -169,9 +178,15 @@ class Agent:
|
||||
return self.last_sent > 0
|
||||
|
||||
def sample(self, entity_id: str, state: str | None, ts: float):
|
||||
if not self.consent.share:
|
||||
share = self.consent.share # without it nothing is collected to send
|
||||
local = self.train_agg is not None
|
||||
if not share and not local:
|
||||
return # the owner has not (or no longer) agreed
|
||||
for a in self.mapping.consented_all(entity_id): # none: not mapped or no consent, never leaves
|
||||
for a in self.mapping.of_entity.get(entity_id, []):
|
||||
up = share and a.consent # to the cloud: the site's and the asset's consent
|
||||
mine = local and trainhist.wanted(a.asset) # the box's own training: never leaves the box
|
||||
if not (up or mine):
|
||||
continue
|
||||
raw = value_of(a, state, self.attrs.get(entity_id))
|
||||
v = number(raw)
|
||||
if v is not None and not a.equals:
|
||||
@@ -180,24 +195,33 @@ class Agent:
|
||||
v = scaled(convert(v, unit, a.unit), a)
|
||||
if v is None:
|
||||
log.warning("%s: unit %s cannot become %s; value dropped", a.asset, unit, a.unit)
|
||||
self.agg.sample(a.asset, a.kind, v, ts)
|
||||
total = self.mapping.part_total.get(a.asset)
|
||||
if up:
|
||||
self.agg.sample(a.asset, a.kind, v, ts)
|
||||
if total:
|
||||
self.sample_total(self.mapping.totals[total], a.asset, v, ts)
|
||||
if local and a.kind == "gauge" and (mine or (total and trainhist.wanted(total))):
|
||||
if mine:
|
||||
self.train_agg.sample(a.asset, a.kind, v, ts)
|
||||
if total and trainhist.wanted(total):
|
||||
self.sample_total(self.mapping.totals[total], a.asset, v, ts, self.train_agg, self.tparts, False)
|
||||
|
||||
def sample_total(self, t, part: str, v: float | None, ts: float):
|
||||
def sample_total(self, t, part: str, v: float | None, ts: float, agg=None, store=None, consented: bool = True):
|
||||
"""A total of parts (pv.power from pv.1.power, pv.2.power: mapping.py)
|
||||
from its consented parts' last values: their sum (the mean for % and
|
||||
°C); unknown while a part is (a counter then waits: a missing part
|
||||
must not look like a meter reset)."""
|
||||
self.parts[part] = v
|
||||
parts = [p for p in t.parts if getattr(self.mapping.by_asset.get(p), "consent", False)]
|
||||
vals = [self.parts.get(p) for p in parts]
|
||||
must not look like a meter reset). agg, store, consented: the box's
|
||||
own training history's (every part, consented or not)."""
|
||||
agg = self.agg if agg is None else agg
|
||||
store = self.parts if store is None else store
|
||||
store[part] = v
|
||||
parts = [p for p in t.parts if not consented or getattr(self.mapping.by_asset.get(p), "consent", False)]
|
||||
vals = [store.get(p) for p in parts]
|
||||
if not vals or None in vals:
|
||||
if t.kind == "gauge":
|
||||
self.agg.sample(t.asset, t.kind, None, ts)
|
||||
agg.sample(t.asset, t.kind, None, ts)
|
||||
return
|
||||
self.agg.sample(t.asset, t.kind, sum(vals) / (len(vals) if t.mean else 1), ts)
|
||||
agg.sample(t.asset, t.kind, sum(vals) / (len(vals) if t.mean else 1), ts)
|
||||
|
||||
def on_entities(self, ev: dict):
|
||||
now = self.clock()
|
||||
@@ -229,6 +253,9 @@ class Agent:
|
||||
except Exception as e: # never keeps the agent from reading
|
||||
log.warning("mapping: %s", e)
|
||||
ids = list(dict.fromkeys(a.entity for a in self.mapping.assets if a.consent)) # one entity may feed several
|
||||
if self.train_agg is not None: # and the box's own training's (never sent: sample())
|
||||
ids = list(dict.fromkeys(ids + [a.entity for a in self.mapping.assets if trainhist.wanted(a.asset)
|
||||
or trainhist.wanted(self.mapping.part_total.get(a.asset, ""))]))
|
||||
await client.subscribe_entities(ids, self.on_entities)
|
||||
log.info("subscribed to %d consented entities (mapping v%d)", len(ids), self.mapping.version)
|
||||
if self.control is not None:
|
||||
@@ -258,6 +285,13 @@ class Agent:
|
||||
def close_window(self, end: float):
|
||||
start = self.agg.start
|
||||
records = self.agg.flush(end)
|
||||
if self.train_agg is not None:
|
||||
tstart, trecs = self.train_agg.start, self.train_agg.flush(end)
|
||||
try: # the box's own training history (trainhist.py): 15-minute means, kept 13 months
|
||||
if trecs:
|
||||
self.control.boxml.store.add_window(tstart, trecs)
|
||||
except Exception as e:
|
||||
log.warning("training history: %s", e)
|
||||
if not records:
|
||||
return
|
||||
seq = self.queue.put("tel", {
|
||||
@@ -327,6 +361,8 @@ class Agent:
|
||||
async def run(self):
|
||||
self.started = time.time()
|
||||
self.agg.begin(self.started)
|
||||
if self.train_agg is not None:
|
||||
self.train_agg.begin(self.started)
|
||||
if self.cfg.mode == "off":
|
||||
log.info("mode off: not reading Home Assistant")
|
||||
await self.stop.wait()
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
"Cantabile" (one per site) under HA's MQTT integration, in the area
|
||||
"Cantabile" (suggested_area, in HA's language: names.py), with
|
||||
availability (offline when the agent stops: MQTT's last will, or said
|
||||
before a clean stop). Read only: nothing in HA changes the agent; the
|
||||
owner's consent is given in the box's setup wizard and panel only (sensitive).
|
||||
before a clean stop). Nothing in HA changes the agent but the owner's
|
||||
preferences (below); the owner's consent is given
|
||||
in the box's setup wizard and panel only (sensitive),
|
||||
and the mode only from the console.
|
||||
|
||||
What it shows (what goes between the cloud or the
|
||||
building and the home, and what the agent does to each device):
|
||||
@@ -36,6 +38,19 @@ building and the home, and what the agent does to each device):
|
||||
actuator its contract target vs its state now;
|
||||
sensors only, unavailable when the bundle is
|
||||
stale (cantabile/<site>/market/availability)
|
||||
the owner's preferences (preferences.py; every mode but off; entity
|
||||
category config, usable in automations; HA's MQTT integration has no
|
||||
time or date entity: times are text in the preferences' text forms):
|
||||
text pref_<room>_schedule "Mo-Fr 22:00-06:00 21 must; ..." (empty: none)
|
||||
number pref_<room>_boost_c, pref_<room>_boost_h, switch pref_<room>_boost
|
||||
switch pref_away, text pref_away_until ("2026-11-10"; empty: until off)
|
||||
text pref_<car>_ready "Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct"
|
||||
number pref_<car>_capacity (kWh, for %), text pref_<car>_needed (one-off)
|
||||
number pref_<battery>_reserve (%), text pref_<battery>_reserve_blocks
|
||||
sensor pref_<control>_result, preferences (advisory and auto, with the
|
||||
plan): per preference met or not and each must's cost
|
||||
Commands on cantabile/<site>/prefs/<object>/set (the owner's, "Home
|
||||
Assistant"); states on cantabile/<site>/prefs and /prefs/result.
|
||||
On a building's own box (flats.json, a signed `flats` bundle from the
|
||||
cloud: bundles.py) each flat is a device of its own ("Flat", via the
|
||||
building's Cantabile device, area "Flats"): its grid power, its
|
||||
@@ -64,7 +79,8 @@ log = logging.getLogger(__name__)
|
||||
INTERVAL = 30 # state at least this often (seconds)
|
||||
RETIRED = (("switch", "share"), ("sensor", "queued"), ("sensor", "last_sent"), ("sensor", "mode"))
|
||||
RECEIVED = 10 # commands listed in HA (sensor received)
|
||||
COMMANDS = ("mode", "control", "release", "limits", "tariff", "model", "building")
|
||||
COMMANDS = ("mode", "control", "release", "limits", "tariff", "model", "building", "preferences")
|
||||
PREF_TOPIC = re.compile(r"^cantabile/[^/]+/prefs/([^/]+)/set$")
|
||||
MAX_FLATS = 200
|
||||
|
||||
|
||||
@@ -86,9 +102,15 @@ class Announcer:
|
||||
self.sent: dict[tuple, str] = {} # (node, component, object) -> config sent (JSON)
|
||||
ctl = getattr(agent, "control", None)
|
||||
self.bundles = ctl.bundles if ctl is not None else Bundles(agent.cfg.state_dir, site)
|
||||
self.pref_cmds: dict[str, tuple[str, str]] = {} # object -> (control id, what): the commands HA may send
|
||||
if ctl is not None:
|
||||
ctl.on_plan.append(self._planned)
|
||||
ctl.on_received.append(self._received)
|
||||
if hasattr(ctl, "on_prefs"):
|
||||
ctl.on_prefs.append(self._prefs_changed)
|
||||
|
||||
async def _prefs_changed(self):
|
||||
self.changed.set()
|
||||
|
||||
@property
|
||||
def lang(self) -> str:
|
||||
@@ -394,6 +416,193 @@ class Announcer:
|
||||
"last": a.get("last"), "shadow": True}
|
||||
return out
|
||||
|
||||
# -- the owner's preferences (preferences.py) -------------------------------
|
||||
def _pref_controls(self):
|
||||
ctl = getattr(self.agent, "control", None)
|
||||
if ctl is None or not hasattr(ctl, "prefs") or ctl.mode() == "off":
|
||||
return None, []
|
||||
return ctl, [c for c in ctl.controls if c.kind in ("heating", "cooling", "ev", "battery")]
|
||||
|
||||
def _prefs_objects(self) -> dict:
|
||||
"""The preferences' entities: settings in every mode but off, their
|
||||
results with the plan (advisory, auto)."""
|
||||
ctl, cs = self._pref_controls()
|
||||
if not cs:
|
||||
self.pref_cmds = {}
|
||||
return {}
|
||||
lang, dev, top = self.lang, self._device(), f"{self.base}/prefs"
|
||||
common = {"state_topic": top, "entity_category": "config", **self._avail(), "device": dev}
|
||||
out, cmds = {}, {}
|
||||
|
||||
def add(comp, obj, cid, what, name, **conf):
|
||||
cmds[obj] = (cid, what)
|
||||
out[(comp, obj)] = {"name": name, "unique_id": f"{self.uid}_{obj}", "command_topic": f"{top}/{obj}/set",
|
||||
"value_template": "{{ value_json['%s'] }}" % obj, **common, **conf}
|
||||
rooms = [c for c in cs if c.kind in ("heating", "cooling")]
|
||||
for c in rooms:
|
||||
lo, hi = (16.0, 32.0) if c.kind == "cooling" else (5.0, 30.0)
|
||||
k, n = _slug(c.id), c.label()
|
||||
add("text", f"pref_{k}_schedule", c.id, "schedule", names.name("pref_schedule_cool" if c.kind == "cooling"
|
||||
else "pref_schedule", lang, n),
|
||||
max=255, mode="text", icon="mdi:calendar-clock")
|
||||
add("number", f"pref_{k}_boost_c", c.id, "boost_c", names.name("pref_boost_c", lang, n), min=lo, max=hi,
|
||||
step=0.5, mode="box", unit_of_measurement="°C", icon="mdi:thermometer-chevron-up")
|
||||
add("number", f"pref_{k}_boost_h", c.id, "boost_h", names.name("pref_boost_h", lang, n), min=0.5, max=24,
|
||||
step=0.5, mode="box", unit_of_measurement="h", icon="mdi:timer-outline")
|
||||
add("switch", f"pref_{k}_boost", c.id, "boost", names.name("pref_boost", lang, n),
|
||||
payload_on="ON", payload_off="OFF", state_on="ON", state_off="OFF", icon="mdi:fire")
|
||||
if rooms:
|
||||
add("switch", "pref_away", "", "away", names.name("pref_away", lang),
|
||||
payload_on="ON", payload_off="OFF", state_on="ON", state_off="OFF", icon="mdi:home-export-outline")
|
||||
add("text", "pref_away_until", "", "away_until", names.name("pref_away_until", lang), max=40, mode="text",
|
||||
icon="mdi:calendar-arrow-right")
|
||||
for c in cs:
|
||||
k, n = _slug(c.id), c.label()
|
||||
if c.kind == "ev":
|
||||
add("text", f"pref_{k}_ready", c.id, "ready", names.name("pref_ready", lang, n), max=255, mode="text",
|
||||
icon="mdi:car-clock")
|
||||
add("number", f"pref_{k}_capacity", c.id, "capacity", names.name("pref_capacity", lang, n), min=1,
|
||||
max=250, step=0.5, mode="box", unit_of_measurement="kWh", icon="mdi:car-battery")
|
||||
add("text", f"pref_{k}_needed", c.id, "needed", names.name("pref_needed", lang, n), max=80, mode="text",
|
||||
icon="mdi:car-arrow-right")
|
||||
elif c.kind == "battery":
|
||||
add("number", f"pref_{k}_reserve", c.id, "reserve", names.name("pref_reserve", lang, n), min=0, max=100,
|
||||
step=1, mode="box", unit_of_measurement="%", icon="mdi:battery-lock")
|
||||
add("text", f"pref_{k}_reserve_blocks", c.id, "reserve_blocks", names.name("pref_reserve_blocks", lang, n),
|
||||
max=255, mode="text", icon="mdi:battery-clock")
|
||||
self.pref_cmds = cmds
|
||||
if ctl.mode() in ("advisory", "auto"):
|
||||
res = {"state_topic": f"{top}/result", "json_attributes_topic": f"{top}/result", **self._avail(), "device": dev}
|
||||
out[("sensor", "preferences")] = {
|
||||
"name": names.name("pref_result", lang), "unique_id": f"{self.uid}_preferences", "icon": "mdi:clipboard-check",
|
||||
"value_template": "{{ value_json.summary }}",
|
||||
"json_attributes_template": "{{ {'results': value_json.lines, 'last_error': value_json.last_error} | tojson }}",
|
||||
**res}
|
||||
for c in cs:
|
||||
k = _slug(c.id)
|
||||
out[("sensor", f"pref_{k}_result")] = {
|
||||
"name": names.name("pref_result_of", lang, c.label()), "unique_id": f"{self.uid}_pref_{k}_result",
|
||||
"icon": "mdi:clipboard-check-outline",
|
||||
"value_template": "{{ value_json.controls['%s'].state }}" % c.id,
|
||||
"json_attributes_template": "{{ value_json.controls['%s'] | tojson }}" % c.id, **res}
|
||||
return out
|
||||
|
||||
def prefs_state(self) -> dict:
|
||||
"""The settings' values as HA shows them (cantabile/<site>/prefs)."""
|
||||
ctl, cs = self._pref_controls()
|
||||
if not cs:
|
||||
return {}
|
||||
from . import preferences as pm
|
||||
p, out = ctl.prefs, {}
|
||||
act = p.active_overrides()
|
||||
for c in cs:
|
||||
k = _slug(c.id)
|
||||
if c.kind in ("heating", "cooling"):
|
||||
out[f"pref_{k}_schedule"] = pm.room_text(p.value(f"room:{c.id}"))
|
||||
b = next((o for o in act if o["kind"] == "boost" and o.get("control") == c.id), None)
|
||||
out[f"pref_{k}_boost_c"] = b["c"] if b else self._boost(p, c.id)["c"]
|
||||
out[f"pref_{k}_boost_h"] = p.ha.get(f"boost_h:{c.id}", 2.0)
|
||||
out[f"pref_{k}_boost"] = "ON" if b else "OFF"
|
||||
elif c.kind == "ev":
|
||||
v = p.value(f"ev:{c.id}") or {}
|
||||
out[f"pref_{k}_ready"] = pm.ev_text(v)
|
||||
out[f"pref_{k}_capacity"] = v.get("capacity_kwh")
|
||||
car = next((o for o in act if o["kind"] == "car" and o.get("control") == c.id), None)
|
||||
out[f"pref_{k}_needed"] = pm.car_text(car, p.tz) if car else ""
|
||||
elif c.kind == "battery":
|
||||
v = p.value(f"battery:{c.id}") or {}
|
||||
out[f"pref_{k}_reserve"] = v.get("reserve_pct", 0.0)
|
||||
out[f"pref_{k}_reserve_blocks"] = pm.battery_text(v)
|
||||
if any(c.kind in ("heating", "cooling") for c in cs):
|
||||
away = next((o for o in act if o["kind"] == "away"), None)
|
||||
out["pref_away"] = "ON" if away else "OFF"
|
||||
out["pref_away_until"] = p.ha.get("away_until", "")
|
||||
return out
|
||||
|
||||
def prefs_result(self) -> dict:
|
||||
"""The plan's feedback per control (cantabile/<site>/prefs/result)."""
|
||||
ctl, cs = self._pref_controls()
|
||||
p = ctl.prefs if ctl is not None else None
|
||||
fb = p.feedback if p is not None else []
|
||||
per = {}
|
||||
for c in cs:
|
||||
mine = [f for f in fb if f.get("control") == c.id]
|
||||
musts = [f for f in mine if f.get("how") == "must" and f.get("met") is not None]
|
||||
state = "none" if not mine else "not met" if any(f.get("met") is False for f in mine) else \
|
||||
"met" if any(f.get("met") for f in mine) else "open"
|
||||
per[c.id] = {"state": state, "lines": [f["text"] for f in mine],
|
||||
"cost_eur_day": round(sum(f.get("cost_eur_day") or 0.0 for f in musts), 2) if musts else None,
|
||||
"results": mine}
|
||||
bad = [f for f in fb if f.get("met") is False]
|
||||
last = (p.errors[-1] if p is not None and p.errors else None)
|
||||
return {"summary": "none" if not fb else f"{len(bad)} not met" if bad else "met",
|
||||
"lines": [f["text"] for f in fb], "controls": per,
|
||||
"last_error": f"{last['item']}: {last['why']}" if last else ""}
|
||||
|
||||
async def pref_message(self, obj: str, payload: bytes):
|
||||
"""A preference set in HA (its entity's command): as the owner's."""
|
||||
ctl = getattr(self.agent, "control", None)
|
||||
if ctl is None or obj not in self.pref_cmds:
|
||||
return
|
||||
cid, what = self.pref_cmds[obj]
|
||||
text = payload.decode(errors="replace").strip()
|
||||
p, by = ctl.prefs, "Home Assistant"
|
||||
msg = None
|
||||
if what == "schedule":
|
||||
msg = {"set": {"item": f"room:{cid}", "value": text or None}}
|
||||
elif what in ("boost_c", "boost_h"):
|
||||
try:
|
||||
p.ha[f"{what}:{cid}"] = float(text)
|
||||
except ValueError:
|
||||
return
|
||||
p.save()
|
||||
b = next((o for o in p.active_overrides() if o["kind"] == "boost" and o.get("control") == cid), None)
|
||||
if b is None:
|
||||
self.changed.set()
|
||||
return
|
||||
msg = {"override": self._boost(p, cid)}
|
||||
elif what == "boost":
|
||||
msg = {"override": self._boost(p, cid)} if text.upper() == "ON" else {"cancel": {"kind": "boost", "control": cid}}
|
||||
elif what == "away":
|
||||
msg = {"override": {"kind": "away", "until_text": p.ha.get("away_until", "")}} if text.upper() == "ON" \
|
||||
else {"cancel": {"kind": "away", "control": ""}}
|
||||
elif what == "away_until":
|
||||
p.ha["away_until"] = text[:40]
|
||||
p.save()
|
||||
if not any(o["kind"] == "away" for o in p.active_overrides()):
|
||||
self.changed.set()
|
||||
return
|
||||
msg = {"override": {"kind": "away", "until_text": text}}
|
||||
elif what in ("ready", "capacity"):
|
||||
v = dict(p.value(f"ev:{cid}") or {"ready": [], "capacity_kwh": None})
|
||||
if what == "capacity":
|
||||
v["capacity_kwh"] = text if text not in ("", "None") else None
|
||||
msg = {"set": {"item": f"ev:{cid}", "value": v}}
|
||||
else:
|
||||
msg = {"set": {"item": f"ev:{cid}", "value": text}} if text else \
|
||||
{"set": {"item": f"ev:{cid}", "value": dict(v, ready=[]) if v.get("capacity_kwh") else None}}
|
||||
elif what == "needed":
|
||||
msg = {"override": {"kind": "car", "control": cid, "text": text}}
|
||||
elif what in ("reserve", "reserve_blocks"):
|
||||
v = dict(p.value(f"battery:{cid}") or {"reserve_pct": 0.0, "blocks": []})
|
||||
if what == "reserve":
|
||||
v["reserve_pct"] = text
|
||||
msg = {"set": {"item": f"battery:{cid}", "value": v}}
|
||||
else:
|
||||
msg = {"set": {"item": f"battery:{cid}", "value": text}} if text else \
|
||||
{"set": {"item": f"battery:{cid}", "value": dict(v, blocks=[])}}
|
||||
if msg is not None:
|
||||
await ctl.prefs_change(msg, "owner", by)
|
||||
self.changed.set()
|
||||
|
||||
def _boost(self, p, cid) -> dict:
|
||||
"""A boost with HA's numbers (their defaults: comfort + 1 °C, 2 h)."""
|
||||
ctl = self.agent.control
|
||||
c = next((c for c in ctl.controls if c.id == cid), None)
|
||||
default = float(c.p("comfort_c")) + (-1.0 if c is not None and c.kind == "cooling" else 1.0) if c else 21.0
|
||||
return {"kind": "boost", "control": cid, "c": p.ha.get(f"boost_c:{cid}", default),
|
||||
"hours": p.ha.get(f"boost_h:{cid}", 2.0)}
|
||||
|
||||
def objects(self) -> dict:
|
||||
"""Every discovery config HA should have now: (node, component,
|
||||
object) -> config; node is the site's or a flat's."""
|
||||
@@ -401,7 +610,7 @@ class Announcer:
|
||||
tariff, model = self.bundles.tariff_now(now), self.bundles.model_info()
|
||||
out = {}
|
||||
for (comp, obj), conf in {**self._configs(), **self._plan_objects(),
|
||||
**self._exchange_objects(tariff, model),
|
||||
**self._exchange_objects(tariff, model), **self._prefs_objects(),
|
||||
**self._market_objects(self.bundles.market_now(now))}.items():
|
||||
out[(self.uid, comp, obj)] = conf
|
||||
out.update(self._flat_objects(self.bundles.flat_list()))
|
||||
@@ -495,6 +704,11 @@ class Announcer:
|
||||
await c.publish(f"{self.base}/model", json.dumps(model), retain=True)
|
||||
if ctl is not None and ctl.mode() != "off":
|
||||
await c.publish(f"{self.base}/actuators", json.dumps(ctl.actuators(), default=str), retain=True)
|
||||
prefs = self.prefs_state()
|
||||
if prefs:
|
||||
await c.publish(f"{self.base}/prefs", json.dumps(prefs, default=str), retain=True)
|
||||
if ctl.mode() in ("advisory", "auto"):
|
||||
await c.publish(f"{self.base}/prefs/result", json.dumps(self.prefs_result(), default=str), retain=True)
|
||||
if ctl is not None and ctl.received:
|
||||
# kept (retained) across restarts until new ones come
|
||||
await c.publish(f"{self.base}/received", json.dumps({"commands": self.received_list(ctl)}), retain=True)
|
||||
@@ -540,19 +754,35 @@ class Announcer:
|
||||
|
||||
async def _grid_orders(self, c):
|
||||
"""The EEBUS bridge's grid order (gridorder.py), to the
|
||||
controller: the grid operator's limit as a higher-rank decision."""
|
||||
controller: the grid operator's limit as a higher-rank decision; and
|
||||
the preferences HA sets (pref_message)."""
|
||||
take = getattr(self.agent, "grid_order", None)
|
||||
if take is None:
|
||||
return
|
||||
prefs = getattr(getattr(self.agent, "control", None), "prefs", None) is not None
|
||||
|
||||
def on(topic, payload):
|
||||
if topic == gridorder.TOPIC and payload:
|
||||
if topic == gridorder.TOPIC and payload and take is not None:
|
||||
asyncio.ensure_future(take(payload))
|
||||
return
|
||||
m = PREF_TOPIC.match(topic)
|
||||
if m and topic.startswith(f"{self.base}/prefs/") and prefs:
|
||||
asyncio.ensure_future(self._pref_safe(m.group(1), payload))
|
||||
c.on_message = on
|
||||
if take is not None:
|
||||
try:
|
||||
await c.subscribe(gridorder.TOPIC)
|
||||
except Exception as e:
|
||||
log.info("mqtt: grid orders not subscribed: %s", e)
|
||||
if prefs:
|
||||
try:
|
||||
await c.subscribe(f"{self.base}/prefs/+/set")
|
||||
except Exception as e:
|
||||
log.info("mqtt: preferences not subscribed: %s", e)
|
||||
|
||||
async def _pref_safe(self, obj, payload):
|
||||
try:
|
||||
await self.pref_message(obj, payload)
|
||||
except Exception as e:
|
||||
log.warning("preferences from HA (%s): %s", obj, e)
|
||||
|
||||
def _password(self) -> str:
|
||||
if self.password_file:
|
||||
|
||||
@@ -56,6 +56,40 @@ async def backfill(client: HAClient, mapping, queue, since: datetime, until: dat
|
||||
return batches
|
||||
|
||||
|
||||
async def collect(client: HAClient, mapping, since: datetime, until: datetime, period: str = "hour",
|
||||
want=None, consented: bool = True) -> dict[str, list[dict]]:
|
||||
"""HA's statistics of the gauges (want(asset): which; consented: only
|
||||
those with consent) as {asset: [{"start": unix, "mean"}]} in the assets'
|
||||
units: for the box's own training history (trainhist.py, every mapped
|
||||
gauge: nothing of it is queued or sent). A total of parts (pv.power from
|
||||
pv.1.power, ...) has no entity of its own: left out."""
|
||||
assets = {a.entity: a for a in mapping.assets
|
||||
if (a.consent or not consented) and a.kind == "gauge" and not a.attribute and not a.equals
|
||||
and (want is None or want(a.asset))}
|
||||
if not assets:
|
||||
return {}
|
||||
meta = await client.send("recorder/list_statistic_ids", statistic_type="mean")
|
||||
units = {m["statistic_id"]: m.get("statistics_unit_of_measurement") for m in meta}
|
||||
ids = [e for e in assets if e in units]
|
||||
out: dict[str, list[dict]] = {}
|
||||
start = since
|
||||
while start < until and ids:
|
||||
end = min(start + CHUNK, until)
|
||||
res = await client.send("recorder/statistics_during_period", timeout=300, start_time=start.isoformat(),
|
||||
end_time=end.isoformat(), statistic_ids=ids, period=period, types=["mean"])
|
||||
for eid, rows in (res or {}).items():
|
||||
a, unit = assets.get(eid), units.get(eid)
|
||||
if a is None:
|
||||
continue
|
||||
for r in rows:
|
||||
v = number(r.get("mean"))
|
||||
v = None if v is None else scaled(convert(v, unit, a.unit), a)
|
||||
if v is not None:
|
||||
out.setdefault(a.asset, []).append({"start": _ts(r["start"]), "mean": v})
|
||||
start = end
|
||||
return out
|
||||
|
||||
|
||||
def parse_since(s: str) -> datetime:
|
||||
"""An ISO date/time (UTC unless it says otherwise) or days back, e.g. 30d."""
|
||||
if s.endswith("d") and s[:-1].isdigit():
|
||||
|
||||
@@ -0,0 +1,809 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The box trains its own models (docs/methods.md B10; step 2 of the
|
||||
distributed models).
|
||||
|
||||
Nightly (cfg.train_at, default 03:30 in Home Assistant's time zone) from
|
||||
the agent's own training history (trainhist.py, 13 months, recent weeks weighing more): per series
|
||||
(load, pv, and base_load where the box plans devices) the cloud's own code
|
||||
(trainlib.train_series: how, gbm, pvphys; byte-identical copies of
|
||||
building/'s trainlib.py, fcmodels.py, pvcalib.py, rcfit.py) fitted on the
|
||||
window before the held-out one and scored on it, on the weather as it was
|
||||
forecast where the box kept it (else observed, and the run says so); the
|
||||
rooms' R and C (trainlib.fit_rooms).
|
||||
|
||||
Never in the controller's way: each series is a child process (python -m
|
||||
cantabile_edge.boxtrain <job> <result>) at the lowest priority (nice 19,
|
||||
one LightGBM thread), one at a time, killed past its wall time
|
||||
(train_series_s) or its resident memory (train_mem_mb), not started while
|
||||
less than train_min_free_mb is available; the agent's loop only awaits
|
||||
it. The data are read in a worker thread.
|
||||
|
||||
Which model the box plans on (inputs._model_series), per step: the box's
|
||||
own model where it is in force, else the cloud's series, else the cloud's
|
||||
model on the box (localml.py), else the hour-of-week. In force means: on
|
||||
the night's held-out window it beat the model the box had (the champion;
|
||||
trainlib's rule) AND the cloud's model in force (the bundle's "models",
|
||||
scored on the same window): the cloud's model is the standing benchmark
|
||||
until the cloud's teacher model exists. Without a cloud model to compare,
|
||||
a box model other than the hour-of-week that beats the hour-of-week. A
|
||||
model in force whose live error over 7 days (the meter against what was
|
||||
planned on, 2 days at least) is worse than the cloud's series' leaves
|
||||
force until it wins again.
|
||||
|
||||
Upward (the plan's "ml", controller.public_plan): the forecasts the box
|
||||
planned on (hourly), the held-out errors per kind, the live error per
|
||||
source, and the training's own figures (seconds, peak memory)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import numpy as np
|
||||
|
||||
from . import fcmodels, localml, pvforecast, trainhist
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SLOT_S = 900
|
||||
NAMES = {"load": "load_w", "pv": "pv_w", "base_load": "base_load_w"} # the bundle's names (boxmodel.NAMES)
|
||||
SERIES = {v: k for k, v in NAMES.items()}
|
||||
CTL_KINDS = ("heating", "cooling", "ev") # trainlib.CONTROLLED_POWER's keys
|
||||
ROOM_DAYS = 42 # the rooms on the cloud's window (trainer: 6 weeks)
|
||||
LIVE_DAYS = 7
|
||||
LIVE_MIN = 2 * 96 # live errors count from 2 days of slots
|
||||
RUNS_KEPT = 30
|
||||
PUBLISH_EVERY_S = 3600 # the forecasts upward at most hourly (the plan comes every 15 min)
|
||||
|
||||
|
||||
def _f(v):
|
||||
return None if v is None or (isinstance(v, float) and not math.isfinite(v)) else float(v)
|
||||
|
||||
|
||||
def _clean(v):
|
||||
"""JSON without NaN, numpy scalars or private keys (lgbtrees' cache)."""
|
||||
if isinstance(v, float):
|
||||
return v if math.isfinite(v) else None
|
||||
if isinstance(v, dict):
|
||||
return {k: _clean(x) for k, x in v.items() if not str(k).startswith("_")}
|
||||
if isinstance(v, (list, tuple)):
|
||||
return [_clean(x) for x in v]
|
||||
if isinstance(v, np.generic):
|
||||
return _clean(v.item())
|
||||
if isinstance(v, np.ndarray):
|
||||
return _clean(v.tolist())
|
||||
return v
|
||||
|
||||
|
||||
def parse_hhmm(s: str) -> tuple[int, int]:
|
||||
try:
|
||||
h, m = (int(x) for x in str(s).split(":")[:2])
|
||||
if 0 <= h < 24 and 0 <= m < 60:
|
||||
return h, m
|
||||
except ValueError:
|
||||
pass
|
||||
return 3, 30
|
||||
|
||||
|
||||
def next_run(now: float, at: str, tz: str) -> float:
|
||||
"""The next train_at (local) after now."""
|
||||
zone = ZoneInfo(tz or "UTC")
|
||||
h, m = parse_hhmm(at)
|
||||
d = datetime.fromtimestamp(now, timezone.utc).astimezone(zone)
|
||||
t = d.replace(hour=h, minute=m, second=0, microsecond=0)
|
||||
if t.timestamp() <= now:
|
||||
t = (d + timedelta(days=1)).replace(hour=h, minute=m, second=0, microsecond=0)
|
||||
return t.timestamp()
|
||||
|
||||
|
||||
def mem_available_mb() -> float | None:
|
||||
try:
|
||||
with open("/proc/meminfo") as f:
|
||||
for line in f:
|
||||
if line.startswith("MemAvailable:"):
|
||||
return int(line.split()[1]) / 1024.0
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def rss_mb(pid: int) -> float | None:
|
||||
try:
|
||||
with open(f"/proc/{pid}/status") as f:
|
||||
for line in f:
|
||||
if line.startswith("VmRSS:"):
|
||||
return int(line.split()[1]) / 1024.0
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def hours_json(hours: list) -> list:
|
||||
return [[int(h.end.timestamp()), h.solar_kwh_m2, h.cloud_pct, h.temp_c] for h in hours]
|
||||
|
||||
|
||||
def hours_from(rows) -> list:
|
||||
return [pvforecast.Hour(datetime.fromtimestamp(int(r[0]), timezone.utc), r[1], r[2], r[3]) for r in rows or []]
|
||||
|
||||
|
||||
def arrays(series: dict[int, float], t: np.ndarray) -> np.ndarray:
|
||||
return np.array([series.get(int(k), np.nan) for k in t.tolist()], dtype=float)
|
||||
|
||||
|
||||
def net_series(raw: dict[str, np.ndarray], n: int) -> np.ndarray | None:
|
||||
"""A member's draw per slot (the coordinator's core.member_use_w): grid
|
||||
import less export where metered, else load less PV and battery."""
|
||||
if "grid.import_power" in raw:
|
||||
exp = raw.get("grid.export_power")
|
||||
return raw["grid.import_power"] - (np.nan_to_num(exp) if exp is not None else 0.0)
|
||||
if "load.power" in raw:
|
||||
out = raw["load.power"].copy()
|
||||
for a in ("pv.power", "battery.power"):
|
||||
if a in raw:
|
||||
out = out - np.nan_to_num(raw[a])
|
||||
return out
|
||||
return None
|
||||
|
||||
|
||||
def score(m: dict | None, key: str) -> float | None:
|
||||
v = (m or {}).get(key)
|
||||
return float(v) if isinstance(v, (int, float)) and math.isfinite(v) else None
|
||||
|
||||
|
||||
# ---- the child process: one job -------------------------------------------------
|
||||
|
||||
def run_job(job: dict) -> dict:
|
||||
"""One series (or the rooms), as the cloud trains it (trainlib). Pure:
|
||||
the parent wrote everything it needs."""
|
||||
from . import trainlib
|
||||
t0 = time.monotonic()
|
||||
t = np.asarray(job["t"], dtype=np.int64)
|
||||
s = job["site"]
|
||||
site = fcmodels.Site(s.get("tz") or "UTC", s.get("lat"), s.get("lon"), s.get("kwp"), s.get("tilt", 30.0),
|
||||
s.get("azimuth", 180.0), s.get("pvgis_monthly"), s.get("state"))
|
||||
w = fcmodels.weather(t, hours_from(job.get("hours")), site.lat, site.lon)
|
||||
out: dict = {"series": job["series"]}
|
||||
if job["series"] == "rooms":
|
||||
assets = {a: np.array([np.nan if v is None else v for v in vals], dtype=float) for a, vals in job["assets"].items()}
|
||||
fits = trainlib.fit_rooms(t, assets, w)
|
||||
tout, p, ghi = trainlib.room_inputs(assets, w)
|
||||
for room, f in fits.items():
|
||||
c = (job.get("cloud_rooms") or {}).get(room)
|
||||
theta = assets.get(f"room.{room}.temp")
|
||||
if c and p is not None and theta is not None and c.get("power", "thermal") == f.get("power"):
|
||||
from . import rcfit
|
||||
b = rcfit.DT_H / c["c_kwh_per_k"]
|
||||
gain, solar = c.get("gain_w") or 0.0, c.get("solar_m2") or 0.0
|
||||
gain = sum(gain) / len(gain) if isinstance(gain, list) else gain
|
||||
f["cloud_rmse_k"] = rcfit.sim_rmse(theta, tout, p / 1000.0, np.nan_to_num(ghi) / 1000.0, c["r_k_per_kw"],
|
||||
c["c_kwh_per_k"], solar * b, gain * b / 1000.0)
|
||||
out["rooms"] = _clean(fits)
|
||||
else:
|
||||
series = job["series"]
|
||||
y = np.array([np.nan if v is None else v for v in job["y"]], dtype=float)
|
||||
wt = fcmodels.weather(t, hours_from(job["test_hours"]), site.lat, site.lon) if job.get("test_hours") else None
|
||||
res = trainlib.train_series(series, t, y, w, site, job.get("champion"), w_test=wt)
|
||||
key = trainlib.score_key(res)
|
||||
cloud = trainlib.score_params(job["cloud"], series, t, y, w, site, wt) if job.get("cloud") else None
|
||||
out.update(key=key, window=list(res.window), n_train=res.n_train, n_test=res.n_test, note=res.note,
|
||||
champion=res.champion, baseline=res.baseline, cloud=cloud,
|
||||
runs=[{"kind": r.kind, "metrics": r.metrics, "note": r.note, "chosen": r.chosen} for r in res.runs],
|
||||
chosen=res.chosen.kind if res.chosen else None,
|
||||
params=fcmodels.box_params(res.chosen.params) if res.chosen else None)
|
||||
if job.get("want_predictions") and res.chosen: # the parity test
|
||||
out["predictions"] = fcmodels.predict(res.chosen.params, series, t, w, site).tolist()
|
||||
out["seconds"] = round(time.monotonic() - t0, 2)
|
||||
try: # this process's own peak (VmHWM: getrusage's maxrss keeps the parent's across exec)
|
||||
with open("/proc/self/status") as f:
|
||||
out["peak_mb"] = round(int(f.read().split("VmHWM:")[1].split()[0]) / 1024.0, 1)
|
||||
except (OSError, IndexError, ValueError):
|
||||
pass
|
||||
return _clean(out)
|
||||
|
||||
|
||||
def child_main(argv: list[str]) -> int:
|
||||
try:
|
||||
os.nice(19)
|
||||
except (OSError, AttributeError):
|
||||
pass
|
||||
with open(argv[0]) as f:
|
||||
job = json.load(f)
|
||||
out = run_job(job)
|
||||
tmp = argv[1] + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(out, f, separators=(",", ":"))
|
||||
os.replace(tmp, argv[1])
|
||||
return 0
|
||||
|
||||
|
||||
# ---- the box's own models and the nightly job --------------------------------------
|
||||
|
||||
class StoreHistory:
|
||||
"""inputs.History's interface over one series of the training history
|
||||
(the lag of the box's own models: the right series, not the agent's
|
||||
base load for all)."""
|
||||
|
||||
def __init__(self, name: str, values: dict[int, float]):
|
||||
self.slots = {name: {k: [v, 1] for k, v in values.items()}}
|
||||
|
||||
def mean(self, name: str, slot: int):
|
||||
s = self.slots.get(name, {}).get(slot)
|
||||
return s[0] if s else None
|
||||
|
||||
|
||||
class BoxTraining:
|
||||
def __init__(self, cfg, store: trainhist.Store, bundles, kinds=None, clock=time.time, child_cmd=None):
|
||||
self.cfg, self.store, self.bundles, self.clock = cfg, store, bundles, clock
|
||||
self.kinds = kinds or (lambda: set()) # the planned devices' kinds (CTL_KINDS) now
|
||||
self.path = os.path.join(cfg.state_dir, "boxmodels.json")
|
||||
self.child_cmd = child_cmd or [sys.executable, "-m", "cantabile_edge.boxtrain"]
|
||||
self.state = self._load()
|
||||
self.running = False
|
||||
self.progress = ""
|
||||
self._models: dict = {} # (name, trained_at) -> localml.Model
|
||||
self._fc_cache: dict = {} # (name, start) -> forecast
|
||||
self._hist_cache: dict = {}
|
||||
self.last_published = 0.0
|
||||
self.enabled = bool(getattr(cfg, "train", True))
|
||||
# LightGBM is imported in the child only (the agent's own memory stays
|
||||
# as it was); here only whether it is there at all
|
||||
import importlib.util
|
||||
self.lightgbm = importlib.util.find_spec("lightgbm") is not None
|
||||
if not self.lightgbm:
|
||||
log.warning("LightGBM not in this image: the box's gbm fits fail, the other kinds train")
|
||||
|
||||
# -- the state ---------------------------------------------------------------
|
||||
def _load(self) -> dict:
|
||||
try:
|
||||
with open(self.path) as f:
|
||||
d = json.load(f)
|
||||
if isinstance(d, dict) and d.get("v") == 1:
|
||||
d.setdefault("series", {})
|
||||
d.setdefault("rooms", {})
|
||||
d.setdefault("runs", [])
|
||||
return d
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return {"v": 1, "series": {}, "rooms": {}, "runs": [], "last": None}
|
||||
|
||||
def save(self):
|
||||
tmp = self.path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(_clean(self.state), f, separators=(",", ":"))
|
||||
os.replace(tmp, self.path)
|
||||
|
||||
def in_force(self, series: str) -> dict | None:
|
||||
e = self.state["series"].get(series)
|
||||
return e if e and e.get("in_force") and e.get("params") else None
|
||||
|
||||
# -- what the planner asks -------------------------------------------------------
|
||||
def model(self, name: str):
|
||||
"""The box's own model for a bundle series name (load_w, pv_w,
|
||||
base_load_w) as localml runs it, or None (none in force)."""
|
||||
e = self.in_force(SERIES.get(name, ""))
|
||||
if e is None:
|
||||
return None
|
||||
key = (name, e.get("trained_at"))
|
||||
m = self._models.get(key)
|
||||
if m is None:
|
||||
try:
|
||||
m = localml.Model(name, {"params": e["params"], "site": e.get("site") or {}})
|
||||
except (ValueError, TypeError, KeyError) as ex:
|
||||
log.warning("the box's own %s model cannot run: %s", name, ex)
|
||||
e["in_force"], e["why"] = False, f"cannot run: {ex}"[:200]
|
||||
return None
|
||||
self._models = {k: v for k, v in self._models.items() if k[0] != name}
|
||||
self._models[key] = m
|
||||
return m
|
||||
|
||||
def history_for(self, series: str, start: int) -> StoreHistory:
|
||||
"""The series' 8 days before start from the training history."""
|
||||
key = (series, start)
|
||||
h = self._hist_cache.get(key)
|
||||
if h is None:
|
||||
since = start - 8 * 86400
|
||||
raw = {a: v for a, v in self.store.series(since, start).items()}
|
||||
t = np.arange(since, start, SLOT_S, dtype=np.int64)
|
||||
arr = self.actual(series, t, {a: arrays(v, t) for a, v in raw.items()})
|
||||
vals = {} if arr is None else {int(k): float(v) for k, v in zip(t.tolist(), arr.tolist()) if not math.isnan(v)}
|
||||
h = StoreHistory(localml.HISTORY[NAMES.get(series, "load_w")], vals)
|
||||
self._hist_cache = {key: h}
|
||||
return h
|
||||
|
||||
def forecast(self, name: str, start: int, steps: int, ha_forecast, weather_entity: str = ""):
|
||||
"""The box's own forecast of a series (localml.predict on its own
|
||||
model, HA's weather, else the cloud's bundle weather): (values with
|
||||
None, info) or None without a model in force. Once per quarter hour."""
|
||||
m = self.model(name)
|
||||
if m is None:
|
||||
return None
|
||||
e = self.in_force(SERIES[name])
|
||||
key = (name, start, e.get("trained_at"))
|
||||
hit = self._fc_cache.get(key)
|
||||
if hit is None:
|
||||
b = self.bundles.model.get() if self.bundles is not None else None
|
||||
b = b if isinstance(b, dict) and b.get("site") == getattr(self.bundles, "site", None) else {}
|
||||
try:
|
||||
hit = localml.predict(m, [start + k * SLOT_S for k in range(steps)], self.history_for(SERIES[name], start),
|
||||
ha_forecast, b.get("weather"), weather_entity, b.get("issued"))
|
||||
except Exception as ex: # never stops a plan: the next source then
|
||||
log.warning("the box's own %s model: %s", name, ex)
|
||||
hit = ([None] * steps, {"kind": m.kind, "weather": "", "steps": 0, "error": str(ex)[:200]})
|
||||
self._fc_cache = {k: v for k, v in self._fc_cache.items() if k[1] == start}
|
||||
self._fc_cache[key] = hit
|
||||
return hit
|
||||
|
||||
def correct(self, series: str, start: int, values: list) -> tuple[list, float]:
|
||||
"""The intraday correction (fcmodels.intraday) of what the box plans
|
||||
on: the last 3 hours' forecast as planned on ("used", before any
|
||||
correction) against the meter. series: plan (the load) or pv."""
|
||||
since = start - int(fcmodels.INTRADAY_WINDOW_H * 3600)
|
||||
fc = self.store.forecasts(series, since, start).get("used")
|
||||
if not fc:
|
||||
return list(values), 0.0
|
||||
t = np.arange(since, start, SLOT_S, dtype=np.int64)
|
||||
raw = {a: arrays(v, t) for a, v in self.store.series(since, start).items()}
|
||||
y = self.actual(series, t, raw)
|
||||
if y is None:
|
||||
return list(values), 0.0
|
||||
out, e = fcmodels.intraday("pv" if series == "pv" else "load", np.asarray(values, dtype=float), arrays(fc, t), y)
|
||||
return [float(v) for v in out], e
|
||||
|
||||
def room_rc(self, control_id: str, area: str = "", only_one: bool = False) -> dict | None:
|
||||
"""A room the box fitted itself and put in force (it simulated the
|
||||
room better than the cloud's fit on the same data, or the cloud has
|
||||
none): as bundles.room_rc gives the cloud's."""
|
||||
rooms = {k: v for k, v in (self.state.get("rooms") or {}).items() if v.get("in_force")}
|
||||
import re
|
||||
key = next((k for k in (control_id, re.sub(r"_(heating|cooling|heat|cool)$", "", control_id or ""), area or "")
|
||||
if k and k in rooms), None)
|
||||
if key is None and only_one and len(rooms) == 1:
|
||||
key = next(iter(rooms))
|
||||
if key is None:
|
||||
return None
|
||||
r = rooms[key]
|
||||
return {"r_k_per_kw": float(r["r_k_per_kw"]), "c_kwh_per_k": float(r["c_kwh_per_k"]),
|
||||
"power": r.get("power") or "thermal", "gain_w": float(r.get("gain_w") or 0.0),
|
||||
"solar_m2": min(max(float(r.get("solar_m2") or 0.0), 0.0), 50.0)}
|
||||
|
||||
# -- the series and their actual values -------------------------------------------
|
||||
def train_series_list(self) -> list[str]:
|
||||
out = ["load", "pv"]
|
||||
if self.kinds():
|
||||
out.insert(1, "base_load")
|
||||
return out
|
||||
|
||||
def actual(self, series: str, t: np.ndarray, raw: dict[str, np.ndarray]) -> np.ndarray | None:
|
||||
"""What was metered for a series: load.power, pv.power, the base
|
||||
load (trainlib.base_load with the planned devices' kinds), the
|
||||
member's net draw; for "plan" the series the box plans on."""
|
||||
from . import trainlib
|
||||
if series == "plan":
|
||||
series = "base_load" if self.kinds() else "load"
|
||||
if series == "base_load":
|
||||
return trainlib.base_load(raw, set(self.kinds()))
|
||||
if series == "net":
|
||||
return net_series(raw, len(t))
|
||||
a = trainlib.SITE_SERIES.get(series)
|
||||
return raw.get(a) if a else None
|
||||
|
||||
# -- live errors ----------------------------------------------------------------
|
||||
def record(self, series: str, start: int, cands: dict[str, list], now: float):
|
||||
"""The forecasts per source the box had for the next 24 h (the
|
||||
live error against the meter, trainhist fc_slots). series: plan
|
||||
(the load the box plans on), pv or net."""
|
||||
try:
|
||||
self.store.put_forecasts(series, start, {k: v for k, v in cands.items() if v}, now)
|
||||
except Exception as e: # never stops a plan
|
||||
log.warning("forecasts per source: %s", e)
|
||||
|
||||
def week_mean(self, series: str, start: int, steps: int = 96) -> list[float | None]:
|
||||
"""The coordinator's member forecast (methods.md B7) as the box
|
||||
sees it: the same quarter hour's mean over the last 7 days."""
|
||||
since = start - 7 * 86400
|
||||
t = np.arange(since, start, SLOT_S, dtype=np.int64)
|
||||
raw = {a: arrays(v, t) for a, v in self.store.series(since, start).items()}
|
||||
y = self.actual(series, t, raw)
|
||||
if y is None:
|
||||
return [None] * steps
|
||||
by: dict[int, list[float]] = {}
|
||||
for ts, v in zip(t.tolist(), y.tolist()):
|
||||
if not math.isnan(v):
|
||||
by.setdefault((ts % 86400) // SLOT_S, []).append(v)
|
||||
return [(sum(by[q]) / len(by[q])) if (q := ((start + k * SLOT_S) % 86400) // SLOT_S) in by else None
|
||||
for k in range(steps)]
|
||||
|
||||
def live_errors(self, series: str, now: float, days: int = LIVE_DAYS) -> dict[str, dict]:
|
||||
"""Per source the error of what was forecast against the meter
|
||||
over the last days: {"nmae", "mae", "n"}."""
|
||||
until = int(now // SLOT_S * SLOT_S)
|
||||
since = until - days * 86400
|
||||
fc = self.store.forecasts(series, since, until)
|
||||
if not fc:
|
||||
return {}
|
||||
t = np.arange(since, until, SLOT_S, dtype=np.int64)
|
||||
raw = {a: arrays(v, t) for a, v in self.store.series(since, until).items()}
|
||||
y = self.actual(series, t, raw)
|
||||
if y is None:
|
||||
return {}
|
||||
out = {}
|
||||
for src, vals in fc.items():
|
||||
p = arrays(vals, t)
|
||||
m = fcmodels.metrics(y, p)
|
||||
if m["n"]:
|
||||
out[src] = {"nmae": _f(m["nmae"]), "mae": _f(m["mae"]), "n": m["n"]}
|
||||
return out
|
||||
|
||||
# -- the nightly job ----------------------------------------------------------------
|
||||
async def run(self, stop: asyncio.Event, tz=lambda: "UTC", location=lambda: {}, client=lambda: None,
|
||||
weather_entity=lambda: ""):
|
||||
"""Each night at train_at: train (never more than once at a time)."""
|
||||
if not self.enabled:
|
||||
log.info("the box's own training is off (train: false)")
|
||||
await stop.wait()
|
||||
return
|
||||
while not stop.is_set():
|
||||
wait = max(1.0, next_run(self.clock(), self.cfg.train_at, tz()) - self.clock())
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), wait)
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
try:
|
||||
await self.train_once(tz(), location(), client(), weather_entity())
|
||||
except Exception as e: # the next night again
|
||||
log.exception("the box's training: %s", e)
|
||||
|
||||
def site(self, tz: str, location: dict) -> fcmodels.Site:
|
||||
"""Where and what: the cloud's model's site where the bundle has one
|
||||
(its kWp, orientation and PVGIS), else Home Assistant's location."""
|
||||
for name in ("pv_w", "load_w", "base_load_w"):
|
||||
m = self.bundles.box_model(name) if self.bundles is not None and hasattr(self.bundles, "box_model") else None
|
||||
if m is not None and m.site.lat is not None:
|
||||
return fcmodels.Site(tz or m.site.tz, m.site.lat, m.site.lon, m.site.kwp, m.site.tilt, m.site.azimuth,
|
||||
m.site.pvgis_monthly)
|
||||
return fcmodels.Site(tz or "UTC", (location or {}).get("latitude"), (location or {}).get("longitude"))
|
||||
|
||||
def metered_since(self) -> datetime | None:
|
||||
v = (getattr(self.cfg, "pv_metered_since", "") or "").strip()
|
||||
if not v:
|
||||
return None
|
||||
try:
|
||||
t = datetime.fromisoformat(v.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
log.warning("pv_metered_since %r: not ISO 8601", v)
|
||||
return None
|
||||
return t if t.tzinfo else t.replace(tzinfo=timezone.utc)
|
||||
|
||||
def prepare(self, now: float, tz: str, location: dict) -> tuple[list[dict], dict]:
|
||||
"""The jobs (in a worker thread: SQLite and numpy, no await)."""
|
||||
from . import trainlib
|
||||
until = int(now // SLOT_S * SLOT_S)
|
||||
since = until - int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS)) * 86400
|
||||
t = np.arange(since, until, SLOT_S, dtype=np.int64)
|
||||
stored = self.store.series(since, until)
|
||||
raw = {a: arrays(v, t) for a, v in stored.items()}
|
||||
site = self.site(tz, location)
|
||||
obs = self.store.training_hours(since - 3600, until + 3600)
|
||||
fc = self.store.weather_hours(since - 3600, until + 3600, "forecast")
|
||||
w = fcmodels.weather(t, obs, site.lat, site.lon)
|
||||
pv_since = self.metered_since()
|
||||
jobs, info = [], {"slots": int(sum((~np.isnan(v)).sum() for v in raw.values())), "weather_hours": len(obs),
|
||||
"forecast_hours": len(fc), "site": asdict(site)}
|
||||
for series in self.train_series_list():
|
||||
y = self.actual(series, t, raw)
|
||||
if y is None or not (~np.isnan(y)).any():
|
||||
continue
|
||||
y = trainlib.cut_before(series, t, y, pv_since)
|
||||
own = self.state["series"].get(series) or {}
|
||||
champ = own.get("params")
|
||||
if champ is not None and pv_since is not None and series in trainlib.SINCE_SERIES \
|
||||
and (own.get("window") or [0])[0] and own["window"][0] < pv_since.timestamp():
|
||||
champ = None # learned from PV before the cut: any new model replaces it
|
||||
cloud = self.bundles.box_model(NAMES[series]) if self.bundles is not None and hasattr(self.bundles, "box_model") else None
|
||||
w_test, on = trainlib.held_out_weather(t, y, w, fc, site.lat, site.lon)
|
||||
jobs.append({"series": series, "t": t.tolist(), "y": [None if math.isnan(v) else float(v) for v in y.tolist()],
|
||||
"hours": hours_json(obs), "test_hours": hours_json(w_test.hours) if w_test is not None else None,
|
||||
"site": asdict(site), "champion": champ, "scored_on": on,
|
||||
"cloud": _clean(cloud.params) if cloud is not None else None,
|
||||
"cloud_model": ((self.bundles.model.get() or {}).get("models") or {}).get(NAMES[series], {}).get("model")
|
||||
if cloud is not None else None})
|
||||
r0 = int(np.searchsorted(t, until - ROOM_DAYS * 86400))
|
||||
rooms_raw = {a: v[r0:] for a, v in raw.items() if a.startswith("room.") or a in trainlib.ROOM_INPUTS}
|
||||
if any(a.startswith("room.") for a in rooms_raw):
|
||||
m = self.bundles.model.get() if self.bundles is not None else None
|
||||
cloud_rooms = (m or {}).get("rooms") if isinstance(m, dict) and m.get("site") == getattr(self.bundles, "site", None) else {}
|
||||
jobs.append({"series": "rooms", "t": t[r0:].tolist(), "site": asdict(site),
|
||||
"hours": hours_json(obs), "cloud_rooms": cloud_rooms or {},
|
||||
"assets": {a: [None if math.isnan(x) else float(x) for x in v.tolist()] for a, v in rooms_raw.items()}})
|
||||
return jobs, info
|
||||
|
||||
async def child(self, job: dict) -> tuple[dict | None, dict]:
|
||||
"""One job in a child process within the caps: (result or None, how
|
||||
it went: seconds, peak_mb, killed)."""
|
||||
d = self.cfg.state_dir
|
||||
jp, op = os.path.join(d, "train-job.json"), os.path.join(d, "train-result.json")
|
||||
await asyncio.to_thread(_write_json, jp, job)
|
||||
try:
|
||||
os.unlink(op)
|
||||
except OSError:
|
||||
pass
|
||||
env = dict(os.environ, OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1", MKL_NUM_THREADS="1")
|
||||
cap_s = float(getattr(self.cfg, "train_series_s", 300.0))
|
||||
cap_mb = float(getattr(self.cfg, "train_mem_mb", 600))
|
||||
t0 = time.monotonic()
|
||||
proc = await asyncio.create_subprocess_exec(*self.child_cmd, jp, op, env=env, stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE)
|
||||
peak, killed = 0.0, ""
|
||||
err_task = asyncio.ensure_future(proc.stderr.read(64 << 10))
|
||||
while True:
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), 0.25)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
r = rss_mb(proc.pid)
|
||||
if r is not None:
|
||||
peak = max(peak, r)
|
||||
if r is not None and r > cap_mb:
|
||||
killed = f"over its memory cap ({r:.0f} MB > {cap_mb:.0f} MB)"
|
||||
elif time.monotonic() - t0 > cap_s:
|
||||
killed = f"over its wall time ({cap_s:.0f} s)"
|
||||
if killed:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
break
|
||||
err = b""
|
||||
try:
|
||||
err = await asyncio.wait_for(err_task, 2)
|
||||
except (asyncio.TimeoutError, Exception):
|
||||
pass
|
||||
how = {"seconds": round(time.monotonic() - t0, 2), "peak_mb": round(peak, 1), "killed": killed}
|
||||
res = None
|
||||
if not killed:
|
||||
try:
|
||||
with open(op) as f:
|
||||
res = json.load(f)
|
||||
how["peak_mb"] = max(how["peak_mb"], float(res.get("peak_mb") or 0.0))
|
||||
except (OSError, ValueError):
|
||||
how["error"] = (err.decode(errors="replace").strip().splitlines() or [f"exit {proc.returncode}"])[-1][:300]
|
||||
for p in (jp, op):
|
||||
try:
|
||||
os.unlink(p)
|
||||
except OSError:
|
||||
pass
|
||||
return res, how
|
||||
|
||||
async def train_once(self, tz: str = "UTC", location: dict | None = None, client=None, weather_entity: str = "") -> dict:
|
||||
"""One night's training; the summary (also in state["last"])."""
|
||||
if self.running:
|
||||
return {"skipped": "already running"}
|
||||
self.running = True
|
||||
t_start = self.clock()
|
||||
summary: dict = {"at": int(t_start), "series": {}, "rooms": None, "notes": []}
|
||||
try:
|
||||
if client is not None:
|
||||
for what, fn in (("statistics", self.fill_from_ha), ("weather", self.weather_from_ha)):
|
||||
try:
|
||||
n = await fn(client, weather_entity) if what == "weather" else await fn(client)
|
||||
if n:
|
||||
summary["notes"].append(f"{n} {what} slots/hours from Home Assistant")
|
||||
except Exception as e: # HA's history is a help, not a need
|
||||
log.info("training: %s from Home Assistant: %s", what, e)
|
||||
jobs, info = await asyncio.to_thread(self.prepare, t_start, tz, location or {})
|
||||
summary["data"] = info
|
||||
for job in jobs:
|
||||
free = mem_available_mb()
|
||||
need = float(getattr(self.cfg, "train_min_free_mb", 300))
|
||||
name = job["series"]
|
||||
if free is not None and free < need:
|
||||
summary["notes"].append(f"{name}: not trained, {free:.0f} MB available (at least {need:.0f})")
|
||||
continue
|
||||
self.progress = name
|
||||
res, how = await self.child(job)
|
||||
if name == "rooms":
|
||||
summary["rooms"] = {"how": how, **self.take_rooms(res, how)}
|
||||
else:
|
||||
summary["series"][name] = self.take(name, job, res, how, t_start)
|
||||
self.state["last"] = summary
|
||||
self.state["runs"] = (self.state.get("runs", []) + [_runlog(summary)])[-RUNS_KEPT:]
|
||||
await asyncio.to_thread(self.save)
|
||||
await asyncio.to_thread(self.store.prune, self.clock(), int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS)))
|
||||
log.info("the box's training: %s in %.0f s", ", ".join(f"{k} {v.get('result')}" for k, v in summary["series"].items())
|
||||
or "nothing to train", self.clock() - t_start)
|
||||
finally:
|
||||
self.running, self.progress = False, ""
|
||||
self._models, self._fc_cache = {}, {}
|
||||
return summary
|
||||
|
||||
def take(self, series: str, job: dict, res: dict | None, how: dict, now: float) -> dict:
|
||||
"""A series' result: the new model in force only if it beats the
|
||||
box's model (trainlib) and the cloud's (score on the same window)."""
|
||||
out = {"how": how, "scored_on": job.get("scored_on", "")}
|
||||
if res is None:
|
||||
out["result"] = "failed: " + (how.get("killed") or how.get("error") or "no result")
|
||||
return out
|
||||
key = res.get("key") or "nmae"
|
||||
runs = {r["kind"]: score(r.get("metrics"), key) for r in res.get("runs") or [] if r.get("metrics")}
|
||||
cloud = score(res.get("cloud"), key)
|
||||
own_now = score(res.get("champion"), key)
|
||||
out.update(key=key, runs=runs, cloud=cloud, champion=own_now, n_test=res.get("n_test"), note=res.get("note", ""))
|
||||
cur = self.state["series"].get(series)
|
||||
chosen = res.get("chosen")
|
||||
|
||||
def beats(kind, v) -> bool:
|
||||
"""In force: better than the cloud's model on the window; without
|
||||
one, any kind but the hour-of-week (it beat the hour-of-week)."""
|
||||
return (cloud is None and kind != "how") or (cloud is not None and v is not None and v < cloud)
|
||||
|
||||
def vs() -> str:
|
||||
return f"the cloud's model {cloud:.3g}" if cloud is not None else "no cloud model on the box to compare"
|
||||
|
||||
new = runs.get(chosen) if chosen else None
|
||||
if chosen and res.get("params") and (beats(chosen, new) or cur is None):
|
||||
# a new model: it beat the box's own (trainlib) and, to be in force, the cloud's
|
||||
cur = {"kind": chosen, "params": res["params"], "site": job["site"], "trained_at": int(now),
|
||||
"window": res.get("window"), "score": new, "key": key, "in_force": beats(chosen, new)}
|
||||
self.state["series"][series] = cur
|
||||
out["result"] = (("new model in force: " if cur["in_force"] else "trained, not in force: ")
|
||||
+ f"{chosen} {key} {new:.3g}, {vs()}"
|
||||
+ (f", the box's own {own_now:.3g}" if own_now is not None else ""))
|
||||
elif cur is not None:
|
||||
# the box's model stays (nothing beat it, or the new one did not beat
|
||||
# the cloud's): in force while it beats the cloud's on this window
|
||||
stays = own_now if own_now is not None else cur.get("score")
|
||||
cur["in_force"] = beats(cur.get("kind"), stays)
|
||||
if own_now is not None:
|
||||
cur["score"] = own_now
|
||||
out["result"] = (f"the box's {cur.get('kind')} stays" + (f" ({key} {stays:.3g}, {vs()})" if stays is not None else "")
|
||||
+ ("" if cur["in_force"] else ": not in force"))
|
||||
if chosen and new is not None:
|
||||
out["result"] += f"; the new {chosen} ({new:.3g}) does not beat {vs()}"
|
||||
else:
|
||||
out["result"] = "nothing trained: " + (res.get("note") or "no model")
|
||||
if cur is not None:
|
||||
cur.update(cloud=cloud, runs=runs, scored_on=job.get("scored_on", ""), n_test=res.get("n_test"),
|
||||
checked_at=int(now), cloud_model=job.get("cloud_model"))
|
||||
# the live error decides too: worse than the cloud's series over a week, out of force
|
||||
live = self.live_errors("plan" if series == ("base_load" if self.kinds() else "load") else series, now)
|
||||
mine, theirs = live.get("own"), live.get("cloud")
|
||||
if cur.get("in_force") and mine and theirs and mine["n"] >= LIVE_MIN and theirs["n"] >= LIVE_MIN \
|
||||
and mine["nmae"] is not None and theirs["nmae"] is not None and mine["nmae"] > theirs["nmae"]:
|
||||
cur["in_force"] = False
|
||||
out["result"] += (f"; out of force: live nMAE {mine['nmae']:.3g} vs the cloud's series "
|
||||
f"{theirs['nmae']:.3g} over {LIVE_DAYS} days")
|
||||
cur["why"] = out["result"][:300]
|
||||
out["in_force"] = cur["in_force"]
|
||||
return out
|
||||
|
||||
def take_rooms(self, res: dict | None, how: dict) -> dict:
|
||||
if res is None:
|
||||
return {"result": "failed: " + (how.get("killed") or how.get("error") or "no result")}
|
||||
out = {}
|
||||
rooms = self.state.setdefault("rooms", {})
|
||||
for room, f in (res.get("rooms") or {}).items():
|
||||
fitted = f.get("status") == "fitted"
|
||||
cloud = f.get("cloud_rmse_k")
|
||||
better = fitted and (cloud is None or (f.get("rmse_k") is not None and f["rmse_k"] < cloud))
|
||||
f["in_force"] = bool(better)
|
||||
f["why"] = ("fitted, simulates the room better than " + (f"the cloud's fit ({f['rmse_k']} K vs {cloud:.3f} K)"
|
||||
if cloud is not None else "the defaults (no cloud fit to compare)")) if better else \
|
||||
(f"fitted, but the cloud's fit simulates it better ({cloud:.3f} K vs {f.get('rmse_k')} K)" if fitted
|
||||
else f.get("note", "default"))
|
||||
rooms[room] = f
|
||||
out[room] = {"in_force": f["in_force"], "why": f["why"]}
|
||||
return {"result": out}
|
||||
|
||||
# -- Home Assistant's history ------------------------------------------------------
|
||||
async def fill_from_ha(self, client) -> int:
|
||||
"""Slots the agent never saw, from HA's statistics (backfill.collect),
|
||||
once a day: 5-minute means for HA's last 10 days, hourly before."""
|
||||
mapping = getattr(self, "mapping", None)
|
||||
if mapping is None:
|
||||
return 0
|
||||
from . import backfill
|
||||
now = datetime.fromtimestamp(self.clock(), timezone.utc)
|
||||
days = int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS))
|
||||
n = 0
|
||||
for period, a, b in (("hour", now - timedelta(days=days), now - timedelta(days=9)),
|
||||
("5minute", now - timedelta(days=9), now)):
|
||||
got = await backfill.collect(client, mapping, a, b, period, trainhist.wanted, consented=False)
|
||||
for asset, rows in got.items():
|
||||
n += await asyncio.to_thread(self.store.fill, asset, trainhist.stats_to_slots(rows, period),
|
||||
"ha-" + period)
|
||||
return n
|
||||
|
||||
async def weather_from_ha(self, client, entity: str) -> int:
|
||||
"""Observed hours the box lacks, from HA's recorded weather entity
|
||||
(temperature and cloud cover at each hour's end; no irradiation)."""
|
||||
if not entity:
|
||||
return 0
|
||||
now = self.clock()
|
||||
days = int(getattr(self.cfg, "train_days", trainhist.KEEP_DAYS))
|
||||
spans = await asyncio.to_thread(self.store.missing_observed, now - days * 86400, now)
|
||||
if not spans:
|
||||
return 0
|
||||
a = datetime.fromtimestamp(spans[0][0], timezone.utc)
|
||||
res = await client.send("history/history_during_period", timeout=120, start_time=a.isoformat(),
|
||||
end_time=datetime.fromtimestamp(now, timezone.utc).isoformat(), entity_ids=[entity],
|
||||
minimal_response=False, no_attributes=False, significant_changes_only=False)
|
||||
hours = ha_history_hours((res or {}).get(entity) or [], spans)
|
||||
return await asyncio.to_thread(self.store.put_weather, "observed", "ha", hours, now)
|
||||
|
||||
# -- upward ------------------------------------------------------------------------
|
||||
def status(self, now: float, forecasts: dict | None = None) -> dict:
|
||||
"""The plan's "ml": the training's figures and the error per source
|
||||
(and, at most hourly, the forecasts the box planned on)."""
|
||||
series = {}
|
||||
for s, e in (self.state.get("series") or {}).items():
|
||||
series[s] = {"kind": e.get("kind"), "in_force": bool(e.get("in_force")), "trained_at": e.get("trained_at"),
|
||||
"checked_at": e.get("checked_at"), "key": e.get("key"), "held_out": e.get("runs") or {},
|
||||
"held_out_box": e.get("score"), "held_out_cloud": e.get("cloud"), "scored_on": e.get("scored_on", ""),
|
||||
"n_test": e.get("n_test"), "why": (e.get("why") or "")[:300]}
|
||||
live = {}
|
||||
for s in ("plan", "pv", "net"):
|
||||
le = self.live_errors(s, now)
|
||||
if le:
|
||||
live[s] = le
|
||||
last = self.state.get("last") or {}
|
||||
out = {"v": 1, "series": series, "live": live, "live_days": LIVE_DAYS,
|
||||
"planned": "base_load" if self.kinds() else "load",
|
||||
"rooms": {k: {"in_force": bool(v.get("in_force")), "r_k_per_kw": v.get("r_k_per_kw"),
|
||||
"c_kwh_per_k": v.get("c_kwh_per_k"), "rmse_k": v.get("rmse_k"),
|
||||
"cloud_rmse_k": v.get("cloud_rmse_k")} for k, v in (self.state.get("rooms") or {}).items()},
|
||||
"train": {"enabled": self.enabled, "lightgbm": self.lightgbm, "at": getattr(self.cfg, "train_at", ""),
|
||||
"last": last.get("at"), "running": self.running,
|
||||
"per_series": {k: v.get("how") for k, v in (last.get("series") or {}).items()},
|
||||
"notes": (last.get("notes") or [])[:5]}}
|
||||
if forecasts and now - self.last_published >= PUBLISH_EVERY_S:
|
||||
out["forecasts"] = forecasts
|
||||
self.last_published = now
|
||||
return _clean(out)
|
||||
|
||||
|
||||
def _write_json(path: str, d: dict):
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(d, f, separators=(",", ":"))
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def _runlog(summary: dict) -> dict:
|
||||
return {"at": summary.get("at"), "series": {k: {"result": v.get("result", "")[:200], "how": v.get("how")}
|
||||
for k, v in (summary.get("series") or {}).items()}}
|
||||
|
||||
|
||||
def ha_history_hours(states: list[dict], spans: list[tuple[int, int]]) -> list:
|
||||
"""A weather entity's recorded states → per hour end in the spans the
|
||||
temperature and cloud cover in force then (the last state before)."""
|
||||
pts = []
|
||||
for s in states:
|
||||
a = s.get("a") or s.get("attributes") or {}
|
||||
ts = s.get("lu") or s.get("lc") or s.get("last_updated") or s.get("last_changed")
|
||||
try:
|
||||
ts = float(ts) if isinstance(ts, (int, float)) else datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp()
|
||||
except ValueError:
|
||||
continue
|
||||
temp, cloud = a.get("temperature"), a.get("cloud_coverage")
|
||||
pts.append((ts, float(temp) if isinstance(temp, (int, float)) else None,
|
||||
float(cloud) if isinstance(cloud, (int, float)) else None))
|
||||
pts.sort()
|
||||
out, j = [], -1
|
||||
for a, b in spans:
|
||||
for end in range(a + 3600, b + 1, 3600):
|
||||
while j + 1 < len(pts) and pts[j + 1][0] <= end:
|
||||
j += 1
|
||||
if j >= 0 and (pts[j][1] is not None or pts[j][2] is not None) and end - pts[j][0] <= 3 * 3600:
|
||||
out.append(pvforecast.Hour(datetime.fromtimestamp(end, timezone.utc), None, pts[j][2], pts[j][1]))
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(child_main(sys.argv[1:]))
|
||||
@@ -440,6 +440,19 @@ class Bundles:
|
||||
out.append((float(f["values"][d.weekday() * 24 + d.hour]), "profile"))
|
||||
return out
|
||||
|
||||
def profile(self, name: str, start: int, step_s: int, steps: int, tz: str) -> list[float] | None:
|
||||
"""The model's hour-of-week profile of a series for every step (the
|
||||
live error per source, boxtrain.py), or None."""
|
||||
m = self.model.get()
|
||||
if m is None or m.get("site") not in (self.site, "*"):
|
||||
return None
|
||||
f = (m.get("forecasts") or {}).get(name)
|
||||
if f is None:
|
||||
return None
|
||||
zone = ZoneInfo(tz or "UTC")
|
||||
return [float(f["values"][d.weekday() * 24 + d.hour]) for d in
|
||||
(datetime.fromtimestamp(start + k * step_s, timezone.utc).astimezone(zone) for k in range(steps))]
|
||||
|
||||
def box_model(self, name: str):
|
||||
"""The bundle's own model for a series (localml.Model), or None: no
|
||||
model bundle for this site, none for the series, or one the box
|
||||
|
||||
@@ -62,6 +62,19 @@ class Config:
|
||||
override_hold_s: int = 7200 # hands off a control the owner changed by hand
|
||||
min_switch_s: int = 600 # anti-cycling: a switch stays on/off at least this long
|
||||
max_actions_per_hour: int = 12 # per control
|
||||
# The box trains its own models (boxtrain.py, methods.md B10): nightly at
|
||||
# train_at (local time, HA's time zone), one series at a time in a child
|
||||
# process at the lowest priority, each at most train_series_s and
|
||||
# train_mem_mb (resident) and only with train_min_free_mb available; the
|
||||
# history kept and trained on: train_days. pv_metered_since (ISO 8601, UTC; as the
|
||||
# cloud's PV_METERED_SINCE): metered PV before it does not count.
|
||||
train: bool = True
|
||||
train_at: str = "03:30"
|
||||
train_series_s: float = 300.0
|
||||
train_mem_mb: int = 600
|
||||
train_min_free_mb: int = 300
|
||||
train_days: int = 396 # 13 months: recent weeks weigh more (fcmodels.recency_weights)
|
||||
pv_metered_since: str = ""
|
||||
|
||||
|
||||
def _read_secret(path: str) -> str:
|
||||
@@ -94,6 +107,10 @@ def load(path: str | None, require_site: bool = True) -> Config:
|
||||
cfg.dashboards = False
|
||||
if env.get("CANTABILE_PROPOSE") in ("1", "true", "on"):
|
||||
cfg.propose = True
|
||||
if env.get("CANTABILE_TRAIN") in ("0", "false", "off"):
|
||||
cfg.train = False
|
||||
cfg.train_at = env.get("CANTABILE_TRAIN_AT", cfg.train_at)
|
||||
cfg.pv_metered_since = env.get("CANTABILE_PV_METERED_SINCE", cfg.pv_metered_since)
|
||||
if token_file:
|
||||
cfg.ha_token = _read_secret(token_file)
|
||||
# Inside a Supervisor add-on the Supervisor proxies Core's API and hands
|
||||
|
||||
@@ -24,6 +24,15 @@ Commands (from the console through the link, which checked the signature):
|
||||
"floor"?, "limit"? (empty: none), "until"?}
|
||||
building {"v", "id", "building", "level", "control", "value", "from",
|
||||
"until", "why"}: the building's requests (building.py)
|
||||
preferences {"msg": JSON of preferences.apply_msg's {"set": {"item",
|
||||
"value"}}, "level": "installer|operator", "by"}: an installer's or
|
||||
Cantabile's setting within the owner's limits (preferences.py);
|
||||
the owner's own come from Home Assistant and the panel
|
||||
|
||||
The owner's preferences (preferences.py): into every model state (a room's
|
||||
schedule, a car's ready-by, a battery's reserve, the overrides), a plan at
|
||||
once after a change (not held back by steady.py), their feedback and the
|
||||
settings with the plan to the cloud (public_plan "preferences").
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
@@ -41,8 +50,10 @@ from .executor import Executor, building_msg
|
||||
from .gridorder import GridOrders
|
||||
from .guard import Guard
|
||||
from .policy import Policy
|
||||
from .preferences import Preferences
|
||||
from .switchover import Switchover
|
||||
from .inputs import Collector, History
|
||||
from . import localml
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -73,6 +84,7 @@ class Mode:
|
||||
class Controller:
|
||||
def __init__(self, cfg, mapping, consent, sink, clock=time.time):
|
||||
self.cfg, self.mapping, self.sink, self.clock = cfg, mapping, sink, clock
|
||||
self.consent = consent
|
||||
self.controls = mapping.controls
|
||||
self.inputs = mapping.inputs
|
||||
self.mode = Mode(cfg.state_dir, cfg.mode)
|
||||
@@ -108,6 +120,25 @@ class Controller:
|
||||
self.on_plan = [] # callbacks (the announcer) when a plan is made
|
||||
self.received: list[dict] = [] # the last commands and bundles (shown in HA, announce.py)
|
||||
self.on_received = [] # callbacks (the announcer) for each
|
||||
self.prefs = Preferences(cfg.state_dir, clock) # the owner's preferences (preferences.py)
|
||||
self.on_prefs = [] # callbacks (the announcer) when they change
|
||||
try:
|
||||
self.prefs.write_view(self.controls, self.mode())
|
||||
except Exception as e:
|
||||
log.warning("preferences view: %s", e)
|
||||
# the box trains its own models (boxtrain.py, methods.md B10): its
|
||||
# history, the nightly job, and its models for the planner (bundles.own)
|
||||
self.boxml = None
|
||||
if getattr(cfg, "train", True):
|
||||
try:
|
||||
from .boxtrain import BoxTraining
|
||||
from .trainhist import Store
|
||||
self.boxml = BoxTraining(cfg, Store(os.path.join(cfg.state_dir, "training.db")), self.bundles,
|
||||
kinds=self.planned_kinds, clock=clock)
|
||||
self.bundles.own = self.boxml
|
||||
except Exception as e: # never stops the controller
|
||||
log.warning("the box's own training is off: %s", e)
|
||||
self.boxml = None
|
||||
|
||||
# -- Home Assistant ----------------------------------------------------
|
||||
def outside_entity(self) -> str:
|
||||
@@ -181,6 +212,7 @@ class Controller:
|
||||
try:
|
||||
cfg = await client.send("get_config")
|
||||
self.tz = (cfg or {}).get("time_zone") or "UTC"
|
||||
self.prefs.tz = self.tz
|
||||
lat, lon = (cfg or {}).get("latitude"), (cfg or {}).get("longitude")
|
||||
if isinstance(lat, (int, float)) and isinstance(lon, (int, float)):
|
||||
self.location = {"latitude": float(lat), "longitude": float(lon)}
|
||||
@@ -215,6 +247,14 @@ class Controller:
|
||||
value = st.get("s")
|
||||
self.guard.observe(c.id, eid, value, now)
|
||||
|
||||
def planned_kinds(self) -> set[str]:
|
||||
"""The kinds of the devices the box plans whose power is inside the
|
||||
load (heating, cooling, EV): its base load is trained where any is
|
||||
metered (inputs.controlled_power), as the planner uses it."""
|
||||
if not (getattr(self.inputs, "controlled_power", None) or []):
|
||||
return set()
|
||||
return {c.kind for c in self.controls if c.kind in ("heating", "cooling", "ev")}
|
||||
|
||||
# -- plans ---------------------------------------------------------------
|
||||
def model_state(self) -> dict:
|
||||
now = self.clock()
|
||||
@@ -223,13 +263,15 @@ class Controller:
|
||||
# and of a thermostat that heats and cools only its active side
|
||||
self.switch_sides()
|
||||
controls = [c for c in self.controls if self.policy.plannable(c) and self.switchover.active(c)]
|
||||
return inputs_mod.model_state(now, controls, self.inputs, self.col, self.bundles,
|
||||
state = inputs_mod.model_state(now, controls, self.inputs, self.col, self.bundles,
|
||||
self.history, self.tz, self.forecast, self.guard.extra_limits,
|
||||
building=self.building.for_model(start, inputs_mod.STEP_S, inputs_mod.STEPS),
|
||||
location=self.location, shade_ctx=self.shade_ctx(),
|
||||
grid_order=self.gridorders.for_model(start, inputs_mod.STEP_S, inputs_mod.STEPS),
|
||||
grid_max_w=getattr(self.cfg, "grid_max_w", None),
|
||||
ev_phases=self.ev_phases())
|
||||
# the owner's preferences per step (preferences.py)
|
||||
return self.prefs.apply(state, {c.id: c for c in controls}, self.col, now)
|
||||
|
||||
async def make_plan(self) -> dict | None:
|
||||
self.forecast = []
|
||||
@@ -239,6 +281,7 @@ class Controller:
|
||||
except Exception as e:
|
||||
log.info("weather forecast: %s", e)
|
||||
state = self.model_state()
|
||||
cands, net_load = state.pop("_cands", None) or {}, state.pop("_net_load", None)
|
||||
plan = await solve(state, self.cfg.solve_timeout_s)
|
||||
if "error" in plan:
|
||||
self.plan_error = plan["error"]
|
||||
@@ -247,12 +290,25 @@ class Controller:
|
||||
calm, self.calm = self.calm, False
|
||||
if calm:
|
||||
steady.carry_quarter(plan, self.plan, self.clock())
|
||||
# changes that do not matter stay out of it (deadband, hold, gain: steady.py)
|
||||
# what the plan does for each preference, and what each must costs
|
||||
self.prefs.take_plan(plan, self.clock())
|
||||
# changes that do not matter stay out of it (deadband, hold, gain: steady.py);
|
||||
# the owner's preference changes are not held back
|
||||
steady.stabilise(plan, state, [c for c in self.controls if c.id in plan.get("devices", {})], self.col,
|
||||
self.guard.last_change, self.clock())
|
||||
self.guard.last_change, self.clock(), free=self.prefs.free())
|
||||
self.prefs.write_view(self.controls, self.mode())
|
||||
plan["mode"] = self.mode()
|
||||
plan["id"] = f"{self.cfg.site_id[:8]}-{plan['made']}"
|
||||
plan["model"] = self.bundles.model_id()
|
||||
if self.boxml is not None:
|
||||
try: # SQLite and numpy: in a worker thread, the loop goes on
|
||||
ml = await asyncio.to_thread(self.ml_after_plan, state, plan, cands, net_load)
|
||||
# upward only with the owner's share consent: without it the box
|
||||
# trains for itself and nothing of it leaves (no forecasts, no errors)
|
||||
if getattr(self.consent, "share", False):
|
||||
plan["ml"] = ml
|
||||
except Exception as e:
|
||||
log.warning("the box's model figures: %s", e)
|
||||
self.plan, self.plan_error = plan, ""
|
||||
self.executor.plan = plan
|
||||
self.executor.inputs_ok = self.inputs_fresh()
|
||||
@@ -276,7 +332,82 @@ class Controller:
|
||||
# for the building (shed_w/grid_w per step, 24 h): absent from older agents
|
||||
"flex": plan.get("flex", {}),
|
||||
# the market's contracts in the plan (shadow): planned kWh and saving vs the grid
|
||||
"market": plan.get("market", [])}
|
||||
"market": plan.get("market", []),
|
||||
# the owner's preferences: per level who set what and when, the
|
||||
# overrides, and per preference met or not and each must's cost
|
||||
"preferences": self.prefs.public(),
|
||||
# the box's own models (boxtrain.py): held-out and live errors per source,
|
||||
# hourly the forecasts it planned on; absent from older agents
|
||||
**({"ml": plan["ml"]} if plan.get("ml") else {})}
|
||||
|
||||
# -- the owner's preferences (preferences.py) -------------------------------
|
||||
async def prefs_change(self, msg: dict, level: str, by: str) -> tuple[bool, str]:
|
||||
"""A change of the preferences (Home Assistant's entities and the
|
||||
panel: the owner's; the console: an installer's or operator's): a
|
||||
plan with it at once (not held back: steady.py), shown in HA."""
|
||||
ok, why = self.prefs.apply_msg(msg, {c.id: c for c in self.controls}, self.tz, level, by)
|
||||
log.info("preferences (%s, %s): %s %s", by, level, "applied" if ok else "refused", why)
|
||||
await self._prefs_changed()
|
||||
return ok, why
|
||||
|
||||
async def _prefs_changed(self):
|
||||
self.prefs.write_view(self.controls, self.mode())
|
||||
self.wake.set()
|
||||
for cb in self.on_prefs:
|
||||
try:
|
||||
await cb()
|
||||
except Exception as e:
|
||||
log.warning("preferences: %s", e)
|
||||
|
||||
async def _prefs_watch(self, stop):
|
||||
"""The panel's changes (prefs-inbox/), and overrides that expire."""
|
||||
while not stop.is_set():
|
||||
try:
|
||||
n = self.prefs.take_inbox({c.id: c for c in self.controls}, self.tz)
|
||||
if n or self.prefs.prune(self.clock()):
|
||||
await self._prefs_changed()
|
||||
except Exception as e:
|
||||
log.warning("preferences: %s", e)
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), PREFS_POLL_S)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
def ml_after_plan(self, state: dict, plan: dict, cands: dict, net_load) -> dict:
|
||||
"""After a plan (in a worker thread): the weather the box forecast
|
||||
with and each source's forecast kept for the training and the live
|
||||
error (trainhist.py), and the figures for the plan's "ml"
|
||||
(boxtrain.status), with, at most hourly, the forecasts it planned on:
|
||||
the load (as planned: the base load where it plans devices), PV and
|
||||
the member's net draw (the plan's import less export where it acts,
|
||||
auto; else the metered load's forecast less PV) for the building."""
|
||||
b, now, start = self.boxml, self.clock(), int(state["start"])
|
||||
st = b.store
|
||||
if self.forecast:
|
||||
st.put_weather("forecast", "ha", localml.ha_hours(self.forecast), now)
|
||||
m = self.bundles.model.get()
|
||||
if isinstance(m, dict) and m.get("site") == self.cfg.site_id:
|
||||
if m.get("weather"):
|
||||
st.put_weather("forecast", "cloud", localml.cloud_hours(m["weather"]), m.get("issued"))
|
||||
if m.get("weather_past"):
|
||||
st.put_weather("observed", "cloud", localml.cloud_hours(m["weather_past"]), now)
|
||||
for key, series in (("load", "plan"), ("pv", "pv")):
|
||||
if cands.get(key):
|
||||
b.record(series, start, cands[key], now)
|
||||
auto = plan.get("mode") == "auto"
|
||||
net = (plan.get("flex") or {}).get("net_w") if auto else \
|
||||
([round(lo - pv) for lo, pv in zip(net_load, state["pv_w"])] if net_load else None)
|
||||
if net:
|
||||
b.record("net", start, {"own": net, "week_mean": b.week_mean("net", start, len(net))}, now)
|
||||
fsrc = (state.get("sources") or {}).get("forecast") or {}
|
||||
ups = {k: {"start": start, "step_s": state["step_s"], "values": [round(float(v), 1) for v in state[f"{k}_w"]],
|
||||
"source": (fsrc.get(k) or {}).get("source") or (state.get("sources") or {}).get(k, "")[:80]}
|
||||
for k in ("load", "pv")}
|
||||
ups["load"]["series"] = "base_load" if b.kinds() else "load"
|
||||
if net:
|
||||
ups["net"] = {"start": start, "step_s": state["step_s"], "values": [float(v) for v in net],
|
||||
"source": "plan" if auto else "forecast"}
|
||||
return b.status(now, ups)
|
||||
|
||||
def inputs_fresh(self) -> bool:
|
||||
"""The plan's room temperatures and the battery's state have a value
|
||||
@@ -341,7 +472,12 @@ class Controller:
|
||||
await stop.wait()
|
||||
return
|
||||
tasks = [asyncio.create_task(self._planner(stop)), asyncio.create_task(self._actor(stop)),
|
||||
asyncio.create_task(self._bundle_watch(stop))]
|
||||
asyncio.create_task(self._bundle_watch(stop)), asyncio.create_task(self._prefs_watch(stop))]
|
||||
if self.boxml is not None:
|
||||
# the nightly training (boxtrain.py): child processes, the loop only awaits them
|
||||
tasks.append(asyncio.create_task(self.boxml.run(
|
||||
stop, tz=lambda: self.tz, location=lambda: self.location, client=lambda: self.client,
|
||||
weather_entity=lambda: self.inputs.weather or "")))
|
||||
await stop.wait()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
@@ -472,6 +608,18 @@ class Controller:
|
||||
return ok, msg
|
||||
if cmd == "building":
|
||||
return await self.building_command(args, now)
|
||||
if cmd == "preferences":
|
||||
# the console's: an installer's or Cantabile's, within the owner's limits
|
||||
level = args.get("level", "")
|
||||
if level not in ("installer", "operator"):
|
||||
return False, "level: installer or operator (the owner's own come from Home Assistant or the panel)"
|
||||
try:
|
||||
msg = json.loads(args.get("msg") or "{}")
|
||||
except ValueError:
|
||||
return False, "msg: JSON"
|
||||
if not isinstance(msg, dict) or "set" not in msg:
|
||||
return False, "msg: {\"set\": {\"item\", \"value\"}} (overrides are the owner's)"
|
||||
return await self.prefs_change(msg, level, f"{level}:{str(args.get('by') or 'console')[:60]}")
|
||||
return False, f"unknown command {cmd!r}"
|
||||
|
||||
async def building_command(self, args: dict, now: float) -> tuple[bool, str]:
|
||||
@@ -668,6 +816,7 @@ async def solve(state: dict, timeout: float) -> dict:
|
||||
|
||||
|
||||
PLAN_AT_S = 5.0 # a plan this long after a quarter hour: its first step is the quarter just begun
|
||||
PREFS_POLL_S = 2.0 # how often the panel's preference changes are looked for
|
||||
|
||||
|
||||
def plan_wait(now: float, interval: float, at: float = PLAN_AT_S) -> float:
|
||||
|
||||
@@ -10,15 +10,21 @@ metered). The kinds:
|
||||
week (168 values; the box's fallback format, ml/model_bundle.py).
|
||||
The baseline every other kind has to beat.
|
||||
gbm LightGBM on calendar (local hour, weekday, a day off: weekend or
|
||||
a German public holiday), the hour-of-week value, the same slot a
|
||||
a German public holiday, the special day's type as a categorical
|
||||
feature, the yearly rhythm: day of year as sine and cosine, the
|
||||
day's daylight hours), the hour-of-week value, the same slot a
|
||||
week before, the weather (temperature, irradiance, cloud cover)
|
||||
and, for PV, the sun and the physical prior. Small (150 trees, 15
|
||||
leaves, one thread): hundreds of sites a night.
|
||||
leaves, one thread): hundreds of sites a night. Recent weeks weigh
|
||||
more (recency_weights); a model's own feature list is kept with
|
||||
it, so a model trained before a feature was added still runs.
|
||||
pvphys PV only: the physical irradiance model (pvforecast.py, shared
|
||||
with the Mieterstrom pricing) times the site's calibration
|
||||
(pvcalib.py).
|
||||
|
||||
A model is a JSON-able dict ("params"); fit_* makes it, predict() uses it.
|
||||
intraday() carries the last hours' error against the meter into the next
|
||||
hours (the box every 15 minutes; methods.md B10).
|
||||
The weather per slot comes from hourly records (pvforecast.Hour: the hour
|
||||
ENDING at `end`, solar in kWh/m² over that hour, cloud %, temp °C)."""
|
||||
from __future__ import annotations
|
||||
@@ -45,8 +51,23 @@ GBM_PARAMS = dict(objective="regression", learning_rate=0.05, num_leaves=15, min
|
||||
bagging_freq=1, feature_fraction=0.9, lambda_l2=1.0, num_threads=1, verbose=-1, seed=1)
|
||||
GBM_ROUNDS = 150
|
||||
MIN_GBM_SLOTS = 96 # a day of data at least
|
||||
LOAD_FEATURES = ["hod", "dow", "off", "how", "lag7", "temp", "ghi", "cloud", "hdd"]
|
||||
PV_FEATURES = ["hod", "how", "lag7", "temp", "ghi", "cloud", "elev", "az", "phys"]
|
||||
LOAD_FEATURES = ["hod", "dow", "off", "how", "lag7", "temp", "ghi", "cloud", "hdd", "special", "doy_sin", "doy_cos",
|
||||
"daylight"]
|
||||
PV_FEATURES = ["hod", "how", "lag7", "temp", "ghi", "cloud", "elev", "az", "phys", "doy_sin", "doy_cos", "daylight"]
|
||||
FEATURES_V1 = {"load": ["hod", "dow", "off", "how", "lag7", "temp", "ghi", "cloud", "hdd"], # before 2026-11-04
|
||||
"pv": ["hod", "how", "lag7", "temp", "ghi", "cloud", "elev", "az", "phys"]}
|
||||
CATEGORICAL = ("special",)
|
||||
# linear leaves (LightGBM linear_tree: each leaf a linear model on its
|
||||
# branch's features, so the forecast keeps rising with heating degrees past
|
||||
# the coldest day trained on): for these series only where it measured better
|
||||
# (measured 2026-11-04, building's synthetic site: a cold snap 12 K below the
|
||||
# coldest day trained on, held-out nMAE 29.5 % -> 6.4 %; an ordinary winter
|
||||
# week 3.8 % -> 3.3 %; methods.md B10)
|
||||
LINEAR_SERIES: tuple = ("load", "base_load")
|
||||
# recency (methods.md B10): a slot's weight max(0.5^(age/30 d), 0.25 x the
|
||||
# season's likeness (cos of the age over a year, last year's same weeks
|
||||
# count), 0.05)
|
||||
HALF_LIFE_D, SEASON_W, FLOOR_W = 30.0, 0.25, 0.05
|
||||
|
||||
|
||||
# ---- the calendar -------------------------------------------------------------
|
||||
@@ -74,12 +95,91 @@ def holidays_de(year: int) -> frozenset:
|
||||
return frozenset(fixed + moving)
|
||||
|
||||
|
||||
def calendar(t: np.ndarray, tz: str) -> dict[str, np.ndarray]:
|
||||
# The special days' types (one categorical feature, "special"; the trees
|
||||
# combine it with the hour of day). A type the site has under 2 days of in
|
||||
# its training data counts as normal (fit_gbm: params["special_kept"]): the
|
||||
# cloud's teacher model will give such a type its shape later.
|
||||
SPECIAL = ("normal", "christmas_eve", "christmas_day", "boxing_day", "new_years_eve", "new_years_day", "easter",
|
||||
"carnival", "bridge_day", "school_holiday", "dst_change", "holiday")
|
||||
SPECIAL_CODE = {k: i for i, k in enumerate(SPECIAL)}
|
||||
# carnival Monday and Tuesday where it is kept (a site's state unknown: everywhere)
|
||||
CARNIVAL_STATES = frozenset({"NW", "RP", "HE", "BW", "BY", "SL"})
|
||||
# School holidays per German state: {state: [(first day, last day), ...]}.
|
||||
# A hook: no compact source in the repository yet (the KMK's yearly tables);
|
||||
# empty, so no day is typed school_holiday until it is filled.
|
||||
SCHOOL_HOLIDAYS: dict[str, list[tuple[date, date]]] = {}
|
||||
|
||||
|
||||
def _last_sunday(year: int, month: int) -> date:
|
||||
d = date(year, month + 1, 1) - timedelta(days=1) if month < 12 else date(year, 12, 31)
|
||||
return d - timedelta(days=(d.weekday() + 1) % 7)
|
||||
|
||||
|
||||
@lru_cache(maxsize=64)
|
||||
def special_days(year: int, state: str | None = None) -> dict:
|
||||
"""{date: type code} for the year's special days (SPECIAL); the rest
|
||||
normal."""
|
||||
e = _easter(year)
|
||||
hol = holidays_de(year) | holidays_de(year - 1) | holidays_de(year + 1)
|
||||
out: dict[date, int] = {}
|
||||
if state in SCHOOL_HOLIDAYS:
|
||||
for a, b in SCHOOL_HOLIDAYS[state]:
|
||||
d = a
|
||||
while d <= b:
|
||||
if d.year == year:
|
||||
out[d] = SPECIAL_CODE["school_holiday"]
|
||||
d += timedelta(days=1)
|
||||
for m in (3, 10):
|
||||
out[_last_sunday(year, m)] = SPECIAL_CODE["dst_change"]
|
||||
d = date(year, 1, 1)
|
||||
while d.year == year: # bridge days: a working day between a holiday and a weekend
|
||||
if d.weekday() < 5 and d not in hol:
|
||||
prev, nxt = d - timedelta(days=1), d + timedelta(days=1)
|
||||
off_prev, off_next = prev.weekday() >= 5 or prev in hol, nxt.weekday() >= 5 or nxt in hol
|
||||
if (prev in hol and off_next) or (nxt in hol and off_prev):
|
||||
out[d] = SPECIAL_CODE["bridge_day"]
|
||||
d += timedelta(days=1)
|
||||
for x in holidays_de(year):
|
||||
out[x] = SPECIAL_CODE["holiday"]
|
||||
if state is None or state in CARNIVAL_STATES:
|
||||
out[e - timedelta(days=48)] = out[e - timedelta(days=47)] = SPECIAL_CODE["carnival"]
|
||||
for x in (e - timedelta(days=2), e, e + timedelta(days=1)):
|
||||
out[x] = SPECIAL_CODE["easter"]
|
||||
out.update({date(year, 12, 24): SPECIAL_CODE["christmas_eve"], date(year, 12, 25): SPECIAL_CODE["christmas_day"],
|
||||
date(year, 12, 26): SPECIAL_CODE["boxing_day"], date(year, 12, 31): SPECIAL_CODE["new_years_eve"],
|
||||
date(year, 1, 1): SPECIAL_CODE["new_years_day"]})
|
||||
return out
|
||||
|
||||
|
||||
def daylight_h(doy: np.ndarray, lat: float | None) -> np.ndarray:
|
||||
"""The day's hours of daylight at a latitude (the sun's declination by
|
||||
day of year; polar day and night clipped), NaN without a location."""
|
||||
if lat is None:
|
||||
return np.full(len(doy), np.nan)
|
||||
decl = np.radians(23.44) * np.sin(2 * np.pi * (284 + doy) / 365.0)
|
||||
x = np.clip(-np.tan(np.radians(lat)) * np.tan(decl), -1.0, 1.0)
|
||||
return 24.0 * np.degrees(np.arccos(x)) / 180.0
|
||||
|
||||
|
||||
def recency_weights(t: np.ndarray, end: int | None = None) -> np.ndarray:
|
||||
"""A slot's weight by its age before end (the last slot): recent weeks
|
||||
weigh most, last year's same season still counts (HALF_LIFE_D,
|
||||
SEASON_W, FLOOR_W)."""
|
||||
if not len(t):
|
||||
return np.zeros(0)
|
||||
end = int(t[-1]) if end is None else int(end)
|
||||
age = (end - t.astype(float)) / 86400.0
|
||||
return np.maximum.reduce([0.5 ** (age / HALF_LIFE_D), SEASON_W * np.maximum(np.cos(2 * np.pi * age / 365.25), 0.0),
|
||||
np.full(len(t), FLOOR_W)])
|
||||
|
||||
|
||||
def calendar(t: np.ndarray, tz: str, state: str | None = None) -> dict[str, np.ndarray]:
|
||||
"""Per slot: local hour of day (fractional), weekday (Monday 0), hour of
|
||||
the week, day of year, and whether it is a day off."""
|
||||
the week, day of year, whether it is a day off, and the special day's
|
||||
type (SPECIAL's code)."""
|
||||
zone = ZoneInfo(tz or "UTC")
|
||||
n = len(t)
|
||||
hod, dow, how, doy, off = (np.zeros(n) for _ in range(5))
|
||||
hod, dow, how, doy, off, special = (np.zeros(n) for _ in range(6))
|
||||
cache: dict[int, tuple] = {}
|
||||
for i, ts in enumerate(t.tolist()):
|
||||
hour_key = int(ts) // 3600
|
||||
@@ -87,12 +187,13 @@ def calendar(t: np.ndarray, tz: str) -> dict[str, np.ndarray]:
|
||||
if c is None:
|
||||
d = datetime.fromtimestamp(hour_key * 3600, timezone.utc).astimezone(zone)
|
||||
c = (d.hour, d.weekday(), d.timetuple().tm_yday,
|
||||
1.0 if d.weekday() >= 5 or d.date() in holidays_de(d.year) else 0.0)
|
||||
1.0 if d.weekday() >= 5 or d.date() in holidays_de(d.year) else 0.0,
|
||||
special_days(d.year, state).get(d.date(), 0))
|
||||
cache[hour_key] = c
|
||||
h, wd, dy, of = c
|
||||
h, wd, dy, of, sp = c
|
||||
hod[i] = h + (int(ts) % 3600) / 3600.0
|
||||
dow[i], how[i], doy[i], off[i] = wd, wd * 24 + h, dy, of
|
||||
return {"hod": hod, "dow": dow, "how": how, "doy": doy, "off": off}
|
||||
dow[i], how[i], doy[i], off[i], special[i] = wd, wd * 24 + h, dy, of, sp
|
||||
return {"hod": hod, "dow": dow, "how": how, "doy": doy, "off": off, "special": special}
|
||||
|
||||
|
||||
# ---- the weather per slot -------------------------------------------------------
|
||||
@@ -183,14 +284,16 @@ def metrics(y: np.ndarray, p: np.ndarray) -> dict:
|
||||
|
||||
# ---- how: the hour-of-week profile -----------------------------------------------
|
||||
|
||||
def fit_how(t: np.ndarray, y: np.ndarray, tz: str) -> dict:
|
||||
def fit_how(t: np.ndarray, y: np.ndarray, tz: str, weights: np.ndarray | None = None) -> dict:
|
||||
"""weights: per slot (recency_weights); None: all alike."""
|
||||
cal = calendar(t, tz)
|
||||
vals: list[float | None] = []
|
||||
have = ~np.isnan(y)
|
||||
overall = float(y[have].mean()) if have.any() else 0.0
|
||||
wt = np.ones(len(y)) if weights is None else np.asarray(weights, dtype=float)
|
||||
overall = float(np.average(y[have], weights=wt[have])) if have.any() and wt[have].sum() > 0 else 0.0
|
||||
for h in range(168):
|
||||
m = have & (cal["how"] == h)
|
||||
vals.append(float(y[m].mean()) if m.any() else None)
|
||||
vals.append(float(np.average(y[m], weights=wt[m])) if m.any() and wt[m].sum() > 0 else None)
|
||||
out = []
|
||||
for h, v in enumerate(vals):
|
||||
if v is None: # the same hour on the other weekdays, else the mean
|
||||
@@ -217,27 +320,65 @@ def _lag7(t: np.ndarray, y: np.ndarray, hist_t: np.ndarray | None, hist_y: np.nd
|
||||
|
||||
|
||||
def features(series: str, t: np.ndarray, w: Weather, how_p: dict, tz: str, lag7: np.ndarray,
|
||||
phys: np.ndarray | None) -> tuple[np.ndarray, list[str]]:
|
||||
cal = calendar(t, tz)
|
||||
phys: np.ndarray | None, names: list[str] | None = None, lat: float | None = None,
|
||||
state: str | None = None, kept: list | None = None) -> tuple[np.ndarray, list[str]]:
|
||||
"""The feature matrix; names: a model's own list (default: the
|
||||
series' current one). kept: the special days' types the model learnt
|
||||
(the others count as normal)."""
|
||||
cal = calendar(t, tz, state)
|
||||
special = cal["special"]
|
||||
if kept is not None:
|
||||
special = np.where(np.isin(special, np.asarray(list(kept) + [0], dtype=float)), special, 0.0)
|
||||
ang = 2 * np.pi * (cal["doy"] - 1) / 365.25
|
||||
cols = {"hod": cal["hod"], "dow": cal["dow"], "off": cal["off"], "doy": cal["doy"],
|
||||
"how": how_values(how_p, t, tz), "lag7": lag7, "temp": w.temp, "ghi": w.ghi, "cloud": w.cloud,
|
||||
"hdd": np.maximum(15.0 - w.temp, 0.0), "elev": w.elev, "az": w.az,
|
||||
"phys": phys if phys is not None else np.full(len(t), np.nan)}
|
||||
names = PV_FEATURES if series == "pv" else LOAD_FEATURES
|
||||
"phys": phys if phys is not None else np.full(len(t), np.nan), "special": special,
|
||||
"doy_sin": np.sin(ang), "doy_cos": np.cos(ang), "daylight": daylight_h(cal["doy"], lat)}
|
||||
names = list(names) if names is not None else (PV_FEATURES if series == "pv" else LOAD_FEATURES)
|
||||
return np.column_stack([cols[c] for c in names]), names
|
||||
|
||||
|
||||
def special_kept(t: np.ndarray, y: np.ndarray, tz: str, state: str | None = None, at_least: int = 2) -> list[int]:
|
||||
"""The special days' types with at least 2 metered days in (t, y)."""
|
||||
cal = calendar(t, tz, state)
|
||||
have = ~np.isnan(y)
|
||||
days: dict[int, set] = {}
|
||||
# a local day by its local midnight (a day spans two UTC dates)
|
||||
for sp, ts, h in zip(cal["special"][have].tolist(), t[have].tolist(), cal["hod"][have].tolist()):
|
||||
if sp:
|
||||
days.setdefault(int(sp), set()).add(round((int(ts) - h * 3600) / 86400))
|
||||
return sorted(k for k, v in days.items() if len(v) >= at_least)
|
||||
|
||||
|
||||
def fit_gbm(series: str, t: np.ndarray, y: np.ndarray, w: Weather, tz: str, lag7: np.ndarray,
|
||||
phys: np.ndarray | None) -> dict | None:
|
||||
phys: np.ndarray | None, lat: float | None = None, state: str | None = None,
|
||||
weights: np.ndarray | None = None, linear: bool | None = None,
|
||||
names: list[str] | None = None) -> dict | None:
|
||||
"""weights: per slot (recency_weights; None: all alike); linear: leaves
|
||||
with linear models (default: series in LINEAR_SERIES); names: the
|
||||
features (default: the series' current list)."""
|
||||
import lightgbm as lgb
|
||||
have = ~np.isnan(y)
|
||||
if have.sum() < MIN_GBM_SLOTS:
|
||||
return None
|
||||
how_p = fit_how(t, y, tz)
|
||||
X, names = features(series, t, w, how_p, tz, lag7, phys)
|
||||
m = lgb.train(GBM_PARAMS, lgb.Dataset(X[have], y[have], feature_name=names), num_boost_round=GBM_ROUNDS)
|
||||
return {"kind": "gbm", "model": m.model_to_string(), "features": names, "how": how_p["values"], "tz": tz,
|
||||
how_p = fit_how(t, y, tz, weights)
|
||||
kept = special_kept(t, y, tz, state)
|
||||
X, names = features(series, t, w, how_p, tz, lag7, phys, names, lat, state, kept)
|
||||
params = dict(GBM_PARAMS)
|
||||
if linear if linear is not None else series in LINEAR_SERIES:
|
||||
params["linear_tree"] = True
|
||||
cat = [c for c in CATEGORICAL if c in names]
|
||||
ds = lgb.Dataset(X[have], y[have], feature_name=names, categorical_feature=cat or "auto",
|
||||
weight=None if weights is None else np.asarray(weights, dtype=float)[have])
|
||||
m = lgb.train(params, ds, num_boost_round=GBM_ROUNDS)
|
||||
out = {"kind": "gbm", "model": m.model_to_string(), "features": names, "how": how_p["values"], "tz": tz,
|
||||
"cap_w": float(np.nanpercentile(y[have], 99.9)) * 1.2 if have.any() else None}
|
||||
if "special" in names:
|
||||
out["special_kept"] = kept
|
||||
if params.get("linear_tree"):
|
||||
out["linear"] = True
|
||||
return out
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
@@ -269,11 +410,13 @@ def box_params(params: dict) -> dict:
|
||||
|
||||
|
||||
def predict_gbm(params: dict, series: str, t: np.ndarray, w: Weather, lag7: np.ndarray,
|
||||
phys: np.ndarray | None) -> np.ndarray:
|
||||
phys: np.ndarray | None, lat: float | None = None, state: str | None = None) -> np.ndarray:
|
||||
tz = params.get("tz", "UTC")
|
||||
X, names = features(series, t, w, {"values": params["how"]}, tz, lag7, phys)
|
||||
if names != params["features"]:
|
||||
raise ValueError("gbm: the features changed since it was trained")
|
||||
names = params["features"]
|
||||
if not isinstance(names, list) or any(n not in FEATURE_NAMES for n in names):
|
||||
raise ValueError("gbm: features this code does not know")
|
||||
X, names = features(series, t, w, {"values": params["how"]}, tz, lag7, phys, names, lat, state,
|
||||
params.get("special_kept"))
|
||||
p = np.asarray(_predictor(params).predict(X), dtype=float)
|
||||
p = np.maximum(p, 0.0)
|
||||
if params.get("cap_w"):
|
||||
@@ -283,6 +426,9 @@ def predict_gbm(params: dict, series: str, t: np.ndarray, w: Weather, lag7: np.n
|
||||
return p
|
||||
|
||||
|
||||
FEATURE_NAMES = frozenset(LOAD_FEATURES) | frozenset(PV_FEATURES) | {"doy"}
|
||||
|
||||
|
||||
# ---- one entry point -----------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
@@ -296,6 +442,7 @@ class Site:
|
||||
tilt: float = 30.0
|
||||
azimuth: float = 180.0
|
||||
pvgis_monthly: list | None = None # PVGIS's kWh per kWp per month (pvcalib.pvgis_monthly)
|
||||
state: str | None = None # the German state (NW, BY, ...): carnival, school holidays; None: unknown
|
||||
|
||||
|
||||
def phys_prior(t: np.ndarray, w: Weather, s: Site) -> np.ndarray | None:
|
||||
@@ -309,12 +456,13 @@ def phys_prior(t: np.ndarray, w: Weather, s: Site) -> np.ndarray | None:
|
||||
def fit(kind: str, series: str, t: np.ndarray, y: np.ndarray, w: Weather, s: Site,
|
||||
hist_t: np.ndarray | None = None, hist_y: np.ndarray | None = None) -> dict | None:
|
||||
"""A model of this kind on (t, y), or None where the kind cannot be made
|
||||
(too little data, PV physics without a location or weather)."""
|
||||
(too little data, PV physics without a location or weather). Recent
|
||||
slots weigh more (recency_weights; pvcalib has its own)."""
|
||||
if kind == "how":
|
||||
return fit_how(t, y, s.tz) if (~np.isnan(y)).any() else None
|
||||
return fit_how(t, y, s.tz, recency_weights(t)) if (~np.isnan(y)).any() else None
|
||||
if kind == "gbm":
|
||||
prior = phys_prior(t, w, s) if series == "pv" else None
|
||||
return fit_gbm(series, t, y, w, s.tz, _lag7(t, y, hist_t, hist_y), prior)
|
||||
return fit_gbm(series, t, y, w, s.tz, _lag7(t, y, hist_t, hist_y), prior, s.lat, s.state, recency_weights(t))
|
||||
if kind == "pvphys":
|
||||
if series != "pv":
|
||||
return None
|
||||
@@ -339,9 +487,9 @@ def predict(params: dict, series: str, t: np.ndarray, w: Weather, s: Site,
|
||||
if kind == "gbm":
|
||||
prior = phys_prior(t, w, s) if series == "pv" else None
|
||||
nan = np.full(len(t), np.nan)
|
||||
return predict_gbm(params, series, t, w, _lag7(t, nan, hist_t, hist_y), prior)
|
||||
return predict_gbm(params, series, t, w, _lag7(t, nan, hist_t, hist_y), prior, s.lat, s.state)
|
||||
if kind == "pvphys":
|
||||
ss = Site(s.tz, s.lat, s.lon, s.kwp, params.get("tilt", s.tilt), params.get("azimuth", s.azimuth))
|
||||
ss = Site(s.tz, s.lat, s.lon, s.kwp, params.get("tilt", s.tilt), params.get("azimuth", s.azimuth), state=s.state)
|
||||
prior = phys_prior(t, w, ss)
|
||||
if prior is None:
|
||||
return np.full(len(t), np.nan)
|
||||
@@ -374,3 +522,41 @@ def choose(challengers: dict[str, float | None], champion: float | None) -> str
|
||||
if champion is not None and math.isfinite(champion) and scored[best] >= champion:
|
||||
return None
|
||||
return best
|
||||
|
||||
|
||||
# ---- the intraday correction ------------------------------------------------------
|
||||
|
||||
INTRADAY_WINDOW_H = 3.0 # the recent error over the last 3 hours,
|
||||
INTRADAY_RECENT_H = 1.0 # the latest weighing most (half-life 1 h: an evening that starts at 18:00 shows by 19:00)
|
||||
INTRADAY_HALF_LIFE_H = 2.0 # fading with the step ahead
|
||||
INTRADAY_PV_RATIO = (0.5, 1.5) # PV: the metered over the forecast energy, capped
|
||||
|
||||
|
||||
def intraday(series: str, pred: np.ndarray, past_pred: np.ndarray, past_y: np.ndarray,
|
||||
half_life_h: float = INTRADAY_HALF_LIFE_H) -> tuple[np.ndarray, float]:
|
||||
"""The forecast for the next steps (pred, the first the slot under way
|
||||
or the next) moved by the last hours' error against the meter
|
||||
(past_pred, past_y: the same slots, NaN where unknown; at least an hour
|
||||
of both; the latest weighing most, half-life 1 h), fading with the step
|
||||
ahead (half-life 2 h): load additive (W),
|
||||
PV multiplicative (the ratio of energies, capped 0.5..1.5; none at
|
||||
night). Returns the corrected forecast and the error carried (W, or the
|
||||
PV ratio less 1). It catches any unusual day: a special day, guests,
|
||||
illness."""
|
||||
pred = np.asarray(pred, dtype=float)
|
||||
past_pred, past_y = np.asarray(past_pred, dtype=float), np.asarray(past_y, dtype=float)
|
||||
m = ~(np.isnan(past_pred) | np.isnan(past_y))
|
||||
if m.sum() < 4:
|
||||
return pred, 0.0
|
||||
decay = 0.5 ** (np.arange(len(pred)) * (SLOT_S / 3600.0) / half_life_h)
|
||||
# the past slots' weights: the latest most (the last slot ends where pred starts)
|
||||
age_h = (len(past_y) - np.arange(len(past_y)) - 0.5) * (SLOT_S / 3600.0)
|
||||
wt = np.where(m, 0.5 ** (age_h / INTRADAY_RECENT_H), 0.0)
|
||||
if series == "pv":
|
||||
e_pred, e_y = float(np.sum(wt * np.nan_to_num(past_pred))), float(np.sum(wt * np.nan_to_num(past_y)))
|
||||
if e_pred < 50.0 * float(wt.sum()): # under 50 W on average: night or dusk, nothing to scale
|
||||
return pred, 0.0
|
||||
r = min(max(e_y / e_pred, INTRADAY_PV_RATIO[0]), INTRADAY_PV_RATIO[1]) - 1.0
|
||||
return np.maximum(pred * (1.0 + r * decay), 0.0), r
|
||||
e = float(np.sum(wt * np.nan_to_num(past_y - past_pred)) / wt.sum())
|
||||
return np.maximum(pred + e * decay, 0.0), e
|
||||
@@ -37,6 +37,7 @@ READ_ONLY = frozenset({
|
||||
"config/area_registry/list",
|
||||
"recorder/statistics_during_period",
|
||||
"recorder/list_statistic_ids",
|
||||
"history/history_during_period", # the weather entity's past, for the box's own training (boxtrain.py)
|
||||
"ping",
|
||||
})
|
||||
|
||||
|
||||
@@ -163,37 +163,74 @@ def base_load(col: Collector, inputs) -> float | None:
|
||||
return max(0.0, sum(vals) - ctl)
|
||||
|
||||
|
||||
FC_SOURCES = {"own": "the box's own model", "cloud": "the cloud's series", "box": "the cloud's model on the box",
|
||||
"hour_of_week": "the hour-of-week profile"}
|
||||
|
||||
|
||||
def _model_series(m, start, bundles, history, tz, box) -> tuple[list[float], dict] | None:
|
||||
"""The model bundle's series m for the horizon: the cloud's forecast
|
||||
where its series covers a step, else the box's own model (localml.py;
|
||||
"""A model series m for the horizon, per step the best fresh source
|
||||
(methods.md B10): the box's own model where it is in force and has a
|
||||
value (boxtrain.py: bundles.own), else the cloud's forecast where its
|
||||
series covers the step, else the cloud's model on the box (localml.py;
|
||||
box: HA's weather forecast and its entity) where it has a value, else
|
||||
the hour-of-week profile; and how many steps each gave."""
|
||||
the hour-of-week profile; how many steps each gave, the source that
|
||||
gave most ("source"), and every source's values ("_cands": the live
|
||||
error per source, taken out before the plan)."""
|
||||
trainer = getattr(bundles, "own", None) if bundles is not None else None
|
||||
mine = None
|
||||
if trainer is not None and box is not None:
|
||||
try:
|
||||
mine = trainer.forecast(m, start, STEPS, box[0], box[1])
|
||||
except Exception as e: # never stops a plan
|
||||
log.warning("the box's own %s model: %s", m, e)
|
||||
if not hasattr(bundles, "forecast_parts"):
|
||||
s = bundles.forecast(m, start, STEP_S, STEPS, tz)
|
||||
return None if s is None else (s, {})
|
||||
parts = bundles.forecast_parts(m, start, STEP_S, STEPS, tz)
|
||||
if parts is None:
|
||||
# no cloud model for it: the box's own alone, where it covers the horizon
|
||||
if mine is None or any(v is None for v in mine[0]):
|
||||
return None
|
||||
vals = [float(v) for v in mine[0]]
|
||||
return vals, {"series": m, "own": STEPS, "cloud": 0, "box": 0, "hour_of_week": 0, "own_model": mine[1].get("kind"),
|
||||
"source": "own", "_cands": {"own": vals}}
|
||||
vals = [v for v, _ in parts]
|
||||
info = {"series": m, "cloud": sum(src == "series" for _, src in parts), "box": 0, "hour_of_week": 0}
|
||||
info = {"series": m, "own": 0, "cloud": 0, "box": 0, "hour_of_week": 0}
|
||||
cloud_series = [v if src == "series" else None for v, src in parts]
|
||||
own = None
|
||||
if box is not None and info["cloud"] < STEPS:
|
||||
if box is not None and (trainer is not None or not all(v is not None for v in cloud_series)):
|
||||
own = bundles.box_forecast(m, start, STEPS, history, box[0], box[1])
|
||||
for k, (v, src) in enumerate(parts):
|
||||
if src == "series":
|
||||
continue
|
||||
o = mine[0][k] if mine is not None else None
|
||||
b = own[0][k] if own is not None else None
|
||||
if b is not None:
|
||||
if o is not None:
|
||||
vals[k] = o
|
||||
info["own"] += 1
|
||||
elif src == "series":
|
||||
info["cloud"] += 1
|
||||
elif b is not None:
|
||||
vals[k] = b
|
||||
info["box"] += 1
|
||||
else:
|
||||
info["hour_of_week"] += 1
|
||||
if mine is not None:
|
||||
info["own_model"] = mine[1].get("kind")
|
||||
if mine[1].get("weather"):
|
||||
info["own_weather"] = mine[1]["weather"]
|
||||
if own is not None:
|
||||
info["box_model"] = own[1].get("kind")
|
||||
if own[1].get("weather"):
|
||||
info["box_weather"] = own[1]["weather"]
|
||||
if own[1].get("error"):
|
||||
info["box_error"] = own[1]["error"]
|
||||
info["source"] = max(("own", "cloud", "box", "hour_of_week"), key=lambda k: info[k])
|
||||
if trainer is not None:
|
||||
info["_cands"] = {"own": list(mine[0]) if mine is not None else None, "cloud": cloud_series,
|
||||
"cloud_model": list(own[0]) if own is not None else None,
|
||||
"hour_of_week": (bundles.profile(m, start, STEP_S, STEPS, tz) if hasattr(bundles, "profile")
|
||||
else [v if src == "profile" else None for v, src in parts])}
|
||||
if not info["own"]:
|
||||
info.pop("own")
|
||||
return vals, info
|
||||
|
||||
|
||||
@@ -207,6 +244,9 @@ def _series(name, start, bundles, history, tz, now_value, default, model_names=N
|
||||
if got is not None:
|
||||
s, n = got
|
||||
src = "model" if m == name else f"model ({m})"
|
||||
if n.get("own"):
|
||||
src += (f"; the box's own {n.get('own_model')} model (trained on the box) for {n['own']} of {STEPS} steps"
|
||||
f" (weather: {n.get('own_weather') or 'none'})")
|
||||
if n.get("box"):
|
||||
src += (f"; the box's own {n.get('box_model')} model for {n['box']} of {STEPS} steps"
|
||||
f" (weather: {n.get('box_weather') or 'none'})")
|
||||
@@ -326,6 +366,39 @@ def model_state(now: float, controls, inputs, col: Collector, bundles, history,
|
||||
bundles.forecast("load_w", start, STEP_S, 1, tz) is not None:
|
||||
load_src += " (the model's load_w includes the planned devices: not used)"
|
||||
pv, pv_src = _series("pv_w", start, bundles, history, tz, None, 0.0, None, box, fc_src["pv"])
|
||||
# the forecasts per source for the live error (boxtrain.py), not for the plan
|
||||
cands = {k: v.pop("_cands") for k, v in fc_src.items() if "_cands" in v}
|
||||
net_load = None
|
||||
intraday = {}
|
||||
trainer = getattr(bundles, "own", None)
|
||||
if trainer is not None:
|
||||
# the intraday correction (fcmodels.intraday, methods.md B10): the last
|
||||
# 3 hours' error of what was planned on against the meter, carried into
|
||||
# the next hours and fading (half-life 2 h); load in W, PV as a ratio
|
||||
for key, series in (("load", "plan"), ("pv", "pv")):
|
||||
base = load if key == "load" else pv
|
||||
c = cands.setdefault(key, {})
|
||||
c["used"] = list(base)
|
||||
try:
|
||||
new, e = trainer.correct(series, start, base)
|
||||
except Exception as ex: # never stops a plan
|
||||
log.warning("intraday correction %s: %s", key, ex)
|
||||
continue
|
||||
c["corrected"] = list(new)
|
||||
intraday[key] = round(float(e), 3)
|
||||
if key == "load":
|
||||
load = new
|
||||
else:
|
||||
pv = new
|
||||
if trainer is not None:
|
||||
# the house as metered (devices in), for the member's net draw the box
|
||||
# publishes upward where it does not act itself (controller.forecasts_up)
|
||||
if ctl:
|
||||
nl: dict = {}
|
||||
got = _series("load_w", start, bundles, history, tz, None, 500.0, ("load_w",), box, nl)
|
||||
net_load = got[0] if got[1].startswith("model") else None
|
||||
else:
|
||||
net_load = load
|
||||
t_out = _hourly(weather_forecast or [], "temperature", start, STEPS)
|
||||
t_src = "weather forecast"
|
||||
measured = col.value(inputs.outside_temp) if inputs.outside_temp else None
|
||||
@@ -390,6 +463,7 @@ def model_state(now: float, controls, inputs, col: Collector, bundles, history,
|
||||
"sources": {"load": load_src, "pv": pv_src, "t_out": t_src, "model": bundles.model_id(),
|
||||
"sky": "weather forecast" if clouds else "clear", "rooms": rooms_src, "ghi": ghi_src,
|
||||
"ev_phases": phases_src,
|
||||
**({"intraday": intraday} if intraday else {}),
|
||||
# per series the steps from the cloud's forecast, the box's own model
|
||||
# (its kind and weather) and the hour-of-week profile (methods.md B9)
|
||||
"forecast": {k: v for k, v in fc_src.items() if v}},
|
||||
@@ -398,6 +472,9 @@ def model_state(now: float, controls, inputs, col: Collector, bundles, history,
|
||||
"market": bundles.market_for_model(start, STEP_S, STEPS, now) if hasattr(bundles, "market_for_model") else []}
|
||||
if grid_max_w:
|
||||
out["grid_max_w"] = float(grid_max_w)
|
||||
if cands or net_load is not None:
|
||||
out["_cands"] = cands # taken out by the controller before the plan
|
||||
out["_net_load"] = net_load
|
||||
# a shade's moves by rules on the sun and its room (shades.py); the
|
||||
# optimiser takes them as they are
|
||||
from . import shades
|
||||
@@ -437,6 +514,11 @@ def _room_rc(c, params: dict, bundles, only_one: bool, now: float) -> str:
|
||||
heat is cop * P). Into params; where they came from."""
|
||||
if "r_k_per_kw" in c.params or "c_kwh_per_k" in c.params:
|
||||
return "mapping"
|
||||
# the box's own fit where it is in force (boxtrain.py), else the cloud's
|
||||
own = getattr(bundles, "own", None)
|
||||
rc = own.room_rc(c.id, str(c.params.get("area") or ""), only_one) if own is not None else None
|
||||
src = "box model" if rc is not None else "model"
|
||||
if rc is None:
|
||||
get = getattr(bundles, "room_rc", None)
|
||||
rc = get(c.id, str(c.params.get("area") or ""), only_one, now) if get else None
|
||||
if rc is None:
|
||||
@@ -456,7 +538,7 @@ def _room_rc(c, params: dict, bundles, only_one: bool, now: float) -> str:
|
||||
params["internal_gain_w"] = gain
|
||||
if "solar_m2" not in c.params:
|
||||
params["solar_m2"] = solar
|
||||
return "model"
|
||||
return src
|
||||
|
||||
|
||||
def _quantities(c) -> list[str]:
|
||||
|
||||
@@ -17,7 +17,19 @@ are.
|
||||
Trees(enc, n_features).predict(rows) sums the leaves in LightGBM's order
|
||||
with its decision rule (tree.h NumericalDecision), so it gives LightGBM's
|
||||
raw score to the last bit (measured 2026-11-04: max
|
||||
relative difference 0.0 over 192 slots)."""
|
||||
relative difference 0.0 over 192 slots).
|
||||
|
||||
lgbm-trees-v2 (2026-11-04, methods.md B10), written only for a booster
|
||||
that needs it (a categorical split or linear leaves; else v1, which older
|
||||
agents read): per tree uint16 L, uint8 flags (bit 0: linear leaves); the
|
||||
L-1 nodes as v1, decision bit 0 set for a categorical split, whose
|
||||
threshold is then the index of its category set; uint16 the number of
|
||||
sets, each uint8 n and n uint16 categories (those going left); the L
|
||||
leaf values; with linear leaves per leaf float64 constant, uint8 n and n
|
||||
(uint8 feature, float64 coefficient). Categorical rule (tree.h
|
||||
CategoricalDecision): NaN or a negative value right, else left if the
|
||||
integer is in the set. A linear leaf: constant plus the coefficients
|
||||
times the values in LightGBM's order; a NaN among them: the leaf's value."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
@@ -26,6 +38,7 @@ import struct
|
||||
import zlib
|
||||
|
||||
ENCODING = "lgbm-trees-v1"
|
||||
ENCODING_V2 = "lgbm-trees-v2"
|
||||
NODE = struct.Struct("<BBhhd")
|
||||
MISSING = {"None": 0, "Zero": 1, "NaN": 2}
|
||||
ZERO = 1.0000000180025095e-35 # LightGBM's kZeroThreshold (1e-35f)
|
||||
@@ -36,44 +49,86 @@ def _ref(n: dict) -> int:
|
||||
return int(n["split_index"]) if "split_index" in n else ~int(n.get("leaf_index", 0))
|
||||
|
||||
|
||||
def pack_tree(root: dict) -> bytes:
|
||||
"""One tree (dump_model()'s tree_info[i]["tree_structure"])."""
|
||||
if "split_index" not in root: # no split: one leaf
|
||||
return struct.pack("<Hd", 1, float(root["leaf_value"]))
|
||||
def _walk(root: dict) -> tuple[dict[int, dict], dict[int, dict]]:
|
||||
nodes: dict[int, dict] = {}
|
||||
leaves: dict[int, float] = {}
|
||||
leaves: dict[int, dict] = {}
|
||||
stack = [root]
|
||||
while stack:
|
||||
n = stack.pop()
|
||||
if "split_index" in n:
|
||||
if n.get("decision_type", "<=") != "<=":
|
||||
raise ValueError("a categorical split: not supported")
|
||||
if n.get("decision_type", "<=") not in ("<=", "=="):
|
||||
raise ValueError(f"a split {n.get('decision_type')!r}: not supported")
|
||||
nodes[int(n["split_index"])] = n
|
||||
stack += [n["left_child"], n["right_child"]]
|
||||
else:
|
||||
leaves[int(n["leaf_index"])] = float(n["leaf_value"])
|
||||
leaves[int(n.get("leaf_index", 0))] = n
|
||||
k = len(leaves)
|
||||
if sorted(nodes) != list(range(k - 1)) or sorted(leaves) != list(range(k)):
|
||||
raise ValueError("a tree's node or leaf indices are not contiguous")
|
||||
return nodes, leaves
|
||||
|
||||
|
||||
def _needs_v2(dump: dict) -> bool:
|
||||
for t in dump["tree_info"]:
|
||||
nodes, leaves = _walk(t["tree_structure"])
|
||||
if any(n.get("decision_type") == "==" for n in nodes.values()) or any("leaf_coeff" in x for x in leaves.values()):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def pack_tree(root: dict, v2: bool = False) -> bytes:
|
||||
"""One tree (dump_model()'s tree_info[i]["tree_structure"])."""
|
||||
if "split_index" not in root and not v2: # no split: one leaf
|
||||
return struct.pack("<Hd", 1, float(root["leaf_value"]))
|
||||
nodes, leaves = _walk(root)
|
||||
k = len(leaves)
|
||||
linear = v2 and any("leaf_coeff" in x for x in leaves.values())
|
||||
out = [struct.pack("<H", k)]
|
||||
if v2:
|
||||
out.append(struct.pack("<B", 1 if linear else 0))
|
||||
cats: list[list[int]] = []
|
||||
for i in range(k - 1):
|
||||
n = nodes[i]
|
||||
dec = (2 if n.get("default_left") else 0) | (MISSING[n.get("missing_type", "None")] << 2)
|
||||
out.append(NODE.pack(int(n["split_feature"]), dec, _ref(n["left_child"]), _ref(n["right_child"]),
|
||||
float(n["threshold"])))
|
||||
out.append(struct.pack(f"<{k}d", *(leaves[i] for i in range(k))))
|
||||
threshold = n["threshold"]
|
||||
if n.get("decision_type") == "==":
|
||||
if not v2:
|
||||
raise ValueError("a categorical split: lgbm-trees-v2")
|
||||
dec |= 1
|
||||
cats.append([int(c) for c in str(threshold).split("||") if c != ""])
|
||||
threshold = len(cats) - 1
|
||||
out.append(NODE.pack(int(n["split_feature"]), dec, _ref(n["left_child"]), _ref(n["right_child"]), float(threshold)))
|
||||
if v2:
|
||||
out.append(struct.pack("<H", len(cats)))
|
||||
for c in cats:
|
||||
if len(c) > 255 or any(not 0 <= x < 65536 for x in c):
|
||||
raise ValueError("a categorical split: at most 255 categories below 65536")
|
||||
out.append(struct.pack(f"<B{len(c)}H", len(c), *c))
|
||||
out.append(struct.pack(f"<{k}d", *(float(leaves[i]["leaf_value"]) for i in range(k))))
|
||||
if linear:
|
||||
for i in range(k):
|
||||
x = leaves[i]
|
||||
feats, coeff = [int(f) for f in x.get("leaf_features") or []], [float(c) for c in x.get("leaf_coeff") or []]
|
||||
if len(feats) != len(coeff) or len(feats) > 255:
|
||||
raise ValueError("a linear leaf")
|
||||
out.append(struct.pack("<dB", float(x.get("leaf_const", 0.0)), len(feats)))
|
||||
for f, c in zip(feats, coeff):
|
||||
out.append(struct.pack("<Bd", f, c))
|
||||
return b"".join(out)
|
||||
|
||||
|
||||
def pack(dump: dict) -> dict:
|
||||
"""A booster's dump_model() as lgbm-trees-v1."""
|
||||
"""A booster's dump_model() as lgbm-trees-v1, or v2 where it has
|
||||
categorical splits or linear leaves."""
|
||||
if dump.get("num_class", 1) != 1 or dump.get("num_tree_per_iteration", 1) != 1:
|
||||
raise ValueError("one tree per round only")
|
||||
obj = str(dump.get("objective") or "").split(" ")[0]
|
||||
if obj != "regression":
|
||||
raise ValueError(f"objective {obj!r}: regression only (no output transform)")
|
||||
raw = b"".join(pack_tree(t["tree_structure"]) for t in dump["tree_info"])
|
||||
return {"enc": ENCODING, "n": len(dump["tree_info"]), "data": base64.b64encode(zlib.compress(raw, 9)).decode()}
|
||||
v2 = _needs_v2(dump)
|
||||
raw = b"".join(pack_tree(t["tree_structure"], v2) for t in dump["tree_info"])
|
||||
return {"enc": ENCODING_V2 if v2 else ENCODING, "n": len(dump["tree_info"]),
|
||||
"data": base64.b64encode(zlib.compress(raw, 9)).decode()}
|
||||
|
||||
|
||||
class Trees:
|
||||
@@ -81,8 +136,9 @@ class Trees:
|
||||
leaf (no loops), features in range, finite leaves."""
|
||||
|
||||
def __init__(self, enc: dict, n_features: int):
|
||||
if not isinstance(enc, dict) or enc.get("enc") != ENCODING:
|
||||
raise ValueError(f"trees: {ENCODING} only")
|
||||
if not isinstance(enc, dict) or enc.get("enc") not in (ENCODING, ENCODING_V2):
|
||||
raise ValueError(f"trees: {ENCODING} or {ENCODING_V2} only")
|
||||
v2 = enc["enc"] == ENCODING_V2
|
||||
z = zlib.decompressobj()
|
||||
try:
|
||||
raw = z.decompress(base64.b64decode(str(enc.get("data")), validate=True), MAX_RAW)
|
||||
@@ -100,6 +156,13 @@ class Trees:
|
||||
off += 2
|
||||
if not 1 <= k <= MAX_LEAVES:
|
||||
raise ValueError("trees: leaves")
|
||||
linear = False
|
||||
if v2:
|
||||
(flags,) = struct.unpack_from("<B", raw, off)
|
||||
off += 1
|
||||
if flags & ~1:
|
||||
raise ValueError("trees: flags")
|
||||
linear = bool(flags & 1)
|
||||
nodes = []
|
||||
for i in range(k - 1):
|
||||
f, dec, left, right, threshold = NODE.unpack_from(raw, off)
|
||||
@@ -107,14 +170,44 @@ class Trees:
|
||||
for c in (left, right):
|
||||
if not (i < c < k - 1 or -k <= c < 0):
|
||||
raise ValueError("trees: a child out of range")
|
||||
if f >= n_features or dec & 1 or (dec >> 2) & 3 > 2 or math.isnan(threshold):
|
||||
if f >= n_features or (dec & 1 and not v2) or (dec >> 2) & 3 > 2 or math.isnan(threshold):
|
||||
raise ValueError("trees: a split")
|
||||
nodes.append((f, threshold, bool(dec & 2), (dec >> 2) & 3, left, right))
|
||||
nodes.append([f, threshold, bool(dec & 2), (dec >> 2) & 3, left, right, bool(dec & 1)])
|
||||
if v2:
|
||||
(ncat,) = struct.unpack_from("<H", raw, off)
|
||||
off += 2
|
||||
sets = []
|
||||
for _ in range(ncat):
|
||||
(m,) = struct.unpack_from("<B", raw, off)
|
||||
sets.append(frozenset(struct.unpack_from(f"<{m}H", raw, off + 1)))
|
||||
off += 1 + 2 * m
|
||||
for nd in nodes:
|
||||
if nd[6]:
|
||||
if not 0 <= nd[1] < len(sets) or nd[1] != int(nd[1]):
|
||||
raise ValueError("trees: a category set")
|
||||
nd[1] = sets[int(nd[1])]
|
||||
nodes = [tuple(nd) for nd in nodes]
|
||||
leaves = struct.unpack_from(f"<{k}d", raw, off)
|
||||
off += 8 * k
|
||||
if not all(math.isfinite(v) for v in leaves):
|
||||
raise ValueError("trees: a leaf")
|
||||
self.trees.append((nodes, leaves))
|
||||
lin = None
|
||||
if linear:
|
||||
lin = []
|
||||
for _ in range(k):
|
||||
const, m = struct.unpack_from("<dB", raw, off)
|
||||
off += 9
|
||||
terms = []
|
||||
for _ in range(m):
|
||||
f, c = struct.unpack_from("<Bd", raw, off)
|
||||
off += 9
|
||||
if f >= n_features or not math.isfinite(c):
|
||||
raise ValueError("trees: a linear leaf")
|
||||
terms.append((f, c))
|
||||
if not math.isfinite(const):
|
||||
raise ValueError("trees: a linear leaf")
|
||||
lin.append((const, terms))
|
||||
self.trees.append((nodes, leaves, lin))
|
||||
if len(self.trees) > MAX_TREES:
|
||||
raise ValueError("trees: too many")
|
||||
except struct.error as e:
|
||||
@@ -128,21 +221,31 @@ class Trees:
|
||||
split knows NaN; the split's missing values (zero or NaN, by its
|
||||
type) go its default way; the leaves summed in order."""
|
||||
total = 0.0
|
||||
for nodes, leaves in self.trees:
|
||||
if not nodes:
|
||||
total += leaves[0]
|
||||
continue
|
||||
i = 0
|
||||
for nodes, leaves, lin in self.trees:
|
||||
i = 0 if nodes else ~0
|
||||
while i >= 0:
|
||||
f, threshold, default_left, missing, left, right = nodes[i]
|
||||
f, threshold, default_left, missing, left, right, cat = nodes[i]
|
||||
v = x[f]
|
||||
if cat:
|
||||
i = right if v != v or v < 0 or int(v) not in threshold else left
|
||||
continue
|
||||
if v != v and missing != 2:
|
||||
v = 0.0
|
||||
if (missing == 1 and -ZERO <= v <= ZERO) or (missing == 2 and v != v):
|
||||
i = left if default_left else right
|
||||
else:
|
||||
i = left if v <= threshold else right
|
||||
if lin is None:
|
||||
total += leaves[~i]
|
||||
continue
|
||||
out = lin[~i][0]
|
||||
for f, c in lin[~i][1]:
|
||||
v = x[f]
|
||||
if v != v:
|
||||
out = leaves[~i]
|
||||
break
|
||||
out += c * v
|
||||
total += out
|
||||
return total
|
||||
|
||||
def predict(self, rows) -> list[float]:
|
||||
|
||||
@@ -45,7 +45,7 @@ SERIES = {"load_w": "load", "pv_w": "pv", "base_load_w": "base_load"} # the bu
|
||||
# is the base load already (the house without the planned devices), which
|
||||
# is the load where the box plans none (the only case its load_w is used)
|
||||
HISTORY = {"load_w": "load_w", "base_load_w": "load_w", "pv_w": "pv_w"}
|
||||
SITE_FIELDS = {"tz": str, "lat": float, "lon": float, "kwp": float, "tilt": float, "azimuth": float}
|
||||
SITE_FIELDS = {"tz": str, "lat": float, "lon": float, "kwp": float, "tilt": float, "azimuth": float, "state": str}
|
||||
|
||||
|
||||
def _num(v) -> bool:
|
||||
@@ -86,8 +86,12 @@ class Model:
|
||||
if not _numbers(p.get("values"), 168):
|
||||
raise ValueError("how: 168 numbers")
|
||||
elif self.kind == "gbm":
|
||||
want = fcmodels.PV_FEATURES if self.series == "pv" else fcmodels.LOAD_FEATURES
|
||||
if p.get("features") != want or not _numbers(p.get("how"), 168) or "model" in p:
|
||||
# the features it was trained with: today's, or those before the
|
||||
# special days and the yearly rhythm (fcmodels.FEATURES_V1)
|
||||
key = "pv" if self.series == "pv" else "load"
|
||||
want = fcmodels.PV_FEATURES if key == "pv" else fcmodels.LOAD_FEATURES
|
||||
if p.get("features") not in (want, fcmodels.FEATURES_V1[key]) or not _numbers(p.get("how"), 168) \
|
||||
or "model" in p or not all(isinstance(k, int) for k in p.get("special_kept") or []):
|
||||
raise ValueError(f"gbm: the features {', '.join(want)}, 168 hour-of-week values, trees")
|
||||
if p.get("cap_w") is not None and not _num(p["cap_w"]):
|
||||
raise ValueError("gbm: cap_w")
|
||||
|
||||
@@ -171,6 +171,21 @@ NAMES = {
|
||||
"control": {"en": "Control", "de": "Gerät"},
|
||||
"value": {"en": "Value", "de": "Wert"},
|
||||
"why": {"en": "Why", "de": "Warum"},
|
||||
# the owner's preferences (preferences.py; announce.py)
|
||||
"pref_schedule": {"en": "{0}: minimum temperatures", "de": "{0}: Mindesttemperaturen"},
|
||||
"pref_schedule_cool": {"en": "{0}: maximum temperatures", "de": "{0}: Höchsttemperaturen"},
|
||||
"pref_boost_c": {"en": "{0}: boost temperature", "de": "{0}: Boost-Temperatur"},
|
||||
"pref_boost_h": {"en": "{0}: boost hours", "de": "{0}: Boost-Dauer"},
|
||||
"pref_boost": {"en": "{0}: boost now", "de": "{0}: Boost jetzt"},
|
||||
"pref_away": {"en": "Away", "de": "Abwesend"},
|
||||
"pref_away_until": {"en": "Away until", "de": "Abwesend bis"},
|
||||
"pref_ready": {"en": "{0}: ready by", "de": "{0}: fertig bis"},
|
||||
"pref_capacity": {"en": "{0}: battery capacity", "de": "{0}: Akkukapazität"},
|
||||
"pref_needed": {"en": "{0}: needed once at", "de": "{0}: einmalig gebraucht um"},
|
||||
"pref_reserve": {"en": "{0}: reserve", "de": "{0}: Reserve"},
|
||||
"pref_reserve_blocks": {"en": "{0}: reserve by time", "de": "{0}: Reserve nach Uhrzeit"},
|
||||
"pref_result": {"en": "Preferences", "de": "Präferenzen"},
|
||||
"pref_result_of": {"en": "{0}: preferences", "de": "{0}: Präferenzen"},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -48,7 +48,18 @@ The building's requests (building.py, state["building"]): a load limit on
|
||||
the grid import and a thermostat cap are constraints with a penalty (binding
|
||||
far above any price, flexible with a price), a shift request a soft target
|
||||
(the window's import lowered by its kWh against the house's own net load).
|
||||
|
||||
The owner's preferences (preferences.py, a device's "pref"): a room's
|
||||
target per step from its schedule (in place of the occupied hours) at the
|
||||
comfort cost, and a "must" as a floor (heating) or limit (cooling) with the
|
||||
penalty MUST_EUR_K, inside the bounds and under a building's thermostat cap; a
|
||||
car's ready-by time and energy (a must at EV_MUST_EUR_KWH, a prefer valued
|
||||
at its max_ct per kWh with min_kwh at EV_SHORT_EUR_KWH); a battery's
|
||||
reserve as its floor per step. The plan's "prefs" says per preference
|
||||
whether it is met; with state["prefs_cost"] its "prefs_cost" the cost of
|
||||
each must (pref_costs: the plan made again without it).
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
@@ -67,6 +78,13 @@ COMFORT_EUR_KH = 2.0 # EUR per K and hour below (heating) or above (cooling
|
||||
# while someone is home: above any price (0.30 let a room cool by 1.3 K
|
||||
# at a 40 ct peak); pre-heating in cheap hours still pays
|
||||
EV_SHORT_EUR_KWH = 1.0 # EUR per kWh the car lacks at departure
|
||||
EV_MUST_EUR_KWH = 5.0 # EUR per kWh a car lacks at a "must" ready-by (preferences.py): above any price,
|
||||
# below a building's binding load limit (50) and the grid operator's (500)
|
||||
MUST_EUR_K = 20.0 # EUR per K and slot across a room's "must" (preferences.py): 80 EUR per K·h, so the
|
||||
# heat a room holds (C kWh/K) is not worth letting it fall even at 3 EUR/kWh; it stays
|
||||
# inside the bounds and under a building's cap, so those still win
|
||||
COST_SOLVE_S = 8.0 # s at most per extra plan for a must's cost (pref_costs)
|
||||
COST_MAX = 6 # musts costed per plan at most
|
||||
LIGHT_EUR = 0.002 # EUR per step for a closed cover while someone is home in daylight
|
||||
CHANGE_EUR_KW = 0.002 # EUR per kW a device's power changes from one step to the next: smooth
|
||||
# plans (equal prices would otherwise give any order, and a charger or
|
||||
@@ -245,7 +263,13 @@ class Model:
|
||||
soc0 = cap * min(max(float(st["soc_pct"]), 0.0), 100.0) / 100.0
|
||||
mean_price = sum(self.s["price_import"]) / self.T
|
||||
soc, prev, ch, dis = [], soc0, [], []
|
||||
pref = d.get("pref") # the owner's reserve per step (preferences.py)
|
||||
floors = []
|
||||
for t in range(self.T):
|
||||
lo_t = lo_pct
|
||||
if pref and not (pref.get("off") is not None and pref["group"][t] == pref["off"]):
|
||||
lo_t = min(max(lo_pct, pref["floor_pct"][t]), hi_pct)
|
||||
floors.append(lo_t)
|
||||
c = self.var(0.0, max_ch)
|
||||
x = self.var(0.0, max_dis)
|
||||
z = self.var(0.0, 1.0, integer=True)
|
||||
@@ -255,7 +279,7 @@ class Model:
|
||||
under = self.var(cost=PENALTY_LIMIT / cap * 100) # Wh below the floor, per % of the battery
|
||||
over = self.var(cost=PENALTY_LIMIT / cap * 100)
|
||||
self.h.addConstr(e - prev - self.dt * eff * c + self.dt / eff * x == 0)
|
||||
self.h.addConstr(e + under >= lo_pct / 100.0 * cap)
|
||||
self.h.addConstr(e + under >= lo_t / 100.0 * cap)
|
||||
self.h.addConstr(e - over <= hi_pct / 100.0 * cap)
|
||||
self.flows[t] += [(-1, c), (1, x)]
|
||||
soc.append(e)
|
||||
@@ -267,7 +291,7 @@ class Model:
|
||||
self.h.addConstr(prev + short >= min(soc0, hi_pct / 100.0 * cap))
|
||||
self.smooth([c - x for c, x in zip(ch, dis)])
|
||||
self.devices[d["id"]] = ("battery", {"ch": ch, "dis": dis, "soc": soc, "cap": cap, "max_dis": max_dis,
|
||||
"floor_wh": lo_pct / 100.0 * cap})
|
||||
"floor_wh": lo_pct / 100.0 * cap, "floors": floors})
|
||||
|
||||
def room(self, d, cooling: bool, gains):
|
||||
p, st, b = d["params"], d["state"], d["bounds"]
|
||||
@@ -284,6 +308,8 @@ class Model:
|
||||
internal = self.hourly(p.get("internal_gain_w") or 0.0)
|
||||
solar = float(p.get("solar_m2") or 0.0)
|
||||
ghi = self.s.get("ghi_w_m2") or [0.0] * self.T
|
||||
pref = d.get("pref") # the owner's schedule and overrides (preferences.py)
|
||||
cap = (self.s.get("building") or {}).get("thermostat_cap")
|
||||
for t in range(self.T):
|
||||
if switched:
|
||||
u = self.var(0.0, 1.0, integer=True)
|
||||
@@ -302,7 +328,23 @@ class Model:
|
||||
self.h.addConstr(tt - (1 - a) * prev - sign * self.dt / c * cop / 1000.0 * pw
|
||||
- self.dt / c / 1000.0 * gain == a * self.s["t_out"][t] + self.dt / c / 1000.0 * fixed)
|
||||
occ = self.s["occupied"][t + 1] if t + 1 < self.T else self.s["occupied"][t]
|
||||
target = min(max(comfort if occ else other, lo), hi)
|
||||
want, must = (comfort if occ else other), None
|
||||
if pref:
|
||||
want, must = pref["target"][t], pref["must"][t]
|
||||
if pref.get("off") is not None and pref["group"][t] == pref["off"]:
|
||||
want, must = pref["base"][t], None # pref_costs: the plan without this group
|
||||
target = min(max(want, lo), hi)
|
||||
capped = bool(cap) and cap["from_step"] <= t < cap["to_step"]
|
||||
if must is not None:
|
||||
# a bound inside the bounds in force, and under a building's cap
|
||||
mv = min(max(must, lo), hi)
|
||||
if capped:
|
||||
mv = max(mv, cap["value"]) if cooling else min(mv, cap["value"])
|
||||
cross = self.var(cost=MUST_EUR_K)
|
||||
if cooling:
|
||||
self.h.addConstr(tt - cross <= mv)
|
||||
else:
|
||||
self.h.addConstr(tt + cross >= mv)
|
||||
miss = self.var(cost=COMFORT_EUR_KH * self.dt)
|
||||
under = self.var(cost=PENALTY_LIMIT)
|
||||
over = self.var(cost=PENALTY_LIMIT)
|
||||
@@ -312,8 +354,7 @@ class Model:
|
||||
self.h.addConstr(tt + miss >= target)
|
||||
self.h.addConstr(tt + under >= lo)
|
||||
self.h.addConstr(tt - over <= hi)
|
||||
cap = (self.s.get("building") or {}).get("thermostat_cap")
|
||||
if cap and cap["from_step"] <= t < cap["to_step"]:
|
||||
if capped:
|
||||
miss_cap = self.var(cost=cap["penalty"] * self.dt)
|
||||
if cooling:
|
||||
self.h.addConstr(tt + miss_cap >= cap["value"]) # cooling: not below the cap
|
||||
@@ -345,6 +386,11 @@ class Model:
|
||||
imin = max(float(p["min_current_a"]), b["current_a"][0])
|
||||
dep = departure_step(self.s, str(st.get("departure") or p["departure"]))
|
||||
need = float(st["need_wh"]) if st.get("need_wh") is not None else float(p["need_wh"])
|
||||
pref = d.get("pref") # the owner's ready-by (preferences.py)
|
||||
if pref:
|
||||
dep, need = int(pref["dep_step"]), float(pref["need_wh"])
|
||||
if pref.get("off"):
|
||||
need = 0.0
|
||||
powers, total = [], Lin()
|
||||
for t in range(self.T):
|
||||
if t >= dep or imax < imin:
|
||||
@@ -357,10 +403,21 @@ class Model:
|
||||
self.flows[t].append((-1, pw))
|
||||
powers.append(pw)
|
||||
total += self.dt * pw
|
||||
short = self.var(cost=EV_SHORT_EUR_KWH / 1000.0)
|
||||
if pref and pref["how"] == "prefer":
|
||||
# a kWh is worth max_ct to the owner (without one: today's), min_kwh at any price
|
||||
worth = EV_SHORT_EUR_KWH if pref.get("max_ct") is None else max(0.0, pref["max_ct"]) / 100.0
|
||||
short = self.var(cost=worth / 1000.0)
|
||||
self.h.addConstr(total + short >= need)
|
||||
mwh = min(float(pref.get("min_wh") or 0.0), need)
|
||||
if mwh > 0:
|
||||
short_min = self.var(cost=EV_SHORT_EUR_KWH / 1000.0)
|
||||
self.h.addConstr(total + short_min >= mwh)
|
||||
else:
|
||||
short = self.var(cost=(EV_MUST_EUR_KWH if pref else EV_SHORT_EUR_KWH) / 1000.0)
|
||||
self.h.addConstr(total + short >= need)
|
||||
self.smooth([p for p in powers if p is not None])
|
||||
self.devices[d["id"]] = ("ev", {"power": powers, "volts": volts, "dep": dep})
|
||||
self.devices[d["id"]] = ("ev", {"power": powers, "volts": volts, "dep": dep,
|
||||
"pmax": imax * volts if imax >= imin else 0.0})
|
||||
|
||||
def cover(self, d, room_ids):
|
||||
p, b = d["params"], d["bounds"]
|
||||
@@ -463,16 +520,198 @@ def plan(s: dict, time_limit: float = 30.0) -> dict:
|
||||
if e is not None and (not entries or _key(entries[-1]) != _key(e)):
|
||||
entries.append({"t": start + t * step, **e})
|
||||
devices[d["id"]] = entries
|
||||
energy = sum(m.eur_wh(t, pi[t]) * x(m.gin[t]) - m.eur_wh(t, s["price_export"][t]) * x(m.gout[t])
|
||||
for t in range(T))
|
||||
market = market_plan(s, m, x)
|
||||
energy += sum(c["cost_eur"] for c in market)
|
||||
energy = energy_of(s, m, x, market)
|
||||
out = {"v": 1, "made": int(time.time()), "start": start, "step_s": step, "status": status,
|
||||
"solve_s": round(time.time() - t0, 2), "devices": devices, "flex": flexibility(m, x),
|
||||
"cost_eur": round(energy, 2), "baseline_eur": round(baseline(s, m, x), 2),
|
||||
"price_note": s.get("price_note", ""), "sources": s.get("sources", {})}
|
||||
if market:
|
||||
out["market"] = market
|
||||
if any(d.get("pref") for d in s["devices"]):
|
||||
out["prefs"] = pref_feedback(s, m, x)
|
||||
if s.get("prefs_cost"):
|
||||
out["prefs_cost"] = pref_costs(s, energy, t0 + time_limit)
|
||||
return out
|
||||
|
||||
|
||||
def energy_of(s, m, x, market=None) -> float:
|
||||
"""The plan's cost for power (EUR): the grid's import less its feed-in,
|
||||
and the market's contracts."""
|
||||
pi, pe = s["price_import"], s["price_export"]
|
||||
e = sum(m.eur_wh(t, pi[t]) * x(m.gin[t]) - m.eur_wh(t, pe[t]) * x(m.gout[t]) for t in range(m.T))
|
||||
return e + sum(c["cost_eur"] for c in (market if market is not None else market_plan(s, m, x)))
|
||||
|
||||
|
||||
def _when(s, ts: float) -> str:
|
||||
return datetime.fromtimestamp(ts, timezone.utc).astimezone(ZoneInfo(s.get("tz") or "UTC")).strftime("%a %H:%M")
|
||||
|
||||
|
||||
TOL_K = 0.3 # a room's target counts as met within this
|
||||
TOL_PCT = 0.5 # a battery's reserve
|
||||
TOL_WH = 50.0 # a car's energy
|
||||
|
||||
|
||||
def pref_feedback(s, m, x) -> list[dict]:
|
||||
"""Per preference (a room's block, boost or away; a car's next ready-by;
|
||||
a battery's reserve): met or not in the plan, and in a few words why.
|
||||
Fields: control, group, kind, how, label, met (None: not in the
|
||||
plan's 48 h, or not plugged in), detail, and per kind the numbers."""
|
||||
out = []
|
||||
start, step, T = s["start"], s["step_s"], m.T
|
||||
for d in s["devices"]:
|
||||
pref = d.get("pref")
|
||||
if not pref:
|
||||
continue
|
||||
kind, v = m.devices.get(d["id"], (d["kind"], None))
|
||||
if kind in ("heating", "cooling") and v:
|
||||
out += _room_feedback(s, d, pref, v, x, kind == "cooling")
|
||||
elif kind == "battery" and v:
|
||||
for gi, g in enumerate(pref["groups"]):
|
||||
ts = [t for t in range(T) if pref["group"][t] == gi]
|
||||
f = {"control": d["id"], "group": g["group"], "kind": "battery", "how": "must", "label": g["text"]}
|
||||
if not ts:
|
||||
out.append(dict(f, met=None, detail="not in the next 48 h"))
|
||||
continue
|
||||
soc = [100.0 * x(v["soc"][t]) / v["cap"] for t in ts]
|
||||
worst = min(range(len(ts)), key=lambda k: soc[k] - v["floors"][ts[k]])
|
||||
miss = v["floors"][ts[worst]] - soc[worst]
|
||||
met = miss <= TOL_PCT
|
||||
f.update(met=met, min_pct=round(min(soc), 1), floor_pct=round(v["floors"][ts[worst]], 1))
|
||||
if not met:
|
||||
f["detail"] = f"{soc[worst]:.0f} % at {_when(s, start + (ts[worst] + 1) * step)}"
|
||||
if d["state"].get("soc_pct") is not None and float(d["state"]["soc_pct"]) < v["floors"][ts[0]]:
|
||||
f["detail"] += f" (starts at {float(d['state']['soc_pct']):.0f} %)"
|
||||
out.append(f)
|
||||
elif kind == "ev":
|
||||
out.append(_ev_feedback(s, d, pref, v, x, m))
|
||||
return out
|
||||
|
||||
|
||||
def _room_feedback(s, d, pref, v, x, cooling) -> list[dict]:
|
||||
out = []
|
||||
start, step, T = s["start"], s["step_s"], len(pref["target"])
|
||||
lo, hi, lo_by, hi_by = (d["bounds"]["temp_c"] + ["device", "device"])[:4]
|
||||
cap = (s.get("building") or {}).get("thermostat_cap")
|
||||
sign = -1.0 if cooling else 1.0
|
||||
for gi, g in enumerate(pref["groups"]):
|
||||
ts = [t for t in range(T) if pref["group"][t] == gi]
|
||||
f = {"control": d["id"], "group": g["group"], "kind": g["kind"], "how": g["how"], "label": g["text"]}
|
||||
if not ts:
|
||||
out.append(dict(f, met=None, detail="not in the next 48 h"))
|
||||
continue
|
||||
notes = []
|
||||
want = []
|
||||
for t in ts:
|
||||
w0 = pref["must"][t] if g["how"] == "must" and pref["must"][t] is not None else pref["target"][t]
|
||||
w = min(max(w0, lo), hi)
|
||||
by = (hi_by if not cooling else lo_by) if sign * (w0 - w) > 1e-9 else None
|
||||
if cap and cap["from_step"] <= t < cap["to_step"]:
|
||||
wc = max(w, cap["value"]) if cooling else min(w, cap["value"])
|
||||
if wc != w:
|
||||
by, w = "building", wc
|
||||
if by and by not in notes:
|
||||
notes.append(by)
|
||||
want.append(w)
|
||||
temps = [x(v["temp"][t]) for t in ts]
|
||||
gap = [sign * (w - tt) for w, tt in zip(want, temps)]
|
||||
k = max(range(len(ts)), key=lambda i: gap[i])
|
||||
met = gap[k] <= TOL_K
|
||||
f.update(met=met, first=start + ts[0] * step, worst_c=round(temps[k], 1), want_c=round(want[k], 1))
|
||||
bits = []
|
||||
if notes:
|
||||
held = min(want) if not cooling else max(want)
|
||||
bits.append(f"held at {held:g} °C by the {notes[0]}'s limit")
|
||||
f["held_by"] = notes[0]
|
||||
if not met:
|
||||
bits.append(f"{temps[k]:.1f} °C at {_when(s, start + (ts[k] + 1) * step)}")
|
||||
if bits:
|
||||
f["detail"] = "; ".join(bits)
|
||||
out.append(f)
|
||||
return out
|
||||
|
||||
|
||||
def _ev_feedback(s, d, pref, v, x, m) -> dict:
|
||||
start, step = s["start"], s["step_s"]
|
||||
f = {"control": d["id"], "group": pref["group"], "kind": "ev", "how": pref["how"],
|
||||
"label": f"car {pref['text']}", "at": pref["at"], "need_kwh": round(pref["need_wh"] / 1000.0, 2)}
|
||||
plug = f"; plugged in at {_when(s, pref['plugged_at'])}" if pref.get("plugged_at") else ""
|
||||
if pref.get("note"):
|
||||
f["note"] = pref["note"]
|
||||
if v is None:
|
||||
return dict(f, met=None if pref["how"] == "prefer" else False, detail="not plugged in")
|
||||
pw = [x(p) if p is not None else 0.0 for p in v["power"]]
|
||||
dep = min(v["dep"], len(pw))
|
||||
wh = sum(m.dt * p for p in pw[:dep])
|
||||
short = max(0.0, pref["need_wh"] - wh)
|
||||
pi = s["price_import"]
|
||||
f["planned_kwh"] = round(wh / 1000.0, 2)
|
||||
f["charge_eur"] = round(sum(m.eur_wh(t, pi[t]) * pw[t] for t in range(dep)), 2)
|
||||
reached = ""
|
||||
if pref.get("soc_pct") is not None and pref.get("capacity_wh"):
|
||||
pct = min(100.0, float(pref["soc_pct"]) + 100.0 * wh / pref["capacity_wh"])
|
||||
f["reached_pct"] = round(pct, 0)
|
||||
reached = f"{pct:.0f} %"
|
||||
else:
|
||||
reached = f"{wh / 1000.0:.1f} kWh"
|
||||
when = _when(s, pref["at"])
|
||||
if pref.get("beyond"):
|
||||
f.update(met=None, detail=f"{reached} by the end of the plan (the ready-by is later){plug}")
|
||||
return f
|
||||
if pref["how"] == "prefer":
|
||||
f["met"] = short <= max(TOL_WH, 0.01 * pref["need_wh"])
|
||||
f["detail"] = f"{reached} by {when}" + ("" if f["met"] else
|
||||
f" ({short / 1000.0:.1f} kWh less: dearer than the price wanted)") + plug
|
||||
return f
|
||||
f["met"] = short <= max(TOL_WH, 0.01 * pref["need_wh"])
|
||||
if f["met"]:
|
||||
f["detail"] = f"{reached} by {when}{plug}"
|
||||
return f
|
||||
# a must it cannot meet: by how much, and when it could be ready at full power from now
|
||||
f["short_kwh"] = round(short / 1000.0, 2)
|
||||
rate = v.get("pmax") or 0.0
|
||||
if rate > 0:
|
||||
n = math.ceil(pref["need_wh"] / (rate * m.dt) - 1e-9)
|
||||
f["ready_at"] = int(start + n * step)
|
||||
can = f"; ready at full power by {_when(s, f['ready_at'])}"
|
||||
else:
|
||||
can = "; it may not charge now (a limit holds it at 0 A)"
|
||||
f["detail"] = f"{reached} by {when}, short by {short / 1000.0:.1f} kWh{can}{plug}"
|
||||
return f
|
||||
|
||||
|
||||
def pref_costs(s, energy: float, deadline: float) -> dict:
|
||||
"""Each must's cost (EUR a day): the plan's cost for power less that of
|
||||
the plan made again without it (a room's block at its setback, a
|
||||
battery's reserve at its floor before); a car's must: what its charging
|
||||
costs (no second plan). At most COST_MAX, each within COST_SOLVE_S and
|
||||
the time left."""
|
||||
out = {}
|
||||
days = s["steps"] * s["step_s"] / 86400.0
|
||||
n = 0
|
||||
for i, d in enumerate(s["devices"]):
|
||||
pref = d.get("pref")
|
||||
if not pref or d["kind"] not in ("heating", "cooling", "battery"):
|
||||
continue
|
||||
for gi, g in enumerate(pref["groups"]):
|
||||
if g["how"] != "must" or gi not in pref["group"]:
|
||||
continue
|
||||
left = deadline - time.time() - 1.0
|
||||
if n >= COST_MAX or left < 1.0:
|
||||
return out
|
||||
n += 1
|
||||
s2 = copy.deepcopy(s)
|
||||
s2["prefs_cost"] = False
|
||||
s2["devices"][i]["pref"]["off"] = gi
|
||||
try:
|
||||
m2 = Model(s2)
|
||||
m2.solve(min(COST_SOLVE_S, left))
|
||||
val = m2.h.getSolution().col_value
|
||||
e2 = energy_of(s2, m2, lambda v: val[v.index])
|
||||
except Exception: # noqa: BLE001 - a cost is not worth failing the plan
|
||||
continue
|
||||
c = (energy - e2) / days
|
||||
out[g["group"]] = round(c if abs(c) >= 0.005 else 0.0, 2)
|
||||
return out
|
||||
|
||||
|
||||
@@ -546,7 +785,9 @@ def flexibility(m, x) -> dict:
|
||||
for t in range(min(n, len(cols or []))):
|
||||
if cols[t] is not None and hasattr(cols[t], "index"): # an EV's steps after departure are empty
|
||||
shed[t] += max(x(cols[t]), 0.0)
|
||||
return {"grid_w": grid, "shed_w": [round(min(sh, g)) for sh, g in zip(shed, grid)]}
|
||||
# net_w: import less export, the member's draw as planned (the box's forecast of it upward, boxtrain.py)
|
||||
return {"grid_w": grid, "shed_w": [round(min(sh, g)) for sh, g in zip(shed, grid)],
|
||||
"net_w": [round(x(m.gin[t]) - x(m.gout[t])) for t in range(n)]}
|
||||
|
||||
|
||||
def _charge_why(pi, t) -> str:
|
||||
@@ -607,6 +848,14 @@ def _entry(d, kind, v, t, x, s, pi, grid_w=(1e9, 1e9)):
|
||||
temp = x(v["temp"][t])
|
||||
pw = x(v["power"][t])
|
||||
ahead = s["occupied"][min(t + 4, s["steps"] - 1)]
|
||||
pref = d.get("pref")
|
||||
if pref:
|
||||
# a schedule: "someone home" is a target above the setback ahead
|
||||
# (below the setup, cooling); its target in place of comfort/setback
|
||||
sign = -1 if kind == "cooling" else 1
|
||||
tg, k0 = pref["target"], max(t - 1, 0)
|
||||
occ = sign * (tg[k0] - pref["base"][k0]) > 0
|
||||
ahead = any(sign * (tg[k] - tg[k0]) > 0 for k in range(t, min(t + 4, len(tg))))
|
||||
if kind == "heating" and v["switched"]:
|
||||
on = pw > 0.5 * v["pmax"]
|
||||
why = ("pre-heat before someone is home" if on and not occ and ahead else
|
||||
@@ -630,6 +879,10 @@ def _entry(d, kind, v, t, x, s, pi, grid_w=(1e9, 1e9)):
|
||||
occ1 = s["occupied"][min(t + 1, s["steps"] - 1)]
|
||||
p = d["params"]
|
||||
want = float(p["comfort_c"]) if occ1 else float(p["setback_c"])
|
||||
if pref:
|
||||
want = float(pref["target"][t])
|
||||
if pref["must"][t] is not None:
|
||||
want = max(want, float(pref["must"][t]))
|
||||
floor_t = min(max(math.floor(want * 2) / 2, math.ceil(lo * 2) / 2), math.floor(hi * 2) / 2)
|
||||
return {"set": min(target, floor_t), "why": "setback: away or night" if not occ else "warm enough"}
|
||||
return {"set": target, "why": f"pre-heat while power is cheap ({pi[t]:.1f} ct)" if not occ and ahead
|
||||
|
||||
@@ -0,0 +1,975 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The owner's preferences for the plan ("I may want
|
||||
my car to be fully charged in any case at 8 AM ... I want my home to be min
|
||||
21 degree in night, 19 degree in day").
|
||||
|
||||
Per control, kept in <state dir>/preferences.json:
|
||||
room:<id> a heating room's minimum / a cooling room's maximum per
|
||||
weekday and time: blocks {days, from, to, c, how}
|
||||
ev:<id> ready-by times per weekday {days, at, pct | kwh, how,
|
||||
max_ct, min_kwh}, and the car's capacity_kwh (for %)
|
||||
battery:<id> a reserve (min state of charge, %) and optional blocks
|
||||
{days, from, to, pct}: the owner's floor
|
||||
and overrides that win over them until they expire:
|
||||
boost {control, c, from, until} a room to c (a must) for some hours
|
||||
away {from, until} every room at its setback/setup
|
||||
car {control, at, pct | kwh, how} the car needed at a one-off time
|
||||
|
||||
"must": a bound (a floor or limit in the plan: MUST_EUR_K, EV_MUST_EUR_KWH,
|
||||
inside the bounds in force: the grid operator's, the building's and the
|
||||
owner's limits still win). "prefer": today's comfort cost (rooms), or a
|
||||
value per kWh (a car: charge where a kWh costs at most max_ct, and at least
|
||||
min_kwh at any price). A room without a schedule keeps today's occupied
|
||||
hours (inputs.py) and comfort/setback targets.
|
||||
|
||||
Who sets them, and who wins (limits.py's spirit): the owner (Home
|
||||
Assistant, the Hub's panel: level "owner"), an installer or Cantabile's
|
||||
operators (the console, level "installer" / "operator": only within the
|
||||
owner's limits, never above them). Each level's setting is kept; the
|
||||
highest level's is in force, so the owner's own changes win and an
|
||||
installer's comes back when the owner clears theirs. Overrides are the
|
||||
owner's only. Every change is logged with who, when and what.
|
||||
|
||||
Where they come in: Home Assistant's entities (announce.py, the text
|
||||
formats below), the panel (an inbox directory, prefs-inbox/, the panel
|
||||
writes one JSON file per change; the agent writes preferences-view.json
|
||||
for it), the console (a signed command "preferences"). They go to the
|
||||
cloud with the plan (controller.public_plan), with the feedback per
|
||||
preference: met or not, and the cost of each must (the plan made again
|
||||
without it: optimizer.pref_costs).
|
||||
|
||||
Text formats (an HA text entity holds 255 characters), blocks separated by
|
||||
";": days are Mo Tu We Th Fr Sa Su (or Mo Di Mi Do Fr Sa So), ranges
|
||||
"Mo-Fr", lists "Sa,Su", or daily / weekdays / weekend.
|
||||
room Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19; Sa,Su 07:00-23:00 20
|
||||
ev Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct min 5kWh
|
||||
battery daily 17:00-21:00 50%
|
||||
car 2026-11-05 07:00 80% must (or "07:00 80%": the next 07:00)
|
||||
away 2026-11-10 (back that day; or with a time)
|
||||
A block from 22:00 to 06:00 belongs to the day it starts. Times are the
|
||||
home's wall clock (HA's time zone), so a schedule holds across the change
|
||||
to and from summer time.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DAYS = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
|
||||
DAY_NAMES = {"mo": 0, "mon": 0, "monday": 0, "montag": 0,
|
||||
"tu": 1, "tue": 1, "tuesday": 1, "di": 1, "dienstag": 1,
|
||||
"we": 2, "wed": 2, "wednesday": 2, "mi": 2, "mittwoch": 2,
|
||||
"th": 3, "thu": 3, "thursday": 3, "do": 3, "donnerstag": 3,
|
||||
"fr": 4, "fri": 4, "friday": 4, "freitag": 4,
|
||||
"sa": 5, "sat": 5, "saturday": 5, "samstag": 5,
|
||||
"su": 6, "sun": 6, "sunday": 6, "so": 6, "sonntag": 6}
|
||||
DAY_SETS = {"daily": list(range(7)), "all": list(range(7)), "täglich": list(range(7)), "taeglich": list(range(7)),
|
||||
"weekdays": list(range(5)), "werktags": list(range(5)), "weekend": [5, 6], "wochenende": [5, 6]}
|
||||
HOW = ("must", "prefer")
|
||||
HOW_WORDS = {"must": "must", "muss": "must", "prefer": "prefer", "bevorzugt": "prefer", "gern": "prefer"}
|
||||
LEVELS = ("operator", "installer", "owner") # low to high: the highest set is in force
|
||||
RANK = {lvl: i + 1 for i, lvl in enumerate(LEVELS)}
|
||||
KINDS = {"heating": "room", "cooling": "room", "ev": "ev", "battery": "battery"}
|
||||
RANGE_C = {"heating": (5.0, 30.0), "cooling": (16.0, 32.0)} # controls.RANGE
|
||||
MAX_BLOCKS = 16
|
||||
MAX_OVERRIDES = 20
|
||||
MAX_LOG = 200
|
||||
AWAY_MAX_S = 60 * 86400 # away without a date: at most this long
|
||||
BOOST_MAX_H = 24
|
||||
COST_EVERY_S = 3600 # the musts' costs are made again at most this often (unless something changed)
|
||||
HHMM = re.compile(r"^([01]?\d|2[0-4])(?::([0-5]\d))?$")
|
||||
|
||||
|
||||
# -- small parsers -------------------------------------------------------------
|
||||
def hhmm(v, end=False) -> str:
|
||||
""""7", "07:00", "7:30" -> "07:00"/"07:30"; "24:00" only as an end."""
|
||||
m = HHMM.match(str(v).strip())
|
||||
if not m:
|
||||
raise ValueError(f"{v!r} is not a time (HH:MM)")
|
||||
h, mi = int(m.group(1)), int(m.group(2) or 0)
|
||||
if h == 24 and (mi or not end):
|
||||
raise ValueError(f"{v!r}: 24:00 only as the end of a block")
|
||||
return f"{h:02d}:{mi:02d}"
|
||||
|
||||
|
||||
def minutes(s: str) -> int:
|
||||
h, m = s.split(":")
|
||||
return int(h) * 60 + int(m)
|
||||
|
||||
|
||||
def parse_days(v) -> list[int]:
|
||||
""""Mo-Fr", "Sa,Su", "daily", [0, 1], "mon-fri" -> sorted weekday numbers (Mon 0)."""
|
||||
if isinstance(v, (list, tuple)):
|
||||
out = sorted({int(x) for x in v})
|
||||
if not out or out[0] < 0 or out[-1] > 6:
|
||||
raise ValueError("days: weekday numbers 0 (Monday) to 6 (Sunday)")
|
||||
return out
|
||||
s = str(v).strip().lower()
|
||||
if s in DAY_SETS:
|
||||
return list(DAY_SETS[s])
|
||||
out = set()
|
||||
for part in re.split(r"[,/ ]+", s):
|
||||
if not part:
|
||||
continue
|
||||
if part in DAY_SETS:
|
||||
out.update(DAY_SETS[part])
|
||||
continue
|
||||
a, _, b = part.partition("-")
|
||||
if a not in DAY_NAMES or (b and b not in DAY_NAMES):
|
||||
raise ValueError(f"{part!r} is not a day (Mo Tu We Th Fr Sa Su, a range Mo-Fr, or daily)")
|
||||
i, j = DAY_NAMES[a], DAY_NAMES[b] if b else DAY_NAMES[a]
|
||||
k = i
|
||||
while True:
|
||||
out.add(k)
|
||||
if k == j:
|
||||
break
|
||||
k = (k + 1) % 7
|
||||
if not out:
|
||||
raise ValueError("days: none given")
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def days_text(days: list[int]) -> str:
|
||||
d = sorted(set(days))
|
||||
if d == list(range(7)):
|
||||
return "daily"
|
||||
runs, cur = [], [d[0]]
|
||||
for x in d[1:]:
|
||||
if x == cur[-1] + 1:
|
||||
cur.append(x)
|
||||
else:
|
||||
runs.append(cur)
|
||||
cur = [x]
|
||||
runs.append(cur)
|
||||
return ",".join(DAYS[r[0]][:2] if len(r) == 1 else f"{DAYS[r[0]][:2]}-{DAYS[r[-1]][:2]}" if len(r) > 2
|
||||
else f"{DAYS[r[0]][:2]},{DAYS[r[1]][:2]}" for r in runs)
|
||||
|
||||
|
||||
def _num(v, what, lo, hi) -> float:
|
||||
try:
|
||||
x = float(str(v).replace(",", ".").strip())
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError(f"{what}: {v!r} is not a number") from None
|
||||
if not math.isfinite(x) or not lo <= x <= hi:
|
||||
raise ValueError(f"{what}: {x:g} is outside {lo:g}..{hi:g}")
|
||||
return x
|
||||
|
||||
|
||||
def _how(v, default="prefer") -> str:
|
||||
h = HOW_WORDS.get(str(v or default).strip().lower())
|
||||
if h is None:
|
||||
raise ValueError(f"how: must or prefer, not {v!r}")
|
||||
return h
|
||||
|
||||
|
||||
def _fmt(x: float) -> str:
|
||||
return f"{x:g}"
|
||||
|
||||
|
||||
# -- checked values (every source comes through these) --------------------------
|
||||
def check_room(v: dict, kind: str) -> dict:
|
||||
lo, hi = RANGE_C["cooling" if kind == "cooling" else "heating"]
|
||||
blocks = (v or {}).get("blocks")
|
||||
if not isinstance(blocks, list) or len(blocks) > MAX_BLOCKS:
|
||||
raise ValueError(f"blocks: a list of at most {MAX_BLOCKS}")
|
||||
out = []
|
||||
for b in blocks:
|
||||
out.append({"days": parse_days(b.get("days", "daily")), "from": hhmm(b.get("from", "00:00")),
|
||||
"to": hhmm(b.get("to", "24:00"), end=True),
|
||||
"c": _num(b.get("c"), "temperature", lo, hi), "how": _how(b.get("how"))})
|
||||
return {"blocks": out}
|
||||
|
||||
|
||||
def check_ev(v: dict) -> dict:
|
||||
v = v or {}
|
||||
ready = v.get("ready") or []
|
||||
if not isinstance(ready, list) or len(ready) > MAX_BLOCKS:
|
||||
raise ValueError(f"ready: a list of at most {MAX_BLOCKS}")
|
||||
out = {"ready": [], "capacity_kwh": None}
|
||||
if v.get("capacity_kwh") not in (None, ""):
|
||||
out["capacity_kwh"] = _num(v["capacity_kwh"], "capacity", 1, 250)
|
||||
for r in ready:
|
||||
out["ready"].append(_ev_target(r, {"days": parse_days(r.get("days", "daily")), "at": hhmm(r.get("at", ""))}))
|
||||
return out
|
||||
|
||||
|
||||
def _ev_target(r: dict, into: dict) -> dict:
|
||||
pct, kwh = r.get("pct"), r.get("kwh")
|
||||
if (pct in (None, "")) == (kwh in (None, "")):
|
||||
raise ValueError("a car's target: % or kWh, one of them")
|
||||
into["pct"] = _num(pct, "target %", 1, 100) if pct not in (None, "") else None
|
||||
into["kwh"] = _num(kwh, "target kWh", 0.1, 250) if kwh not in (None, "") else None
|
||||
into["how"] = _how(r.get("how"))
|
||||
into["max_ct"] = _num(r["max_ct"], "max price", -100, 500) if r.get("max_ct") not in (None, "") else None
|
||||
into["min_kwh"] = _num(r["min_kwh"], "min kWh", 0, 250) if r.get("min_kwh") not in (None, "") else None
|
||||
if into["how"] == "must":
|
||||
into["max_ct"] = into["min_kwh"] = None # a must charges at any price
|
||||
return into
|
||||
|
||||
|
||||
def check_battery(v: dict) -> dict:
|
||||
v = v or {}
|
||||
blocks = v.get("blocks") or []
|
||||
if not isinstance(blocks, list) or len(blocks) > MAX_BLOCKS:
|
||||
raise ValueError(f"blocks: a list of at most {MAX_BLOCKS}")
|
||||
out = {"reserve_pct": _num(v.get("reserve_pct", 0) or 0, "reserve", 0, 100), "blocks": []}
|
||||
for b in blocks:
|
||||
out["blocks"].append({"days": parse_days(b.get("days", "daily")), "from": hhmm(b.get("from", "00:00")),
|
||||
"to": hhmm(b.get("to", "24:00"), end=True), "pct": _num(b.get("pct"), "reserve", 0, 100)})
|
||||
return out
|
||||
|
||||
|
||||
def check(key: str, value: dict, kind: str) -> dict:
|
||||
what = key.split(":", 1)[0]
|
||||
if what == "room":
|
||||
return check_room(value, kind)
|
||||
if what == "ev":
|
||||
return check_ev(value)
|
||||
if what == "battery":
|
||||
return check_battery(value)
|
||||
raise ValueError(f"no preference {key!r}")
|
||||
|
||||
|
||||
# -- text forms (Home Assistant's text entities) -----------------------------------
|
||||
def _split(text: str) -> list[list[str]]:
|
||||
return [p.split() for p in re.split(r"[;\n]+", str(text or "")) if p.strip()]
|
||||
|
||||
|
||||
def _days_and_rest(words: list[str]) -> tuple[list[int], list[str]]:
|
||||
"""A block's leading days, else daily."""
|
||||
try:
|
||||
return parse_days(words[0]), words[1:]
|
||||
except ValueError:
|
||||
return list(range(7)), words
|
||||
|
||||
|
||||
def _range(w: str) -> tuple[str, str]:
|
||||
a, sep, b = w.partition("-")
|
||||
if not sep:
|
||||
raise ValueError(f"{w!r}: a time range, e.g. 22:00-06:00")
|
||||
return hhmm(a), hhmm(b, end=True)
|
||||
|
||||
|
||||
def parse_room_text(text: str, kind: str) -> dict:
|
||||
blocks = []
|
||||
for words in _split(text):
|
||||
days, rest = _days_and_rest(words)
|
||||
if len(rest) < 2:
|
||||
raise ValueError(f"{' '.join(words)!r}: days, a time range and a temperature (e.g. Mo-Fr 22:00-06:00 21 must)")
|
||||
a, b = _range(rest[0])
|
||||
c = rest[1].strip("≥≤>=<°Cc")
|
||||
how = rest[2] if len(rest) > 2 else "prefer"
|
||||
blocks.append({"days": days, "from": a, "to": b, "c": c, "how": how})
|
||||
return check_room({"blocks": blocks}, kind)
|
||||
|
||||
|
||||
def room_text(v: dict | None) -> str:
|
||||
return "; ".join(f"{days_text(b['days'])} {b['from']}-{b['to']} {_fmt(b['c'])}" + (" must" if b["how"] == "must" else "")
|
||||
for b in (v or {}).get("blocks") or [])
|
||||
|
||||
|
||||
def _target_words(words: list[str]) -> dict:
|
||||
""""80%", "20kWh", "20 kWh", "must", "max 25ct", "<25ct", "min 5kWh"."""
|
||||
out, i = {}, 0
|
||||
ws = [w.lower() for w in words]
|
||||
while i < len(ws):
|
||||
w = ws[i]
|
||||
nxt = ws[i + 1] if i + 1 < len(ws) else ""
|
||||
if w in HOW_WORDS:
|
||||
out["how"] = w
|
||||
elif w.endswith("%"):
|
||||
out["pct"] = w[:-1]
|
||||
elif w.endswith("kwh") and w[:-3]:
|
||||
out["kwh"] = w[:-3]
|
||||
elif nxt == "kwh" and "pct" not in out and "kwh" not in out:
|
||||
out["kwh"] = w
|
||||
i += 1
|
||||
elif w in ("max", "<", "bis") or (w.startswith("<") and w[1:]):
|
||||
v = w[1:] if w.startswith("<") and w[1:] else nxt
|
||||
if not (w.startswith("<") and w[1:]):
|
||||
i += 1
|
||||
out["max_ct"] = v.removesuffix("ct")
|
||||
elif w == "min":
|
||||
v = nxt
|
||||
i += 1
|
||||
if v.endswith("kwh"):
|
||||
out["min_kwh"] = v[:-3]
|
||||
elif i + 1 < len(ws) and ws[i + 1] == "kwh":
|
||||
out["min_kwh"] = v
|
||||
i += 1
|
||||
else:
|
||||
out["min_kwh"] = v
|
||||
else:
|
||||
raise ValueError(f"{w!r}: not understood (a target like 80% or 20kWh, must/prefer, max 25ct, min 5kWh)")
|
||||
i += 1
|
||||
return out
|
||||
|
||||
|
||||
def parse_ev_text(text: str, capacity_kwh=None) -> dict:
|
||||
ready = []
|
||||
for words in _split(text):
|
||||
days, rest = _days_and_rest(words)
|
||||
if not rest:
|
||||
raise ValueError(f"{' '.join(words)!r}: a time and a target (e.g. Mo-Fr 08:00 100% must)")
|
||||
ready.append({"days": days, "at": hhmm(rest[0]), **_target_words(rest[1:])})
|
||||
return check_ev({"ready": ready, "capacity_kwh": capacity_kwh})
|
||||
|
||||
|
||||
def _target_text(r: dict) -> str:
|
||||
s = f"{_fmt(r['pct'])}%" if r.get("pct") is not None else f"{_fmt(r['kwh'])}kWh"
|
||||
s += " must" if r["how"] == "must" else ""
|
||||
if r.get("max_ct") is not None:
|
||||
s += f" max {_fmt(r['max_ct'])}ct"
|
||||
if r.get("min_kwh") is not None:
|
||||
s += f" min {_fmt(r['min_kwh'])}kWh"
|
||||
return s
|
||||
|
||||
|
||||
def ev_text(v: dict | None) -> str:
|
||||
return "; ".join(f"{days_text(r['days'])} {r['at']} {_target_text(r)}" for r in (v or {}).get("ready") or [])
|
||||
|
||||
|
||||
def parse_battery_text(text: str, reserve_pct=0.0) -> dict:
|
||||
blocks = []
|
||||
for words in _split(text):
|
||||
days, rest = _days_and_rest(words)
|
||||
if len(rest) < 2:
|
||||
raise ValueError(f"{' '.join(words)!r}: days, a time range and a reserve (e.g. daily 17:00-21:00 50%)")
|
||||
a, b = _range(rest[0])
|
||||
blocks.append({"days": days, "from": a, "to": b, "pct": rest[1].rstrip("%")})
|
||||
return check_battery({"reserve_pct": reserve_pct, "blocks": blocks})
|
||||
|
||||
|
||||
def battery_text(v: dict | None) -> str:
|
||||
return "; ".join(f"{days_text(b['days'])} {b['from']}-{b['to']} {_fmt(b['pct'])}%" for b in (v or {}).get("blocks") or [])
|
||||
|
||||
|
||||
def local_when(text: str, zone: ZoneInfo, now: float, default_time: str | None = None) -> float:
|
||||
""""2026-11-05 07:00", "2026-11-05" (default_time, else 00:00), "07:00"
|
||||
(the next one) -> unix time, in the home's time zone."""
|
||||
words = str(text).split()
|
||||
if not words:
|
||||
raise ValueError("a date or time is needed")
|
||||
d, t = None, None
|
||||
for w in words:
|
||||
if re.match(r"^\d{4}-\d{2}-\d{2}$", w):
|
||||
d = date.fromisoformat(w)
|
||||
else:
|
||||
t = hhmm(w)
|
||||
t = t or default_time or "00:00"
|
||||
h, mi = (int(x) for x in t.split(":"))
|
||||
if d is None:
|
||||
n = datetime.fromtimestamp(now, zone)
|
||||
cand = _wall(n.date(), h, mi, zone)
|
||||
if cand <= now:
|
||||
cand = _wall(n.date() + timedelta(days=1), h, mi, zone)
|
||||
return cand
|
||||
return _wall(d, h, mi, zone)
|
||||
|
||||
|
||||
def _wall(d: date, h: int, mi: int, zone: ZoneInfo) -> float:
|
||||
"""A wall-clock time's unix time; one that does not exist (the hour
|
||||
skipped in spring) moves on by the gap, as a clock does (02:30 is 03:30
|
||||
summer time: PEP 495's fold 0 takes the offset before the change); one
|
||||
that comes twice (autumn) is the first."""
|
||||
naive = datetime(d.year, d.month, d.day, h % 24, mi) + timedelta(days=h // 24)
|
||||
return naive.replace(tzinfo=zone).timestamp()
|
||||
|
||||
|
||||
def parse_car_text(text: str, zone: ZoneInfo, now: float) -> dict | None:
|
||||
"""The one-off "car needed" text: None when empty (cancel)."""
|
||||
words = str(text or "").split()
|
||||
if not words:
|
||||
return None
|
||||
when = [w for w in words if re.match(r"^\d{4}-\d{2}-\d{2}$", w) or HHMM.match(w)]
|
||||
rest = [w for w in words if w not in when]
|
||||
if not when:
|
||||
raise ValueError("a time (and a date): e.g. 2026-11-05 07:00 80% must")
|
||||
at = local_when(" ".join(when), zone, now, default_time="07:00")
|
||||
tgt = _target_words(rest)
|
||||
tgt.setdefault("how", "must")
|
||||
if "pct" not in tgt and "kwh" not in tgt:
|
||||
tgt["pct"] = 100
|
||||
return _ev_target(tgt, {"kind": "car", "at": at})
|
||||
|
||||
|
||||
# -- the store --------------------------------------------------------------------
|
||||
class Preferences:
|
||||
def __init__(self, state_dir: str, clock=time.time):
|
||||
self.dir, self.clock = state_dir, clock
|
||||
self.tz = "UTC" # the home's (HA's), for times in words
|
||||
self.path = os.path.join(state_dir, "preferences.json")
|
||||
self.inbox = os.path.join(state_dir, "prefs-inbox")
|
||||
self.view_path = os.path.join(state_dir, "preferences-view.json")
|
||||
self.items: dict[str, dict[str, dict]] = {} # key -> level -> {"value", "by", "at"}
|
||||
self.overrides: list[dict] = []
|
||||
self.ha: dict = {} # HA's helper values: boost_c/boost_h per room, away_until text
|
||||
self.log: list[dict] = []
|
||||
self.changed_at = 0.0 # last change (the plan's costs, steady's free controls)
|
||||
self.dirty: set[str] = set() # controls changed since the last plan (steady.py: not held back)
|
||||
self.costs: dict = {"at": 0, "of": 0.0, "eur_day": {}} # the musts' costs, as of a plan
|
||||
self.feedback: list[dict] = []
|
||||
self.errors: list[dict] = [] # the last refused changes (the panel shows them)
|
||||
try:
|
||||
with open(self.path) as f:
|
||||
raw = json.load(f)
|
||||
self.items = {k: {lvl: e for lvl, e in v.items() if lvl in RANK} for k, v in (raw.get("items") or {}).items()}
|
||||
self.overrides = list(raw.get("overrides") or [])
|
||||
self.ha = dict(raw.get("ha") or {})
|
||||
self.log = list(raw.get("log") or [])[-MAX_LOG:]
|
||||
self.costs = raw.get("costs") or self.costs
|
||||
self.changed_at = float(raw.get("changed_at") or 0)
|
||||
except (OSError, ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
def save(self):
|
||||
os.makedirs(self.dir, exist_ok=True)
|
||||
tmp = self.path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump({"v": 1, "items": self.items, "overrides": self.overrides, "ha": self.ha, "log": self.log,
|
||||
"costs": self.costs, "changed_at": self.changed_at}, f)
|
||||
os.replace(tmp, self.path)
|
||||
|
||||
# -- reading -------------------------------------------------------------
|
||||
def effective(self, key: str) -> dict | None:
|
||||
"""The setting in force: the highest level's ({"value", "by", "at", "level"})."""
|
||||
per = self.items.get(key) or {}
|
||||
for lvl in reversed(LEVELS):
|
||||
if lvl in per:
|
||||
return dict(per[lvl], level=lvl)
|
||||
return None
|
||||
|
||||
def value(self, key: str) -> dict | None:
|
||||
e = self.effective(key)
|
||||
return e["value"] if e else None
|
||||
|
||||
def active_overrides(self, now: float | None = None) -> list[dict]:
|
||||
now = self.clock() if now is None else now
|
||||
return [o for o in self.overrides if _end(o) > now]
|
||||
|
||||
def prune(self, now: float | None = None) -> bool:
|
||||
"""Expired overrides out; True when one went (a plan without it)."""
|
||||
now = self.clock() if now is None else now
|
||||
keep = [o for o in self.overrides if _end(o) > now]
|
||||
gone = [o for o in self.overrides if _end(o) <= now]
|
||||
if not gone:
|
||||
return False
|
||||
self.overrides = keep
|
||||
for o in gone:
|
||||
self._log(now, o.get("by", ""), "owner", f"override:{o['kind']}", "expired", o.get("control", ""))
|
||||
if o.get("control"):
|
||||
self.dirty.add(o["control"])
|
||||
self.save()
|
||||
return True
|
||||
|
||||
# -- changing --------------------------------------------------------------
|
||||
def _log(self, now, by, level, key, what, control=""):
|
||||
self.log = (self.log + [{"at": int(now), "by": by, "level": level, "item": key, "what": str(what)[:200],
|
||||
**({"control": control} if control else {})}])[-MAX_LOG:]
|
||||
|
||||
def _refused(self, now, by, key, why):
|
||||
self.errors = (self.errors + [{"at": int(now), "by": by, "item": key, "why": why}])[-10:]
|
||||
return False, why
|
||||
|
||||
def set(self, key: str, value: dict | None, by: str, level: str, control, now: float | None = None) -> tuple[bool, str]:
|
||||
"""A setting at a level (None: that level's cleared). control: the
|
||||
control it is for (its kind; its owner limits for lower levels)."""
|
||||
now = self.clock() if now is None else now
|
||||
if level not in RANK:
|
||||
return self._refused(now, by, key, f"level: one of {', '.join(LEVELS)}")
|
||||
what, _, cid = key.partition(":")
|
||||
if control is None or control.id != cid or KINDS.get(control.kind) != what:
|
||||
return self._refused(now, by, key, f"no {what} control {cid!r}")
|
||||
if value is None:
|
||||
if level in (self.items.get(key) or {}):
|
||||
del self.items[key][level]
|
||||
if not self.items[key]:
|
||||
del self.items[key]
|
||||
self._log(now, by, level, key, "cleared", cid)
|
||||
else:
|
||||
try:
|
||||
value = check(key, value, control.kind)
|
||||
if level != "owner":
|
||||
within_owner(key, value, control)
|
||||
except ValueError as e:
|
||||
return self._refused(now, by, key, str(e))
|
||||
self.items.setdefault(key, {})[level] = {"value": value, "by": by, "at": int(now)}
|
||||
self._log(now, by, level, key, describe(key, value, control.kind), cid)
|
||||
self.changed_at = now
|
||||
self.dirty.add(cid)
|
||||
self.save()
|
||||
top = self.effective(key)
|
||||
note = "" if top is None or top["level"] == level else f" (the {top['level']}'s own setting is in force)"
|
||||
return True, f"{key}: {'cleared' if value is None else describe(key, value, control.kind)}{note}"
|
||||
|
||||
def add_override(self, o: dict, by: str, controls: dict, now: float | None = None) -> tuple[bool, str]:
|
||||
"""An override (the owner's): boost, away or car; one of each kind
|
||||
per control (a new one replaces it)."""
|
||||
now = self.clock() if now is None else now
|
||||
kind = o.get("kind")
|
||||
try:
|
||||
if kind == "boost":
|
||||
c = controls.get(o.get("control", ""))
|
||||
if c is None or c.kind not in ("heating", "cooling"):
|
||||
raise ValueError(f"no room {o.get('control')!r}")
|
||||
lo, hi = RANGE_C[c.kind]
|
||||
hours = _num(o.get("hours", 2), "hours", 0.25, BOOST_MAX_H)
|
||||
o = {"kind": "boost", "control": c.id, "c": _num(o.get("c"), "temperature", lo, hi),
|
||||
"from": int(now), "until": int(o.get("until") or now + hours * 3600)}
|
||||
elif kind == "away":
|
||||
until = int(o.get("until") or now + AWAY_MAX_S)
|
||||
if until <= now or until > now + AWAY_MAX_S:
|
||||
raise ValueError("away: until a time within the next 60 days")
|
||||
o = {"kind": "away", "from": int(o.get("from") or now), "until": until}
|
||||
elif kind == "car":
|
||||
c = controls.get(o.get("control", ""))
|
||||
if c is None or c.kind != "ev":
|
||||
raise ValueError(f"no car charger {o.get('control')!r}")
|
||||
at = int(o.get("at") or 0)
|
||||
if not now < at <= now + 7 * 86400:
|
||||
raise ValueError("car needed: a time within the next 7 days")
|
||||
o = _ev_target(dict(o, how=o.get("how") or "must"), {"kind": "car", "control": c.id, "at": at})
|
||||
else:
|
||||
raise ValueError("override: boost, away or car")
|
||||
except (ValueError, TypeError) as e:
|
||||
return self._refused(now, by, f"override:{kind}", str(e))
|
||||
o.update(id=uuid.uuid4().hex[:8], by=by, set_at=int(now))
|
||||
self.overrides = [x for x in self.overrides if not (x["kind"] == o["kind"] and x.get("control") == o.get("control"))]
|
||||
if len(self.overrides) >= MAX_OVERRIDES:
|
||||
return self._refused(now, by, f"override:{kind}", f"at most {MAX_OVERRIDES} overrides")
|
||||
self.overrides.append(o)
|
||||
self._log(now, by, "owner", f"override:{kind}", override_words(o, self.tz), o.get("control", ""))
|
||||
self._touch(now, o)
|
||||
return True, override_words(o, self.tz)
|
||||
|
||||
def cancel_override(self, kind: str, control: str, by: str, now: float | None = None) -> tuple[bool, str]:
|
||||
now = self.clock() if now is None else now
|
||||
gone = [x for x in self.overrides if x["kind"] == kind and x.get("control", "") == (control or "")]
|
||||
if not gone:
|
||||
return True, "no such override"
|
||||
self.overrides = [x for x in self.overrides if x not in gone]
|
||||
self._log(now, by, "owner", f"override:{kind}", "cancelled", control)
|
||||
for o in gone:
|
||||
self._touch(now, o)
|
||||
return True, f"{kind} cancelled"
|
||||
|
||||
def _touch(self, now, o):
|
||||
self.changed_at = now
|
||||
if o.get("control"):
|
||||
self.dirty.add(o["control"])
|
||||
else:
|
||||
self.dirty.add("*")
|
||||
self.save()
|
||||
|
||||
def free(self) -> set[str]:
|
||||
"""Controls whose next plan is taken as it is (steady.py), once."""
|
||||
d, self.dirty = self.dirty, set()
|
||||
return d
|
||||
|
||||
# -- the panel's inbox ---------------------------------------------------------
|
||||
def take_inbox(self, controls: dict, tz: str) -> int:
|
||||
"""The panel's changes (prefs-inbox/*.json, one per change, in name
|
||||
order): {"by", "set": {"item", "value" | null}} | {"override": {...}}
|
||||
| {"cancel": {"kind", "control"}}; each applied as the owner's."""
|
||||
try:
|
||||
names = sorted(n for n in os.listdir(self.inbox) if n.endswith(".json"))
|
||||
except OSError:
|
||||
return 0
|
||||
n = 0
|
||||
for name in names:
|
||||
p = os.path.join(self.inbox, name)
|
||||
try:
|
||||
with open(p) as f:
|
||||
msg = json.load(f)
|
||||
except (OSError, ValueError):
|
||||
msg = None
|
||||
try:
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
if isinstance(msg, dict):
|
||||
self.apply_msg(msg, controls, tz, "owner", "panel:" + str(msg.get("by") or "owner")[:40])
|
||||
n += 1
|
||||
return n
|
||||
|
||||
def apply_msg(self, msg: dict, controls: dict, tz: str, level: str, by: str) -> tuple[bool, str]:
|
||||
"""One change in the inbox's form (the panel's, the console's)."""
|
||||
now = self.clock()
|
||||
zone = ZoneInfo(tz or "UTC")
|
||||
if "set" in msg:
|
||||
s = msg["set"] or {}
|
||||
key = str(s.get("item", ""))
|
||||
val = s.get("value")
|
||||
if isinstance(val, str): # a text form
|
||||
try:
|
||||
val = from_text(key, val, controls.get(key.partition(":")[2]),
|
||||
(self.value(key) or {}) if key.startswith(("ev:", "battery:")) else {})
|
||||
except ValueError as e:
|
||||
return self._refused(now, by, key, str(e))
|
||||
return self.set(key, val, by, level, controls.get(key.partition(":")[2]), now)
|
||||
if level != "owner":
|
||||
return self._refused(now, by, "override", "overrides are the owner's (Home Assistant, the panel)")
|
||||
if "override" in msg:
|
||||
o = dict(msg["override"] or {})
|
||||
if o.get("kind") == "car" and isinstance(o.get("text"), str):
|
||||
try:
|
||||
car = parse_car_text(o["text"], zone, now)
|
||||
except ValueError as e:
|
||||
return self._refused(now, by, "override:car", str(e))
|
||||
if car is None:
|
||||
return self.cancel_override("car", o.get("control", ""), by, now)
|
||||
o = dict(car, control=o.get("control", ""))
|
||||
if o.get("kind") == "away" and isinstance(o.get("until_text"), str) and o["until_text"].strip():
|
||||
try:
|
||||
o["until"] = local_when(o["until_text"], zone, now)
|
||||
except ValueError as e:
|
||||
return self._refused(now, by, "override:away", str(e))
|
||||
return self.add_override(o, by, controls, now)
|
||||
if "cancel" in msg:
|
||||
c = msg["cancel"] or {}
|
||||
return self.cancel_override(str(c.get("kind", "")), str(c.get("control", "")), by, now)
|
||||
return self._refused(now, by, "", "set, override or cancel")
|
||||
|
||||
# -- into the plan ---------------------------------------------------------------
|
||||
def apply(self, state: dict, controls: dict, col=None, now: float | None = None) -> dict:
|
||||
"""The model state with each control's preferences per step
|
||||
(d["pref"]: optimizer.py). Controls without any keep today's."""
|
||||
now = self.clock() if now is None else now
|
||||
self.prune(now)
|
||||
zone = ZoneInfo(state.get("tz") or "UTC")
|
||||
T, start, step = state["steps"], state["start"], state["step_s"]
|
||||
ends = [_local(start + (t + 1) * step, zone) for t in range(T)] # a room's or battery's value at a step's end
|
||||
ovs = self.active_overrides(now)
|
||||
for d in state["devices"]:
|
||||
c = controls.get(d["id"])
|
||||
if c is None:
|
||||
continue
|
||||
if d["kind"] in ("heating", "cooling"):
|
||||
p = room_series(d, self.value(f"room:{d['id']}"), ovs, state, ends)
|
||||
elif d["kind"] == "ev":
|
||||
p = ev_need(d, self.value(f"ev:{d['id']}"), ovs, state, zone, now, col, c)
|
||||
elif d["kind"] == "battery":
|
||||
p = battery_series(d, self.value(f"battery:{d['id']}"), ends)
|
||||
else:
|
||||
p = None
|
||||
if p is not None:
|
||||
d["pref"] = p
|
||||
state["prefs_cost"] = bool(any(d.get("pref") for d in state["devices"])) and (
|
||||
self.changed_at > self.costs.get("of", 0) or now - self.costs.get("at", 0) >= COST_EVERY_S)
|
||||
return state
|
||||
|
||||
def take_plan(self, plan: dict, now: float | None = None):
|
||||
"""The plan's feedback (optimizer.pref_feedback), with the musts'
|
||||
costs: the plan's own when it made them, else the last ones."""
|
||||
now = self.clock() if now is None else now
|
||||
made = plan.get("prefs_cost")
|
||||
if made is not None:
|
||||
self.costs = {"at": int(now), "of": max(self.changed_at, 0.0), "eur_day": made}
|
||||
try:
|
||||
self.save()
|
||||
except OSError as e:
|
||||
log.warning("preferences: %s", e)
|
||||
eur = self.costs.get("eur_day") or {}
|
||||
fb = []
|
||||
for f in plan.get("prefs") or []:
|
||||
f = dict(f)
|
||||
if f.get("how") == "must" and f.get("group") in eur:
|
||||
f["cost_eur_day"] = eur[f["group"]]
|
||||
f["cost_as_of"] = self.costs.get("at")
|
||||
f["text"] = feedback_words(f)
|
||||
fb.append(f)
|
||||
self.feedback = fb
|
||||
plan["prefs"] = fb
|
||||
plan.pop("prefs_cost", None)
|
||||
return fb
|
||||
|
||||
# -- out ---------------------------------------------------------------------------
|
||||
def public(self) -> dict:
|
||||
"""For the cloud (with the plan) and the panel: the settings per
|
||||
level with who and when, the overrides in force, the feedback, the
|
||||
last changes."""
|
||||
items = {}
|
||||
for key, per in self.items.items():
|
||||
top = self.effective(key)
|
||||
items[key] = {"in_force": top["level"] if top else None,
|
||||
"levels": {lvl: {"value": e["value"], "by": e["by"], "at": e["at"],
|
||||
"text": text_of(key, e["value"])} for lvl, e in per.items()}}
|
||||
return {"v": 1, "items": items, "overrides": self.active_overrides(), "feedback": self.feedback,
|
||||
"log": self.log[-20:], "changed_at": int(self.changed_at)}
|
||||
|
||||
def write_view(self, controls, mode: str = ""):
|
||||
"""preferences-view.json for the Hub's panel: the controls (id,
|
||||
kind, name, the owner's limits), and public()."""
|
||||
cl = []
|
||||
for c in controls:
|
||||
if c.kind not in KINDS:
|
||||
continue
|
||||
q = {"heating": "temp_c", "cooling": "temp_c", "battery": "soc_pct", "ev": "current_a"}[c.kind]
|
||||
b = c.bounds(q)
|
||||
cl.append({"id": c.id, "kind": c.kind, "name": c.label(), "bounds": [b.lo, b.hi],
|
||||
"setback_c": _p(c, "setback_c"), "setup_c": _p(c, "setup_c"), "comfort_c": _p(c, "comfort_c")})
|
||||
v = dict(self.public(), controls=cl, mode=mode, errors=self.errors, ha=self.ha, made=int(self.clock()))
|
||||
try:
|
||||
tmp = self.view_path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(v, f)
|
||||
os.replace(tmp, self.view_path)
|
||||
except OSError as e:
|
||||
log.warning("preferences view: %s", e)
|
||||
|
||||
|
||||
def _p(c, k):
|
||||
try:
|
||||
return float(c.p(k))
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _end(o: dict) -> float:
|
||||
return float(o.get("until") or o.get("at") or 0)
|
||||
|
||||
|
||||
def _local(t: float, zone: ZoneInfo) -> tuple[int, int]:
|
||||
d = datetime.fromtimestamp(t, timezone.utc).astimezone(zone)
|
||||
return d.weekday(), d.hour * 60 + d.minute
|
||||
|
||||
|
||||
def in_block(b: dict, wd: int, m: int) -> bool:
|
||||
"""A block {days, from, to} covers the wall-clock (weekday, minute)? A
|
||||
block past midnight belongs to the day it starts; from == to: all day."""
|
||||
a, z = minutes(b["from"]), minutes(b["to"])
|
||||
if a < z:
|
||||
return wd in b["days"] and a <= m < z
|
||||
if a == z:
|
||||
return wd in b["days"]
|
||||
return (wd in b["days"] and m >= a) or ((wd - 1) % 7 in b["days"] and m < z)
|
||||
|
||||
|
||||
def within_owner(key: str, value: dict, c):
|
||||
"""An installer's or operator's setting within the owner's limits (the
|
||||
mapping's owner level), never above them."""
|
||||
what = key.split(":", 1)[0]
|
||||
owner = lambda q: (c.limits.get(q) or {}).get("owner") or (None, None) # noqa: E731
|
||||
if what == "room":
|
||||
lo, hi = owner("temp_c")
|
||||
for b in value["blocks"]:
|
||||
if (lo is not None and b["c"] < lo) or (hi is not None and b["c"] > hi):
|
||||
raise ValueError(f"{_fmt(b['c'])} °C is outside the owner's limits "
|
||||
f"{_fmt(lo) if lo is not None else '-'}..{_fmt(hi) if hi is not None else '-'} °C")
|
||||
elif what == "battery":
|
||||
lo, hi = owner("soc_pct")
|
||||
for p in [value["reserve_pct"]] + [b["pct"] for b in value["blocks"]]:
|
||||
if (lo is not None and p < lo) or (hi is not None and p > hi):
|
||||
raise ValueError(f"a reserve of {_fmt(p)} % is outside the owner's limits")
|
||||
|
||||
|
||||
# -- per step ----------------------------------------------------------------------
|
||||
def room_series(d: dict, value: dict | None, ovs: list[dict], state: dict, ends) -> dict | None:
|
||||
"""A room's target per step (its temperature at the step's end), the
|
||||
must's bound (None: none), and the group (a block, a boost, away) it
|
||||
comes from. base: what it would be without the group (setback/setup;
|
||||
today's occupied hours without a schedule)."""
|
||||
cooling = d["kind"] == "cooling"
|
||||
p = d["params"]
|
||||
other = float(p["setup_c"] if cooling else p["setback_c"])
|
||||
comfort = float(p["comfort_c"])
|
||||
blocks = (value or {}).get("blocks") or []
|
||||
mine = [o for o in ovs if (o["kind"] == "boost" and o.get("control") == d["id"]) or o["kind"] == "away"]
|
||||
if not blocks and not mine:
|
||||
return None
|
||||
T, start, step = state["steps"], state["start"], state["step_s"]
|
||||
occ = state.get("occupied") or [1] * T
|
||||
groups = [{"group": f"{d['id']}:{i}", "kind": "block", "how": b["how"], "c": b["c"],
|
||||
"text": f"{days_text(b['days'])} {b['from']}-{b['to']} {'≤' if cooling else '≥'} {_fmt(b['c'])} °C"}
|
||||
for i, b in enumerate(blocks)]
|
||||
gi = {}
|
||||
for o in mine:
|
||||
gi[o["id"]] = len(groups)
|
||||
groups.append({"group": f"{d['id']}:{o['kind']}", "kind": o["kind"], "how": "must" if o["kind"] == "boost" else "prefer",
|
||||
"c": o.get("c", other), "text": override_words(o, state.get("tz")), "until": o.get("until")})
|
||||
target, must, group, base = [], [], [], []
|
||||
pick = min if cooling else max
|
||||
for t in range(T):
|
||||
wd, m = ends[t]
|
||||
tend = start + (t + 1) * step
|
||||
b0 = other if blocks else (comfort if occ[min(t + 1, T - 1)] else other)
|
||||
base.append(b0)
|
||||
boost = next((o for o in mine if o["kind"] == "boost" and o["from"] <= tend < o["until"]), None)
|
||||
away = next((o for o in mine if o["kind"] == "away" and o.get("from", 0) <= tend < o["until"]), None)
|
||||
if boost is not None:
|
||||
target.append(boost["c"])
|
||||
must.append(boost["c"])
|
||||
group.append(gi[boost["id"]])
|
||||
continue
|
||||
if away is not None:
|
||||
target.append(other)
|
||||
must.append(None)
|
||||
group.append(gi[away["id"]])
|
||||
continue
|
||||
hit = [(i, b) for i, b in enumerate(blocks) if in_block(b, wd, m)]
|
||||
if not hit:
|
||||
target.append(b0)
|
||||
must.append(None)
|
||||
group.append(-1)
|
||||
continue
|
||||
tc = pick(b["c"] for _, b in hit)
|
||||
ms = [b["c"] for _, b in hit if b["how"] == "must"]
|
||||
target.append(tc)
|
||||
must.append(pick(ms) if ms else None)
|
||||
# the group: the must that binds, else the block that sets the target
|
||||
g = next((i for i, b in hit if b["how"] == "must" and b["c"] == must[-1]), None) if ms else None
|
||||
group.append(g if g is not None else next(i for i, b in hit if b["c"] == tc))
|
||||
return {"target": target, "must": must, "group": group, "base": base, "groups": groups}
|
||||
|
||||
|
||||
def next_ready(value: dict | None, zone: ZoneInfo, now: float) -> tuple[float, dict] | None:
|
||||
"""The next ready-by time after now and its entry."""
|
||||
best = None
|
||||
today = datetime.fromtimestamp(now, zone).date()
|
||||
for r in (value or {}).get("ready") or []:
|
||||
h, mi = (int(x) for x in r["at"].split(":"))
|
||||
for k in range(8):
|
||||
d = today + timedelta(days=k)
|
||||
if d.weekday() not in r["days"]:
|
||||
continue
|
||||
t = _wall(d, h, mi, zone)
|
||||
if t > now:
|
||||
if best is None or t < best[0]:
|
||||
best = (t, r)
|
||||
break
|
||||
return best
|
||||
|
||||
|
||||
def ev_need(d: dict, value: dict | None, ovs: list[dict], state: dict, zone, now: float, col, c) -> dict | None:
|
||||
"""The car's next ready-by (a one-off override wins while it lasts) as
|
||||
the departure step and the energy needed: kWh as given, or % from the
|
||||
car's state of charge: (target - soc) x capacity."""
|
||||
car = next((o for o in ovs if o["kind"] == "car" and o.get("control") == d["id"]), None)
|
||||
if car is not None:
|
||||
when, r, src = car["at"], car, "override"
|
||||
else:
|
||||
nr = next_ready(value, zone, now)
|
||||
if nr is None:
|
||||
return None
|
||||
(when, r), src = nr, "schedule"
|
||||
start, step, T = state["start"], state["step_s"], state["steps"]
|
||||
dep = int(max(0, min(T, (when - start) // step)))
|
||||
st = d.get("state") or {}
|
||||
soc = st.get("soc_pct")
|
||||
cap = (value or {}).get("capacity_kwh")
|
||||
note = ""
|
||||
if r.get("kwh") is not None:
|
||||
need = r["kwh"] * 1000.0
|
||||
elif soc is not None and cap:
|
||||
need = max(0.0, (r["pct"] - float(soc)) / 100.0 * cap * 1000.0)
|
||||
else:
|
||||
need = float(st["need_wh"]) if st.get("need_wh") is not None else float(d["params"]["need_wh"])
|
||||
note = "no state of charge from the car" if soc is None else "the car's capacity is not set"
|
||||
loc = datetime.fromtimestamp(when, zone)
|
||||
out = {"group": f"{d['id']}:{'car' if src == 'override' else 'ready'}", "source": src, "dep_step": dep,
|
||||
"at": int(when), "at_local": loc.strftime("%a %H:%M"), "need_wh": round(need, 1), "how": r["how"],
|
||||
"max_ct": r.get("max_ct"), "min_wh": (r.get("min_kwh") or 0.0) * 1000.0 if r.get("min_kwh") else 0.0,
|
||||
"pct": r.get("pct"), "kwh": r.get("kwh"), "soc_pct": soc, "capacity_wh": cap * 1000.0 if cap else None,
|
||||
"note": note, "beyond": when > start + T * step}
|
||||
if col is not None and c.entities.get("connected"):
|
||||
st0 = col.states.get(c.entities["connected"]) or {}
|
||||
if st0.get("s") == "on" and st0.get("t"):
|
||||
out["plugged_at"] = int(float(st0["t"]))
|
||||
out["text"] = f"{_target_text(r).replace(' must', '')} by {loc.strftime('%a %H:%M')}" + \
|
||||
(" (one-off)" if src == "override" else "")
|
||||
return out
|
||||
|
||||
|
||||
def battery_series(d: dict, value: dict | None, ends) -> dict | None:
|
||||
if not value:
|
||||
return None
|
||||
base = float(value.get("reserve_pct") or 0.0)
|
||||
groups = [{"group": f"{d['id']}:reserve", "kind": "reserve", "how": "must", "pct": base,
|
||||
"text": f"reserve {_fmt(base)} %"}] if base > 0 else []
|
||||
for i, b in enumerate(value.get("blocks") or []):
|
||||
groups.append({"group": f"{d['id']}:{i}", "kind": "block", "how": "must", "pct": b["pct"],
|
||||
"text": f"reserve {_fmt(b['pct'])} % {days_text(b['days'])} {b['from']}-{b['to']}"})
|
||||
floor, group = [], []
|
||||
off = 1 if base > 0 else 0
|
||||
for wd, m in ends:
|
||||
best, g = base, 0 if base > 0 else -1
|
||||
for i, b in enumerate(value.get("blocks") or []):
|
||||
if in_block(b, wd, m) and b["pct"] > best:
|
||||
best, g = b["pct"], i + off
|
||||
floor.append(best)
|
||||
group.append(g)
|
||||
if not groups:
|
||||
return None
|
||||
return {"floor_pct": floor, "group": group, "groups": groups}
|
||||
|
||||
|
||||
# -- words ----------------------------------------------------------------------------
|
||||
def describe(key: str, value: dict, kind: str = "") -> str:
|
||||
what = key.split(":", 1)[0]
|
||||
if what == "room":
|
||||
return room_text(value) or "no schedule"
|
||||
if what == "ev":
|
||||
s = ev_text(value) or "no ready-by times"
|
||||
return s + (f"; capacity {_fmt(value['capacity_kwh'])} kWh" if value.get("capacity_kwh") else "")
|
||||
if what == "battery":
|
||||
return f"reserve {_fmt(value['reserve_pct'])} %" + (f"; {battery_text(value)}" if value.get("blocks") else "")
|
||||
return json.dumps(value)[:200]
|
||||
|
||||
|
||||
def text_of(key: str, value: dict) -> str:
|
||||
what = key.split(":", 1)[0]
|
||||
return {"room": room_text, "ev": ev_text, "battery": battery_text}.get(what, lambda v: "")(value)
|
||||
|
||||
|
||||
def from_text(key: str, text: str, control, current: dict) -> dict:
|
||||
what = key.split(":", 1)[0]
|
||||
if control is None:
|
||||
raise ValueError(f"no control for {key!r}")
|
||||
if what == "room":
|
||||
return parse_room_text(text, control.kind)
|
||||
if what == "ev":
|
||||
return parse_ev_text(text, current.get("capacity_kwh"))
|
||||
if what == "battery":
|
||||
return parse_battery_text(text, current.get("reserve_pct", 0.0))
|
||||
raise ValueError(f"no preference {key!r}")
|
||||
|
||||
|
||||
def car_text(o: dict, tz: str | None = None) -> str:
|
||||
"""A car override in the text form: "2026-11-05 07:00 80% must"."""
|
||||
when = datetime.fromtimestamp(o["at"], ZoneInfo(tz or "UTC")).strftime("%Y-%m-%d %H:%M")
|
||||
t = _target_text(o)
|
||||
return f"{when} {t}" if o.get("how") == "must" else f"{when} {t} prefer"
|
||||
|
||||
|
||||
def override_words(o: dict, tz: str | None = None) -> str:
|
||||
until = datetime.fromtimestamp(_end(o), ZoneInfo(tz or "UTC")).strftime("%a %d %b %H:%M")
|
||||
if o["kind"] == "boost":
|
||||
return f"boost to {_fmt(o['c'])} °C until {until}"
|
||||
if o["kind"] == "away":
|
||||
return f"away until {until}"
|
||||
return f"car {_target_text(o).replace(' must', '')} at {until}" + (" (must)" if o.get("how") == "must" else "")
|
||||
|
||||
|
||||
def feedback_words(f: dict) -> str:
|
||||
"""One preference's result in words: "Mon-Fri 22:00-06:00 ≥ 21 °C: met,
|
||||
+0.40 €/day"; "car 100% by Thu 08:00: short by 4.2 kWh, ready at full
|
||||
power by 09:15"."""
|
||||
head = f.get("label") or f.get("text") or f.get("group", "")
|
||||
if f.get("met") is None:
|
||||
bits = [f.get("detail") or "not in the plan's 48 h"]
|
||||
elif f["met"]:
|
||||
bits = ["met"] + ([f["detail"]] if f.get("detail") else [])
|
||||
else:
|
||||
bits = ["not met"] + ([f["detail"]] if f.get("detail") else [])
|
||||
if f.get("cost_eur_day") is not None:
|
||||
bits.append(f"{f['cost_eur_day']:+.2f} €/day")
|
||||
return f"{head}: " + ", ".join(bits)
|
||||
@@ -0,0 +1,136 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""A room's thermal resistance R and capacity C from its history
|
||||
(docs/methods.md §6): the planner's first-order model (edge optimizer.py
|
||||
`room`),
|
||||
|
||||
θ_t = (1 − a) θ_{t−1} + a T_out,t + (Δt/C) P_t + γ G_t + c0,
|
||||
a = Δt / (R C),
|
||||
|
||||
written as a regression on 15-minute steps (Δt = 0.25 h):
|
||||
|
||||
θ_t − θ_{t−1} = a (T_out,t − θ_{t−1}) + b P_t + γ G_t + c0, b = Δt / C,
|
||||
|
||||
with P the heating power (kW; thermal where the site meters the heat
|
||||
output, else electric: the planner's COP turns that into heat), G the
|
||||
global irradiance (kW/m², the sun through the windows) and c0 the internal
|
||||
gains (people, appliances). Least squares on every step where all inputs
|
||||
are there; then C = Δt / b, R = Δt / (a C).
|
||||
|
||||
The other two terms go to the planner with R and C (Finding B6-a: R and C
|
||||
were fitted with them; without them the planner expects the room to cool
|
||||
faster than it does), as heat in the same units as P:
|
||||
|
||||
gain_w = 1000 c0 / b the internal gain (W, constant),
|
||||
solar_m2 = γ / b the sun's heat per irradiance (m²: W of heat
|
||||
per W/m² of global irradiance, an effective
|
||||
window area).
|
||||
|
||||
(electric fits: W and m² of electric equivalent; the box multiplies by
|
||||
the COP like C.)
|
||||
|
||||
Sanity before a fit is used (else the planner keeps its defaults, R 10
|
||||
K/kW and C 3 kWh/K): a > 0 and b > 0; R within 1..100 K/kW, C within
|
||||
0.3..50 kWh/K, τ = RC within 3..1000 h; at least a day of steps, the
|
||||
heating on and off in it, 2 K of spread in T_out − θ; and the fitted
|
||||
model simulates the room better than the defaults do (4-hour open-loop
|
||||
runs from every 4th hour: RMSE in K).
|
||||
|
||||
The site's heating power is used for every room (the sims: one zone per
|
||||
home): the rooms of a home with one heat meter each get the whole
|
||||
home's C. Per-room heating power, where metered per control, is the
|
||||
next step."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
|
||||
DT_H = 0.25
|
||||
DEFAULT_R, DEFAULT_C = 10.0, 3.0
|
||||
R_BOUNDS, C_BOUNDS, TAU_BOUNDS = (1.0, 100.0), (0.3, 50.0), (3.0, 1000.0)
|
||||
MIN_STEPS = 96
|
||||
SIM_STEPS = 16
|
||||
SIM_EVERY = 16
|
||||
|
||||
|
||||
def simulate(theta0: float, tout: np.ndarray, p_kw: np.ndarray, ghi_kw: np.ndarray, r: float, c: float,
|
||||
gamma: float = 0.0, c0: float = 0.0) -> np.ndarray:
|
||||
a, b = DT_H / (r * c), DT_H / c
|
||||
out, th = [], theta0
|
||||
for k in range(len(tout)):
|
||||
th = (1 - a) * th + a * tout[k] + b * p_kw[k] + gamma * ghi_kw[k] + c0
|
||||
out.append(th)
|
||||
return np.array(out)
|
||||
|
||||
|
||||
def sim_rmse(theta: np.ndarray, tout: np.ndarray, p_kw: np.ndarray, ghi_kw: np.ndarray, r: float, c: float,
|
||||
gamma: float = 0.0, c0: float = 0.0) -> float | None:
|
||||
"""4-hour open-loop runs from every 4th hour where all inputs exist."""
|
||||
errs = []
|
||||
for s in range(0, len(theta) - SIM_STEPS - 1, SIM_EVERY):
|
||||
seg = slice(s + 1, s + 1 + SIM_STEPS)
|
||||
if np.isnan(theta[s]) or np.isnan(theta[seg]).any() or np.isnan(tout[seg]).any() or np.isnan(p_kw[seg]).any():
|
||||
continue
|
||||
sim = simulate(theta[s], tout[seg], p_kw[seg], np.nan_to_num(ghi_kw[seg]), r, c, gamma, c0)
|
||||
errs.append((sim - theta[seg]) ** 2)
|
||||
if not errs:
|
||||
return None
|
||||
return float(math.sqrt(np.concatenate(errs).mean()))
|
||||
|
||||
|
||||
def fit(theta: np.ndarray, tout: np.ndarray, p_w: np.ndarray, ghi: np.ndarray | None = None,
|
||||
power: str = "thermal") -> dict:
|
||||
"""theta, tout (°C), p_w (W), ghi (W/m²) per 15-minute slot, aligned.
|
||||
Returns {"status": "fitted"|"default", r_k_per_kw, c_kwh_per_k, tau_h,
|
||||
power, n, rmse_k (simulation, fitted), rmse_default_k, r2, note}, and
|
||||
when fitted gain_w and solar_m2 (the module's docstring)."""
|
||||
n_all = len(theta)
|
||||
ghi = np.zeros(n_all) if ghi is None else np.where(np.isnan(ghi), 0.0, ghi)
|
||||
p_kw = p_w / 1000.0
|
||||
g_kw = ghi / 1000.0
|
||||
prev = np.concatenate([[np.nan], theta[:-1]])
|
||||
dy = theta - prev
|
||||
x1 = tout - prev
|
||||
ok = ~(np.isnan(dy) | np.isnan(x1) | np.isnan(p_kw))
|
||||
out = {"status": "default", "r_k_per_kw": DEFAULT_R, "c_kwh_per_k": DEFAULT_C, "tau_h": DEFAULT_R * DEFAULT_C,
|
||||
"power": power, "n": int(ok.sum()), "rmse_k": None, "rmse_default_k": None, "r2": None, "note": ""}
|
||||
if ok.sum() < MIN_STEPS:
|
||||
out["note"] = f"{int(ok.sum())} usable steps (at least {MIN_STEPS})"
|
||||
return out
|
||||
on = p_kw[ok] > 0.05
|
||||
if on.all() or not on.any():
|
||||
out["note"] = "the heating was never " + ("off" if on.all() else "on") + " in the data"
|
||||
return out
|
||||
if np.ptp(x1[ok]) < 2.0:
|
||||
out["note"] = "less than 2 K between outside and inside"
|
||||
return out
|
||||
X = np.column_stack([x1[ok], p_kw[ok], g_kw[ok], np.ones(int(ok.sum()))])
|
||||
coef, *_ = np.linalg.lstsq(X, dy[ok], rcond=None)
|
||||
a, b, gamma, c0 = (float(v) for v in coef)
|
||||
resid = dy[ok] - X @ coef
|
||||
var = float(np.var(dy[ok]))
|
||||
out["r2"] = round(1 - float(np.var(resid)) / var, 3) if var > 0 else None
|
||||
if a <= 0 or b <= 0:
|
||||
out["note"] = f"no physical fit (a {a:.4g}, b {b:.4g})"
|
||||
return out
|
||||
c = DT_H / b
|
||||
r = DT_H / (a * c)
|
||||
tau = r * c
|
||||
out["fit"] = {"a": a, "b": b, "gamma": gamma, "c0": c0}
|
||||
if not (R_BOUNDS[0] <= r <= R_BOUNDS[1] and C_BOUNDS[0] <= c <= C_BOUNDS[1] and TAU_BOUNDS[0] <= tau <= TAU_BOUNDS[1]):
|
||||
out["note"] = f"outside the sanity bounds (R {r:.3g} K/kW, C {c:.3g} kWh/K, τ {tau:.3g} h)"
|
||||
return out
|
||||
fitted = sim_rmse(theta, tout, p_kw, g_kw, r, c, gamma, c0)
|
||||
default = sim_rmse(theta, tout, p_kw, g_kw, DEFAULT_R, DEFAULT_C)
|
||||
out["rmse_default_k"] = round(default, 3) if default is not None else None
|
||||
out["rmse_k"] = round(fitted, 3) if fitted is not None else None
|
||||
if fitted is None or default is None:
|
||||
out["note"] = "too few complete 4-hour stretches to check the fit"
|
||||
return out
|
||||
if fitted >= default:
|
||||
out["note"] = f"no better than the defaults ({fitted:.2f} K vs {default:.2f} K)"
|
||||
return out
|
||||
out.update(status="fitted", r_k_per_kw=round(r, 3), c_kwh_per_k=round(c, 3), tau_h=round(tau, 2),
|
||||
gain_w=round(1000.0 * c0 / b, 1), solar_m2=round(gamma / b, 3),
|
||||
note=f"simulates the room to {fitted:.2f} K (defaults {default:.2f} K)")
|
||||
return out
|
||||
@@ -31,8 +31,11 @@ keeps the value in force, its why says so) unless all hold:
|
||||
Not here, so never held back: the console's and the building's commands,
|
||||
the safe defaults, the grid operator's order (the plan is made again at
|
||||
once within its bounds; a kept value outside the bounds in force is never
|
||||
kept), the owner's own changes (the guard's hands off) and the guard's
|
||||
floors. The entry the plan made is kept in the entry as "planned".
|
||||
kept), the owner's own changes (the guard's hands off; and the owner's
|
||||
preferences, preferences.py: the first plan after a change of a control's
|
||||
preferences or overrides, `free`, is taken as it is for that control) and
|
||||
the guard's floors. The entry the plan made is kept in the entry as
|
||||
"planned".
|
||||
"""
|
||||
import math
|
||||
|
||||
@@ -117,10 +120,13 @@ def _within(bounds, v) -> bool:
|
||||
return (lo is None or v >= lo - 1e-9) and (hi is None or v <= hi + 1e-9)
|
||||
|
||||
|
||||
def stabilise(plan: dict, state: dict, controls, col, last_set=None, now: float | None = None) -> dict:
|
||||
def stabilise(plan: dict, state: dict, controls, col, last_set=None, now: float | None = None,
|
||||
free: set | None = None) -> dict:
|
||||
"""The plan with changes that do not matter kept out (module
|
||||
docstring). last_set(control id, kind "mode"|"set") -> when the agent
|
||||
last changed that (None: not known). In place; returns the plan."""
|
||||
last changed that (None: not known). free: control ids (or "*": all)
|
||||
whose preferences the owner just changed: their entries as planned.
|
||||
In place; returns the plan."""
|
||||
now = plan.get("made", 0) if now is None else now
|
||||
step = plan.get("step_s", 900)
|
||||
end = plan.get("start", now) + step * state.get("steps", 192)
|
||||
@@ -133,6 +139,8 @@ def stabilise(plan: dict, state: dict, controls, col, last_set=None, now: float
|
||||
entries = (plan.get("devices") or {}).get(c.id)
|
||||
if not entries or c.kind not in ("battery", "heating", "cooling", "ev"):
|
||||
continue
|
||||
if free and (c.id in free or "*" in free):
|
||||
continue # the owner's change: not held back
|
||||
cur = current(c, col)
|
||||
if cur is None:
|
||||
continue
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The box's own training history (docs/methods.md B10): what its nightly
|
||||
training (boxtrain.py) learns from, kept in
|
||||
the agent's SQLite (<state dir>/training.db), rolling.
|
||||
|
||||
slots 15-minute means per asset the trainer uses (load.power,
|
||||
pv.power, the planned devices' power, room.<r>.temp, the
|
||||
heating's power, the outdoor thermometer, the grid meter and
|
||||
the battery), from the agent's own 1-minute windows: per slot
|
||||
the mean of the windows' means, as the cloud's trainer reads
|
||||
telemetry (time_bucket('15 minutes') of avg(mean)). A slot the
|
||||
agent did not see can be filled once from Home Assistant's
|
||||
statistics (fill_from_stats: 5-minute where HA still has them,
|
||||
hourly before), never over the agent's own. 13 months (the
|
||||
training weighs recent weeks more and last year's same season
|
||||
still: fcmodels.recency_weights); compact: an asset's number,
|
||||
the slot's number, the sum and count (WITHOUT ROWID; ~20 bytes
|
||||
a row).
|
||||
weather hourly records (pvforecast.Hour's fields) per kind: observed
|
||||
(the cloud's DWD observations shipped in the model bundle,
|
||||
else Home Assistant's recorded weather entity) and forecast
|
||||
(the last forecast made before the hour was over: Home
|
||||
Assistant's, the cloud's), each with its source. 13 months.
|
||||
fc_slots per series, slot and source the last forecast made before the
|
||||
slot began (the box's own model, the cloud's series, the cloud's
|
||||
model on the box, the hour-of-week, the coordinator's week
|
||||
mean): the live error per source against the meter. 14 days.
|
||||
|
||||
Nothing here leaves the site except the error figures the plan carries
|
||||
(controller.public_plan "ml")."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
|
||||
SLOT_S = 900
|
||||
KEEP_DAYS = 396 # 13 months
|
||||
KEEP_FC_DAYS = 14
|
||||
# the assets the trainer reads (building/trainlib.py: SITE_SERIES, ROOM_INPUTS,
|
||||
# CONTROLLED; trainer.load_site), and the meter for the member's net draw
|
||||
ASSETS = {"load.power", "pv.power", "weather.outside_temp", "heat.power", "heatpump.power", "heating.power",
|
||||
"ac.power", "cooling.power", "ev.power", "grid.import_power", "grid.export_power", "battery.power"}
|
||||
ASSET_RE = re.compile(r"^(room\.[^.]+\.temp|(heatpump|heating|ac|cooling|ev)\.[0-9]+\.power)$")
|
||||
OBSERVED_PREF = ("cloud", "ha") # an observed hour: DWD's (with irradiation) before HA's entity
|
||||
FORECAST_PREF = ("ha", "cloud") # a forecast hour: as the box forecasts (localml.merge_hours: HA's first)
|
||||
|
||||
|
||||
def wanted(asset: str) -> bool:
|
||||
return asset in ASSETS or bool(ASSET_RE.match(asset))
|
||||
|
||||
|
||||
def _finite(v) -> bool:
|
||||
return isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)
|
||||
|
||||
|
||||
class Store:
|
||||
"""The training history. Thread-safe (the nightly job reads it in a
|
||||
worker thread while the agent's loop writes): one connection, a lock."""
|
||||
|
||||
def __init__(self, path: str):
|
||||
self.path = path
|
||||
self.lock = threading.Lock()
|
||||
self.db = sqlite3.connect(path, isolation_level=None, check_same_thread=False)
|
||||
self.db.execute("PRAGMA journal_mode=WAL")
|
||||
self.db.execute("PRAGMA synchronous=NORMAL")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS assets (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE)")
|
||||
# slot: the quarter hour's number (unix / 900); src: 0 the agent, 1 HA's 5-minute, 2 HA's hourly statistics
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS slots (aid INTEGER NOT NULL, slot INTEGER NOT NULL, sum REAL NOT NULL, "
|
||||
"n INTEGER NOT NULL, src INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (aid, slot)) WITHOUT ROWID")
|
||||
self._ids = {name: i for i, name in self.db.execute("SELECT id, name FROM assets")}
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS weather (end_ts INTEGER NOT NULL, kind TEXT NOT NULL, source TEXT NOT NULL, "
|
||||
"solar REAL, cloud REAL, temp REAL, issued INTEGER, PRIMARY KEY (end_ts, kind, source))")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS fc_slots (series TEXT NOT NULL, slot INTEGER NOT NULL, source TEXT NOT NULL, "
|
||||
"value REAL NOT NULL, issued INTEGER NOT NULL, PRIMARY KEY (series, slot, source))")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
||||
|
||||
def _aid(self, name: str) -> int:
|
||||
i = self._ids.get(name)
|
||||
if i is None:
|
||||
self.db.execute("INSERT OR IGNORE INTO assets (name) VALUES (?)", (name,))
|
||||
i = self._ids[name] = self.db.execute("SELECT id FROM assets WHERE name = ?", (name,)).fetchone()[0]
|
||||
return i
|
||||
|
||||
def close(self):
|
||||
with self.lock:
|
||||
self.db.close()
|
||||
|
||||
# -- the agent's own samples -------------------------------------------------
|
||||
def add_window(self, start: float, records: list[dict]) -> int:
|
||||
"""A closed 1-minute window (agent.close_window's records): each
|
||||
wanted gauge's mean into its 15-minute slot (by the window's start)."""
|
||||
slot = int(start // SLOT_S)
|
||||
recs = [(r["asset"], float(r["mean"])) for r in records
|
||||
if r.get("kind") == "gauge" and _finite(r.get("mean")) and wanted(r.get("asset", ""))]
|
||||
if not recs:
|
||||
return 0
|
||||
with self.lock:
|
||||
rows = [(self._aid(a), slot, v) for a, v in recs]
|
||||
self.db.executemany(
|
||||
"INSERT INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, 0) ON CONFLICT (aid, slot) "
|
||||
"DO UPDATE SET sum = CASE WHEN src = 0 THEN sum + excluded.sum ELSE excluded.sum END, "
|
||||
"n = CASE WHEN src = 0 THEN n + 1 ELSE 1 END, src = 0", rows)
|
||||
return len(rows)
|
||||
|
||||
def fill(self, asset: str, values: dict[int, float], src: str) -> int:
|
||||
"""Slots the agent did not see (Home Assistant's statistics): never
|
||||
over a slot that has a value."""
|
||||
code = {"agent": 0, "ha-5minute": 1, "ha-hour": 2}.get(src, 2)
|
||||
with self.lock:
|
||||
aid = self._aid(asset)
|
||||
rows = [(aid, int(k) // SLOT_S, float(v), code) for k, v in values.items() if _finite(v) and int(k) % SLOT_S == 0]
|
||||
before = self.db.total_changes
|
||||
self.db.executemany("INSERT OR IGNORE INTO slots (aid, slot, sum, n, src) VALUES (?, ?, ?, 1, ?)", rows)
|
||||
return self.db.total_changes - before
|
||||
|
||||
def assets(self) -> list[str]:
|
||||
with self.lock:
|
||||
return [a for (a,) in self.db.execute("SELECT name FROM assets WHERE id IN (SELECT DISTINCT aid FROM slots)")]
|
||||
|
||||
def series(self, since: int, until: int, assets=None) -> dict[str, dict[int, float]]:
|
||||
"""{asset: {slot: mean}} for slots in [since, until)."""
|
||||
lo, hi = -(-int(since) // SLOT_S), -(-int(until) // SLOT_S)
|
||||
with self.lock:
|
||||
names = {i: n for n, i in self._ids.items()}
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for aid, name in names.items():
|
||||
if assets is not None and name not in assets:
|
||||
continue
|
||||
rows = self.db.execute("SELECT slot, sum / n FROM slots WHERE aid = ? AND slot >= ? AND slot < ?",
|
||||
(aid, lo, hi)).fetchall()
|
||||
if rows:
|
||||
out[name] = {int(s) * SLOT_S: float(v) for s, v in rows}
|
||||
return out
|
||||
|
||||
def span(self) -> tuple[int | None, int | None, int]:
|
||||
"""The first and last slot kept and how many (all assets)."""
|
||||
with self.lock:
|
||||
a, b, n = self.db.execute("SELECT min(slot), max(slot), count(*) FROM slots").fetchone()
|
||||
return (a * SLOT_S if a is not None else None), (b * SLOT_S if b is not None else None), n
|
||||
|
||||
# -- the weather -------------------------------------------------------------
|
||||
def put_weather(self, kind: str, source: str, hours: list, issued: float | None = None) -> int:
|
||||
"""Hourly records (pvforecast.Hour: end, solar_kwh_m2, cloud_pct,
|
||||
temp_c). A forecast only for hours not over when it was made, the
|
||||
newest such kept; an observation replaces what was there."""
|
||||
rows = []
|
||||
for h in hours:
|
||||
end = int(h.end.timestamp())
|
||||
if end % 3600:
|
||||
continue
|
||||
if kind == "forecast" and (issued is None or issued >= end):
|
||||
continue
|
||||
if h.solar_kwh_m2 is None and h.cloud_pct is None and h.temp_c is None:
|
||||
continue
|
||||
rows.append((end, kind, source, h.solar_kwh_m2, h.cloud_pct, h.temp_c, int(issued) if issued else None))
|
||||
with self.lock:
|
||||
self.db.executemany("INSERT OR REPLACE INTO weather (end_ts, kind, source, solar, cloud, temp, issued) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?)", rows)
|
||||
return len(rows)
|
||||
|
||||
def weather_hours(self, since: int, until: int, kind: str) -> list:
|
||||
"""One record per hour ending in (since, until] of this kind, by the
|
||||
kind's preferred source (OBSERVED_PREF, FORECAST_PREF)."""
|
||||
from .pvforecast import Hour
|
||||
pref = OBSERVED_PREF if kind == "observed" else FORECAST_PREF
|
||||
with self.lock:
|
||||
rows = self.db.execute("SELECT end_ts, source, solar, cloud, temp FROM weather WHERE kind = ? AND end_ts > ? "
|
||||
"AND end_ts <= ?", (kind, int(since), int(until))).fetchall()
|
||||
by: dict[int, tuple] = {}
|
||||
for end, src, sol, cl, tc in rows:
|
||||
rank = pref.index(src) if src in pref else len(pref)
|
||||
if end not in by or rank < by[end][0]:
|
||||
by[end] = (rank, sol, cl, tc)
|
||||
return [Hour(datetime.fromtimestamp(e, timezone.utc), *by[e][1:]) for e in sorted(by)]
|
||||
|
||||
def training_hours(self, since: int, until: int) -> list:
|
||||
"""The weather the models learn from: observed, else (an hour never
|
||||
observed) the last forecast for it (wxfetch.hours, prefer observed)."""
|
||||
obs = {int(h.end.timestamp()): h for h in self.weather_hours(since, until, "observed")}
|
||||
for h in self.weather_hours(since, until, "forecast"):
|
||||
obs.setdefault(int(h.end.timestamp()), h)
|
||||
return [obs[k] for k in sorted(obs)]
|
||||
|
||||
def missing_observed(self, since: int, until: int) -> list[tuple[int, int]]:
|
||||
"""Spans of hours without an observation (for Home Assistant's history)."""
|
||||
with self.lock:
|
||||
have = {e for (e,) in self.db.execute("SELECT end_ts FROM weather WHERE kind = 'observed' AND end_ts > ? "
|
||||
"AND end_ts <= ?", (int(since), int(until)))}
|
||||
spans, cur = [], None
|
||||
for e in range(int(since) // 3600 * 3600 + 3600, int(until) + 1, 3600):
|
||||
if e in have:
|
||||
if cur:
|
||||
spans.append(tuple(cur))
|
||||
cur = None
|
||||
elif cur is None:
|
||||
cur = [e - 3600, e]
|
||||
else:
|
||||
cur[1] = e
|
||||
if cur:
|
||||
spans.append(tuple(cur))
|
||||
return spans
|
||||
|
||||
# -- the forecasts per source -------------------------------------------------
|
||||
def put_forecasts(self, series: str, start: int, cands: dict[str, list], issued: float, first: int = 1,
|
||||
steps: int = 96) -> int:
|
||||
"""Per source its values for the slots start + k·900, k in
|
||||
[first, steps): the slot not begun yet (the quarter under way is
|
||||
not a forecast any more)."""
|
||||
rows = []
|
||||
for src, vals in cands.items():
|
||||
for k in range(first, min(steps, len(vals or []))):
|
||||
v = vals[k]
|
||||
if _finite(v):
|
||||
rows.append((series, int(start) + k * SLOT_S, src, float(v), int(issued)))
|
||||
with self.lock:
|
||||
self.db.executemany("INSERT OR REPLACE INTO fc_slots (series, slot, source, value, issued) VALUES (?, ?, ?, ?, ?)",
|
||||
rows)
|
||||
return len(rows)
|
||||
|
||||
def forecasts(self, series: str, since: int, until: int) -> dict[str, dict[int, float]]:
|
||||
with self.lock:
|
||||
rows = self.db.execute("SELECT source, slot, value FROM fc_slots WHERE series = ? AND slot >= ? AND slot < ?",
|
||||
(series, int(since), int(until))).fetchall()
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for src, s, v in rows:
|
||||
out.setdefault(src, {})[int(s)] = float(v)
|
||||
return out
|
||||
|
||||
# -- meta, pruning ---------------------------------------------------------------
|
||||
def get(self, key: str) -> str | None:
|
||||
with self.lock:
|
||||
row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def set(self, key: str, value: str):
|
||||
with self.lock:
|
||||
self.db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, str(value)))
|
||||
|
||||
def prune(self, now: float, days: int = KEEP_DAYS):
|
||||
cut = int(now) - days * 86400
|
||||
with self.lock:
|
||||
self.db.execute("DELETE FROM slots WHERE slot < ?", (cut // SLOT_S,))
|
||||
self.db.execute("DELETE FROM weather WHERE end_ts < ?", (cut,))
|
||||
self.db.execute("DELETE FROM fc_slots WHERE slot < ?", (int(now) - KEEP_FC_DAYS * 86400,))
|
||||
|
||||
def size_bytes(self) -> int:
|
||||
try:
|
||||
return sum(os.path.getsize(self.path + s) for s in ("", "-wal") if os.path.exists(self.path + s))
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def stats_to_slots(rows: list[dict], period: str) -> dict[int, float]:
|
||||
"""Home Assistant's statistics rows ({"start": unix, "mean"}) as
|
||||
15-minute values: 5-minute means averaged per slot, an hourly mean held
|
||||
for its four slots (a step, not a shape: noted where used)."""
|
||||
out: dict[int, list[float]] = {}
|
||||
for r in rows:
|
||||
v = r.get("mean")
|
||||
if not _finite(v):
|
||||
continue
|
||||
s = int(r["start"])
|
||||
if period == "hour":
|
||||
for q in range(4):
|
||||
out.setdefault(s // 3600 * 3600 + q * SLOT_S, []).append(float(v))
|
||||
else:
|
||||
out.setdefault(s // SLOT_S * SLOT_S, []).append(float(v))
|
||||
return {k: sum(v) / len(v) for k, v in out.items()}
|
||||
@@ -0,0 +1,274 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""Training one series and the rooms: the pure part of the trainer
|
||||
(docs/methods.md B5, B6, B10), shared by the cloud's trainer (trainer.py)
|
||||
and the Edge Agent's own nightly training (edge/cantabile_edge/boxtrain.py:
|
||||
a byte-identical copy, building/test_boxmodel.py checks it). numpy,
|
||||
LightGBM (through fcmodels.fit), no database, no network.
|
||||
|
||||
train_series per series the challengers (how, gbm, pvphys) fitted on
|
||||
the window before the held-out one, scored on it with the
|
||||
model in force (the champion), the best put in force only
|
||||
if it beats the champion (fcmodels.choose)
|
||||
fit_rooms each room's R and C (rcfit.py)
|
||||
base_load load.power less the planned devices' power (methods.md A2-a)
|
||||
held_out_weather, scored_on
|
||||
the held-out window on the weather as it was forecast
|
||||
where that is known (methods.md B5-a)
|
||||
cut_before PV_METERED_SINCE: metered PV before it does not count"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
import numpy as np
|
||||
|
||||
try: # on the box: the Edge Agent's copies (edge/cantabile_edge; methods.md B10)
|
||||
from . import fcmodels, rcfit
|
||||
except ImportError:
|
||||
import fcmodels
|
||||
import rcfit
|
||||
|
||||
log = logging.getLogger("trainer")
|
||||
SLOT_S = 900
|
||||
SITE_SERIES = {"load": "load.power", "pv": "pv.power"}
|
||||
# A site's `base_load` series (methods.md A2-a; the box's planner adds the
|
||||
# devices it plans itself, and load.power includes them): load.power less
|
||||
# the power assets of the kinds of control the site's inventory has, as the
|
||||
# Edge Agent does (mapping.CONTROLLED_POWER). A system's total (ev.power),
|
||||
# else the sum of its parts (ev.1.power, ...). The battery is not here:
|
||||
# load.power does not include its charging (sim physics.py: house = load +
|
||||
# heat pump + A/C + EV). Published as base_load_w next to load_w (as
|
||||
# metered: the coordinator's B7 uses it).
|
||||
BASE = "base_load"
|
||||
CONTROLLED_POWER = {"heating": ("heatpump.power", "heating.power"), "cooling": ("ac.power", "cooling.power"),
|
||||
"ev": ("ev.power",)}
|
||||
CONTROLLED = tuple(a for names in CONTROLLED_POWER.values() for a in names)
|
||||
ROOM_INPUTS = ["weather.outside_temp", "heat.power", "heatpump.power"]
|
||||
MIN_SLOTS = 16 # a series with less metered is not trained
|
||||
SINCE_SERIES = ("pv",) # the series PV_METERED_SINCE cuts
|
||||
|
||||
|
||||
def cut_before(series: str, t: np.ndarray, y: np.ndarray, since: datetime | None) -> np.ndarray:
|
||||
"""y without the metered slots before `since` (NaN) for the series it
|
||||
applies to (SINCE_SERIES)."""
|
||||
if since is None or series not in SINCE_SERIES:
|
||||
return y
|
||||
return np.where(t < int(since.timestamp()), np.nan, y)
|
||||
|
||||
|
||||
def scored_on(w_test_hours: int, n_hours: int) -> str:
|
||||
"""What the held-out scores were made on (methods.md B5-a): the weather
|
||||
as it was forecast where it is stored, else the observed weather."""
|
||||
if n_hours <= 0 or w_test_hours <= 0:
|
||||
return "scored on observed weather (no stored forecast for the held-out window: optimistic)"
|
||||
if w_test_hours >= n_hours:
|
||||
return "scored on forecast weather"
|
||||
return (f"scored on forecast weather for {100 * w_test_hours // n_hours} % of the held-out hours, "
|
||||
"observed for the rest")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Run:
|
||||
series: str
|
||||
kind: str
|
||||
metrics: dict
|
||||
params: dict | None = None
|
||||
chosen: bool = False
|
||||
note: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SeriesResult:
|
||||
series: str
|
||||
runs: list[Run] = field(default_factory=list)
|
||||
champion: dict | None = None # metrics of the model in force on the held-out window
|
||||
baseline: dict | None = None
|
||||
chosen: Run | None = None
|
||||
window: tuple = () # (data_from, data_to, test_from, test_to) unix
|
||||
n_train: int = 0
|
||||
n_test: int = 0
|
||||
note: str = ""
|
||||
|
||||
|
||||
def wslice(w: fcmodels.Weather, sl: slice) -> fcmodels.Weather:
|
||||
return fcmodels.Weather(w.temp[sl], w.ghi[sl], w.cloud[sl], w.elev[sl], w.az[sl], w.hours)
|
||||
|
||||
|
||||
# ---- one series, champion against challengers -----------------------------------
|
||||
|
||||
def train_series(series: str, t: np.ndarray, y: np.ndarray, w: fcmodels.Weather, site: fcmodels.Site,
|
||||
champion: dict | None, kinds=None, w_test: fcmodels.Weather | None = None) -> SeriesResult:
|
||||
"""t, y: the series over the training weeks (and the week before, for
|
||||
the lag), w: the weather on the same slots (observed: what the models
|
||||
learn from). Fits each kind on the window before the held-out one,
|
||||
scores it and the champion's params on the held-out window, picks
|
||||
(fcmodels.choose). w_test: the weather the held-out window is scored
|
||||
on, as it was forecast (methods.md B5-a: the published forecasts run on
|
||||
forecasts, so the choice and the drift baseline should too); else w."""
|
||||
wt = w_test if w_test is not None else w
|
||||
res = SeriesResult(series)
|
||||
have = np.flatnonzero(~np.isnan(y))
|
||||
if len(have) < MIN_SLOTS:
|
||||
res.note = f"{len(have)} metered slots (at least {MIN_SLOTS})"
|
||||
return res
|
||||
cut = fcmodels.holdout(t, y)
|
||||
first = int(have[0])
|
||||
tr, te = slice(first, cut), slice(cut, len(t))
|
||||
res.window = (int(t[first]), int(t[cut - 1]) + SLOT_S if cut > first else int(t[first]), int(t[cut]) if cut < len(t) else None,
|
||||
int(t[-1]) + SLOT_S)
|
||||
res.n_train = int((~np.isnan(y[tr])).sum())
|
||||
res.n_test = int((~np.isnan(y[te])).sum())
|
||||
if res.n_train < MIN_SLOTS or res.n_test < 4:
|
||||
res.note = f"too little data to hold a window out ({res.n_train} slots to train, {res.n_test} to test)"
|
||||
return res
|
||||
kinds = kinds or (["how", "gbm", "pvphys"] if series == "pv" else ["how", "gbm"])
|
||||
hist_t, hist_y = t[:cut], y[:cut]
|
||||
for kind in kinds:
|
||||
try:
|
||||
p = fcmodels.fit(kind, series, t[tr], y[tr], wslice(w, tr), site, t[:first], y[:first])
|
||||
except Exception as e: # one kind's failure never stops the others
|
||||
log.exception("fit %s %s", series, kind)
|
||||
res.runs.append(Run(series, kind, {}, None, note=f"failed: {e}"))
|
||||
continue
|
||||
if p is None:
|
||||
why = {"gbm": "under a day of data", "pvphys": "no location or no weather for it"}.get(kind, "no data")
|
||||
res.runs.append(Run(series, kind, {}, None, note=f"not trained: {why}"))
|
||||
continue
|
||||
pred = fcmodels.predict(p, series, t[te], wslice(wt, te), site, hist_t, hist_y)
|
||||
res.runs.append(Run(series, kind, fcmodels.metrics(y[te], pred), p))
|
||||
base = next((r for r in res.runs if r.kind == "how" and r.params), None)
|
||||
res.baseline = base.metrics if base else None
|
||||
if champion is not None:
|
||||
try:
|
||||
pred = fcmodels.predict(champion, series, t[te], wslice(wt, te), site, hist_t, hist_y)
|
||||
res.champion = fcmodels.metrics(y[te], pred)
|
||||
except Exception as e: # a champion that can no longer run is replaced
|
||||
res.champion = None
|
||||
res.note = f"the model in force could not be scored ({e}): replaced"
|
||||
# nMAE where the metered mean is not ~0, else MAE (a held-out night of PV)
|
||||
key = score_key(res)
|
||||
scores = {r.kind: r.metrics.get(key) for r in res.runs if r.params}
|
||||
champ = res.champion.get(key) if res.champion else None
|
||||
best = fcmodels.choose(scores, champ)
|
||||
if best is None and champion is None and scores:
|
||||
best = min((k for k in scores if scores[k] is not None), key=lambda k: scores[k], default=None)
|
||||
if best:
|
||||
res.chosen = next(r for r in res.runs if r.kind == best and r.params)
|
||||
res.chosen.chosen = True
|
||||
elif champ is not None:
|
||||
res.note = (res.note + "; " if res.note else "") + f"the model in force stays ({key} {champ:.3g})"
|
||||
return res
|
||||
|
||||
|
||||
def score_key(res: SeriesResult) -> str:
|
||||
"""The metric a choice is made on: nMAE where the metered mean of the
|
||||
held-out window is not ~0, else MAE (a held-out night of PV)."""
|
||||
return "nmae" if res.baseline and res.baseline.get("nmae") is not None else "mae"
|
||||
|
||||
|
||||
def score_params(params: dict, series: str, t: np.ndarray, y: np.ndarray, w: fcmodels.Weather, site: fcmodels.Site,
|
||||
w_test: fcmodels.Weather | None = None) -> dict | None:
|
||||
"""Another model's metrics on the same held-out window as train_series
|
||||
cuts it (a benchmark: the cloud's model on the box), or None where it
|
||||
cannot run or there is no window."""
|
||||
have = np.flatnonzero(~np.isnan(y))
|
||||
if len(have) < MIN_SLOTS:
|
||||
return None
|
||||
cut = fcmodels.holdout(t, y)
|
||||
wt = w_test if w_test is not None else w
|
||||
te = slice(cut, len(t))
|
||||
try:
|
||||
pred = fcmodels.predict(params, series, t[te], wslice(wt, te), site, t[:cut], y[:cut])
|
||||
except Exception as e:
|
||||
log.warning("benchmark %s: %s", series, e)
|
||||
return None
|
||||
return fcmodels.metrics(y[te], pred)
|
||||
|
||||
|
||||
# ---- the rooms ---------------------------------------------------------------------
|
||||
|
||||
def fit_rooms(t: np.ndarray, assets: dict[str, np.ndarray], w: fcmodels.Weather) -> dict[str, dict]:
|
||||
"""Each room.<r>.temp against the outside (the site's thermometer, else
|
||||
the weather) and the site's heating (thermal heat.power, else the heat
|
||||
pump's electric power)."""
|
||||
out = {}
|
||||
tout = assets.get("weather.outside_temp")
|
||||
if tout is None or np.isnan(tout).all():
|
||||
tout = w.temp
|
||||
else:
|
||||
tout = np.where(np.isnan(tout), w.temp, tout)
|
||||
if "heat.power" in assets:
|
||||
p, power = assets["heat.power"], "thermal"
|
||||
elif "heatpump.power" in assets:
|
||||
p, power = assets["heatpump.power"], "electric"
|
||||
else:
|
||||
p, power = None, "thermal"
|
||||
for a, theta in sorted(assets.items()):
|
||||
if not (a.startswith("room.") and a.endswith(".temp")):
|
||||
continue
|
||||
room = a[len("room."):-len(".temp")]
|
||||
if p is None:
|
||||
out[room] = {**rcfit.fit(theta, tout, np.full(len(theta), np.nan)), "note": "no heating power metered"}
|
||||
continue
|
||||
out[room] = rcfit.fit(theta, tout, p, w.ghi, power)
|
||||
return out
|
||||
|
||||
|
||||
def room_inputs(assets: dict[str, np.ndarray], w: fcmodels.Weather) -> tuple:
|
||||
"""The inputs fit_rooms fits every room on: (outside °C, heating power
|
||||
W or None, irradiance W/m²), for scoring other R and C on the same data."""
|
||||
tout = assets.get("weather.outside_temp")
|
||||
tout = w.temp if tout is None or np.isnan(tout).all() else np.where(np.isnan(tout), w.temp, tout)
|
||||
p = assets.get("heat.power", assets.get("heatpump.power"))
|
||||
return tout, p, w.ghi
|
||||
|
||||
|
||||
# ---- the base load -----------------------------------------------------------------
|
||||
|
||||
def _system(asset: str) -> str:
|
||||
return asset.split(".", 1)[0]
|
||||
|
||||
|
||||
def controlled_power(raw: dict[str, np.ndarray], kinds: set[str], n: int) -> np.ndarray:
|
||||
"""The planned devices' metered power per slot (W): per kind its assets
|
||||
(a system's total, else the sum of its parts <system>.<n>.power), each
|
||||
once; a slot without a value counts 0."""
|
||||
dev = np.zeros(n)
|
||||
for kind in sorted(kinds):
|
||||
for name in CONTROLLED_POWER.get(kind, ()):
|
||||
sysname, q = name.split(".", 1)
|
||||
parts = [a for a in raw if a.startswith(sysname + ".") and a.endswith("." + q)
|
||||
and a[len(sysname) + 1:-len(q) - 1].isdigit()]
|
||||
for a in ([name] if name in raw else sorted(parts)):
|
||||
dev = dev + np.nan_to_num(np.maximum(raw[a], 0.0))
|
||||
return dev
|
||||
|
||||
|
||||
def base_load(raw: dict[str, np.ndarray], kinds: set[str]) -> np.ndarray | None:
|
||||
"""load.power less the planned devices' metered power (controlled_power),
|
||||
never below 0 (methods.md A2-a); NaN where load.power is."""
|
||||
load = raw.get(SITE_SERIES["load"])
|
||||
if load is None:
|
||||
return None
|
||||
return np.where(np.isnan(load), np.nan, np.maximum(load - controlled_power(raw, kinds, len(load)), 0.0))
|
||||
|
||||
|
||||
# ---- the weather the held-out window is scored on -----------------------------------
|
||||
|
||||
def held_out_weather(t: np.ndarray, y: np.ndarray, w: fcmodels.Weather, fc_hours: list, lat: float | None,
|
||||
lon: float | None) -> tuple[fcmodels.Weather | None, str]:
|
||||
"""The weather the held-out window (fcmodels.holdout, as train_series
|
||||
cuts it) is scored on: the observed hours with every hour that was
|
||||
forecast put in their place, and what that was (scored_on). None where
|
||||
nothing was forecast: the observed weather."""
|
||||
if not fc_hours or lat is None or lon is None or (~np.isnan(y)).sum() < MIN_SLOTS:
|
||||
return None, scored_on(0, 0)
|
||||
cut = fcmodels.holdout(t, y)
|
||||
test_ends = {(int(ts) // 3600 + 1) * 3600 for ts in t[cut:].tolist()}
|
||||
fc = {int(h.end.timestamp()): h for h in fc_hours}
|
||||
n_fc = len(test_ends & set(fc))
|
||||
if n_fc == 0:
|
||||
return None, scored_on(0, len(test_ends))
|
||||
merged = {int(h.end.timestamp()): h for h in w.hours} | fc
|
||||
return fcmodels.weather(t, [merged[k] for k in sorted(merged)], lat, lon), scored_on(n_fc, len(test_ends))
|
||||
@@ -10,7 +10,9 @@ package main
|
||||
// a command). Fleet
|
||||
// sends one as a signed command, `tariff` or `model`, with args
|
||||
// {"bundle": base64 of the bundle's JSON bytes, "sig": base64 of their
|
||||
// signature with the BUNDLE key}. The command itself is checked with the
|
||||
// signature with the BUNDLE key}, or, since 2026-11-04 to a link that says
|
||||
// so in its heartbeat (bundle_text), {"bundle_text": the JSON itself, "sig"}:
|
||||
// one base64 layer less on the wire (fleet bundlecap.go). The command itself is checked with the
|
||||
// command key (checkCommand); here the bundle's own signature, its kind, site
|
||||
// and expiry, and that it is not older than the one the agent has. Then the
|
||||
// JSON goes into the agent's state directory (tariff.json, model.json),
|
||||
@@ -37,7 +39,10 @@ var devicesPolicy = func() {
|
||||
}
|
||||
}
|
||||
|
||||
const bundleMax = 256 << 10 // a tariff is a few kB; a model's data well under this
|
||||
// bundleMax: the largest bundle taken, reported in the heartbeat
|
||||
// (bundle_max; fleet sends an older link at most 136 KiB). 512 KiB as text is
|
||||
// ~0.7 MiB on the wire, under NATS's 1 MiB and any 1 MiB WebSocket proxy.
|
||||
const bundleMax = 512 << 10
|
||||
|
||||
// bundleCert: the bundle key's certificate (CANTABILE_BUNDLE_CERT). Read for
|
||||
// each bundle: a box may get it after its link started.
|
||||
@@ -60,7 +65,13 @@ type bundleHead struct {
|
||||
// expired and not issued in the future.
|
||||
func checkBundle(key crypto.PublicKey, kind, site string, args map[string]string, now time.Time) ([]byte, bundleHead, error) {
|
||||
var h bundleHead
|
||||
data, err1 := base64.StdEncoding.DecodeString(args["bundle"])
|
||||
var data []byte
|
||||
var err1 error
|
||||
if t, ok := args["bundle_text"]; ok {
|
||||
data = []byte(t)
|
||||
} else {
|
||||
data, err1 = base64.StdEncoding.DecodeString(args["bundle"])
|
||||
}
|
||||
sig, err2 := base64.StdEncoding.DecodeString(args["sig"])
|
||||
switch {
|
||||
case err1 != nil || err2 != nil || len(data) == 0 || len(sig) == 0:
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
// Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A bundle as text (fleet bundlecap.go, for a link that reports bundle_text)
|
||||
// is taken like the base64 one, the signature over the same bytes; up to
|
||||
// bundleMax (reported in the heartbeat), larger refused; a tampered text refused.
|
||||
func TestBundlesAsTextAndLarge(t *testing.T) {
|
||||
k, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
site := strings.Repeat("ab", 16)
|
||||
now := time.Now()
|
||||
l := &link{id: site, cfg: config{BundleCert: certFile(t, k), AgentState: t.TempDir()}}
|
||||
model := func(id string, issued int64, pad int) map[string]any {
|
||||
return map[string]any{"v": 1, "kind": "model", "id": id, "site": site, "issued": issued,
|
||||
"expires": now.Add(time.Hour).Unix(), "pad": strings.Repeat("x", pad)}
|
||||
}
|
||||
text := func(b map[string]any) map[string]string {
|
||||
a := bundleArgs(t, k, b)
|
||||
data, _ := base64.StdEncoding.DecodeString(a["bundle"])
|
||||
return map[string]string{"bundle_text": string(data), "sig": a["sig"]}
|
||||
}
|
||||
cmd := func(args map[string]string) command { return command{ID: "c", Cmd: "model", Args: args} }
|
||||
if ok, msg := l.bundleCmd(cmd(text(model("m1", now.Unix(), 400<<10))), now); !ok { // over the old 256 KiB
|
||||
t.Fatalf("a 400 KiB model as text: %s", msg)
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(l.cfg.AgentState, "model.json")); len(b) < 400<<10 {
|
||||
t.Fatalf("model.json: %d bytes", len(b))
|
||||
}
|
||||
if ok, msg := l.bundleCmd(cmd(bundleArgs(t, k, model("m2", now.Unix()+1, 300<<10))), now); !ok {
|
||||
t.Fatalf("a 300 KiB model as base64: %s", msg)
|
||||
}
|
||||
if ok, msg := l.bundleCmd(cmd(text(model("m3", now.Unix()+2, bundleMax))), now); ok || !strings.Contains(msg, "larger than") {
|
||||
t.Errorf("over bundleMax: %v %q", ok, msg)
|
||||
}
|
||||
a := text(model("m4", now.Unix()+3, 10))
|
||||
a["bundle_text"] = strings.Replace(a["bundle_text"], `"m4"`, `"m5"`, 1)
|
||||
if ok, msg := l.bundleCmd(cmd(a), now); ok || !strings.HasSuffix(msg, "bad bundle signature") {
|
||||
t.Errorf("tampered text: %v %q", ok, msg)
|
||||
}
|
||||
t.Setenv("CANTABILE_HUB", "0")
|
||||
t.Setenv("CANTABILE_CONSENT_FILE", filepath.Join(t.TempDir(), "none.json"))
|
||||
hb := (&link{started: now, cfg: config{AgentState: t.TempDir()}, loc: &local{}}).heartbeat()
|
||||
if hb["bundle_max"] != bundleMax || hb["bundle_text"] != true || bundleMax < 512<<10 {
|
||||
t.Errorf("heartbeat: %v %v", hb["bundle_max"], hb["bundle_text"])
|
||||
}
|
||||
}
|
||||
@@ -756,6 +756,8 @@ func (l *link) heartbeat() map[string]any {
|
||||
if p := l.pathInfo(); p != nil {
|
||||
hb["path"] = p
|
||||
}
|
||||
// the bundles this link takes (bundles.go): as large, and as text
|
||||
hb["bundle_max"], hb["bundle_text"] = bundleMax, true
|
||||
c := consents(env("CANTABILE_CONSENT_FILE",
|
||||
map[bool]string{true: "/data/cantabile/edge/consent.json", false: "/data/state/consent.json"}[isHub()]))
|
||||
hb["consents"] = c
|
||||
|
||||
@@ -15,6 +15,10 @@ dependencies = [
|
||||
"PyYAML==6.0.2",
|
||||
"highspy==1.11.0", # the optimiser (HiGHS MILP), v1
|
||||
"numpy==2.3.5", # the box's forecast models (localml.py, fcmodels.py): as building/requirements.txt
|
||||
# the box trains its own models (boxtrain.py, methods.md B10; +152 MB in the image):
|
||||
# LightGBM as building/requirements.txt, scipy (LightGBM requires it) as resolved with numpy 2.3.5
|
||||
"lightgbm==4.5.0",
|
||||
"scipy==1.18.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The box trains its own models (boxtrain.py, trainhist.py; methods.md
|
||||
B10): the training history, the nightly job's caps (wall time, memory) and
|
||||
that it never blocks the controller's loop, the box keeping the better
|
||||
model (against its own and the cloud's), the planner taking the box's own
|
||||
model first, the intraday correction, the plan's "ml". The parity with the
|
||||
cloud's trainer on the same data: building/test_boxmodel.py."""
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from cantabile_edge import boxtrain, fcmodels, pvforecast, trainhist
|
||||
from cantabile_edge.bundles import Bundles
|
||||
from cantabile_edge.inputs import Collector, History, model_state
|
||||
|
||||
from test_base_load import site_mapping, watts
|
||||
from test_control_parts import SITE
|
||||
from test_localml import START, bundle, gbm, put, weather
|
||||
|
||||
DAY = 86400
|
||||
|
||||
|
||||
def cfg(tmp_path, **kw):
|
||||
d = dict(state_dir=str(tmp_path), train=True, train_at="03:30", train_series_s=60.0, train_mem_mb=600,
|
||||
train_min_free_mb=0, train_days=60, pv_metered_since="")
|
||||
d.update(kw)
|
||||
return SimpleNamespace(**d)
|
||||
|
||||
|
||||
def store(tmp_path):
|
||||
return trainhist.Store(str(tmp_path / "training.db"))
|
||||
|
||||
|
||||
# -- the history -----------------------------------------------------------------
|
||||
def test_windows_into_quarter_hour_means(tmp_path):
|
||||
s = store(tmp_path)
|
||||
t0 = START
|
||||
for k in range(15): # 15 one-minute windows of the slot
|
||||
s.add_window(t0 + 60 * k, [{"asset": "load.power", "kind": "gauge", "mean": 400.0 + 20 * k},
|
||||
{"asset": "sensor.unrelated", "kind": "gauge", "mean": 1.0},
|
||||
{"asset": "pv.power", "kind": "gauge"}, # unavailable: no mean
|
||||
{"asset": "grid.energy", "kind": "counter", "delta": 3.0}])
|
||||
got = s.series(t0, t0 + 900)
|
||||
assert set(got) == {"load.power"} and got["load.power"][t0] == pytest.approx(540.0)
|
||||
# HA's statistics fill only what the agent did not see; the agent's own replace them
|
||||
assert s.fill("load.power", {t0: 1.0, t0 + 900: 300.0}, "ha-hour") == 1
|
||||
assert s.series(t0, t0 + 1800)["load.power"] == {t0: pytest.approx(540.0), t0 + 900: 300.0}
|
||||
s.add_window(t0 + 900, [{"asset": "load.power", "kind": "gauge", "mean": 700.0}])
|
||||
assert s.series(t0 + 900, t0 + 1800)["load.power"][t0 + 900] == 700.0
|
||||
assert trainhist.wanted("room.living.temp") and trainhist.wanted("ev.2.power") and not trainhist.wanted("ev.power.x")
|
||||
s.prune(t0 + 400 * DAY)
|
||||
assert s.span()[2] == 0
|
||||
|
||||
|
||||
def test_weather_kinds_and_sources(tmp_path):
|
||||
s = store(tmp_path)
|
||||
H = lambda end, cloud: pvforecast.Hour(datetime.fromtimestamp(end, timezone.utc), None, cloud, 5.0) # noqa: E731
|
||||
s.put_weather("forecast", "cloud", [H(START + 3600, 10.0), H(START + 7200, 10.0)], issued=START)
|
||||
s.put_weather("forecast", "ha", [H(START + 3600, 20.0)], issued=START + 100)
|
||||
s.put_weather("forecast", "ha", [H(START + 3600, 99.0)], issued=START + 3600) # made after the hour: no forecast
|
||||
s.put_weather("observed", "ha", [H(START + 7200, 70.0)])
|
||||
s.put_weather("observed", "cloud", [H(START + 7200, 60.0)])
|
||||
fc = s.weather_hours(START, START + 7200, "forecast")
|
||||
assert [h.cloud_pct for h in fc] == [20.0, 10.0] # HA's first, as the box forecasts
|
||||
tr = s.training_hours(START, START + 7200)
|
||||
assert [h.cloud_pct for h in tr] == [20.0, 60.0] # observed (DWD first), else the forecast
|
||||
assert s.missing_observed(START, START + 4 * 3600) == [(START, START + 3600), (START + 7200, START + 4 * 3600)]
|
||||
|
||||
|
||||
def test_ha_history_to_hours():
|
||||
st = [{"a": {"temperature": 3.0, "cloud_coverage": 40}, "lu": START + 100},
|
||||
{"a": {"temperature": 2.0, "cloud_coverage": 90}, "lu": START + 3700}]
|
||||
hs = boxtrain.ha_history_hours(st, [(START, START + 3 * 3600)])
|
||||
assert [(int(h.end.timestamp()) - START, h.temp_c, h.cloud_pct) for h in hs] == \
|
||||
[(3600, 3.0, 40.0), (7200, 2.0, 90.0), (10800, 2.0, 90.0)]
|
||||
|
||||
|
||||
# -- the caps: never in the controller's way -----------------------------------------
|
||||
def slow_child(seconds, mb=0):
|
||||
return [sys.executable, "-c", f"import time; x = bytearray({mb} << 20); [x.__setitem__(i, 1) for i in range(0, len(x), 4096)]; "
|
||||
f"time.sleep({seconds})"]
|
||||
|
||||
|
||||
async def ticking(job, period=0.05):
|
||||
"""Run job while counting ticks of a loop like the controller's."""
|
||||
ticks, done = [], asyncio.Event()
|
||||
|
||||
async def ticker():
|
||||
while not done.is_set():
|
||||
ticks.append(time.monotonic())
|
||||
await asyncio.sleep(period)
|
||||
|
||||
tk = asyncio.create_task(ticker())
|
||||
t0 = time.monotonic()
|
||||
res = await job
|
||||
done.set()
|
||||
await tk
|
||||
gaps = np.diff(ticks) if len(ticks) > 1 else np.array([0.0])
|
||||
return res, time.monotonic() - t0, len(ticks), float(gaps.max())
|
||||
|
||||
|
||||
async def test_a_slow_fit_is_stopped_at_its_wall_time_and_never_blocks(tmp_path):
|
||||
b = boxtrain.BoxTraining(cfg(tmp_path, train_series_s=1.0), store(tmp_path), None, child_cmd=slow_child(30))
|
||||
(res, how), took, n, gap = await ticking(b.child({"series": "load"}))
|
||||
assert res is None and "wall time" in how["killed"]
|
||||
assert 0.9 < took < 5.0
|
||||
assert n >= 15 and gap < 0.5 # the loop ran all along
|
||||
|
||||
|
||||
@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="/proc")
|
||||
async def test_a_hungry_fit_is_stopped_at_its_memory_cap(tmp_path):
|
||||
b = boxtrain.BoxTraining(cfg(tmp_path, train_mem_mb=80), store(tmp_path), None, child_cmd=slow_child(20, 200))
|
||||
(res, how), took, n, gap = await ticking(b.child({"series": "load"}))
|
||||
assert res is None and "memory cap" in how["killed"] and how["peak_mb"] > 80
|
||||
assert took < 15 and gap < 0.5
|
||||
|
||||
|
||||
async def test_low_memory_skips_the_night(tmp_path, monkeypatch):
|
||||
b = boxtrain.BoxTraining(cfg(tmp_path, train_min_free_mb=10 ** 9), store(tmp_path), None)
|
||||
monkeypatch.setattr(boxtrain, "mem_available_mb", lambda: 100.0)
|
||||
monkeypatch.setattr(b, "prepare", lambda *a: ([{"series": "load"}], {}))
|
||||
monkeypatch.setattr(b, "child", lambda job: pytest.fail("trained with too little memory"))
|
||||
out = await b.train_once()
|
||||
assert "100 MB available" in out["notes"][0]
|
||||
|
||||
|
||||
def test_next_run_local_time():
|
||||
now = datetime(2026, 11, 4, 12, 0, tzinfo=timezone.utc).timestamp()
|
||||
nxt = datetime.fromtimestamp(boxtrain.next_run(now, "03:30", "Europe/Berlin"), timezone.utc)
|
||||
assert (nxt.day, nxt.hour, nxt.minute) == (5, 2, 30) # 03:30 CET
|
||||
|
||||
|
||||
# -- the box keeps the better model -----------------------------------------------------
|
||||
def result(chosen, runs, champion=None, cloud=None):
|
||||
return {"key": "nmae", "chosen": chosen, "params": {"kind": chosen, "values": [1.0] * 168} if chosen else None,
|
||||
"runs": [{"kind": k, "metrics": {"nmae": v}} for k, v in runs.items()], "champion": champion and {"nmae": champion},
|
||||
"cloud": cloud and {"nmae": cloud}, "window": [0, 1, 2, 3], "n_test": 672, "note": ""}
|
||||
|
||||
|
||||
def test_the_box_keeps_the_better_model(tmp_path):
|
||||
b = boxtrain.BoxTraining(cfg(tmp_path), store(tmp_path), None)
|
||||
job = {"site": {"tz": "UTC"}, "scored_on": "scored on forecast weather"}
|
||||
# the first model: in force only if it beats the cloud's
|
||||
out = b.take("load", job, result("gbm", {"how": 0.4, "gbm": 0.30}, cloud=0.25), {"seconds": 1}, 1000)
|
||||
assert not out["in_force"] and "not in force" in out["result"]
|
||||
assert b.state["series"]["load"]["kind"] == "gbm" # kept: tomorrow's champion
|
||||
out = b.take("load", job, result("gbm", {"how": 0.4, "gbm": 0.20}, champion=0.30, cloud=0.25), {}, 2000)
|
||||
assert out["in_force"] and b.state["series"]["load"]["trained_at"] == 2000
|
||||
# a new one that beats the box's own but not the cloud's: the old one stays (and is in force while it beats the cloud)
|
||||
out = b.take("load", job, result("gbm", {"how": 0.4, "gbm": 0.19}, champion=0.21, cloud=0.18), {}, 3000)
|
||||
e = b.state["series"]["load"]
|
||||
assert e["trained_at"] == 2000 and not e["in_force"] and "does not beat" in out["result"]
|
||||
# nothing beats the box's own, and it beats the cloud's: in force
|
||||
out = b.take("load", job, result(None, {"how": 0.4, "gbm": 0.3}, champion=0.2, cloud=0.25), {}, 4000)
|
||||
assert b.state["series"]["load"]["in_force"] and b.state["series"]["load"]["trained_at"] == 2000
|
||||
# no cloud model on the box: anything but the hour-of-week that beat it
|
||||
out = b.take("pv", job, result("how", {"how": 0.3}), {}, 5000)
|
||||
assert not out["in_force"]
|
||||
out = b.take("pv", job, result("pvphys", {"how": 0.3, "pvphys": 0.2}, champion=0.3), {}, 6000)
|
||||
assert out["in_force"]
|
||||
# a failed child changes nothing
|
||||
out = b.take("pv", job, None, {"killed": "over its wall time (300 s)"}, 7000)
|
||||
assert out["result"].startswith("failed") and b.state["series"]["pv"]["trained_at"] == 6000
|
||||
|
||||
|
||||
def test_the_live_error_takes_a_model_out_of_force(tmp_path):
|
||||
s = store(tmp_path)
|
||||
now = START + 8 * DAY
|
||||
for k in range(7 * 96):
|
||||
ts = now - 7 * DAY + k * 900
|
||||
s.add_window(ts, [{"asset": "load.power", "kind": "gauge", "mean": 500.0}])
|
||||
s.put_forecasts("plan", ts - 900, {"own": [None, 800.0], "cloud": [None, 520.0]}, ts - 900)
|
||||
b = boxtrain.BoxTraining(cfg(tmp_path), s, None)
|
||||
le = b.live_errors("plan", now)
|
||||
assert le["own"]["nmae"] == pytest.approx(0.6) and le["cloud"]["nmae"] == pytest.approx(0.04) and le["own"]["n"] == 672
|
||||
out = b.take("load", {"site": {}}, result("gbm", {"how": 0.4, "gbm": 0.2}, cloud=0.25), {}, now)
|
||||
assert not out["in_force"] and "live nMAE" in out["result"]
|
||||
|
||||
|
||||
# -- the planner takes the box's own model first ------------------------------------------
|
||||
class Own:
|
||||
"""A stand-in for BoxTraining at the planner: a forecast for load_w."""
|
||||
|
||||
def __init__(self, vals):
|
||||
self.vals = vals
|
||||
|
||||
def forecast(self, name, start, steps, ha, entity=""):
|
||||
return (list(self.vals), {"kind": "gbm", "weather": "Home Assistant's forecast"}) if name == "base_load_w" else None
|
||||
|
||||
def correct(self, series, start, values):
|
||||
return list(values), 0.0
|
||||
|
||||
def room_rc(self, *a):
|
||||
return None
|
||||
|
||||
|
||||
def state(tmp_path, b, ha=None):
|
||||
m = site_mapping(tmp_path, [], [])
|
||||
col = Collector()
|
||||
watts(col, sensor__load=700)
|
||||
return model_state(START, [], m.inputs, col, b, History(str(tmp_path / "h.json")), "UTC", ha)
|
||||
|
||||
|
||||
def test_own_model_first_then_the_cloud(tmp_path):
|
||||
b = put(tmp_path, bundle(models={"base_load_w": gbm()}, wx=weather(30)))
|
||||
b.own = Own([1234.0] * 100 + [None] * 92)
|
||||
s = state(tmp_path, b)
|
||||
f = s["sources"]["forecast"]["load"]
|
||||
assert s["load_w"][0] == 1234.0 and s["load_w"][99] == 1234.0
|
||||
assert s["load_w"][100] != 1234.0 # past the box's own: the cloud's model on the box, the profile
|
||||
assert f["own"] == 100 and f["own_model"] == "gbm" and f["source"] == "own"
|
||||
assert "the box's own gbm model (trained on the box) for 100 of 192 steps" in s["sources"]["load"]
|
||||
c = s["_cands"]["load"]
|
||||
assert set(c) >= {"own", "cloud", "cloud_model", "hour_of_week", "used"}
|
||||
assert c["cloud"][0] == 5000.0 and c["own"][0] == 1234.0 and c["hour_of_week"][0] == 111.0
|
||||
# no own model in force: the cloud's series as before
|
||||
b.own = Own([None] * 192)
|
||||
s = state(tmp_path, b)
|
||||
assert s["load_w"][0] == 5000.0 and "own" not in s["sources"]["forecast"]["load"]
|
||||
|
||||
|
||||
def test_own_model_alone_without_a_cloud_model(tmp_path):
|
||||
b = Bundles(str(tmp_path), SITE)
|
||||
b.own = Own([800.0] * 192)
|
||||
s = state(tmp_path, b)
|
||||
assert s["load_w"] == [800.0] * 192 and s["sources"]["forecast"]["load"]["source"] == "own"
|
||||
|
||||
|
||||
# -- the intraday correction on the box ------------------------------------------------------
|
||||
def test_the_box_carries_the_last_hours_error(tmp_path):
|
||||
s = store(tmp_path)
|
||||
for k in range(12): # the last 3 hours: 300 W above what was planned on
|
||||
ts = START - 3 * 3600 + k * 900
|
||||
s.add_window(ts, [{"asset": "load.power", "kind": "gauge", "mean": 1300.0}])
|
||||
s.put_forecasts("plan", ts - 900, {"used": [None, 1000.0]}, ts - 900)
|
||||
b = boxtrain.BoxTraining(cfg(tmp_path), s, None)
|
||||
out, e = b.correct("plan", START, [1000.0] * 192)
|
||||
assert e == pytest.approx(300.0)
|
||||
assert out[0] == pytest.approx(1300.0) and out[8] == pytest.approx(1150.0) and out[191] == pytest.approx(1000.0, abs=1e-3)
|
||||
assert b.correct("pv", START, [0.0] * 192)[1] == 0.0 # nothing known: unchanged
|
||||
|
||||
|
||||
# -- the plan's "ml" ----------------------------------------------------------------------
|
||||
def test_status_for_the_plan(tmp_path):
|
||||
b = boxtrain.BoxTraining(cfg(tmp_path), store(tmp_path), None)
|
||||
b.take("load", {"site": {}, "scored_on": "x"}, result("gbm", {"how": 0.4, "gbm": 0.2}, cloud=0.25), {"seconds": 2.0}, 1000)
|
||||
fc = {"load": {"start": START, "step_s": 900, "values": [1.0] * 192, "source": "own", "series": "load"}}
|
||||
st = b.status(START, fc)
|
||||
assert st["v"] == 1 and st["series"]["load"]["in_force"] and st["series"]["load"]["held_out"] == {"how": 0.4, "gbm": 0.2}
|
||||
assert st["series"]["load"]["held_out_cloud"] == 0.25 and st["forecasts"] == fc
|
||||
assert "forecasts" not in b.status(START + 900, fc) # at most hourly
|
||||
json.dumps(st, allow_nan=False)
|
||||
assert len(json.dumps(st)) < 8000
|
||||
@@ -0,0 +1,72 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The box trains for itself without the share consent: its training
|
||||
history fills from its own readings either way (they never leave the
|
||||
box), and nothing of it goes upward without that consent (no "ml" in the
|
||||
plan: no forecasts, no error figures). With the consent, as before: the
|
||||
telemetry and the plan's "ml"."""
|
||||
import pytest
|
||||
|
||||
from cantabile_edge import controller as controller_mod
|
||||
from cantabile_edge.agent import Agent
|
||||
from cantabile_edge.config import Config
|
||||
from cantabile_edge.store import Queue
|
||||
|
||||
from test_base_load import gauge, heater, site_mapping
|
||||
from test_control_parts import SITE
|
||||
|
||||
T0 = 1_800_000_000 // 900 * 900
|
||||
|
||||
|
||||
class Sink:
|
||||
def __init__(self):
|
||||
self.published = []
|
||||
|
||||
async def publish(self, kind, data):
|
||||
self.published.append((kind, data))
|
||||
|
||||
|
||||
def agent(tmp_path, share: bool):
|
||||
m = site_mapping(tmp_path, [gauge("sensor.pv", "pv.power")], [heater()])
|
||||
cfg = Config(site_id=SITE, mapping=str(tmp_path / "mapping.yaml"), state_dir=str(tmp_path), dashboards=False)
|
||||
a = Agent(cfg, m, Queue(str(tmp_path / "queue.db"), 100), Sink())
|
||||
a.clock = lambda: T0 + 30
|
||||
if share:
|
||||
a.consent.set(True, "test")
|
||||
a.agg.begin(T0)
|
||||
a.train_agg.begin(T0)
|
||||
a.on_entities({"a": {"sensor.load": {"s": "800", "a": {"unit_of_measurement": "W"}},
|
||||
"sensor.pv": {"s": "300", "a": {"unit_of_measurement": "W"}}}})
|
||||
a.close_window(T0 + 60)
|
||||
return a
|
||||
|
||||
|
||||
@pytest.mark.parametrize("share", [False, True])
|
||||
def test_the_history_fills_with_or_without_the_consent(tmp_path, share):
|
||||
a = agent(tmp_path, share)
|
||||
got = a.control.boxml.store.series(T0, T0 + 900)
|
||||
assert got["load.power"][T0] == pytest.approx(800.0) and got["pv.power"][T0] == pytest.approx(300.0)
|
||||
# the cloud: only with the consent
|
||||
assert a.queue.depth() == (1 if share else 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("share", [False, True])
|
||||
async def test_nothing_goes_upward_without_the_consent(tmp_path, monkeypatch, share):
|
||||
a = agent(tmp_path, share)
|
||||
ctl = a.control
|
||||
|
||||
async def solve(state, timeout):
|
||||
return {"v": 1, "made": T0 + 30, "start": state["start"], "step_s": 900, "status": "Optimal", "solve_s": 0.0,
|
||||
"devices": {}, "flex": {"grid_w": [500] * 96, "shed_w": [0] * 96, "net_w": [500] * 96},
|
||||
"cost_eur": 1.0, "baseline_eur": 1.2, "price_note": "", "sources": state["sources"]}
|
||||
|
||||
monkeypatch.setattr(controller_mod, "solve", solve)
|
||||
ctl.client = object()
|
||||
plan = await ctl.make_plan()
|
||||
published = [d for k, d in ctl.sink.published if k == "plan"]
|
||||
assert plan is not None and len(published) == 1
|
||||
if share:
|
||||
assert published[0]["ml"]["v"] == 1 and "forecasts" in published[0]["ml"]
|
||||
else:
|
||||
assert "ml" not in published[0] and "ml" not in plan
|
||||
# the box keeps its forecasts per source for itself either way (the live error, the intraday correction)
|
||||
assert ctl.boxml.store.forecasts("plan", 0, 2 ** 40)
|
||||
@@ -119,7 +119,8 @@ def test_a_bundle_with_the_new_keys_is_accepted_and_one_without_still_works(tmp_
|
||||
s = state(tmp_path, b, history(tmp_path))
|
||||
assert s["sources"]["load"] == "model (base_load_w)"
|
||||
assert s["load_w"][0] == 5000.0 and s["load_w"][-1] == 111.0 # the series, then the profile
|
||||
assert s["sources"]["forecast"]["load"] == {"series": "base_load_w", "cloud": 32, "box": 0, "hour_of_week": 160}
|
||||
assert s["sources"]["forecast"]["load"] == {"series": "base_load_w", "cloud": 32, "box": 0, "hour_of_week": 160,
|
||||
"source": "hour_of_week"}
|
||||
|
||||
|
||||
def test_a_model_the_box_cannot_run_is_left_out(tmp_path):
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""The owner's preferences (preferences.py): the text forms, the target per
|
||||
step (DST days too), must vs prefer in the plan, a car's ready-by with its
|
||||
state of charge and when it cannot be met, overrides expiring, the levels
|
||||
(the owner's win, an installer within the owner's limits), Home Assistant's
|
||||
entities round trip, the panel's inbox, persistence across a restart."""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
|
||||
from cantabile_edge import preferences as pm
|
||||
from cantabile_edge import steady
|
||||
from cantabile_edge.controls import parse as parse_controls
|
||||
|
||||
BERLIN = ZoneInfo("Europe/Berlin")
|
||||
START = 1793232000 # 2026-10-29 00:00 UTC, a Thursday (01:00 in Berlin)
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self, t):
|
||||
self.t = t
|
||||
|
||||
def __call__(self):
|
||||
return self.t
|
||||
|
||||
|
||||
def controls():
|
||||
raw = [{"id": "living", "kind": "heating", "entities": {"climate": "climate.living"},
|
||||
"limits": {"temp_c": {"owner": [19, 24]}}},
|
||||
{"id": "ac", "kind": "cooling", "entities": {"climate": "climate.ac"}},
|
||||
{"id": "car", "kind": "ev", "entities": {"switch": "switch.ev", "connected": "binary_sensor.ev"}},
|
||||
{"id": "battery", "kind": "battery", "entities": {"setpoint": "number.b"},
|
||||
"limits": {"soc_pct": {"owner": [10, 100]}}}]
|
||||
return {c.id: c for c in parse_controls(raw, set())}
|
||||
|
||||
|
||||
def state(start=START, steps=192, tz="Europe/Berlin", devices=()):
|
||||
return {"start": start, "step_s": 900, "steps": steps, "tz": tz, "occupied": [1] * steps,
|
||||
"devices": list(devices)}
|
||||
|
||||
|
||||
def room_dev(cid="living", kind="heating"):
|
||||
return {"id": cid, "kind": kind, "params": {"comfort_c": 21.0, "setback_c": 17.0, "setup_c": 28.0},
|
||||
"state": {"temp_c": 20.0}, "bounds": {"temp_c": [16, 26, "owner", "owner"]}}
|
||||
|
||||
|
||||
def local(t):
|
||||
return datetime.fromtimestamp(t, timezone.utc).astimezone(BERLIN)
|
||||
|
||||
|
||||
# -- text forms ---------------------------------------------------------------
|
||||
def test_room_text_round_trip():
|
||||
v = pm.parse_room_text("Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19; Sa,So 7-23 20.5", "heating")
|
||||
assert v["blocks"][0] == {"days": [0, 1, 2, 3, 4], "from": "22:00", "to": "06:00", "c": 21.0, "how": "must"}
|
||||
assert v["blocks"][2]["days"] == [5, 6] and v["blocks"][2]["from"] == "07:00" and v["blocks"][2]["how"] == "prefer"
|
||||
t = pm.room_text(v)
|
||||
assert t == "Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19; Sa,Su 07:00-23:00 20.5"
|
||||
assert pm.parse_room_text(t, "heating") == v
|
||||
assert pm.parse_days("daily") == list(range(7)) and pm.parse_days("Di-Do") == [1, 2, 3]
|
||||
assert pm.parse_days("Sa-Mo") == [0, 5, 6] # a range over the week's end
|
||||
for bad in ("Mo-Fr 22:00 21", "Xx 22:00-06:00 21", "Mo 22:00-06:00 40", "Mo 25:00-06:00 21",
|
||||
"Mo 22:00-06:00 21 maybe"):
|
||||
with pytest.raises(ValueError):
|
||||
pm.parse_room_text(bad, "heating")
|
||||
assert len(t) <= 255
|
||||
|
||||
|
||||
def test_ev_and_battery_text():
|
||||
v = pm.parse_ev_text("Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct min 5kWh", 60)
|
||||
assert v["capacity_kwh"] == 60
|
||||
a, b = v["ready"]
|
||||
assert a == {"days": [0, 1, 2, 3, 4], "at": "08:00", "pct": 100.0, "kwh": None, "how": "must",
|
||||
"max_ct": None, "min_kwh": None}
|
||||
assert b["kwh"] == 20 and b["how"] == "prefer" and b["max_ct"] == 25 and b["min_kwh"] == 5
|
||||
assert pm.ev_text(v) == "Mo-Fr 08:00 100% must; Sa,Su 10:00 20kWh max 25ct min 5kWh"
|
||||
with pytest.raises(ValueError):
|
||||
pm.parse_ev_text("Mo 08:00 80% 20kWh") # both
|
||||
bt = pm.parse_battery_text("daily 17:00-21:00 50%", 30)
|
||||
assert bt == {"reserve_pct": 30.0, "blocks": [{"days": list(range(7)), "from": "17:00", "to": "21:00", "pct": 50.0}]}
|
||||
assert pm.battery_text(bt) == "daily 17:00-21:00 50%"
|
||||
|
||||
|
||||
# -- the target per step ------------------------------------------------------------
|
||||
def test_schedule_target_per_step():
|
||||
v = pm.parse_room_text("Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19", "heating")
|
||||
s = state(devices=[room_dev()])
|
||||
ends = [pm._local(START + (t + 1) * 900, BERLIN) for t in range(192)]
|
||||
p = pm.room_series(s["devices"][0], v, [], s, ends)
|
||||
|
||||
def at(y, mo, d, h, mi): # the step whose end is this local time
|
||||
ts = datetime(y, mo, d, h, mi, tzinfo=BERLIN).timestamp()
|
||||
return int((ts - START) // 900) - 1
|
||||
t = at(2026, 10, 29, 3, 0) # Thu 03:00: Wed's night block
|
||||
assert (p["target"][t], p["must"][t]) == (21, 21)
|
||||
t = at(2026, 10, 29, 12, 0) # Thu noon: 19, prefer
|
||||
assert (p["target"][t], p["must"][t]) == (19, None)
|
||||
t = at(2026, 10, 30, 23, 0) # Fri 23:00: Fri's night block
|
||||
assert p["must"][t] == 21
|
||||
t = at(2026, 10, 31, 0, 30) # Sat 00:30: still Fri's night block
|
||||
assert p["must"][t] == 21
|
||||
# a day later: the weekend
|
||||
s2 = state(start=START + 86400, devices=[room_dev()])
|
||||
ends2 = [pm._local(START + 86400 + (t + 1) * 900, BERLIN) for t in range(192)]
|
||||
p2 = pm.room_series(s2["devices"][0], v, [], s2, ends2)
|
||||
t = at(2026, 10, 31, 5, 45) - 96 # Sat 05:45: Fri's night block
|
||||
assert p2["must"][t] == 21
|
||||
t = at(2026, 10, 31, 7, 0) - 96 # Sat 07:00: no block: the setback, nothing must
|
||||
assert (p2["target"][t], p2["must"][t], p2["group"][t]) == (17, None, -1)
|
||||
t = at(2026, 10, 31, 22, 30) - 96 # Sat 22:30: Saturday starts no night block
|
||||
assert (p2["target"][t], p2["must"][t]) == (17, None)
|
||||
assert p["groups"][0]["text"] == "Mo-Fr 22:00-06:00 ≥ 21 °C" and p["groups"][0]["how"] == "must"
|
||||
# without a schedule (and no override): today's behaviour
|
||||
assert pm.room_series(s["devices"][0], None, [], s, ends) is None
|
||||
|
||||
|
||||
def test_schedule_across_the_change_to_winter_time():
|
||||
"""2026-10-25 03:00 CEST -> 02:00 CET: a 25-hour Sunday. A block is on
|
||||
the wall clock: 06:00 is 04:00 UTC on Saturday, 05:00 UTC on Sunday."""
|
||||
start = int(datetime(2026, 10, 24, 0, 0, tzinfo=timezone.utc).timestamp())
|
||||
v = pm.parse_room_text("daily 06:00-22:00 20", "heating")
|
||||
s = state(start=start, devices=[room_dev()])
|
||||
ends = [pm._local(start + (t + 1) * 900, BERLIN) for t in range(192)]
|
||||
p = pm.room_series(s["devices"][0], v, [], s, ends)
|
||||
step = lambda h, m=0, d=24: int((datetime(2026, 10, d, h, m, tzinfo=timezone.utc).timestamp() - start) // 900) - 1 # noqa: E731
|
||||
assert p["target"][step(4, 0)] == 20 and p["target"][step(3, 45)] == 17 # Sat 06:00 CEST
|
||||
assert p["target"][step(5, 0, 25)] == 20 and p["target"][step(4, 45, 25)] == 17 # Sun 06:00 CET
|
||||
assert p["target"][step(21, 0, 25)] == 17 and p["target"][step(20, 45, 25)] == 20 # Sun 22:00 CET
|
||||
# every step: the block's wall-clock rule, by local time
|
||||
for t in range(192):
|
||||
lt = local(start + (t + 1) * 900)
|
||||
assert p["target"][t] == (20 if 6 <= lt.hour < 22 else 17), lt
|
||||
|
||||
|
||||
def test_ready_by_on_the_spring_change():
|
||||
"""2027-03-28 02:00 CET -> 03:00 CEST: a ready-by at 02:30 does not
|
||||
exist that day; it is 03:30 CEST (as a clock moves on)."""
|
||||
v = pm.parse_ev_text("daily 02:30 80%")
|
||||
now = datetime(2027, 3, 28, 0, 0, tzinfo=BERLIN).timestamp()
|
||||
t, r = pm.next_ready(v, BERLIN, now)
|
||||
assert local(t).strftime("%H:%M %Z") == "03:30 CEST"
|
||||
t2, _ = pm.next_ready(v, BERLIN, t + 60)
|
||||
assert local(t2).strftime("%d %H:%M") == "29 02:30"
|
||||
|
||||
|
||||
# -- the plan: must vs prefer, a car's ready-by ---------------------------------------------
|
||||
optimizer = pytest.importorskip("cantabile_edge.optimizer")
|
||||
from test_optimizer import house # noqa: E402
|
||||
|
||||
|
||||
def heated(how):
|
||||
"""A room whose 17:00-21:00 block costs 3 EUR/kWh to hold (a scarce
|
||||
evening); the owner's limit 21.5 °C keeps it from storing heat before."""
|
||||
s = house([{"id": "living", "kind": "heating", "channels": ["climate"],
|
||||
"params": {"r_k_per_kw": 10, "c_kwh_per_k": 3, "max_power_w": 4000, "comfort_c": 21,
|
||||
"setback_c": 17, "cop": 1.0},
|
||||
"state": {"temp_c": 21.0}, "bounds": {"temp_c": [16, 21.5, "owner", "owner"]}}])
|
||||
hour = lambda k: (k // 4 + 1) % 24 # noqa: E731
|
||||
s["price_import"] = [300.0 if 17 <= hour(k) < 21 else p for k, p in enumerate(s["price_import"])]
|
||||
ends = [pm._local(START + (t + 1) * 900, BERLIN) for t in range(192)]
|
||||
v = pm.parse_room_text(f"daily 17:00-21:00 21 {how}", "heating")
|
||||
s["devices"][0]["pref"] = pm.room_series(s["devices"][0], v, [], s, ends)
|
||||
return s
|
||||
|
||||
|
||||
def block_temps(s, p):
|
||||
m = optimizer.Model(s)
|
||||
m.solve(20)
|
||||
val = m.h.getSolution().col_value
|
||||
temps = [val[v.index] for v in m.devices["living"][1]["temp"]]
|
||||
g = s["devices"][0]["pref"]["group"]
|
||||
return [temps[t] for t in range(192) if g[t] == 0]
|
||||
|
||||
|
||||
def test_must_is_hard_and_prefer_soft():
|
||||
s = heated("must")
|
||||
must = block_temps(s, None)
|
||||
assert len(must) == 32 and min(must) >= 20.9, min(must)
|
||||
pre = heated("prefer")
|
||||
assert min(block_temps(pre, None)) < 20.5 # at 3 EUR/kWh comfort gives way
|
||||
# the plan says so, with the must's cost
|
||||
s["prefs_cost"] = True
|
||||
out = optimizer.plan(s, 20)
|
||||
f = out["prefs"][0]
|
||||
assert f["how"] == "must" and f["met"] is True and f["label"] == "daily 17:00-21:00 ≥ 21 °C"
|
||||
assert out["prefs_cost"]["living:0"] > 0.5 # holding 21 at 3 EUR/kWh costs
|
||||
fp = optimizer.plan(heated("prefer"), 20)["prefs"][0]
|
||||
assert fp["met"] is False and "°C at" in fp["detail"]
|
||||
|
||||
|
||||
def test_must_gives_way_to_a_building_cap_and_the_owners_limit():
|
||||
s = heated("must")
|
||||
s["building"] = {"thermostat_cap": {"value": 19.0, "from_step": 0, "to_step": 192, "penalty": 2.0,
|
||||
"level": "flexible"}}
|
||||
out = optimizer.plan(s, 20)
|
||||
f = out["prefs"][0]
|
||||
assert f["met"] is True and f.get("held_by") == "building" and "held at 19 °C" in f["detail"]
|
||||
s2 = heated("must")
|
||||
v = pm.parse_room_text("daily 17:00-21:00 23 must", "heating")
|
||||
ends = [pm._local(START + (t + 1) * 900, BERLIN) for t in range(192)]
|
||||
s2["devices"][0]["pref"] = pm.room_series(s2["devices"][0], v, [], s2, ends)
|
||||
f2 = optimizer.plan(s2, 20)["prefs"][0]
|
||||
assert f2.get("held_by") == "owner" and "21.5" in f2["detail"]
|
||||
|
||||
|
||||
def ev(soc, connected=True):
|
||||
return {"id": "car", "kind": "ev", "channels": ["switch", "current"],
|
||||
"params": {"need_wh": 8000, "departure": "07:00", "min_current_a": 6, "max_current_a": 16,
|
||||
"phases": 1, "voltage": 230},
|
||||
"state": {"connected": connected, "soc_pct": soc}, "bounds": {"current_a": [0, 16, "device", "device"]}}
|
||||
|
||||
|
||||
def with_car(soc, text, capacity=50, connected=True, now=START):
|
||||
s = house([ev(soc, connected)])
|
||||
c = controls()["car"]
|
||||
v = pm.parse_ev_text(text, capacity)
|
||||
s["devices"][0]["pref"] = pm.ev_need(s["devices"][0], v, [], s, BERLIN, now, None, c)
|
||||
return s
|
||||
|
||||
|
||||
def test_ev_ready_by_from_the_state_of_charge():
|
||||
s = with_car(40, "daily 08:00 80% must") # (80 - 40) % of 50 kWh = 20 kWh by Thu 08:00
|
||||
p = s["devices"][0]["pref"]
|
||||
assert p["need_wh"] == 20000 and p["dep_step"] == 28 and p["at_local"] == "Thu 08:00"
|
||||
out = optimizer.plan(s, 20)
|
||||
f = out["prefs"][0]
|
||||
assert f["met"] is True and abs(f["planned_kwh"] - 20) < 0.3 and f["reached_pct"] == 80
|
||||
assert f["detail"].startswith("80 % by Thu 08:00") and f["charge_eur"] > 0
|
||||
|
||||
|
||||
def test_ev_must_infeasible_says_by_how_much_and_when():
|
||||
s = with_car(10, "daily 08:00 100% must", capacity=60) # 54 kWh in 7 h at 3.68 kW: no
|
||||
out = optimizer.plan(s, 20)
|
||||
f = out["prefs"][0]
|
||||
assert f["met"] is False and 27 < f["short_kwh"] < 29.5
|
||||
ready = local(f["ready_at"])
|
||||
assert ready.strftime("%a") == "Thu" and ready.hour >= 15 # 54 kWh / 3.68 kW: ~14.7 h from 01:00
|
||||
assert "short by" in f["detail"] and "ready at full power by Thu" in f["detail"]
|
||||
nc = optimizer.plan(with_car(10, "daily 08:00 100% must", 60, connected=False), 20)["prefs"][0]
|
||||
assert nc["met"] is False and nc["detail"] == "not plugged in"
|
||||
|
||||
|
||||
def test_ev_prefer_up_to_a_price_and_a_minimum():
|
||||
# ready Thu 18:00: the night (12 ct, 01:00-06:00) gives 5 h x 3.68 kW = 18.4 kWh; the day costs 28 ct
|
||||
s = with_car(None, "daily 18:00 30kWh max 20ct")
|
||||
out = optimizer.plan(s, 20)
|
||||
f = out["prefs"][0]
|
||||
assert 17 < f["planned_kwh"] < 19 and f["met"] is False and "dearer" in f["detail"]
|
||||
s = with_car(None, "daily 18:00 30kWh max 20ct min 22kWh")
|
||||
f = optimizer.plan(s, 20)["prefs"][0]
|
||||
assert 21.8 < f["planned_kwh"] < 23
|
||||
s = with_car(None, "daily 18:00 30kWh max 50ct")
|
||||
assert optimizer.plan(s, 20)["prefs"][0]["planned_kwh"] > 29.5
|
||||
|
||||
|
||||
def test_battery_reserve_is_the_floor():
|
||||
from test_optimizer import battery
|
||||
s = house([battery(soc=60, floor=10)])
|
||||
v = pm.check_battery({"reserve_pct": 30, "blocks": [{"days": "daily", "from": "17:00", "to": "21:00", "pct": 55}]})
|
||||
ends = [pm._local(START + (t + 1) * 900, BERLIN) for t in range(192)]
|
||||
s["devices"][0]["pref"] = pm.battery_series(s["devices"][0], v, ends)
|
||||
m = optimizer.Model(s)
|
||||
m.solve(20)
|
||||
val = m.h.getSolution().col_value
|
||||
soc = [val[x.index] / 100.0 for x in m.devices["battery"][1]["soc"]] # % of 10 kWh
|
||||
fl = s["devices"][0]["pref"]["floor_pct"]
|
||||
assert all(soc[t] >= fl[t] - 0.5 for t in range(192))
|
||||
assert min(soc[t] for t in range(192) if fl[t] == 55) >= 54.5
|
||||
assert min(soc) < 50 # outside the evening only 30 holds it
|
||||
out = optimizer.plan(s, 20)
|
||||
assert {f["group"] for f in out["prefs"]} == {"battery:reserve", "battery:0"} and all(f["met"] for f in out["prefs"])
|
||||
|
||||
|
||||
# -- the store: levels, overrides, persistence ------------------------------------------------
|
||||
def test_levels_owner_wins_installer_within_limits(tmp_path):
|
||||
cs = controls()
|
||||
p = pm.Preferences(str(tmp_path), Clock(START))
|
||||
ok, why = p.set("room:living", {"blocks": [{"days": "daily", "from": "22:00", "to": "06:00", "c": 26}]},
|
||||
"installer:bob", "installer", cs["living"])
|
||||
assert not ok and "owner's limits" in why # 26 above the owner's 24
|
||||
ok, _ = p.set("room:living", {"blocks": [{"days": "daily", "from": "22:00", "to": "06:00", "c": 22}]},
|
||||
"installer:bob", "installer", cs["living"])
|
||||
assert ok and p.effective("room:living")["level"] == "installer"
|
||||
ok, why = p.set("room:living", {"blocks": [{"days": "daily", "from": "22:00", "to": "06:00", "c": 26, "how": "must"}]},
|
||||
"Home Assistant", "owner", cs["living"])
|
||||
assert ok and p.effective("room:living")["by"] == "Home Assistant" # the owner may (the plan keeps the bounds)
|
||||
ok, why = p.set("room:living", {"blocks": []}, "installer:bob", "installer", cs["living"])
|
||||
assert ok and "owner's own setting is in force" in why
|
||||
p.set("room:living", None, "panel:alice", "owner", cs["living"])
|
||||
assert p.effective("room:living")["level"] == "installer" # the owner cleared theirs
|
||||
assert [e["by"] for e in p.log] == ["installer:bob", "Home Assistant", "installer:bob", "panel:alice"]
|
||||
assert p.errors[-1]["by"] == "installer:bob"
|
||||
ok, why = p.apply_msg({"override": {"kind": "away"}}, cs, "Europe/Berlin", "installer", "installer:bob")
|
||||
assert not ok and "owner" in why
|
||||
|
||||
|
||||
def test_overrides_win_and_expire(tmp_path):
|
||||
cs = controls()
|
||||
clock = Clock(START)
|
||||
p = pm.Preferences(str(tmp_path), clock)
|
||||
p.set("room:living", pm.parse_room_text("daily 00:00-24:00 19", "heating"), "ha", "owner", cs["living"])
|
||||
ok, _ = p.add_override({"kind": "boost", "control": "living", "c": 23, "hours": 2}, "ha", cs)
|
||||
assert ok
|
||||
s = state(devices=[room_dev()])
|
||||
p.apply(s, cs)
|
||||
pr = s["devices"][0]["pref"]
|
||||
assert pr["target"][0] == 23 and pr["must"][0] == 23 and pr["groups"][pr["group"][0]]["kind"] == "boost"
|
||||
assert pr["target"][8] == 19 # after 2 h: the schedule again
|
||||
assert p.free() == {"living"} and p.free() == set()
|
||||
clock.t += 3 * 3600
|
||||
assert p.prune() is True and p.overrides == [] and p.log[-1]["what"] == "expired"
|
||||
s = state(start=START + 3 * 3600, devices=[room_dev()])
|
||||
p.apply(s, cs)
|
||||
assert s["devices"][0]["pref"]["target"][0] == 19
|
||||
# away: every room at its setback, the schedule's musts suspended
|
||||
p.set("room:living", pm.parse_room_text("daily 00:00-24:00 21 must", "heating"), "ha", "owner", cs["living"])
|
||||
ok, _ = p.apply_msg({"override": {"kind": "away", "until_text": "2026-10-30"}}, cs, "Europe/Berlin", "owner", "ha")
|
||||
assert ok
|
||||
s = state(start=START + 3 * 3600, devices=[room_dev()])
|
||||
p.apply(s, cs)
|
||||
pr = s["devices"][0]["pref"]
|
||||
back = int((datetime(2026, 10, 30, 0, 0, tzinfo=BERLIN).timestamp() - s["start"]) // 900)
|
||||
assert pr["target"][0] == 17 and pr["must"][0] is None
|
||||
assert pr["must"][back] == 21 and pr["target"][back - 2] == 17
|
||||
# a car needed once wins over its schedule until then
|
||||
p.set("ev:car", pm.parse_ev_text("daily 08:00 100% must", 50), "ha", "owner", cs["car"])
|
||||
ok, msg = p.apply_msg({"override": {"kind": "car", "control": "car", "text": "2026-10-29 06:00 60%"}},
|
||||
cs, "Europe/Berlin", "owner", "ha")
|
||||
assert ok, msg
|
||||
s = state(start=START + 3 * 3600, devices=[dict(ev(20), pref=None)])
|
||||
s["devices"][0]["params"]["need_wh"] = 8000
|
||||
p.apply(s, cs, now=START + 3 * 3600)
|
||||
e = s["devices"][0]["pref"]
|
||||
assert e["source"] == "override" and e["need_wh"] == 20000 and e["at_local"] == "Thu 06:00"
|
||||
clock.t = datetime(2026, 10, 29, 6, 1, tzinfo=BERLIN).timestamp()
|
||||
p.prune()
|
||||
s = state(start=int(clock.t // 900 * 900), devices=[ev(20)])
|
||||
p.apply(s, cs)
|
||||
assert s["devices"][0]["pref"]["source"] == "schedule" and s["devices"][0]["pref"]["at_local"] == "Thu 08:00"
|
||||
|
||||
|
||||
def test_persists_across_a_restart_and_the_panels_inbox(tmp_path):
|
||||
cs = controls()
|
||||
p = pm.Preferences(str(tmp_path), Clock(START))
|
||||
os.makedirs(tmp_path / "prefs-inbox")
|
||||
for i, m in enumerate([{"by": "alice", "set": {"item": "battery:battery", "value": {"reserve_pct": 30, "blocks": []}}},
|
||||
{"by": "alice", "set": {"item": "ev:car", "value": "Mo-Fr 07:30 80% must"}},
|
||||
{"by": "alice", "override": {"kind": "boost", "control": "living", "c": 22.5, "hours": 1}},
|
||||
{"by": "alice", "set": {"item": "room:nope", "value": {"blocks": []}}}]):
|
||||
(tmp_path / "prefs-inbox" / f"{i:03d}.json").write_text(json.dumps(m))
|
||||
assert p.take_inbox(cs, "Europe/Berlin") == 4
|
||||
assert os.listdir(tmp_path / "prefs-inbox") == []
|
||||
assert p.errors[-1]["item"] == "room:nope"
|
||||
p.write_view(list(cs.values()), "shadow")
|
||||
view = json.loads((tmp_path / "preferences-view.json").read_text())
|
||||
assert {c["id"] for c in view["controls"]} == {"living", "ac", "car", "battery"}
|
||||
assert view["items"]["ev:car"]["levels"]["owner"]["text"] == "Mo-Fr 07:30 80% must"
|
||||
assert view["items"]["ev:car"]["levels"]["owner"]["by"] == "panel:alice"
|
||||
q = pm.Preferences(str(tmp_path), Clock(START + 60)) # the agent started again
|
||||
assert q.items == p.items and q.overrides == p.overrides and q.log == p.log
|
||||
assert q.value("battery:battery")["reserve_pct"] == 30 and q.active_overrides()[0]["c"] == 22.5
|
||||
|
||||
|
||||
def test_costs_made_again_only_when_due(tmp_path):
|
||||
cs = controls()
|
||||
clock = Clock(START)
|
||||
p = pm.Preferences(str(tmp_path), clock)
|
||||
p.set("room:living", pm.parse_room_text("daily 22:00-06:00 21 must", "heating"), "ha", "owner", cs["living"])
|
||||
s = p.apply(state(devices=[room_dev()]), cs)
|
||||
assert s["prefs_cost"] is True
|
||||
plan = {"prefs": [{"control": "living", "group": "living:0", "how": "must", "met": True, "label": "x"}],
|
||||
"prefs_cost": {"living:0": 0.4}}
|
||||
p.take_plan(plan)
|
||||
assert plan["prefs"][0]["cost_eur_day"] == 0.4 and plan["prefs"][0]["text"] == "x: met, +0.40 €/day"
|
||||
clock.t += 600
|
||||
assert p.apply(state(devices=[room_dev()]), cs)["prefs_cost"] is False # not changed, not an hour yet
|
||||
plan = {"prefs": [{"control": "living", "group": "living:0", "how": "must", "met": True, "label": "x"}]}
|
||||
p.take_plan(plan)
|
||||
assert plan["prefs"][0]["cost_eur_day"] == 0.4 # the last cost, as of then
|
||||
clock.t += 3600
|
||||
assert p.apply(state(devices=[room_dev()]), cs)["prefs_cost"] is True
|
||||
|
||||
|
||||
def test_steady_does_not_hold_back_the_owners_change():
|
||||
c = controls()["living"]
|
||||
col = SimpleNamespace(state=lambda e: "heat", attr=lambda e, a: 20.0)
|
||||
plan = {"made": START, "start": START, "step_s": 900,
|
||||
"devices": {"living": [{"t": START, "set": 21.0, "why": ""}]}}
|
||||
st = {"steps": 192, "devices": [{"id": "living", "params": {}, "bounds": {"temp_c": [16, 26]}}]}
|
||||
held = steady.stabilise(json.loads(json.dumps(plan)), st, [c], col, lambda *_: START - 60, START)
|
||||
assert any("kept" in e["why"] for e in held["devices"]["living"])
|
||||
free = steady.stabilise(json.loads(json.dumps(plan)), st, [c], col, lambda *_: START - 60, START, free={"living"})
|
||||
assert free["devices"]["living"] == plan["devices"]["living"]
|
||||
|
||||
|
||||
# -- Home Assistant's entities ---------------------------------------------------------------
|
||||
def test_ha_entities_round_trip(tmp_path):
|
||||
from test_announce import make
|
||||
a, ann = make(tmp_path)
|
||||
ctl = a.control
|
||||
ctl.tz = ctl.prefs.tz = "Europe/Berlin"
|
||||
objs = ann.objects()
|
||||
keys = {(c, o) for (_, c, o) in objs}
|
||||
for k in [("text", "pref_living_schedule"), ("number", "pref_living_boost_c"), ("switch", "pref_living_boost"),
|
||||
("switch", "pref_away"), ("text", "pref_away_until"), ("text", "pref_car_ready"),
|
||||
("number", "pref_car_capacity"), ("text", "pref_car_needed"), ("number", "pref_battery_reserve"),
|
||||
("text", "pref_battery_reserve_blocks"), ("text", "pref_boiler_schedule")]:
|
||||
assert k in keys, k
|
||||
sched = objs[(ann.uid, "text", "pref_living_schedule")]
|
||||
assert sched["command_topic"] == f"cantabile/{ann.site}/prefs/pref_living_schedule/set" and sched["max"] == 255
|
||||
assert sched["entity_category"] == "config" and sched["default_entity_id"] == "text.cantabile_pref_living_schedule"
|
||||
assert ("sensor", "preferences") not in keys # shadow: no plan, no results in HA
|
||||
assert ann.prefs_state()["pref_living_schedule"] == ""
|
||||
|
||||
def send(obj, text):
|
||||
asyncio.run(ann.pref_message(obj, text.encode()))
|
||||
send("pref_living_schedule", "mo-fr 22-6 21 must; mo-fr 6-22 19")
|
||||
assert ctl.prefs.value("room:living")["blocks"][0]["how"] == "must"
|
||||
assert ann.prefs_state()["pref_living_schedule"] == "Mo-Fr 22:00-06:00 21 must; Mo-Fr 06:00-22:00 19"
|
||||
assert ctl.prefs.effective("room:living")["by"] == "Home Assistant"
|
||||
send("pref_living_schedule", "Mo-Fr 22:00-06:00 99") # refused: as it was
|
||||
assert ann.prefs_state()["pref_living_schedule"].startswith("Mo-Fr 22:00-06:00 21 must")
|
||||
assert ann.prefs_result()["last_error"].startswith("room:living")
|
||||
send("pref_living_boost_c", "23.5")
|
||||
send("pref_living_boost", "ON")
|
||||
st = ann.prefs_state()
|
||||
assert st["pref_living_boost"] == "ON" and st["pref_living_boost_c"] == 23.5
|
||||
send("pref_living_boost", "OFF")
|
||||
assert ann.prefs_state()["pref_living_boost"] == "OFF"
|
||||
send("pref_car_capacity", "60")
|
||||
send("pref_car_ready", "Mo-Fr 08:00 100% must")
|
||||
assert ctl.prefs.value("ev:car") == {"ready": [{"days": [0, 1, 2, 3, 4], "at": "08:00", "pct": 100.0, "kwh": None,
|
||||
"how": "must", "max_ct": None, "min_kwh": None}], "capacity_kwh": 60.0}
|
||||
send("pref_car_needed", "2030-01-01 07:00 80%") # too far ahead: refused
|
||||
assert ann.prefs_state()["pref_car_needed"] == ""
|
||||
when = datetime.fromtimestamp(ctl.prefs.clock() + 86400, BERLIN).strftime("%Y-%m-%d")
|
||||
send("pref_car_needed", f"{when} 07:00 80%")
|
||||
assert ann.prefs_state()["pref_car_needed"] == f"{when} 07:00 80% must"
|
||||
send("pref_car_needed", "")
|
||||
assert ann.prefs_state()["pref_car_needed"] == ""
|
||||
send("pref_battery_reserve", "25")
|
||||
send("pref_battery_reserve_blocks", "daily 17:00-21:00 50%")
|
||||
assert ctl.prefs.value("battery:battery") == {"reserve_pct": 25.0, "blocks": [
|
||||
{"days": list(range(7)), "from": "17:00", "to": "21:00", "pct": 50.0}]}
|
||||
send("pref_away_until", "2030-01-01") # stored; away is off
|
||||
assert ann.prefs_state()["pref_away"] == "OFF" and ann.prefs_state()["pref_away_until"] == "2030-01-01"
|
||||
send("pref_away", "ON") # beyond 60 days: refused
|
||||
assert ann.prefs_state()["pref_away"] == "OFF"
|
||||
send("pref_away_until", "")
|
||||
send("pref_away", "ON") # no date: until switched off
|
||||
assert ann.prefs_state()["pref_away"] == "ON"
|
||||
send("pref_away", "OFF")
|
||||
assert ann.prefs_state()["pref_away"] == "OFF"
|
||||
# with the plan shown (advisory): the results
|
||||
ctl.mode.current = "advisory"
|
||||
ctl.prefs.feedback = [{"control": "living", "group": "living:0", "how": "must", "met": False,
|
||||
"text": "Mo-Fr 22:00-06:00 ≥ 21 °C: not met", "cost_eur_day": 0.4}]
|
||||
keys = {(c, o) for (_, c, o) in ann.objects()}
|
||||
assert ("sensor", "preferences") in keys and ("sensor", "pref_living_result") in keys
|
||||
r = ann.prefs_result()
|
||||
assert r["summary"] == "1 not met" and r["controls"]["living"]["state"] == "not met"
|
||||
assert r["controls"]["car"]["state"] == "none"
|
||||
# who did what, kept across a restart
|
||||
q = pm.Preferences(str(tmp_path))
|
||||
assert q.value("battery:battery")["reserve_pct"] == 25 and all(e["by"] == "Home Assistant" for e in q.log)
|
||||
|
||||
|
||||
def test_console_command_sets_an_installers_level(tmp_path):
|
||||
from test_announce import make
|
||||
a, _ = make(tmp_path)
|
||||
ctl = a.control
|
||||
msg = json.dumps({"set": {"item": "room:living", "value": "daily 22:00-06:00 25"}})
|
||||
ok, why = asyncio.run(ctl.command("preferences", {"msg": msg, "level": "installer", "by": "bob"}))
|
||||
assert not ok and "owner's limits" in why # the owner's 19..24
|
||||
msg = json.dumps({"set": {"item": "room:living", "value": "daily 22:00-06:00 22"}})
|
||||
ok, why = asyncio.run(ctl.command("preferences", {"msg": msg, "level": "installer", "by": "bob"}))
|
||||
assert ok and ctl.prefs.effective("room:living")["by"] == "installer:bob"
|
||||
ok, why = asyncio.run(ctl.command("preferences", {"msg": msg, "level": "owner", "by": "bob"}))
|
||||
assert not ok # the console is never the owner
|
||||
ok, why = asyncio.run(ctl.command("preferences", {"msg": json.dumps({"override": {"kind": "away"}}),
|
||||
"level": "installer"}))
|
||||
assert not ok
|
||||
pub = ctl.public_plan({"id": "x", "made": START, "start": START, "step_s": 900, "mode": "shadow", "devices": {},
|
||||
"cost_eur": 0, "baseline_eur": 0, "status": "Optimal"})
|
||||
pr = pub["preferences"]
|
||||
assert pr["items"]["room:living"]["in_force"] == "installer"
|
||||
assert pr["items"]["room:living"]["levels"]["installer"]["text"] == "daily 22:00-06:00 22"
|
||||
assert pr["log"][-1]["by"] == "installer:bob"
|
||||
Reference in new issue
Block a user