Files
cantabile-ha-addons/src/edge/cantabile_edge/backfill.py
T

99 lines
4.7 KiB
Python

# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
"""History backfill: HA's long-term statistics (hourly mean/min/max for
gauges, the running sum for counters) for consented assets, queued as
"stats" batches and sent like telemetry. Chunks of a week keep each HA
answer and each batch small."""
import logging
from datetime import datetime, timedelta, timezone
from .agent import number
from .ha import HAClient
from .mapping import convert, scaled
log = logging.getLogger(__name__)
CHUNK = timedelta(days=7)
def _ts(v) -> float:
# HA sends epoch milliseconds (newer) or ISO strings (older)
if isinstance(v, (int, float)):
return v / 1000
return datetime.fromisoformat(v).timestamp()
async def backfill(client: HAClient, mapping, queue, since: datetime, until: datetime,
period: str = "hour") -> int:
"""period: hour (kept for ever) or 5minute (HA keeps those ~10 days)."""
assets = {a.entity: a for a in mapping.assets if a.consent}
meta = await client.send("recorder/list_statistic_ids") # no type: both mean and sum
units = {m["statistic_id"]: m.get("statistics_unit_of_measurement") for m in meta}
ids = [e for e in assets if e in units]
log.info("backfill %d of %d consented entities have statistics", len(ids), len(assets))
batches, start = 0, since
while start < until and ids:
end = min(start + CHUNK, until)
res = await client.send("recorder/statistics_during_period", timeout=300,
start_time=start.isoformat(), end_time=end.isoformat(),
statistic_ids=ids, period=period,
types=["mean", "min", "max", "sum", "state"])
records = []
for eid, rows in sorted((res or {}).items()):
a, unit = assets[eid], units.get(eid)
out = []
for r in rows:
row = {"start": _ts(r["start"])}
for k in (("sum", "state") if a.kind == "counter" else ("mean", "min", "max")):
v = number(r.get(k))
row[k] = None if v is None else scaled(convert(v, getattr(a, "unit_in", "") or unit, a.unit), a)
out.append(row)
records.append({"asset": a.asset, "kind": a.kind, "rows": out})
if records:
queue.put("stats", {"period": period, "window_start": start.timestamp(),
"window_end": end.timestamp(), "mapping_version": mapping.version,
"records": records})
batches += 1
start = end
return batches
async def collect(client: HAClient, mapping, since: datetime, until: datetime, period: str = "hour",
want=None, consented: bool = True) -> dict[str, list[dict]]:
"""HA's statistics of the gauges (want(asset): which; consented: only
those with consent) as {asset: [{"start": unix, "mean"}]} in the assets'
units: for the box's own training history (trainhist.py, every mapped
gauge: nothing of it is queued or sent). A total of parts (pv.power from
pv.1.power, ...) has no entity of its own: left out."""
assets = {a.entity: a for a in mapping.assets
if (a.consent or not consented) and a.kind == "gauge" and not a.attribute and not a.equals
and (want is None or want(a.asset))}
if not assets:
return {}
meta = await client.send("recorder/list_statistic_ids", statistic_type="mean")
units = {m["statistic_id"]: m.get("statistics_unit_of_measurement") for m in meta}
ids = [e for e in assets if e in units]
out: dict[str, list[dict]] = {}
start = since
while start < until and ids:
end = min(start + CHUNK, until)
res = await client.send("recorder/statistics_during_period", timeout=300, start_time=start.isoformat(),
end_time=end.isoformat(), statistic_ids=ids, period=period, types=["mean"])
for eid, rows in (res or {}).items():
a, unit = assets.get(eid), units.get(eid)
if a is None:
continue
for r in rows:
v = number(r.get("mean"))
v = None if v is None else scaled(convert(v, getattr(a, "unit_in", "") or unit, a.unit), a)
if v is not None:
out.setdefault(a.asset, []).append({"start": _ts(r["start"]), "mean": v})
start = end
return out
def parse_since(s: str) -> datetime:
"""An ISO date/time (UTC unless it says otherwise) or days back, e.g. 30d."""
if s.endswith("d") and s[:-1].isdigit():
return datetime.now(timezone.utc) - timedelta(days=int(s[:-1]))
d = datetime.fromisoformat(s)
return d if d.tzinfo else d.replace(tzinfo=timezone.utc)