99 lines
4.7 KiB
Python
99 lines
4.7 KiB
Python
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
|
"""History backfill: HA's long-term statistics (hourly mean/min/max for
|
|
gauges, the running sum for counters) for consented assets, queued as
|
|
"stats" batches and sent like telemetry. Chunks of a week keep each HA
|
|
answer and each batch small."""
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from .agent import number
|
|
from .ha import HAClient
|
|
from .mapping import convert, scaled
|
|
|
|
log = logging.getLogger(__name__)
|
|
CHUNK = timedelta(days=7)
|
|
|
|
|
|
def _ts(v) -> float:
|
|
# HA sends epoch milliseconds (newer) or ISO strings (older)
|
|
if isinstance(v, (int, float)):
|
|
return v / 1000
|
|
return datetime.fromisoformat(v).timestamp()
|
|
|
|
|
|
async def backfill(client: HAClient, mapping, queue, since: datetime, until: datetime,
|
|
period: str = "hour") -> int:
|
|
"""period: hour (kept for ever) or 5minute (HA keeps those ~10 days)."""
|
|
assets = {a.entity: a for a in mapping.assets if a.consent}
|
|
meta = await client.send("recorder/list_statistic_ids") # no type: both mean and sum
|
|
units = {m["statistic_id"]: m.get("statistics_unit_of_measurement") for m in meta}
|
|
ids = [e for e in assets if e in units]
|
|
log.info("backfill %d of %d consented entities have statistics", len(ids), len(assets))
|
|
batches, start = 0, since
|
|
while start < until and ids:
|
|
end = min(start + CHUNK, until)
|
|
res = await client.send("recorder/statistics_during_period", timeout=300,
|
|
start_time=start.isoformat(), end_time=end.isoformat(),
|
|
statistic_ids=ids, period=period,
|
|
types=["mean", "min", "max", "sum", "state"])
|
|
records = []
|
|
for eid, rows in sorted((res or {}).items()):
|
|
a, unit = assets[eid], units.get(eid)
|
|
out = []
|
|
for r in rows:
|
|
row = {"start": _ts(r["start"])}
|
|
for k in (("sum", "state") if a.kind == "counter" else ("mean", "min", "max")):
|
|
v = number(r.get(k))
|
|
row[k] = None if v is None else scaled(convert(v, getattr(a, "unit_in", "") or unit, a.unit), a)
|
|
out.append(row)
|
|
records.append({"asset": a.asset, "kind": a.kind, "rows": out})
|
|
if records:
|
|
queue.put("stats", {"period": period, "window_start": start.timestamp(),
|
|
"window_end": end.timestamp(), "mapping_version": mapping.version,
|
|
"records": records})
|
|
batches += 1
|
|
start = end
|
|
return batches
|
|
|
|
|
|
async def collect(client: HAClient, mapping, since: datetime, until: datetime, period: str = "hour",
|
|
want=None, consented: bool = True) -> dict[str, list[dict]]:
|
|
"""HA's statistics of the gauges (want(asset): which; consented: only
|
|
those with consent) as {asset: [{"start": unix, "mean"}]} in the assets'
|
|
units: for the box's own training history (trainhist.py, every mapped
|
|
gauge: nothing of it is queued or sent). A total of parts (pv.power from
|
|
pv.1.power, ...) has no entity of its own: left out."""
|
|
assets = {a.entity: a for a in mapping.assets
|
|
if (a.consent or not consented) and a.kind == "gauge" and not a.attribute and not a.equals
|
|
and (want is None or want(a.asset))}
|
|
if not assets:
|
|
return {}
|
|
meta = await client.send("recorder/list_statistic_ids", statistic_type="mean")
|
|
units = {m["statistic_id"]: m.get("statistics_unit_of_measurement") for m in meta}
|
|
ids = [e for e in assets if e in units]
|
|
out: dict[str, list[dict]] = {}
|
|
start = since
|
|
while start < until and ids:
|
|
end = min(start + CHUNK, until)
|
|
res = await client.send("recorder/statistics_during_period", timeout=300, start_time=start.isoformat(),
|
|
end_time=end.isoformat(), statistic_ids=ids, period=period, types=["mean"])
|
|
for eid, rows in (res or {}).items():
|
|
a, unit = assets.get(eid), units.get(eid)
|
|
if a is None:
|
|
continue
|
|
for r in rows:
|
|
v = number(r.get("mean"))
|
|
v = None if v is None else scaled(convert(v, getattr(a, "unit_in", "") or unit, a.unit), a)
|
|
if v is not None:
|
|
out.setdefault(a.asset, []).append({"start": _ts(r["start"]), "mean": v})
|
|
start = end
|
|
return out
|
|
|
|
|
|
def parse_since(s: str) -> datetime:
|
|
"""An ISO date/time (UTC unless it says otherwise) or days back, e.g. 30d."""
|
|
if s.endswith("d") and s[:-1].isdigit():
|
|
return datetime.now(timezone.utc) - timedelta(days=int(s[:-1]))
|
|
d = datetime.fromisoformat(s)
|
|
return d if d.tzinfo else d.replace(tzinfo=timezone.utc)
|