cantabile_edge, cantabile_modbus, cantabile_eebus 0.1.400: full source
This commit is contained in:
1 parent
343331d234
commit
d5fbcacaa4
176 files changed
+32511
-19
No files matched your search
@@ -0,0 +1,263 @@
|
||||
# Copyright © 2026 Setti Solutions S.R.L, www.setti.md. All rights reserved.
|
||||
"""Uplinks. send() returns True only when the batch is safely stored on the
|
||||
other side; only then the queue deletes it.
|
||||
|
||||
NATS: JetStream publish, whose acknowledgement means the stream stored the
|
||||
message. The message id (site and sequence number) lets JetStream drop a
|
||||
batch sent twice (a lost acknowledgement, then a replay). Subject:
|
||||
<prefix>.<site_id>.tel (the scheme: ctb.site.<site_id>.<kind>).
|
||||
JSON for v0; Protobuf later.
|
||||
|
||||
link: the site's link (edge/link) over a local socket; in production the
|
||||
agent's only uplink (the NATS sink is for tests without a link).
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import nats
|
||||
from nats.js.api import StreamConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
# nats-py logs a traceback for every refused connection; the uploader logs
|
||||
# one line per failed attempt instead.
|
||||
logging.getLogger("nats.aio.client").setLevel(logging.CRITICAL)
|
||||
|
||||
|
||||
async def _quiet(e):
|
||||
pass
|
||||
|
||||
|
||||
def _encode(site: str, seq: int, kind: str, payload: dict) -> bytes:
|
||||
return json.dumps({"v": 1, "site": site, "seq": seq, "type": kind, **payload},
|
||||
separators=(",", ":")).encode()
|
||||
|
||||
|
||||
class FileSink:
|
||||
"""JSON lines; fsync before the acknowledgement."""
|
||||
|
||||
def __init__(self, path: str, site: str):
|
||||
self.path, self.site = path, site
|
||||
|
||||
async def send(self, seq, kind, payload) -> bool:
|
||||
with open(self.path, "ab") as f:
|
||||
f.write(_encode(self.site, seq, kind, payload) + b"\n")
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
return True
|
||||
|
||||
async def publish(self, kind: str, data: dict) -> bool:
|
||||
"""A plan or an action (best effort: no queue, the next plan follows)."""
|
||||
with open(self.path, "ab") as f:
|
||||
f.write(json.dumps({"site": self.site, "type": kind, "data": data}, separators=(",", ":")).encode() + b"\n")
|
||||
return True
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class StdoutSink:
|
||||
def __init__(self, site: str):
|
||||
self.site = site
|
||||
|
||||
async def send(self, seq, kind, payload) -> bool:
|
||||
sys.stdout.write(_encode(self.site, seq, kind, payload).decode() + "\n")
|
||||
sys.stdout.flush()
|
||||
return True
|
||||
|
||||
async def publish(self, kind: str, data: dict) -> bool:
|
||||
sys.stdout.write(json.dumps({"site": self.site, "type": kind, "data": data}) + "\n")
|
||||
sys.stdout.flush()
|
||||
return True
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class NatsSink:
|
||||
def __init__(self, url: str, site: str, prefix: str, ensure_stream: bool = False):
|
||||
self.url, self.site, self.ensure = url, site, ensure_stream
|
||||
self.subject = f"{prefix}.{site}.tel"
|
||||
self.prefix = prefix
|
||||
self.nc = self.js = None
|
||||
|
||||
async def _connect(self):
|
||||
# No reconnect inside nats-py: the uploader retries with its own
|
||||
# backoff and a fresh connection, and keeps everything queued.
|
||||
# nats-py 2.10 retries a first connect forever even so (seen in
|
||||
# tests): the timeout turns that into a failure.
|
||||
self.nc = await asyncio.wait_for(
|
||||
nats.connect(self.url, allow_reconnect=False, connect_timeout=5, max_reconnect_attempts=0,
|
||||
error_cb=_quiet), 10)
|
||||
self.js = self.nc.jetstream(timeout=5)
|
||||
if self.ensure:
|
||||
await self.js.add_stream(StreamConfig(name="CTB_SITE_TEL", subjects=[f"{self.prefix}.*.tel"],
|
||||
duplicate_window=120))
|
||||
|
||||
async def send(self, seq, kind, payload) -> bool:
|
||||
try:
|
||||
if self.nc is None or not self.nc.is_connected:
|
||||
await self._connect()
|
||||
await self.js.publish(self.subject, _encode(self.site, seq, kind, payload),
|
||||
headers={"Nats-Msg-Id": f"{self.site}:{seq}"})
|
||||
return True
|
||||
except Exception as e: # any failure: keep the batch, try later
|
||||
log.warning("nats: batch %d not sent: %s", seq, str(e) or type(e).__name__)
|
||||
await self.close()
|
||||
return False
|
||||
|
||||
async def publish(self, kind: str, data: dict) -> bool:
|
||||
"""Plans and actions on <prefix>.<site>.<kind> (core NATS, no stream)."""
|
||||
try:
|
||||
if self.nc is None or not self.nc.is_connected:
|
||||
await self._connect()
|
||||
await self.nc.publish(f"{self.prefix}.{self.site}.{kind}", json.dumps(data).encode())
|
||||
return True
|
||||
except Exception as e:
|
||||
log.warning("nats: %s not published: %s", kind, str(e) or type(e).__name__)
|
||||
return False
|
||||
|
||||
async def close(self):
|
||||
if self.nc is not None:
|
||||
try:
|
||||
await self.nc.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.nc = self.js = None
|
||||
|
||||
|
||||
class LinkSink:
|
||||
"""The site's link (edge/link, Go) over its local socket: the link holds
|
||||
the site's key and NATS connection; a batch counts as sent once the link
|
||||
says JetStream stored it. Commands the link passes on (mode, control,
|
||||
limits, release; checked and signed by the link) go to on_command, an
|
||||
async (cmd, args) -> (ok, message) the agent sets; plans and actions go
|
||||
out with publish()."""
|
||||
|
||||
def __init__(self, path: str, site: str):
|
||||
self.path, self.site = path, site
|
||||
self.r = self.w = None
|
||||
self.reader = None
|
||||
self.waits: dict[int, asyncio.Future] = {}
|
||||
self.n = 0
|
||||
self.state_wait: asyncio.Future | None = None
|
||||
self.lock: asyncio.Lock | None = None
|
||||
self.on_command = None
|
||||
|
||||
async def _ensure(self):
|
||||
"""Connected, once: the uploader and the announcer (cloud()) share it."""
|
||||
if self.lock is None:
|
||||
self.lock = asyncio.Lock()
|
||||
async with self.lock:
|
||||
if self.w is None or self.w.is_closing() or (self.reader and self.reader.done()):
|
||||
await self.close()
|
||||
await self._connect()
|
||||
|
||||
async def _connect(self):
|
||||
self.r, self.w = await asyncio.wait_for(asyncio.open_unix_connection(self.path, limit=1 << 20), 5)
|
||||
self.w.write(b'{"op":"hello"}\n')
|
||||
await self.w.drain()
|
||||
hello = json.loads(await asyncio.wait_for(self.r.readline(), 5))
|
||||
if hello.get("op") != "hello" or hello.get("site") != self.site:
|
||||
raise RuntimeError(f"the link is for site {hello.get('site')!r}, this agent for {self.site!r}")
|
||||
self.reader = asyncio.create_task(self._read())
|
||||
|
||||
async def _read(self):
|
||||
try:
|
||||
while line := await self.r.readline():
|
||||
m = json.loads(line)
|
||||
if m.get("op") == "ack" and (f := self.waits.pop(m.get("id"), None)) and not f.done():
|
||||
f.set_result(m)
|
||||
elif m.get("op") == "state" and self.state_wait and not self.state_wait.done():
|
||||
self.state_wait.set_result(bool(m.get("ok")))
|
||||
elif m.get("op") == "command":
|
||||
asyncio.create_task(self._command(m))
|
||||
except Exception as e:
|
||||
log.warning("link: %s", str(e) or type(e).__name__)
|
||||
finally:
|
||||
for f in self.waits.values():
|
||||
if not f.done():
|
||||
f.set_exception(ConnectionError("link closed"))
|
||||
self.waits.clear()
|
||||
|
||||
async def _command(self, m: dict):
|
||||
ok, msg = False, "the agent takes no commands"
|
||||
if self.on_command is not None:
|
||||
try:
|
||||
ok, msg = await self.on_command(str(m.get("cmd", "")), dict(m.get("args") or {}))
|
||||
except Exception as e: # an answer always goes back
|
||||
ok, msg = False, f"failed: {e}"
|
||||
log.info("command %s (%s): %s %s", m.get("id"), m.get("cmd"), "done" if ok else "refused", msg)
|
||||
try:
|
||||
self.w.write(json.dumps({"op": "result", "id": m.get("id"), "ok": bool(ok),
|
||||
"message": str(msg)[:500]}).encode() + b"\n")
|
||||
await self.w.drain()
|
||||
except Exception as e:
|
||||
log.warning("link: result %s not sent: %s", m.get("id"), e)
|
||||
|
||||
async def publish(self, kind: str, data: dict) -> bool:
|
||||
"""A plan or an action: the link publishes it (ctb.site.<id>.<kind>),
|
||||
within its limits (a plan every 5 min, 120 actions an hour)."""
|
||||
try:
|
||||
await self._ensure()
|
||||
self.w.write(json.dumps({"op": kind, "data": data}, separators=(",", ":")).encode() + b"\n")
|
||||
await self.w.drain()
|
||||
return True
|
||||
except Exception as e:
|
||||
log.warning("link: %s not sent: %s", kind, str(e) or type(e).__name__)
|
||||
return False
|
||||
|
||||
async def cloud(self) -> bool:
|
||||
"""The link's NATS connection is up (the link's "state")."""
|
||||
try:
|
||||
await self._ensure()
|
||||
self.state_wait = asyncio.get_running_loop().create_future()
|
||||
self.w.write(b'{"op":"state"}\n')
|
||||
await self.w.drain()
|
||||
return await asyncio.wait_for(self.state_wait, 5)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def send(self, seq, kind, payload) -> bool:
|
||||
try:
|
||||
await self._ensure()
|
||||
self.n += 1
|
||||
n, f = self.n, asyncio.get_running_loop().create_future()
|
||||
self.waits[n] = f
|
||||
data = json.loads(_encode(self.site, seq, kind, payload))
|
||||
self.w.write(json.dumps({"op": "tel", "id": n, "msg_id": f"{self.site}:{seq}", "data": data},
|
||||
separators=(",", ":")).encode() + b"\n")
|
||||
await self.w.drain()
|
||||
ack = await asyncio.wait_for(f, 20)
|
||||
if not ack.get("ok"):
|
||||
log.warning("link: batch %d not stored: %s", seq, ack.get("error", "?"))
|
||||
return bool(ack.get("ok"))
|
||||
except Exception as e: # any failure: keep the batch, try later
|
||||
log.warning("link: batch %d not sent: %s", seq, str(e) or type(e).__name__)
|
||||
await self.close()
|
||||
return False
|
||||
|
||||
async def close(self):
|
||||
if self.reader is not None:
|
||||
self.reader.cancel()
|
||||
if self.w is not None:
|
||||
try:
|
||||
self.w.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.r = self.w = self.reader = None
|
||||
|
||||
|
||||
def make(cfg) -> object:
|
||||
s = cfg.sink
|
||||
if s.kind == "file":
|
||||
return FileSink(s.path, cfg.site_id)
|
||||
if s.kind == "stdout":
|
||||
return StdoutSink(cfg.site_id)
|
||||
if s.kind == "link":
|
||||
return LinkSink(s.path, cfg.site_id)
|
||||
if s.kind == "nats":
|
||||
return NatsSink(s.url, cfg.site_id, s.subject_prefix, s.ensure_stream)
|
||||
raise ValueError(f"unknown sink {s.kind}")
|
||||
Reference in new issue
Block a user