cantabile_edge, cantabile_modbus, cantabile_eebus 0.1.676: full source

This commit is contained in:
Setti Solutions S.R.L committed 2026-11-04 10:33:14 +02:00
1 parent 3edd015039
commit d48b9aff87
32 files changed
+4633 -133

No files matched your search

+130 -27
View File
@@ -17,7 +17,19 @@ are.
Trees(enc, n_features).predict(rows) sums the leaves in LightGBM's order
with its decision rule (tree.h NumericalDecision), so it gives LightGBM's
raw score to the last bit (measured 2026-11-04: max
relative difference 0.0 over 192 slots)."""
relative difference 0.0 over 192 slots).
lgbm-trees-v2 (2026-11-04, methods.md B10), written only for a booster
that needs it (a categorical split or linear leaves; else v1, which older
agents read): per tree uint16 L, uint8 flags (bit 0: linear leaves); the
L-1 nodes as v1, decision bit 0 set for a categorical split, whose
threshold is then the index of its category set; uint16 the number of
sets, each uint8 n and n uint16 categories (those going left); the L
leaf values; with linear leaves per leaf float64 constant, uint8 n and n
(uint8 feature, float64 coefficient). Categorical rule (tree.h
CategoricalDecision): NaN or a negative value right, else left if the
integer is in the set. A linear leaf: constant plus the coefficients
times the values in LightGBM's order; a NaN among them: the leaf's value."""
from __future__ import annotations
import base64
@@ -26,6 +38,7 @@ import struct
import zlib
ENCODING = "lgbm-trees-v1"
ENCODING_V2 = "lgbm-trees-v2"
NODE = struct.Struct("<BBhhd")
MISSING = {"None": 0, "Zero": 1, "NaN": 2}
ZERO = 1.0000000180025095e-35 # LightGBM's kZeroThreshold (1e-35f)
@@ -36,44 +49,86 @@ def _ref(n: dict) -> int:
return int(n["split_index"]) if "split_index" in n else ~int(n.get("leaf_index", 0))
def pack_tree(root: dict) -> bytes:
"""One tree (dump_model()'s tree_info[i]["tree_structure"])."""
if "split_index" not in root: # no split: one leaf
return struct.pack("<Hd", 1, float(root["leaf_value"]))
def _walk(root: dict) -> tuple[dict[int, dict], dict[int, dict]]:
nodes: dict[int, dict] = {}
leaves: dict[int, float] = {}
leaves: dict[int, dict] = {}
stack = [root]
while stack:
n = stack.pop()
if "split_index" in n:
if n.get("decision_type", "<=") != "<=":
raise ValueError("a categorical split: not supported")
if n.get("decision_type", "<=") not in ("<=", "=="):
raise ValueError(f"a split {n.get('decision_type')!r}: not supported")
nodes[int(n["split_index"])] = n
stack += [n["left_child"], n["right_child"]]
else:
leaves[int(n["leaf_index"])] = float(n["leaf_value"])
leaves[int(n.get("leaf_index", 0))] = n
k = len(leaves)
if sorted(nodes) != list(range(k - 1)) or sorted(leaves) != list(range(k)):
raise ValueError("a tree's node or leaf indices are not contiguous")
return nodes, leaves
def _needs_v2(dump: dict) -> bool:
for t in dump["tree_info"]:
nodes, leaves = _walk(t["tree_structure"])
if any(n.get("decision_type") == "==" for n in nodes.values()) or any("leaf_coeff" in x for x in leaves.values()):
return True
return False
def pack_tree(root: dict, v2: bool = False) -> bytes:
"""One tree (dump_model()'s tree_info[i]["tree_structure"])."""
if "split_index" not in root and not v2: # no split: one leaf
return struct.pack("<Hd", 1, float(root["leaf_value"]))
nodes, leaves = _walk(root)
k = len(leaves)
linear = v2 and any("leaf_coeff" in x for x in leaves.values())
out = [struct.pack("<H", k)]
if v2:
out.append(struct.pack("<B", 1 if linear else 0))
cats: list[list[int]] = []
for i in range(k - 1):
n = nodes[i]
dec = (2 if n.get("default_left") else 0) | (MISSING[n.get("missing_type", "None")] << 2)
out.append(NODE.pack(int(n["split_feature"]), dec, _ref(n["left_child"]), _ref(n["right_child"]),
float(n["threshold"])))
out.append(struct.pack(f"<{k}d", *(leaves[i] for i in range(k))))
threshold = n["threshold"]
if n.get("decision_type") == "==":
if not v2:
raise ValueError("a categorical split: lgbm-trees-v2")
dec |= 1
cats.append([int(c) for c in str(threshold).split("||") if c != ""])
threshold = len(cats) - 1
out.append(NODE.pack(int(n["split_feature"]), dec, _ref(n["left_child"]), _ref(n["right_child"]), float(threshold)))
if v2:
out.append(struct.pack("<H", len(cats)))
for c in cats:
if len(c) > 255 or any(not 0 <= x < 65536 for x in c):
raise ValueError("a categorical split: at most 255 categories below 65536")
out.append(struct.pack(f"<B{len(c)}H", len(c), *c))
out.append(struct.pack(f"<{k}d", *(float(leaves[i]["leaf_value"]) for i in range(k))))
if linear:
for i in range(k):
x = leaves[i]
feats, coeff = [int(f) for f in x.get("leaf_features") or []], [float(c) for c in x.get("leaf_coeff") or []]
if len(feats) != len(coeff) or len(feats) > 255:
raise ValueError("a linear leaf")
out.append(struct.pack("<dB", float(x.get("leaf_const", 0.0)), len(feats)))
for f, c in zip(feats, coeff):
out.append(struct.pack("<Bd", f, c))
return b"".join(out)
def pack(dump: dict) -> dict:
"""A booster's dump_model() as lgbm-trees-v1."""
"""A booster's dump_model() as lgbm-trees-v1, or v2 where it has
categorical splits or linear leaves."""
if dump.get("num_class", 1) != 1 or dump.get("num_tree_per_iteration", 1) != 1:
raise ValueError("one tree per round only")
obj = str(dump.get("objective") or "").split(" ")[0]
if obj != "regression":
raise ValueError(f"objective {obj!r}: regression only (no output transform)")
raw = b"".join(pack_tree(t["tree_structure"]) for t in dump["tree_info"])
return {"enc": ENCODING, "n": len(dump["tree_info"]), "data": base64.b64encode(zlib.compress(raw, 9)).decode()}
v2 = _needs_v2(dump)
raw = b"".join(pack_tree(t["tree_structure"], v2) for t in dump["tree_info"])
return {"enc": ENCODING_V2 if v2 else ENCODING, "n": len(dump["tree_info"]),
"data": base64.b64encode(zlib.compress(raw, 9)).decode()}
class Trees:
@@ -81,8 +136,9 @@ class Trees:
leaf (no loops), features in range, finite leaves."""
def __init__(self, enc: dict, n_features: int):
if not isinstance(enc, dict) or enc.get("enc") != ENCODING:
raise ValueError(f"trees: {ENCODING} only")
if not isinstance(enc, dict) or enc.get("enc") not in (ENCODING, ENCODING_V2):
raise ValueError(f"trees: {ENCODING} or {ENCODING_V2} only")
v2 = enc["enc"] == ENCODING_V2
z = zlib.decompressobj()
try:
raw = z.decompress(base64.b64decode(str(enc.get("data")), validate=True), MAX_RAW)
@@ -100,6 +156,13 @@ class Trees:
off += 2
if not 1 <= k <= MAX_LEAVES:
raise ValueError("trees: leaves")
linear = False
if v2:
(flags,) = struct.unpack_from("<B", raw, off)
off += 1
if flags & ~1:
raise ValueError("trees: flags")
linear = bool(flags & 1)
nodes = []
for i in range(k - 1):
f, dec, left, right, threshold = NODE.unpack_from(raw, off)
@@ -107,14 +170,44 @@ class Trees:
for c in (left, right):
if not (i < c < k - 1 or -k <= c < 0):
raise ValueError("trees: a child out of range")
if f >= n_features or dec & 1 or (dec >> 2) & 3 > 2 or math.isnan(threshold):
if f >= n_features or (dec & 1 and not v2) or (dec >> 2) & 3 > 2 or math.isnan(threshold):
raise ValueError("trees: a split")
nodes.append((f, threshold, bool(dec & 2), (dec >> 2) & 3, left, right))
nodes.append([f, threshold, bool(dec & 2), (dec >> 2) & 3, left, right, bool(dec & 1)])
if v2:
(ncat,) = struct.unpack_from("<H", raw, off)
off += 2
sets = []
for _ in range(ncat):
(m,) = struct.unpack_from("<B", raw, off)
sets.append(frozenset(struct.unpack_from(f"<{m}H", raw, off + 1)))
off += 1 + 2 * m
for nd in nodes:
if nd[6]:
if not 0 <= nd[1] < len(sets) or nd[1] != int(nd[1]):
raise ValueError("trees: a category set")
nd[1] = sets[int(nd[1])]
nodes = [tuple(nd) for nd in nodes]
leaves = struct.unpack_from(f"<{k}d", raw, off)
off += 8 * k
if not all(math.isfinite(v) for v in leaves):
raise ValueError("trees: a leaf")
self.trees.append((nodes, leaves))
lin = None
if linear:
lin = []
for _ in range(k):
const, m = struct.unpack_from("<dB", raw, off)
off += 9
terms = []
for _ in range(m):
f, c = struct.unpack_from("<Bd", raw, off)
off += 9
if f >= n_features or not math.isfinite(c):
raise ValueError("trees: a linear leaf")
terms.append((f, c))
if not math.isfinite(const):
raise ValueError("trees: a linear leaf")
lin.append((const, terms))
self.trees.append((nodes, leaves, lin))
if len(self.trees) > MAX_TREES:
raise ValueError("trees: too many")
except struct.error as e:
@@ -128,21 +221,31 @@ class Trees:
split knows NaN; the split's missing values (zero or NaN, by its
type) go its default way; the leaves summed in order."""
total = 0.0
for nodes, leaves in self.trees:
if not nodes:
total += leaves[0]
continue
i = 0
for nodes, leaves, lin in self.trees:
i = 0 if nodes else ~0
while i >= 0:
f, threshold, default_left, missing, left, right = nodes[i]
f, threshold, default_left, missing, left, right, cat = nodes[i]
v = x[f]
if cat:
i = right if v != v or v < 0 or int(v) not in threshold else left
continue
if v != v and missing != 2:
v = 0.0
if (missing == 1 and -ZERO <= v <= ZERO) or (missing == 2 and v != v):
i = left if default_left else right
else:
i = left if v <= threshold else right
total += leaves[~i]
if lin is None:
total += leaves[~i]
continue
out = lin[~i][0]
for f, c in lin[~i][1]:
v = x[f]
if v != v:
out = leaves[~i]
break
out += c * v
total += out
return total
def predict(self, rows) -> list[float]: