1import router from "./routes.js";
2import User from "../../seed/models/user.js";
3import Node from "../../seed/models/node.js";
4
5export async function init(core) {
6  core.llm.registerFailoverResolver(async (userId, rootId) => {
7    // Tree-level stack first (tree owner's backups apply to everyone)
8    let treeStack = [];
9    if (rootId) {
10      const root = await Node.findById(rootId).select("metadata").lean();
11      const rootMeta = root?.metadata instanceof Map ? Object.fromEntries(root.metadata) : (root?.metadata || {});
12      treeStack = rootMeta.llm?.failoverStack || [];
13    }
14
15    // User-level stack (personal fallbacks)
16    const user = await User.findById(userId).select("metadata").lean();
17    const userMeta = user?.metadata instanceof Map ? Object.fromEntries(user.metadata) : (user?.metadata || {});
18    const userStack = userMeta.llm?.failoverStack || [];
19
20    // Deduplicate: tree stack wins position
21    const seen = new Set();
22    const combined = [];
23    for (const id of [...treeStack, ...userStack]) {
24      if (!seen.has(id)) { seen.add(id); combined.push(id); }
25    }
26    return combined;
27  });
28
29  return { router };
30}
31

1export default {
2  name: "llm-failover",
3  version: "1.0.1",
4  builtFor: "TreeOS",
5  description:
6    "Failover stack management for LLM connections. Push backup connections " +
7    "onto per-user and per-tree stacks (max 10 each). When the primary " +
8    "connection fails (429, 500, 502, 503, 504, timeout), the conversation " +
9    "system walks the tree-level stack first, then the user-level stack, " +
10    "until one succeeds. Tree-level failover lets tree owners configure " +
11    "backup connections that apply to everyone in that tree.",
12
13  needs: {
14    services: ["llm"],
15    models: ["User", "Node"],
16  },
17
18  optional: {
19    extensions: ["html-rendering"],
20  },
21
22  provides: {
23    models: {},
24    routes: "./routes.js",
25    tools: false,
26    jobs: false,
27    orchestrator: false,
28    energyActions: {},
29    sessionTypes: {},
30
31    cli: [
32      // User-level failover
33      { command: "llm failover", scope: ["home"], description: "Show your failover stack", method: "GET", endpoint: "/user/:userId/llm-failover" },
34      { command: "llm failover-push <connectionId>", scope: ["home"], description: "Add a connection to your failover stack", method: "POST", endpoint: "/user/:userId/llm-failover", body: ["connectionId"] },
35      { command: "llm failover-pop", scope: ["home"], description: "Remove last connection from your failover stack", method: "DELETE", endpoint: "/user/:userId/llm-failover" },
36      // Tree-level failover
37      { command: "llm tree-failover", scope: ["tree"], description: "Show tree failover stack", method: "GET", endpoint: "/root/:rootId/llm-failover" },
38      { command: "llm tree-failover-push <connectionId>", scope: ["tree"], description: "Add a connection to tree failover stack", method: "POST", endpoint: "/root/:rootId/llm-failover", body: ["connectionId"] },
39      { command: "llm tree-failover-pop", scope: ["tree"], description: "Remove last connection from tree failover stack", method: "DELETE", endpoint: "/root/:rootId/llm-failover" },
40    ],
41  },
42};
43

1import express from "express";
2import User from "../../seed/models/user.js";
3import Node from "../../seed/models/node.js";
4import LlmConnection from "../../seed/models/llmConnection.js";
5import authenticate from "../../seed/middleware/authenticate.js";
6import { sendOk, sendError, ERR } from "../../seed/protocol.js";
7import { getUserMeta, setUserMeta } from "../../seed/tree/userMetadata.js";
8
9const router = express.Router();
10const MAX_STACK = 10;
11
12// ── User-Level Failover ──────────────────────────────────────────────────
13
14router.get("/user/:userId/llm-failover", authenticate, async (req, res) => {
15  try {
16    const user = await User.findById(req.userId).select("metadata").lean();
17    const meta = user?.metadata instanceof Map ? Object.fromEntries(user.metadata) : (user?.metadata || {});
18    const stack = meta.llm?.failoverStack || [];
19    sendOk(res, { stack });
20  } catch (err) {
21    sendError(res, 500, ERR.INTERNAL, err.message);
22  }
23});
24
25router.post("/user/:userId/llm-failover", authenticate, async (req, res) => {
26  try {
27    const { connectionId } = req.body;
28    if (!connectionId) return sendError(res, 400, ERR.INVALID_INPUT, "connectionId required");
29
30    const user = await User.findById(req.userId);
31    if (!user) return sendError(res, 404, ERR.USER_NOT_FOUND, "User not found");
32
33    const conn = await LlmConnection.findOne({ _id: connectionId, userId: req.userId }).lean();
34    if (!conn) return sendError(res, 404, ERR.NODE_NOT_FOUND, "Connection not found or not yours");
35
36    if (user.llmDefault === connectionId) {
37      return sendError(res, 400, ERR.INVALID_INPUT, "That is already your default connection. Failover is for backups.");
38    }
39
40    const llmMeta = getUserMeta(user, "llm") || {};
41    const stack = llmMeta.failoverStack || [];
42
43    if (stack.includes(connectionId)) return sendError(res, 400, ERR.INVALID_INPUT, "Already in failover stack");
44    if (stack.length >= MAX_STACK) return sendError(res, 400, ERR.INVALID_INPUT, `Failover stack full (max ${MAX_STACK})`);
45    stack.push(connectionId);
46
47    llmMeta.failoverStack = stack;
48    setUserMeta(user, "llm", llmMeta);
49    await user.save();
50
51    sendOk(res, { stack, added: conn.name || connectionId });
52  } catch (err) {
53    sendError(res, 500, ERR.INTERNAL, err.message);
54  }
55});
56
57router.delete("/user/:userId/llm-failover/:connectionId", authenticate, async (req, res) => {
58  try {
59    const user = await User.findById(req.userId);
60    if (!user) return sendError(res, 404, ERR.USER_NOT_FOUND, "User not found");
61
62    const llmMeta = getUserMeta(user, "llm") || {};
63    const stack = llmMeta.failoverStack || [];
64    const idx = stack.indexOf(req.params.connectionId);
65    if (idx === -1) return sendError(res, 404, ERR.NODE_NOT_FOUND, "Not in stack");
66    stack.splice(idx, 1);
67
68    llmMeta.failoverStack = stack;
69    setUserMeta(user, "llm", llmMeta);
70    await user.save();
71
72    sendOk(res, { removed: req.params.connectionId, stack });
73  } catch (err) {
74    sendError(res, 500, ERR.INTERNAL, err.message);
75  }
76});
77
78router.delete("/user/:userId/llm-failover", authenticate, async (req, res) => {
79  try {
80    const user = await User.findById(req.userId);
81    if (!user) return sendError(res, 404, ERR.USER_NOT_FOUND, "User not found");
82
83    const llmMeta = getUserMeta(user, "llm") || {};
84    const stack = llmMeta.failoverStack || [];
85    const removed = stack.pop();
86
87    llmMeta.failoverStack = stack;
88    setUserMeta(user, "llm", llmMeta);
89    await user.save();
90
91    sendOk(res, { removed, stack });
92  } catch (err) {
93    sendError(res, 500, ERR.INTERNAL, err.message);
94  }
95});
96
97// ── Tree-Level Failover ──────────────────────────────────────────────────
98
99router.get("/root/:rootId/llm-failover", authenticate, async (req, res) => {
100  try {
101    const root = await Node.findById(req.params.rootId).select("rootOwner metadata").lean();
102    if (!root) return sendError(res, 404, ERR.TREE_NOT_FOUND, "Root not found");
103    if (!root.rootOwner) return sendError(res, 400, ERR.INVALID_INPUT, "Node is not a root");
104
105    const meta = root.metadata instanceof Map ? Object.fromEntries(root.metadata) : (root.metadata || {});
106    const stack = meta.llm?.failoverStack || [];
107    sendOk(res, { stack });
108  } catch (err) {
109    sendError(res, 500, ERR.INTERNAL, err.message);
110  }
111});
112
113router.post("/root/:rootId/llm-failover", authenticate, async (req, res) => {
114  try {
115    const { connectionId } = req.body;
116    if (!connectionId) return sendError(res, 400, ERR.INVALID_INPUT, "connectionId required");
117
118    const root = await Node.findById(req.params.rootId).select("rootOwner llmDefault metadata").lean();
119    if (!root) return sendError(res, 404, ERR.TREE_NOT_FOUND, "Root not found");
120    if (!root.rootOwner) return sendError(res, 400, ERR.INVALID_INPUT, "Node is not a root");
121    if (root.rootOwner.toString() !== req.userId.toString()) {
122      return sendError(res, 403, ERR.FORBIDDEN, "Only the root owner can manage tree failover");
123    }
124
125    const conn = await LlmConnection.findOne({ _id: connectionId, userId: req.userId }).lean();
126    if (!conn) return sendError(res, 404, ERR.NODE_NOT_FOUND, "Connection not found or not yours");
127
128    if (root.llmDefault === connectionId) {
129      return sendError(res, 400, ERR.INVALID_INPUT, "That is already the tree default. Failover is for backups.");
130    }
131
132    const meta = root.metadata instanceof Map ? Object.fromEntries(root.metadata) : (root.metadata || {});
133    const llmMeta = meta.llm || {};
134    const stack = llmMeta.failoverStack || [];
135
136    if (stack.includes(connectionId)) return sendError(res, 400, ERR.INVALID_INPUT, "Already in tree failover stack");
137    if (stack.length >= MAX_STACK) return sendError(res, 400, ERR.INVALID_INPUT, `Tree failover stack full (max ${MAX_STACK})`);
138    stack.push(connectionId);
139
140    await Node.findByIdAndUpdate(req.params.rootId, {
141      $set: { "metadata.llm.failoverStack": stack },
142    });
143
144    const { clearUserClientCache } = await import("../../seed/llm/conversation.js");
145    clearUserClientCache(req.userId);
146
147    sendOk(res, { stack, added: conn.name || connectionId });
148  } catch (err) {
149    sendError(res, 500, ERR.INTERNAL, err.message);
150  }
151});
152
153router.delete("/root/:rootId/llm-failover/:connectionId", authenticate, async (req, res) => {
154  try {
155    const root = await Node.findById(req.params.rootId).select("rootOwner metadata").lean();
156    if (!root) return sendError(res, 404, ERR.TREE_NOT_FOUND, "Root not found");
157    if (!root.rootOwner) return sendError(res, 400, ERR.INVALID_INPUT, "Node is not a root");
158    if (root.rootOwner.toString() !== req.userId.toString()) {
159      return sendError(res, 403, ERR.FORBIDDEN, "Only the root owner can manage tree failover");
160    }
161
162    const meta = root.metadata instanceof Map ? Object.fromEntries(root.metadata) : (root.metadata || {});
163    const llmMeta = meta.llm || {};
164    const stack = llmMeta.failoverStack || [];
165    const idx = stack.indexOf(req.params.connectionId);
166    if (idx === -1) return sendError(res, 404, ERR.NODE_NOT_FOUND, "Not in tree failover stack");
167    stack.splice(idx, 1);
168
169    await Node.findByIdAndUpdate(req.params.rootId, {
170      $set: { "metadata.llm.failoverStack": stack },
171    });
172
173    const { clearUserClientCache } = await import("../../seed/llm/conversation.js");
174    clearUserClientCache(req.userId);
175
176    sendOk(res, { removed: req.params.connectionId, stack });
177  } catch (err) {
178    sendError(res, 500, ERR.INTERNAL, err.message);
179  }
180});
181
182router.delete("/root/:rootId/llm-failover", authenticate, async (req, res) => {
183  try {
184    const root = await Node.findById(req.params.rootId).select("rootOwner metadata").lean();
185    if (!root) return sendError(res, 404, ERR.TREE_NOT_FOUND, "Root not found");
186    if (!root.rootOwner) return sendError(res, 400, ERR.INVALID_INPUT, "Node is not a root");
187    if (root.rootOwner.toString() !== req.userId.toString()) {
188      return sendError(res, 403, ERR.FORBIDDEN, "Only the root owner can manage tree failover");
189    }
190
191    const meta = root.metadata instanceof Map ? Object.fromEntries(root.metadata) : (root.metadata || {});
192    const llmMeta = meta.llm || {};
193    const stack = llmMeta.failoverStack || [];
194    const removed = stack.pop();
195
196    await Node.findByIdAndUpdate(req.params.rootId, {
197      $set: { "metadata.llm.failoverStack": stack },
198    });
199
200    const { clearUserClientCache } = await import("../../seed/llm/conversation.js");
201    clearUserClientCache(req.userId);
202
203    sendOk(res, { removed, stack });
204  } catch (err) {
205    sendError(res, 500, ERR.INTERNAL, err.message);
206  }
207});
208
209export default router;
210