1/**
2 * Trace Core
3 *
4 * Follow one concept through the entire tree chronologically.
5 * Multi-step pipeline: search across all branches, then AI synthesis.
6 * Uses OrchestratorRuntime for abort, locking, and step tracking.
7 */
8
9import log from "../../seed/log.js";
10import Node from "../../seed/models/node.js";
11import Note from "../../seed/models/note.js";
12import { CONTENT_TYPE } from "../../seed/protocol.js";
13import { getDescendantIds } from "../../seed/tree/treeFetch.js";
14import { OrchestratorRuntime } from "../../seed/orchestrators/runtime.js";
15
16let _metadata = null;
17export function configure({ metadata }) { _metadata = metadata; }
18
19let LLM_PRIORITY;
20try {
21  ({ LLM_PRIORITY } = await import("../../seed/llm/conversation.js"));
22} catch {
23  LLM_PRIORITY = { INTERACTIVE: 2 };
24}
25
26// ─────────────────────────────────────────────────────────────────────────
27// SEARCH
28// ─────────────────────────────────────────────────────────────────────────
29
30/**
31 * Expand query words using codebook dictionary entries if available.
32 */
33async function expandQuery(query, nodeId, userId) {
34  const queryWords = query.toLowerCase().split(/\s+/).filter(w => w.length > 2);
35
36  try {
37    const { getExtension } = await import("../loader.js");
38    const codebook = getExtension("codebook");
39    if (!codebook?.exports?.getDictionary) return queryWords;
40
41    const dictionary = await codebook.exports.getDictionary(nodeId, userId);
42    if (!dictionary || Object.keys(dictionary).length === 0) return queryWords;
43
44    const expanded = new Set(queryWords);
45    for (const [shorthand, meaning] of Object.entries(dictionary)) {
46      const shortLower = shorthand.toLowerCase();
47      const meaningLower = (meaning || "").toLowerCase();
48      // If any query word matches a shorthand or its meaning, expand with the other
49      if (queryWords.some(w => shortLower.includes(w) || meaningLower.includes(w))) {
50        // Add words from the meaning
51        meaningLower.split(/\s+/).filter(w => w.length > 2).forEach(w => expanded.add(w));
52        expanded.add(shortLower);
53      }
54    }
55
56    return [...expanded];
57  } catch {
58    return queryWords;
59  }
60}
61
62/**
63 * Find all notes matching the query across the entire tree, chronologically.
64 */
65async function searchNotes(rootId, queryWords, opts = {}) {
66  // Get all node IDs in this tree
67  const descendantIds = await getDescendantIds(rootId, { maxResults: 10000 });
68  const allIds = [rootId, ...descendantIds];
69
70  // Build filter
71  const noteQuery = {
72    nodeId: { $in: allIds },
73    contentType: CONTENT_TYPE.TEXT,
74  };
75  if (opts.since) noteQuery.createdAt = { $gte: opts.since };
76  if (opts.userId) noteQuery.userId = opts.userId;
77
78  const notes = await Note.find(noteQuery)
79    .select("_id nodeId content createdAt userId")
80    .sort({ createdAt: 1 }) // chronological
81    .lean();
82
83  // Score each note
84  const matches = [];
85  const minScore = opts.minScore || 0.3;
86
87  for (const n of notes) {
88    const content = (n.content || "").toLowerCase();
89    const matchCount = queryWords.filter(w => content.includes(w)).length;
90    if (matchCount === 0) continue;
91    const score = matchCount / queryWords.length;
92    if (score < minScore) continue;
93
94    matches.push({
95      noteId: String(n._id),
96      nodeId: String(n.nodeId),
97      content: n.content.slice(0, 300),
98      date: n.createdAt,
99      userId: n.userId,
100      score,
101    });
102  }
103
104  return matches;
105}
106
107/**
108 * Run semantic search if embed is available and merge results.
109 */
110async function addSemanticResults(matches, rootId, query, userId) {
111  try {
112    const { getExtension } = await import("../loader.js");
113    const embed = getExtension("embed");
114    if (!embed?.exports?.findSimilar || !embed?.exports?.generateEmbedding) return matches;
115
116    const queryVector = await embed.exports.generateEmbedding(query, userId);
117    if (!queryVector) return matches;
118
119    const similar = await embed.exports.findSimilar(queryVector, rootId, {
120      similarityThreshold: 0.7,
121      maxResults: 50,
122    });
123
124    if (!similar || similar.length === 0) return matches;
125
126    // Merge: add semantic results not already in text matches
127    const existingNotes = new Set(matches.map(m => m.noteId));
128    for (const s of similar) {
129      const noteId = s.noteId || null;
130      if (noteId && existingNotes.has(noteId)) {
131        // Boost existing match score
132        const existing = matches.find(m => m.noteId === noteId);
133        if (existing) existing.score = Math.min(existing.score + 0.2, 1.0);
134      } else {
135        matches.push({
136          noteId: noteId || `embed:${s.nodeId}`,
137          nodeId: s.nodeId,
138          content: (s.snippet || s.content || "").slice(0, 300),
139          date: s.date || null,
140          userId: s.userId || null,
141          score: s.similarity || 0.7,
142        });
143      }
144    }
145
146    // Re-sort chronologically (semantic results may not have dates)
147    matches.sort((a, b) => {
148      if (!a.date) return 1;
149      if (!b.date) return -1;
150      return new Date(a.date) - new Date(b.date);
151    });
152
153    return matches;
154  } catch (err) {
155    log.debug("Trace", `Semantic search failed: ${err.message}`);
156    return matches;
157  }
158}
159
160// ─────────────────────────────────────────────────────────────────────────
161// MAIN PIPELINE
162// ─────────────────────────────────────────────────────────────────────────
163
164/**
165 * Trace a concept through the entire tree chronologically.
166 *
167 * @param {string} rootId - tree root
168 * @param {string} query - concept to trace
169 * @param {string} userId
170 * @param {string} username
171 * @param {object} opts - { since, minScore, maxResults }
172 */
173export async function runTrace(rootId, query, userId, username, opts = {}) {
174  const rt = new OrchestratorRuntime({
175    rootId,
176    userId,
177    username: username || "system",
178    visitorId: `trace:${userId}:${rootId}:${Date.now()}`,
179    sessionType: "TRACE",
180    description: `Tracing: ${query}`,
181    modeKeyForLlm: "tree:trace",
182    lockNamespace: "trace",
183    lockKey: `trace:${rootId}`,
184    llmPriority: LLM_PRIORITY?.INTERACTIVE || 2,
185  });
186
187  const ok = await rt.init(query);
188  if (!ok) {
189    return { error: "Trace already running on this tree" };
190  }
191
192  try {
193    // Step 1: Expand query with codebook
194    const queryWords = await expandQuery(query, rootId, userId);
195
196    rt.trackStep("tree:trace", {
197      input: { phase: "expand-query", originalQuery: query },
198      output: { expandedWords: queryWords.length },
199      startTime: Date.now(),
200      endTime: Date.now(),
201    });
202
203    if (rt.aborted) {
204      rt.setError("Trace cancelled", "tree:trace");
205      return { error: "Trace cancelled" };
206    }
207
208    // Step 2: Search all notes chronologically
209    const startSearch = Date.now();
210    let matches = await searchNotes(rootId, queryWords, {
211      since: opts.since ? new Date(opts.since) : null,
212      userId: opts.filterUserId || null,
213      minScore: opts.minScore || 0.3,
214    });
215
216    // Add semantic results if embed is available
217    matches = await addSemanticResults(matches, rootId, query, userId);
218
219    // Cap results
220    const maxResults = opts.maxResults || 100;
221    if (matches.length > maxResults) {
222      matches = matches.slice(0, maxResults);
223    }
224
225    rt.trackStep("tree:trace", {
226      input: { phase: "search", queryWords: queryWords.length },
227      output: { matchCount: matches.length },
228      startTime: startSearch,
229      endTime: Date.now(),
230    });
231
232    if (matches.length === 0) {
233      rt.setResult("No matches found", "tree:trace");
234      return {
235        query,
236        matches: 0,
237        origin: null,
238        touchpoints: [],
239        currentState: "No notes found referencing this concept.",
240        unresolved: [],
241        threadLength: null,
242        crossBranch: false,
243      };
244    }
245
246    if (rt.aborted) {
247      rt.setError("Trace cancelled", "tree:trace");
248      return { error: "Trace cancelled" };
249    }
250
251    // Step 3: Enrich with node names
252    const nodeIds = [...new Set(matches.map(m => m.nodeId))];
253    const nodes = await Node.find({ _id: { $in: nodeIds } }).select("_id name parent").lean();
254    const nodeMap = new Map(nodes.map(n => [String(n._id), n]));
255
256    for (const m of matches) {
257      const node = nodeMap.get(m.nodeId);
258      m.nodeName = node?.name || "unknown";
259    }
260
261    // Detect if thread crosses branches (different parent chains)
262    const parentSet = new Set(nodes.map(n => n.parent ? String(n.parent) : null).filter(Boolean));
263    const crossBranch = parentSet.size > 1;
264
265    // Step 4: AI synthesis via runStep
266    const threadText = matches
267      .map(m => {
268        const dateStr = m.date ? new Date(m.date).toISOString().slice(0, 10) : "unknown";
269        return `[${dateStr}] ${m.nodeName}: "${m.content}"`;
270      })
271      .join("\n");
272
273    let parsed = null;
274    try {
275      const result = await rt.runStep("tree:trace", {
276        prompt: `Trace the thread "${query}" through this tree.
277
278${matches.length} notes found across ${nodeIds.length} nodes, chronologically:
279
280${threadText}
281
282Trace how this concept evolved. Where did it start? How did it change at each stop? What's the current state? What's unresolved?
283
284Return ONLY JSON:
285{
286  "origin": { "nodeId": "...", "nodeName": "...", "date": "...", "summary": "..." },
287  "touchpoints": [{ "nodeId": "...", "nodeName": "...", "date": "...", "what": "..." }],
288  "currentState": "where this thread stands now",
289  "unresolved": ["open questions or incomplete work"],
290  "threadLength": "timespan from first to last",
291  "crossBranch": ${crossBranch}
292}`,
293      });
294      parsed = result?.parsed;
295    } catch (err) {
296      log.debug("Trace", `Synthesis failed: ${err.message}`);
297    }
298
299    // Step 5: Build result
300    const currentState = parsed?.currentState || "See touchpoints for chronological thread.";
301    const traceResult = {
302      query,
303      answer: currentState,
304      matches: matches.length,
305      nodesVisited: nodeIds.length,
306      origin: parsed?.origin || { nodeId: matches[0]?.nodeId, nodeName: matches[0]?.nodeName, date: matches[0]?.date, summary: "First occurrence" },
307      touchpoints: parsed?.touchpoints || matches.slice(0, 20).map(m => ({ nodeId: m.nodeId, nodeName: m.nodeName, date: m.date, what: m.content.slice(0, 100) })),
308      currentState,
309      unresolved: parsed?.unresolved || [],
310      threadLength: parsed?.threadLength || null,
311      crossBranch,
312      tracedAt: new Date().toISOString(),
313    };
314
315    // Write to metadata for working memory
316    try {
317      const rootNode = await Node.findById(rootId);
318      if (rootNode) {
319        const meta = _metadata.getExtMeta(rootNode, "trace") || {};
320        const history = meta.history || [];
321        history.unshift({
322          query,
323          matches: matches.length,
324          nodesVisited: nodeIds.length,
325          crossBranch,
326          tracedAt: traceResult.tracedAt,
327        });
328        meta.history = history.slice(0, 10);
329        meta.lastTrace = traceResult;
330        await _metadata.setExtMeta(rootNode, "trace", meta);
331      }
332    } catch (err) {
333      log.debug("Trace", `Failed to write trace metadata: ${err.message}`);
334    }
335
336    rt.setResult(traceResult.currentState || `Traced ${matches.length} mentions across ${nodeIds.length} nodes`, "tree:trace");
337    return traceResult;
338
339  } catch (err) {
340    rt.setError(err.message, "tree:trace");
341    throw err;
342  } finally {
343    await rt.cleanup();
344  }
345}
346
347// ─────────────────────────────────────────────────────────────────────────
348// MAP ACCESS
349// ─────────────────────────────────────────────────────────────────────────
350
351export async function getTraceMap(nodeId) {
352  const node = await Node.findById(nodeId).select("metadata").lean();
353  if (!node) return null;
354  const meta = node.metadata instanceof Map
355    ? node.metadata.get("trace") || {}
356    : node.metadata?.trace || {};
357  return meta.lastTrace || null;
358}
359

1import log from "../../seed/log.js";
2import tools from "./tools.js";
3import traceMode from "./modes/trace.js";
4import { configure, runTrace, getTraceMap } from "./core.js";
5
6export async function init(core) {
7  configure({ metadata: core.metadata });
8  core.modes.registerMode("tree:trace", traceMode, "trace");
9
10  // enrichContext: inject last trace summary if fresh
11  core.hooks.register("enrichContext", async ({ context, node, meta }) => {
12    const traceMeta = meta?.trace;
13    if (!traceMeta?.lastTrace) return;
14
15    // Only inject if trace is fresh (last 7 days)
16    if (traceMeta.lastTrace.tracedAt) {
17      const age = Date.now() - new Date(traceMeta.lastTrace.tracedAt).getTime();
18      if (age > 7 * 24 * 60 * 60 * 1000) return;
19    }
20
21    context.recentTrace = {
22      query: traceMeta.lastTrace.query,
23      matches: traceMeta.lastTrace.matches,
24      currentState: traceMeta.lastTrace.currentState,
25      crossBranch: traceMeta.lastTrace.crossBranch,
26    };
27  }, "trace");
28
29  const { default: router } = await import("./routes.js");
30
31  log.verbose("Trace", "Trace loaded");
32
33  return {
34    router,
35    tools,
36    exports: {
37      runTrace,
38      getTraceMap,
39    },
40  };
41}
42

1export default {
2  name: "trace",
3  version: "1.0.1",
4  builtFor: "treeos-intelligence",
5  description:
6    "Follow one thread through the entire tree. Every node it touched, in order. " +
7    "Not broad search like scout. Not downward exploration like explore. One concept, " +
8    "every note that references it across the whole tree, chronologically. Where did " +
9    "it start? How did it evolve at each stop? What's the current state? What's unresolved?",
10
11  needs: {
12    services: ["hooks", "llm", "metadata", "session"],
13    models: ["Node", "Note"],
14  },
15
16  optional: {
17    extensions: ["embed", "codebook", "long-memory", "inverse-tree"],
18  },
19
20  provides: {
21    models: {},
22    routes: "./routes.js",
23    tools: true,
24    jobs: false,
25    orchestrator: false,
26    energyActions: {},
27    sessionTypes: {
28      TRACE: "trace",
29    },
30
31    hooks: {
32      fires: [],
33      listens: ["enrichContext"],
34    },
35
36    cli: [
37      {
38        command: "trace [concept...]", scope: ["tree"],
39        description: "Follow a concept through the tree",
40        method: "POST",
41        endpoint: "/node/:nodeId/trace",
42        subcommands: {
43          map: {
44            method: "GET",
45            endpoint: "/node/:nodeId/trace/map",
46            description: "Show last trace as node map",
47          },
48        },
49      },
50    ],
51  },
52};
53

1export default {
2  emoji: "🧵",
3  label: "Trace",
4  bigMode: "tree",
5  hidden: true,
6  toolNames: ["trace-query", "trace-map"],
7  buildSystemPrompt() {
8    return `You trace concepts through a tree chronologically. You receive timestamped notes from across the tree that reference a concept. Your job is to reconstruct the narrative: where the concept originated, how it evolved, where it stands now, and what remains unresolved.
9
10Return ONLY JSON:
11{
12  "origin": { "nodeId": "...", "nodeName": "...", "date": "...", "summary": "..." },
13  "touchpoints": [{ "nodeId": "...", "nodeName": "...", "date": "...", "what": "..." }],
14  "currentState": "where this thread stands now",
15  "unresolved": ["open questions or incomplete work"],
16  "threadLength": "timespan from first to last",
17  "crossBranch": true/false
18}`;
19  },
20};
21

1import express from "express";
2import authenticate from "../../seed/middleware/authenticate.js";
3import { sendOk, sendError, ERR } from "../../seed/protocol.js";
4import { runTrace, getTraceMap } from "./core.js";
5
6const router = express.Router();
7
8// POST /node/:nodeId/trace - trace a concept through the tree
9router.post("/node/:nodeId/trace", authenticate, async (req, res) => {
10  try {
11    const { query, since } = req.body || {};
12    if (!query || typeof query !== "string") {
13      return sendError(res, 400, ERR.INVALID_INPUT, "query is required");
14    }
15
16    // Resolve root for trace (traces the whole tree from root)
17    let rootId = req.params.nodeId;
18    try {
19      const { resolveRootNode } = await import("../../seed/tree/treeFetch.js");
20      const root = await resolveRootNode(rootId);
21      if (root?._id) rootId = root._id;
22    } catch {
23      // Use nodeId as root
24    }
25
26    const result = await runTrace(rootId, query, req.userId, req.username || "system", { since });
27    if (result.error) {
28      return sendError(res, 409, ERR.RESOURCE_CONFLICT, result.error);
29    }
30    sendOk(res, result);
31  } catch (err) {
32    sendError(res, 500, ERR.INTERNAL, err.message);
33  }
34});
35
36// GET /node/:nodeId/trace/map - last trace map
37router.get("/node/:nodeId/trace/map", authenticate, async (req, res) => {
38  try {
39    const map = await getTraceMap(req.params.nodeId);
40    if (!map) return sendOk(res, { message: "No trace map at this position." });
41    sendOk(res, map);
42  } catch (err) {
43    sendError(res, 500, ERR.INTERNAL, err.message);
44  }
45});
46
47export default router;
48

1import { z } from "zod";
2import { runTrace, getTraceMap } from "./core.js";
3
4export default [
5  {
6    name: "trace-query",
7    description:
8      "Follow one concept through the entire tree chronologically. Finds every note " +
9      "that references the concept across all branches, ordered by time. Shows where " +
10      "it started, how it evolved, and what's unresolved.",
11    schema: {
12      nodeId: z.string().describe("The tree root or starting node."),
13      query: z.string().describe("The concept to trace."),
14      since: z.string().optional().describe("Time filter: ISO date or relative (7d, 30d)."),
15      userId: z.string().describe("Injected by server. Ignore."),
16      username: z.string().optional().describe("Injected by server. Ignore."),
17      chatId: z.string().nullable().optional().describe("Injected by server. Ignore."),
18      sessionId: z.string().nullable().optional().describe("Injected by server. Ignore."),
19    },
20    annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
21    handler: async ({ nodeId, query, since, userId, username }) => {
22      try {
23        const result = await runTrace(nodeId, query, userId, username || "system", { since });
24        if (result.error) {
25          return { content: [{ type: "text", text: result.error }] };
26        }
27
28        // Human-readable output for AI to relay
29        const parts = [];
30        if (result.origin?.summary) {
31          parts.push(`Origin: ${result.origin.nodeName} (${result.origin.date || "unknown date"}) - ${result.origin.summary}`);
32        }
33        if (result.touchpoints?.length > 0) {
34          parts.push(`\nThread (${result.matches} mentions across ${result.nodesVisited} nodes${result.crossBranch ? ", cross-branch" : ""}):`);
35          for (const tp of result.touchpoints.slice(0, 10)) {
36            parts.push(`  ${tp.nodeName} (${tp.date || "?"}) - ${tp.what}`);
37          }
38          if (result.touchpoints.length > 10) parts.push(`  ...and ${result.touchpoints.length - 10} more`);
39        }
40        if (result.currentState) parts.push(`\nCurrent state: ${result.currentState}`);
41        if (result.unresolved?.length > 0) parts.push(`\nUnresolved: ${result.unresolved.join("; ")}`);
42        if (result.threadLength) parts.push(`\nThread span: ${result.threadLength}`);
43
44        return {
45          content: [{ type: "text", text: parts.join("\n") || "No trace data found." }],
46        };
47      } catch (err) {
48        return { content: [{ type: "text", text: `Trace failed: ${err.message}` }] };
49      }
50    },
51  },
52  {
53    name: "trace-map",
54    description: "Show the last trace run at this position. The thread map without re-tracing.",
55    schema: {
56      nodeId: z.string().describe("The node to check."),
57      userId: z.string().describe("Injected by server. Ignore."),
58      chatId: z.string().nullable().optional().describe("Injected by server. Ignore."),
59      sessionId: z.string().nullable().optional().describe("Injected by server. Ignore."),
60    },
61    annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
62    handler: async ({ nodeId }) => {
63      try {
64        const map = await getTraceMap(nodeId);
65        if (!map) return { content: [{ type: "text", text: "No trace map at this position. Run trace-query first." }] };
66        return { content: [{ type: "text", text: JSON.stringify(map, null, 2) }] };
67      } catch (err) {
68        return { content: [{ type: "text", text: `Failed: ${err.message}` }] };
69      }
70    },
71  },
72];
73