1/**
2 * Evolve Core
3 *
4 * Two phases:
5 * 1. Pattern detection: afterNote and afterLLMCall record behavioral signals
6 *    in a rolling window. When a pattern repeats enough times, it becomes
7 *    a detected pattern stored on the land root metadata.
8 *
9 * 2. Proposal generation: a background job reads detected patterns, searches
10 *    Horizon for matching extensions, and either suggests installation or
11 *    generates a spec for an extension that doesn't exist yet.
12 */
13
14import log from "../../seed/log.js";
15import Node from "../../seed/models/node.js";
16import Note from "../../seed/models/note.js";
17import { SYSTEM_ROLE, CONTENT_TYPE } from "../../seed/protocol.js";
18import { parseJsonSafe } from "../../seed/orchestrators/helpers.js";
19import { v4 as uuidv4 } from "uuid";
20
21let _runChat = null;
22let _metadata = null;
23export function setRunChat(fn) { _runChat = fn; }
24export function configure({ metadata }) { _metadata = metadata; }
25
26const MAX_PATTERNS = 30;
27const MAX_PROPOSALS = 20;
28const MIN_OCCURRENCES = 10; // pattern must appear this many times before action
29const WINDOW_SIZE = 200;    // rolling signal window
30
31// ─────────────────────────────────────────────────────────────────────────
32// SIGNAL RECORDING
33// ─────────────────────────────────────────────────────────────────────────
34
35// In-memory rolling window of behavioral signals
36let signalWindow = [];
37
38/**
39 * Record a behavioral signal from a note or LLM interaction.
40 */
41export function recordSignal(signal) {
42  signalWindow.push({ ...signal, ts: Date.now() });
43  if (signalWindow.length > WINDOW_SIZE) {
44    signalWindow = signalWindow.slice(-WINDOW_SIZE);
45  }
46}
47
48/**
49 * Detect behavioral patterns from the signal window.
50 * Returns array of { type, description, count, examples }.
51 */
52function detectPatterns() {
53  if (signalWindow.length < MIN_OCCURRENCES) return [];
54
55  const patterns = [];
56
57  // Pattern: notes containing dollar amounts (values extension gap)
58  const dollarNotes = signalWindow.filter(s =>
59    s.type === "note" && s.content && /\$\d+|\d+\s*dollars?/i.test(s.content)
60  );
61  if (dollarNotes.length >= MIN_OCCURRENCES) {
62    patterns.push({
63      type: "numeric-values",
64      description: "Notes frequently contain dollar amounts or numeric values",
65      count: dollarNotes.length,
66      suggestedExtension: "values",
67      examples: dollarNotes.slice(-3).map(s => s.content?.slice(0, 80)),
68    });
69  }
70
71  // Pattern: notes containing dates/schedules
72  const dateNotes = signalWindow.filter(s =>
73    s.type === "note" && s.content && /\b(monday|tuesday|wednesday|thursday|friday|saturday|sunday|tomorrow|next week|deadline|due|by \d|schedule)\b/i.test(s.content)
74  );
75  if (dateNotes.length >= MIN_OCCURRENCES) {
76    patterns.push({
77      type: "scheduling",
78      description: "Notes frequently reference dates, deadlines, or schedules",
79      count: dateNotes.length,
80      suggestedExtension: "schedules",
81      examples: dateNotes.slice(-3).map(s => s.content?.slice(0, 80)),
82    });
83  }
84
85  // Pattern: notes containing URLs
86  const urlNotes = signalWindow.filter(s =>
87    s.type === "note" && s.content && /https?:\/\/[^\s]+/i.test(s.content)
88  );
89  if (urlNotes.length >= MIN_OCCURRENCES) {
90    patterns.push({
91      type: "url-content",
92      description: "Notes frequently contain URLs that could be auto-extracted",
93      count: urlNotes.length,
94      suggestedExtension: null, // no existing extension for this
95      examples: urlNotes.slice(-3).map(s => s.content?.slice(0, 80)),
96    });
97  }
98
99  // Pattern: AI frequently says "I don't have information about..."
100  const silenceResponses = signalWindow.filter(s =>
101    s.type === "llm-response" && s.hadAnswer === false
102  );
103  if (silenceResponses.length >= MIN_OCCURRENCES) {
104    patterns.push({
105      type: "knowledge-gap",
106      description: "AI frequently cannot answer questions at certain positions",
107      count: silenceResponses.length,
108      suggestedExtension: "learn",
109      examples: silenceResponses.slice(-3).map(s => s.query?.slice(0, 80)),
110    });
111  }
112
113  // Pattern: user frequently switches between two branches
114  const navSignals = signalWindow.filter(s => s.type === "navigation");
115  if (navSignals.length >= 20) {
116    const pathPairs = new Map();
117    for (let i = 1; i < navSignals.length; i++) {
118      const from = navSignals[i - 1].nodeId;
119      const to = navSignals[i].nodeId;
120      if (from && to && from !== to) {
121        const key = [from, to].sort().join(":");
122        pathPairs.set(key, (pathPairs.get(key) || 0) + 1);
123      }
124    }
125    for (const [pair, count] of pathPairs) {
126      if (count >= 5) {
127        patterns.push({
128          type: "frequent-path",
129          description: `User frequently navigates between the same two positions (${count} times)`,
130          count,
131          suggestedExtension: "channels",
132          examples: [pair],
133        });
134        break; // only report the most frequent
135      }
136    }
137  }
138
139  return patterns;
140}
141
142// ─────────────────────────────────────────────────────────────────────────
143// PATTERN STORAGE
144// ─────────────────────────────────────────────────────────────────────────
145
146async function getLandRoot() {
147  return Node.findOne({ systemRole: SYSTEM_ROLE.LAND_ROOT });
148}
149
150/**
151 * Run detection from the signal window and store results.
152 * Called by the background job.
153 */
154export async function detectAndStorePatterns() {
155  const patterns = detectPatterns();
156  if (patterns.length > 0) {
157    await storePatterns(patterns);
158  }
159  return patterns;
160}
161
162/**
163 * Write detected patterns to land root metadata.
164 */
165export async function storePatterns(patterns) {
166  const landRoot = await getLandRoot();
167  if (!landRoot) return;
168
169  const meta = _metadata.getExtMeta(landRoot, "evolve") || {};
170  const existing = meta.patterns || [];
171  const dismissed = new Set((meta.dismissed || []).map(d => d.type));
172
173  // Merge: update existing patterns, add new ones, skip dismissed
174  for (const p of patterns) {
175    if (dismissed.has(p.type)) continue;
176    const idx = existing.findIndex(e => e.type === p.type);
177    if (idx >= 0) {
178      existing[idx].count = p.count;
179      existing[idx].lastSeen = new Date().toISOString();
180      existing[idx].examples = p.examples;
181    } else {
182      existing.push({
183        id: uuidv4(),
184        ...p,
185        firstSeen: new Date().toISOString(),
186        lastSeen: new Date().toISOString(),
187        status: "detected", // detected, proposed, approved, dismissed
188      });
189    }
190  }
191
192  meta.patterns = existing.slice(0, MAX_PATTERNS);
193  await _metadata.setExtMeta(landRoot, "evolve", meta);
194}
195
196// ─────────────────────────────────────────────────────────────────────────
197// PROPOSAL GENERATION
198// ─────────────────────────────────────────────────────────────────────────
199
200/**
201 * For each detected pattern, either suggest an existing extension
202 * or generate a spec for a new one.
203 */
204export async function generateProposals() {
205  const landRoot = await getLandRoot();
206  if (!landRoot) return [];
207
208  const meta = _metadata.getExtMeta(landRoot, "evolve") || {};
209  const patterns = (meta.patterns || []).filter(p => p.status === "detected" && p.count >= MIN_OCCURRENCES);
210
211  if (patterns.length === 0) return [];
212  if (!_runChat) return [];
213
214  // Check which extensions are installed
215  let installedNames = new Set();
216  try {
217    const { getLoadedManifests } = await import("../../extensions/loader.js");
218    installedNames = new Set(getLoadedManifests().map(m => m.name));
219  } catch {}
220
221  // Check Horizon for matching extensions
222  let registryExts = [];
223  try {
224    const horizonUrl = process.env.HORIZON_URL?.split(",")[0]?.trim();
225    if (horizonUrl) {
226      const res = await fetch(`${horizonUrl}/extensions?limit=100`, {
227        signal: AbortSignal.timeout(10000),
228      });
229      if (res.ok) {
230        const data = await res.json();
231        registryExts = data.extensions || [];
232      }
233    }
234  } catch {}
235
236  const registryNames = new Set(registryExts.map(e => e.name));
237  const proposals = meta.proposals || [];
238
239  for (const pattern of patterns) {
240    // Already proposed?
241    if (proposals.some(p => p.patternType === pattern.type)) continue;
242
243    if (pattern.suggestedExtension) {
244      // Known extension suggestion
245      if (installedNames.has(pattern.suggestedExtension)) {
246        // Already installed. Pattern might be a false positive. Skip.
247        pattern.status = "resolved";
248        continue;
249      }
250
251      const inRegistry = registryNames.has(pattern.suggestedExtension);
252      proposals.push({
253        id: uuidv4(),
254        patternType: pattern.type,
255        type: "install",
256        extensionName: pattern.suggestedExtension,
257        inRegistry,
258        reason: pattern.description,
259        occurrences: pattern.count,
260        createdAt: new Date().toISOString(),
261        status: "pending",
262      });
263      pattern.status = "proposed";
264    } else {
265      // No known extension. Generate a spec via AI.
266      try {
267        const prompt =
268          `A tree user exhibits this behavioral pattern:\n` +
269          `Pattern: ${pattern.description}\n` +
270          `Occurrences: ${pattern.count}\n` +
271          `Examples: ${(pattern.examples || []).join("; ")}\n\n` +
272          `No existing extension handles this. Design one.\n\n` +
273          `Return ONLY JSON following this format:\n` +
274          `{\n` +
275          `  "name": "extension-name",\n` +
276          `  "description": "one sentence",\n` +
277          `  "hooks": { "listens": ["afterNote"], "fires": [] },\n` +
278          `  "tools": [{ "name": "tool-name", "description": "what it does" }],\n` +
279          `  "cli": [{ "command": "cmd-name", "description": "what it does" }],\n` +
280          `  "enrichContext": "what it injects into AI context",\n` +
281          `  "needs": { "services": ["hooks"], "models": ["Node"] },\n` +
282          `  "rationale": "why this extension would help"\n` +
283          `}`;
284
285        const { answer } = await _runChat({
286          userId: "SYSTEM",
287          username: "evolve",
288          message: prompt,
289          mode: "tree:respond",
290          rootId: null,
291          slot: "evolve",
292        });
293
294        const spec = answer ? parseJsonSafe(answer) : null;
295        if (spec && spec.name) {
296          proposals.push({
297            id: uuidv4(),
298            patternType: pattern.type,
299            type: "spec",
300            spec,
301            reason: pattern.description,
302            occurrences: pattern.count,
303            createdAt: new Date().toISOString(),
304            status: "pending",
305          });
306          pattern.status = "proposed";
307        }
308      } catch (err) {
309        log.debug("Evolve", `Spec generation failed for ${pattern.type}: ${err.message}`);
310      }
311    }
312  }
313
314  meta.proposals = proposals.slice(0, MAX_PROPOSALS);
315  meta.patterns = meta.patterns; // preserve updates
316  await _metadata.setExtMeta(landRoot, "evolve", meta);
317
318  return proposals.filter(p => p.status === "pending");
319}
320
321// ─────────────────────────────────────────────────────────────────────────
322// READ / MANAGE
323// ─────────────────────────────────────────────────────────────────────────
324
325export async function getPatterns() {
326  const landRoot = await getLandRoot();
327  if (!landRoot) return [];
328  const meta = _metadata.getExtMeta(landRoot, "evolve") || {};
329  return (meta.patterns || []).filter(p => p.status !== "dismissed" && p.status !== "resolved");
330}
331
332export async function getProposals() {
333  const landRoot = await getLandRoot();
334  if (!landRoot) return [];
335  const meta = _metadata.getExtMeta(landRoot, "evolve") || {};
336  return (meta.proposals || []).filter(p => p.status === "pending");
337}
338
339export async function dismissPattern(patternId) {
340  const landRoot = await getLandRoot();
341  if (!landRoot) return null;
342
343  const meta = _metadata.getExtMeta(landRoot, "evolve") || {};
344  const pattern = (meta.patterns || []).find(p => p.id === patternId);
345  if (!pattern) return null;
346
347  pattern.status = "dismissed";
348  if (!meta.dismissed) meta.dismissed = [];
349  meta.dismissed.push({ type: pattern.type, dismissedAt: new Date().toISOString() });
350
351  // Also dismiss any proposals for this pattern
352  for (const p of (meta.proposals || [])) {
353    if (p.patternType === pattern.type) p.status = "dismissed";
354  }
355
356  await _metadata.setExtMeta(landRoot, "evolve", meta);
357  return pattern;
358}
359
360export async function approveProposal(proposalId) {
361  const landRoot = await getLandRoot();
362  if (!landRoot) return null;
363
364  const meta = _metadata.getExtMeta(landRoot, "evolve") || {};
365  const proposal = (meta.proposals || []).find(p => p.id === proposalId);
366  if (!proposal) return null;
367
368  proposal.status = "approved";
369  proposal.approvedAt = new Date().toISOString();
370
371  await _metadata.setExtMeta(landRoot, "evolve", meta);
372  return proposal;
373}
374

1import log from "../../seed/log.js";
2import tools from "./tools.js";
3import {
4  configure, setRunChat, recordSignal, detectAndStorePatterns, generateProposals,
5  getPatterns, getProposals,
6} from "./core.js";
7
8let _jobTimer = null;
9
10export async function init(core) {
11  configure({ metadata: core.metadata });
12  core.llm.registerRootLlmSlot("evolve");
13  const BG = core.llm.LLM_PRIORITY.BACKGROUND;
14  setRunChat(async (opts) => {
15    if (opts.userId && opts.userId !== "SYSTEM" && !await core.llm.userHasLlm(opts.userId)) return { answer: null };
16    return core.llm.runChat({ ...opts, llmPriority: BG });
17  });
18
19  // afterNote: record note content patterns
20  core.hooks.register("afterNote", async ({ note, nodeId, userId, contentType, action }) => {
21    if (contentType !== "text" || action !== "create") return;
22    if (!userId || userId === "SYSTEM") return;
23
24    recordSignal({
25      type: "note",
26      content: (note?.content || "").slice(0, 200),
27      nodeId,
28      userId,
29    });
30  }, "evolve");
31
32  // afterLLMCall: record interaction patterns
33  core.hooks.register("afterLLMCall", async ({ userId, rootId, mode, hasToolCalls }) => {
34    if (!userId || userId === "SYSTEM") return;
35
36    recordSignal({
37      type: "llm-response",
38      hadToolCalls: !!hasToolCalls,
39      mode: mode || "unknown",
40      rootId,
41      userId,
42    });
43  }, "evolve");
44
45  const { default: router } = await import("./routes.js");
46
47  log.verbose("Evolve", "Evolve loaded");
48
49  return {
50    router,
51    tools,
52    jobs: [
53      {
54        name: "evolve-cycle",
55        start: () => {
56          _jobTimer = setInterval(async () => {
57            try {
58              await detectAndStorePatterns();
59              await generateProposals();
60            } catch (err) {
61              log.debug("Evolve", `Cycle failed: ${err.message}`);
62            }
63          }, 6 * 60 * 60 * 1000);
64          if (_jobTimer.unref) _jobTimer.unref();
65        },
66        stop: () => {
67          if (_jobTimer) { clearInterval(_jobTimer); _jobTimer = null; }
68        },
69      },
70    ],
71    exports: {
72      getPatterns,
73      getProposals,
74      generateProposals,
75    },
76  };
77}
78

1export default {
2  name: "evolve",
3  version: "1.0.1",
4  builtFor: "treeos-intelligence",
5  description:
6    "The tree imagines what it could become. Watches the gap between what users do and " +
7    "what extensions handle. When users repeatedly do something manually that could be " +
8    "automated, evolve notices. For patterns matching existing directory extensions: " +
9    "suggest installation. For patterns matching nothing in the directory: generate an " +
10    "extension spec. The tree doesn't code. It specs. The spec follows EXTENSION_FORMAT.md. " +
11    "A developer reads it and builds from it. The operator always decides.",
12
13  needs: {
14    services: ["hooks", "llm", "metadata"],
15    models: ["Node", "Note"],
16  },
17
18  optional: {
19    extensions: [
20      "gap-detection",
21      "intent",
22      "evolution",
23      "inverse-tree",
24      "competence",
25    ],
26  },
27
28  provides: {
29    models: {},
30    routes: "./routes.js",
31    tools: true,
32    jobs: true,
33    orchestrator: false,
34    energyActions: {},
35    sessionTypes: {},
36
37    hooks: {
38      fires: [],
39      listens: ["afterNote", "afterLLMCall"],
40    },
41
42    cli: [
43      {
44        command: "evolve [action] [args...]",
45        scope: ["land"],
46        description: "Detected patterns and extension proposals. Actions: proposals, dismiss <id>, approve <id>",
47        method: "GET",
48        endpoint: "/land/evolve",
49        subcommands: {
50          proposals: {
51            method: "GET",
52            endpoint: "/land/evolve/proposals",
53            description: "Generated extension specs",
54          },
55          dismiss: {
56            method: "POST",
57            endpoint: "/land/evolve/dismiss",
58            args: ["id"],
59            description: "Dismiss a detected pattern",
60          },
61          approve: {
62            method: "POST",
63            endpoint: "/land/evolve/approve",
64            args: ["id"],
65            description: "Mark a proposal for building",
66          },
67        },
68      },
69    ],
70  },
71};
72

1import express from "express";
2import authenticate from "../../seed/middleware/authenticate.js";
3import { sendOk, sendError, ERR } from "../../seed/protocol.js";
4import { getPatterns, getProposals, dismissPattern, approveProposal } from "./core.js";
5
6const router = express.Router();
7
8// GET /land/evolve - patterns and proposals summary
9router.get("/land/evolve", authenticate, async (req, res) => {
10  try {
11    const patterns = await getPatterns();
12    const proposals = await getProposals();
13    sendOk(res, { patterns, proposals });
14  } catch (err) {
15    sendError(res, 500, ERR.INTERNAL, err.message);
16  }
17});
18
19// GET /land/evolve/proposals - just proposals
20router.get("/land/evolve/proposals", authenticate, async (req, res) => {
21  try {
22    const proposals = await getProposals();
23    sendOk(res, { proposals });
24  } catch (err) {
25    sendError(res, 500, ERR.INTERNAL, err.message);
26  }
27});
28
29// POST /land/evolve/dismiss - dismiss a pattern
30router.post("/land/evolve/dismiss", authenticate, async (req, res) => {
31  try {
32    const { id } = req.body;
33    if (!id) return sendError(res, 400, ERR.INVALID_INPUT, "id is required");
34    const result = await dismissPattern(id);
35    if (!result) return sendError(res, 404, ERR.NODE_NOT_FOUND, "Pattern not found");
36    sendOk(res, result);
37  } catch (err) {
38    sendError(res, 500, ERR.INTERNAL, err.message);
39  }
40});
41
42// POST /land/evolve/approve - approve a proposal
43router.post("/land/evolve/approve", authenticate, async (req, res) => {
44  try {
45    const { id } = req.body;
46    if (!id) return sendError(res, 400, ERR.INVALID_INPUT, "id is required");
47    const result = await approveProposal(id);
48    if (!result) return sendError(res, 404, ERR.NODE_NOT_FOUND, "Proposal not found");
49    sendOk(res, result);
50  } catch (err) {
51    sendError(res, 500, ERR.INTERNAL, err.message);
52  }
53});
54
55export default router;
56

1import { z } from "zod";
2import { getPatterns, getProposals, dismissPattern, approveProposal, generateProposals } from "./core.js";
3
4export default [
5  {
6    name: "evolve-status",
7    description:
8      "Show detected behavioral patterns and pending extension proposals. " +
9      "The tree noticed what users do that no extension handles.",
10    schema: {
11      userId: z.string().describe("Injected by server. Ignore."),
12    },
13    annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true },
14    handler: async () => {
15      try {
16        const patterns = await getPatterns();
17        const proposals = await getProposals();
18
19        if (patterns.length === 0 && proposals.length === 0) {
20          return { content: [{ type: "text", text: "No patterns detected yet. The tree needs more usage data." }] };
21        }
22
23        const lines = [];
24        if (patterns.length > 0) {
25          lines.push(`Detected patterns (${patterns.length}):`);
26          for (const p of patterns) {
27            lines.push(`  [${p.id?.slice(0, 8)}] ${p.description} (${p.count}x, ${p.status})`);
28          }
29        }
30        if (proposals.length > 0) {
31          lines.push(`\nProposals (${proposals.length}):`);
32          for (const p of proposals) {
33            if (p.type === "install") {
34              lines.push(`  [${p.id?.slice(0, 8)}] Install ${p.extensionName}${p.inRegistry ? " (in registry)" : " (not in registry)"}: ${p.reason}`);
35            } else {
36              lines.push(`  [${p.id?.slice(0, 8)}] New: "${p.spec?.name}" - ${p.spec?.description || p.reason}`);
37            }
38          }
39        }
40
41        return { content: [{ type: "text", text: lines.join("\n") }] };
42      } catch (err) {
43        return { content: [{ type: "text", text: `Failed: ${err.message}` }] };
44      }
45    },
46  },
47  {
48    name: "evolve-dismiss",
49    description: "Dismiss a detected pattern. The tree won't suggest it again.",
50    schema: {
51      patternId: z.string().describe("The pattern ID to dismiss."),
52      userId: z.string().describe("Injected by server. Ignore."),
53    },
54    annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true },
55    handler: async ({ patternId }) => {
56      try {
57        const result = await dismissPattern(patternId);
58        if (!result) return { content: [{ type: "text", text: "Pattern not found." }] };
59        return { content: [{ type: "text", text: `Dismissed: ${result.description}` }] };
60      } catch (err) {
61        return { content: [{ type: "text", text: `Failed: ${err.message}` }] };
62      }
63    },
64  },
65  {
66    name: "evolve-approve",
67    description: "Approve a proposal for building. Marks it as accepted.",
68    schema: {
69      proposalId: z.string().describe("The proposal ID to approve."),
70      userId: z.string().describe("Injected by server. Ignore."),
71    },
72    annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true },
73    handler: async ({ proposalId }) => {
74      try {
75        const result = await approveProposal(proposalId);
76        if (!result) return { content: [{ type: "text", text: "Proposal not found." }] };
77        if (result.type === "install") {
78          return { content: [{ type: "text", text: `Approved: install ${result.extensionName}. Use land-ext-install to install it.` }] };
79        }
80        return { content: [{ type: "text", text: `Approved: "${result.spec?.name}" spec. Share it or build it.` }] };
81      } catch (err) {
82        return { content: [{ type: "text", text: `Failed: ${err.message}` }] };
83      }
84    },
85  },
86];
87