1/**
2 * Competence Core
3 *
4 * Tracks which queries found answers and which found silence.
5 * Builds a competence boundary map from accumulated query history.
6 * No LLM calls. Purely observational.
7 */
8
9import log from "../../seed/log.js";
10import Node from "../../seed/models/node.js";
11
12let _metadata = null;
13export function configure({ metadata }) { _metadata = metadata; }
14
15const MAX_QUERIES = 100;
16
17// ─────────────────────────────────────────────────────────────────────────
18// RECORDING
19// ─────────────────────────────────────────────────────────────────────────
20
21/**
22 * Heuristic: did the AI's response actually answer the user's query?
23 * Checks for tool usage, note references, and hedging language.
24 */
25function detectAnswer(response) {
26  if (!response) return { hadAnswer: false, confidence: 0 };
27  const text = (response || "").toLowerCase();
28
29  // Signals of answering: tool calls happened, notes referenced, specific content cited
30  const answerSignals = [
31    /\baccording to\b/,
32    /\bnotes? (show|indicate|say|mention)/,
33    /\bhere('s| is) what/,
34    /\bfound\b.*\b(note|entry|record)/,
35    /\byou (wrote|created|recorded|noted)/,
36    /\bon \d{4}-\d{2}-\d{2}/,  // date references suggest citing real content
37  ];
38
39  // Signals of not answering: hedging, suggesting to look elsewhere
40  const silenceSignals = [
41    /\bi don't (have|see|find) (information|data|notes?|anything)/,
42    /\bno (information|data|notes?|records?) (about|on|for|regarding)/,
43    /\bi('m| am) not (sure|able|finding)/,
44    /\byou (might|could|should) (try|check|look)/,
45    /\bthere('s| is) no (data|information|content)/,
46    /\bi (can't|cannot) find/,
47  ];
48
49  let answerScore = 0;
50  let silenceScore = 0;
51
52  for (const p of answerSignals) {
53    if (p.test(text)) answerScore++;
54  }
55  for (const p of silenceSignals) {
56    if (p.test(text)) silenceScore++;
57  }
58
59  if (answerScore > silenceScore) {
60    return { hadAnswer: true, confidence: Math.min(0.5 + answerScore * 0.15, 1.0) };
61  }
62  if (silenceScore > 0) {
63    return { hadAnswer: false, confidence: Math.min(0.5 + silenceScore * 0.15, 1.0) };
64  }
65
66  // Ambiguous: assume answered if response is substantial
67  return { hadAnswer: text.length > 200, confidence: 0.4 };
68}
69
70/**
71 * Record a query result on a node.
72 */
73export async function recordQuery(nodeId, query, hadAnswer, confidence, userId) {
74  try {
75    const node = await Node.findById(nodeId);
76    if (!node) return;
77
78    const meta = _metadata.getExtMeta(node, "competence") || {};
79    if (!meta.queries) meta.queries = [];
80
81    meta.queries.push({
82      query: (query || "").slice(0, 200),
83      hadAnswer,
84      confidence,
85      timestamp: Date.now(),
86      userId,
87    });
88
89    // Cap rolling array
90    if (meta.queries.length > MAX_QUERIES) {
91      meta.queries = meta.queries.slice(-MAX_QUERIES);
92    }
93
94    // Recompute topics
95    const computed = computeCompetence(meta.queries);
96    meta.strongTopics = computed.strongTopics;
97    meta.weakTopics = computed.weakTopics;
98    meta.answerRate = computed.answerRate;
99    meta.lastUpdated = Date.now();
100
101    await _metadata.setExtMeta(node, "competence", meta);
102  } catch (err) {
103    log.debug("Competence", `recordQuery failed: ${err.message}`);
104  }
105}
106
107/**
108 * Process an afterLLMCall event to detect competence.
109 */
110export function processLLMCall({ nodeId, userId, message, answer }) {
111  if (!nodeId || !message) return;
112
113  // Only process user queries (not system prompts, not tool responses)
114  if (!userId || userId === "SYSTEM") return;
115
116  const { hadAnswer, confidence } = detectAnswer(answer);
117  recordQuery(nodeId, message, hadAnswer, confidence, userId);
118}
119
120// ─────────────────────────────────────────────────────────────────────────
121// COMPUTATION
122// ─────────────────────────────────────────────────────────────────────────
123
124/**
125 * Extract strong and weak topics from query history via word frequency.
126 */
127function computeCompetence(queries) {
128  if (!queries || queries.length === 0) {
129    return { strongTopics: [], weakTopics: [], answerRate: 0 };
130  }
131
132  const answered = queries.filter(q => q.hadAnswer);
133  const unanswered = queries.filter(q => !q.hadAnswer);
134  const answerRate = queries.length > 0 ? answered.length / queries.length : 0;
135
136  // Word frequency in answered vs unanswered queries
137  const stopWords = new Set(["the", "a", "an", "is", "are", "was", "were", "what", "how", "when", "where", "why", "who", "can", "do", "does", "did", "will", "would", "should", "could", "have", "has", "had", "been", "being", "this", "that", "these", "those", "for", "with", "about", "from", "into", "but", "and", "not", "any", "all", "some", "more", "most", "you", "your", "my", "me", "its"]);
138
139  function extractWords(queryList) {
140    const freq = new Map();
141    for (const q of queryList) {
142      const words = (q.query || "").toLowerCase().split(/\s+/).filter(w => w.length > 2 && !stopWords.has(w));
143      for (const w of words) {
144        freq.set(w, (freq.get(w) || 0) + 1);
145      }
146    }
147    return freq;
148  }
149
150  const answeredFreq = extractWords(answered);
151  const unansweredFreq = extractWords(unanswered);
152
153  // Strong topics: words that appear frequently in answered queries but rarely in unanswered
154  const strongTopics = [...answeredFreq.entries()]
155    .filter(([word, count]) => count >= 2 && (!unansweredFreq.has(word) || unansweredFreq.get(word) < count))
156    .sort((a, b) => b[1] - a[1])
157    .slice(0, 10)
158    .map(([word]) => word);
159
160  // Weak topics: words that appear frequently in unanswered queries but rarely in answered
161  const weakTopics = [...unansweredFreq.entries()]
162    .filter(([word, count]) => count >= 2 && (!answeredFreq.has(word) || answeredFreq.get(word) < count))
163    .sort((a, b) => b[1] - a[1])
164    .slice(0, 10)
165    .map(([word]) => word);
166
167  return { strongTopics, weakTopics, answerRate };
168}
169
170// ─────────────────────────────────────────────────────────────────────────
171// READ
172// ─────────────────────────────────────────────────────────────────────────
173
174/**
175 * Get the competence map for a node.
176 */
177export async function getCompetence(nodeId) {
178  const node = await Node.findById(nodeId).select("metadata").lean();
179  if (!node) return null;
180
181  const meta = node.metadata instanceof Map
182    ? node.metadata.get("competence") || {}
183    : node.metadata?.competence || {};
184
185  return {
186    totalQueries: (meta.queries || []).length,
187    answered: (meta.queries || []).filter(q => q.hadAnswer).length,
188    unanswered: (meta.queries || []).filter(q => !q.hadAnswer).length,
189    answerRate: meta.answerRate || 0,
190    strongTopics: meta.strongTopics || [],
191    weakTopics: meta.weakTopics || [],
192    lastUpdated: meta.lastUpdated || null,
193  };
194}
195

1import log from "../../seed/log.js";
2import tools from "./tools.js";
3import { configure, processLLMCall, getCompetence } from "./core.js";
4
5export async function init(core) {
6  configure({ metadata: core.metadata });
7  // afterLLMCall: track which modes produced tool calls (proxy for finding answers)
8  core.hooks.register("afterLLMCall", async ({ userId, rootId, mode, hasToolCalls }) => {
9    if (!rootId || !userId || userId === "SYSTEM") return;
10
11    try {
12      processLLMCall({ rootId, userId, mode, hasToolCalls });
13    } catch (err) {
14      log.debug("Competence", `afterLLMCall processing failed: ${err.message}`);
15    }
16  }, "competence");
17
18  // enrichContext: tell the AI what it can and can't help with
19  core.hooks.register("enrichContext", async ({ context, node, meta }) => {
20    const comp = meta?.competence;
21    if (!comp?.queries?.length || comp.queries.length < 10) return; // need enough data
22
23    const strong = comp.strongTopics?.slice(0, 5) || [];
24    const weak = comp.weakTopics?.slice(0, 5) || [];
25
26    if (strong.length > 0 || weak.length > 0) {
27      context.competence = {
28        canHelpWith: strong,
29        noDataOn: weak,
30        answerRate: comp.answerRate,
31      };
32    }
33  }, "competence");
34
35  const { default: router } = await import("./routes.js");
36
37  log.verbose("Competence", "Competence loaded");
38
39  return {
40    router,
41    tools,
42    exports: {
43      getCompetence,
44    },
45  };
46}
47

1export default {
2  name: "competence",
3  version: "1.0.1",
4  builtFor: "treeos-intelligence",
5  description:
6    "The tree knows where its knowledge ends. Tracks which queries found answers " +
7    "and which found silence. Over time builds a map of the tree's competence boundary. " +
8    "The AI injects: I can help with X, Y, Z at this branch. I don't have information " +
9    "about A or B. Honest about limits instead of hallucinating.",
10
11  needs: {
12    services: ["hooks", "metadata"],
13    models: ["Node"],
14  },
15
16  optional: {
17    extensions: ["embed", "explore", "gap-detection"],
18  },
19
20  provides: {
21    models: {},
22    routes: "./routes.js",
23    tools: true,
24    jobs: false,
25    orchestrator: false,
26    energyActions: {},
27    sessionTypes: {},
28
29    hooks: {
30      fires: [],
31      listens: ["afterLLMCall", "enrichContext"],
32    },
33
34    cli: [
35      {
36        command: "competence", scope: ["tree"],
37        description: "Knowledge boundaries at this position",
38        method: "GET",
39        endpoint: "/node/:nodeId/competence",
40      },
41    ],
42  },
43};
44

1import express from "express";
2import authenticate from "../../seed/middleware/authenticate.js";
3import { sendOk, sendError, ERR } from "../../seed/protocol.js";
4import { getCompetence } from "./core.js";
5
6const router = express.Router();
7
8// GET /node/:nodeId/competence
9router.get("/node/:nodeId/competence", authenticate, async (req, res) => {
10  try {
11    const comp = await getCompetence(req.params.nodeId);
12    if (!comp || comp.totalQueries === 0) {
13      return sendOk(res, { message: "No competence data yet.", totalQueries: 0 });
14    }
15    sendOk(res, comp);
16  } catch (err) {
17    sendError(res, 500, ERR.INTERNAL, err.message);
18  }
19});
20
21export default router;
22

1import { z } from "zod";
2import { getCompetence } from "./core.js";
3
4export default [
5  {
6    name: "competence-status",
7    description:
8      "Knowledge boundaries at this position. Shows what topics the tree can help " +
9      "with and what it has no data on. Based on accumulated query history.",
10    schema: {
11      nodeId: z.string().describe("The node to check competence for."),
12      userId: z.string().describe("Injected by server. Ignore."),
13      chatId: z.string().nullable().optional().describe("Injected by server. Ignore."),
14      sessionId: z.string().nullable().optional().describe("Injected by server. Ignore."),
15    },
16    annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
17    handler: async ({ nodeId }) => {
18      try {
19        const comp = await getCompetence(nodeId);
20        if (!comp || comp.totalQueries === 0) {
21          return { content: [{ type: "text", text: "No competence data yet. The tree needs more queries to map its knowledge boundaries." }] };
22        }
23        return {
24          content: [{
25            type: "text",
26            text: JSON.stringify({
27              totalQueries: comp.totalQueries,
28              answered: comp.answered,
29              unanswered: comp.unanswered,
30              answerRate: `${(comp.answerRate * 100).toFixed(0)}%`,
31              canHelpWith: comp.strongTopics,
32              noDataOn: comp.weakTopics,
33            }, null, 2),
34          }],
35        };
36      } catch (err) {
37        return { content: [{ type: "text", text: `Failed: ${err.message}` }] };
38      }
39    },
40  },
41];
42