feat: port contexts layer and MCP parity

This commit is contained in:
“BeeRad”
2026-04-10 19:41:26 +10:00
parent a8c5506fb7
commit 35f9ecf89c
62 changed files with 4206 additions and 1224 deletions
+45 -40
View File
@@ -5,6 +5,7 @@ import { paperExtractTool } from '@/tools/other/paperExtract';
import { formatNodeForChat } from '@/tools/infrastructure/nodeFormatter';
import { summarizeTranscript } from './transcriptSummarizer';
import { eventBroadcaster } from '@/services/events';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
export type QuickAddMode = 'link' | 'text';
@@ -242,7 +243,7 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin
`Attempted pipeline: ${type}`,
].join('\n');
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
@@ -251,11 +252,16 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin
source,
link: url,
metadata: {
source: 'quick-add-link-fallback',
attempted_pipeline: type,
extraction_failed: true,
extraction_error: message,
refined_at: new Date().toISOString(),
type: 'website',
state: 'not_processed',
captured_method: 'quick_add_link_fallback',
captured_by: 'human',
source_metadata: {
attempted_pipeline: type,
extraction_failed: true,
extraction_error: message,
refined_at: new Date().toISOString(),
},
},
}),
});
@@ -294,8 +300,13 @@ async function handleNoteQuickAdd(rawInput: string, task: string, userDescriptio
title,
source: content,
metadata: {
source: 'quick-add-note',
refined_at: new Date().toISOString(),
type: 'note',
state: 'not_processed',
captured_method: 'quick_add_note',
captured_by: 'human',
source_metadata: {
refined_at: new Date().toISOString(),
},
},
};
@@ -303,7 +314,7 @@ async function handleNoteQuickAdd(rawInput: string, task: string, userDescriptio
nodePayload.description = userDescription.trim();
}
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(nodePayload),
@@ -356,48 +367,42 @@ async function handleChatTranscriptQuickAdd(rawInput: string, task: string): Pro
? ['Where things felt stuck:', ...stickingPoints.map((item) => `- ${item}`)].join('\n')
: null;
const contentParts = [
`Overview:\n${baseSummary}`,
];
if (intentLine) {
contentParts.push(`What you were trying to do:\n${intentLine}`);
}
if (progressLine) {
contentParts.push(`Where you made progress:\n${progressLine}`);
}
if (stickingSection) {
contentParts.push(stickingSection);
}
if (highlightSection) contentParts.push(highlightSection);
if (followUpSection) contentParts.push(followUpSection);
const content = contentParts.join('\n\n');
const title = deriveChatTitle(transcript, summaryResult.subject);
const wordCount = transcript.split(/\s+/).filter(Boolean).length;
const compactSummary = baseSummary.replace(/\s+/g, ' ').trim();
const whyDetail = intentLine
? `It belongs in the graph because it preserves context around ${intentLine.toLowerCase()}.`
: 'It belongs in the graph because it preserves context from this conversation.';
const statusDetail = 'It has not been reviewed yet.';
const nodeDescription = `${compactSummary} ${whyDetail} ${statusDetail}`.slice(0, 500);
const metadata = {
source: 'quick-add-chat',
summary_subject: summaryResult.subject,
summary_intent: summaryResult.intent,
summary_progress: summaryResult.progress,
highlights: summaryResult.highlights ?? [],
open_questions: summaryResult.openQuestions ?? [],
participants: summaryResult.participants ?? [],
sticking_points: summaryResult.stickingPoints ?? [],
transcript_length_chars: transcript.length,
transcript_length_words: wordCount,
transcript_truncated_for_summary: summaryResult.truncated ?? false,
summary_generated_at: new Date().toISOString(),
type: 'chat',
state: 'not_processed',
captured_method: 'quick_add_chat',
captured_by: 'human',
source_metadata: {
summary_subject: summaryResult.subject,
summary_intent: summaryResult.intent,
summary_progress: summaryResult.progress,
highlights: summaryResult.highlights ?? [],
open_questions: summaryResult.openQuestions ?? [],
participants: summaryResult.participants ?? [],
sticking_points: summaryResult.stickingPoints ?? [],
transcript_length_chars: transcript.length,
transcript_length_words: wordCount,
transcript_truncated_for_summary: summaryResult.truncated ?? false,
summary_generated_at: new Date().toISOString(),
},
};
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title,
source: transcript,
description: content,
description: nodeDescription,
metadata,
}),
});
+199 -38
View File
@@ -1,72 +1,233 @@
import { getSQLiteClient } from '@/services/database/sqlite-client';
import { getAutoContextSettings } from '@/services/settings/autoContextSettings';
interface AutoContextRow {
id: number;
title: string | null;
updated_at: string;
edge_count: number | null;
}
export interface AutoContextSummary {
id: number;
title: string;
edgeCount: number;
description: string;
updatedAt: string;
edgeCount: number;
}
export interface ContextAnchorSummary {
id: number;
title: string;
description: string;
updatedAt: string;
edgeCount: number;
}
export interface PromptContextSummary {
id: number;
name: string;
description: string | null;
icon: string | null;
count: number;
anchor: ContextAnchorSummary | null;
}
function truncate(value: string | null | undefined, maxChars: number): string {
const trimmed = (value || '').trim();
if (trimmed.length <= maxChars) return trimmed;
return `${trimmed.slice(0, maxChars)}...`;
}
function fetchAutoContextRows(limit: number): AutoContextSummary[] {
const db = getSQLiteClient();
const rows = db
.query<AutoContextRow>(
`
SELECT n.id,
n.title,
n.updated_at,
COUNT(DISTINCT e.id) AS edge_count
FROM nodes n
LEFT JOIN edges e
ON (e.from_node_id = n.id OR e.to_node_id = n.id)
WHERE 1=1
GROUP BY n.id
ORDER BY edge_count DESC, n.updated_at DESC
LIMIT ?
`,
[limit]
)
.rows;
const rows = db.query<{
id: number;
title: string | null;
description: string | null;
updated_at: string;
edge_count: number | null;
}>(
`
SELECT n.id,
n.title,
n.description,
n.updated_at,
COUNT(DISTINCT e.id) AS edge_count
FROM nodes n
LEFT JOIN edges e
ON (e.from_node_id = n.id OR e.to_node_id = n.id)
GROUP BY n.id
ORDER BY edge_count DESC, n.updated_at DESC, n.id ASC
LIMIT ?
`,
[limit]
).rows;
return rows.map((row) => ({
id: row.id,
title: row.title || 'Untitled node',
description: row.description || '',
updatedAt: row.updated_at,
edgeCount: Number(row.edge_count ?? 0),
}));
}
export function getAutoContextSummaries(limit = 10): AutoContextSummary[] {
const settings = getAutoContextSettings();
if (!settings.autoContextEnabled) {
return [];
}
export function getHubNodes(limit = 5): AutoContextSummary[] {
return fetchAutoContextRows(limit);
}
export function buildAutoContextBlock(limit = 10): string | null {
const summaries = getAutoContextSummaries(limit);
export function getContextSummaries(limit = 12): PromptContextSummary[] {
const db = getSQLiteClient();
const rows = db.query<{
id: number;
name: string;
description: string | null;
icon: string | null;
count: number;
anchor_id: number | null;
anchor_title: string | null;
anchor_description: string | null;
anchor_updated_at: string | null;
anchor_edge_count: number | null;
}>(`
WITH context_counts AS (
SELECT c.id, c.name, c.description, c.icon, COUNT(n.id) AS count
FROM contexts c
LEFT JOIN nodes n ON n.context_id = c.id
GROUP BY c.id
),
ranked_anchors AS (
SELECT
c.id AS context_id,
n.id AS node_id,
n.title,
n.description,
n.updated_at,
COUNT(e.id) AS edge_count,
ROW_NUMBER() OVER (
PARTITION BY c.id
ORDER BY COUNT(e.id) DESC, n.updated_at DESC, n.id ASC
) AS anchor_rank
FROM contexts c
LEFT JOIN nodes n ON n.context_id = c.id
LEFT JOIN edges e ON (e.from_node_id = n.id OR e.to_node_id = n.id)
GROUP BY c.id, n.id
)
SELECT
cc.id,
cc.name,
cc.description,
cc.icon,
cc.count,
ra.node_id AS anchor_id,
ra.title AS anchor_title,
ra.description AS anchor_description,
ra.updated_at AS anchor_updated_at,
ra.edge_count AS anchor_edge_count
FROM context_counts cc
LEFT JOIN ranked_anchors ra
ON ra.context_id = cc.id
AND ra.anchor_rank = 1
ORDER BY cc.name COLLATE NOCASE ASC
LIMIT ?
`, [limit]).rows;
return rows.map((row) => ({
id: row.id,
name: row.name,
description: row.description ?? null,
icon: row.icon ?? null,
count: Number(row.count ?? 0),
anchor: row.anchor_id == null ? null : {
id: Number(row.anchor_id),
title: row.anchor_title || 'Untitled node',
description: row.anchor_description || '',
updatedAt: row.anchor_updated_at || '',
edgeCount: Number(row.anchor_edge_count ?? 0),
},
}));
}
export function buildContextsBlock(limit = 12): string | null {
const contexts = getContextSummaries(limit);
if (contexts.length === 0) {
return null;
}
const lines: string[] = [
'User Contexts',
'Use contexts as the primary scope layer. Dimensions are secondary metadata and filters.',
'',
];
contexts.forEach((context, index) => {
const description = truncate(context.description, 140) || 'No description.';
const iconPrefix = context.icon ? `${context.icon} ` : '';
lines.push(`${index + 1}. ${iconPrefix}${context.name} (${context.count} nodes)`);
lines.push(` ${description}`);
});
return lines.join('\n');
}
export function buildContextAnchorsBlock(limit = 12): string | null {
const contexts = getContextSummaries(limit).filter((context) => context.anchor);
if (contexts.length === 0) {
return null;
}
const lines: string[] = [
'Context Anchors',
'Each context anchor is the highest-edge node in that context. Use it as the starting graph waypoint for that scope.',
'',
];
contexts.forEach((context, index) => {
const anchor = context.anchor!;
lines.push(`${index + 1}. ${context.name}: [NODE:${anchor.id}:"${anchor.title}"] (${anchor.edgeCount} edges)`);
if (anchor.description) {
lines.push(` ${truncate(anchor.description, 160)}`);
}
});
return lines.join('\n');
}
export function buildHubNodesBlock(limit = 5): string | null {
const summaries = getHubNodes(limit);
if (summaries.length === 0) {
return null;
}
const lines: string[] = [
'=== BACKGROUND CONTEXT ===',
'Top 10 most-connected nodes (important knowledge hubs). Use queryNodes/getNodesById if relevant.',
'Global Hub Diagnostics',
'These are secondary graph diagnostics only. Do not use them as the primary scope layer when contexts are available.',
'',
];
for (const summary of summaries) {
lines.push(`[NODE:${summary.id}:"${summary.title}"] (edges: ${summary.edgeCount})`);
}
summaries.forEach((summary, i) => {
lines.push(`${i + 1}. [NODE:${summary.id}:"${summary.title}"] (${summary.edgeCount} edges)`);
if (summary.description) {
lines.push(` ${truncate(summary.description, 140)}`);
}
});
return lines.join('\n');
}
export function getAutoContextSummaries(limit = 5): AutoContextSummary[] {
const settings = getAutoContextSettings();
if (!settings.autoContextEnabled) {
return [];
}
return getHubNodes(limit);
}
export function buildAutoContextBlock(limit = 12): string | null {
const settings = getAutoContextSettings();
if (!settings.autoContextEnabled) {
return null;
}
const sections = [
buildContextsBlock(limit),
buildContextAnchorsBlock(limit),
buildHubNodesBlock(5),
].filter(Boolean);
return sections.length > 0 ? sections.join('\n\n') : null;
}
+168
View File
@@ -0,0 +1,168 @@
import { getSQLiteClient } from '@/services/database/sqlite-client';
type ContextCandidate = {
id: number;
name: string;
description: string | null;
count: number;
anchor_title: string | null;
anchor_description: string | null;
};
type InferContextInput = {
title?: string | null;
description?: string | null;
source?: string | null;
dimensions?: string[] | null;
metadata?: unknown;
};
const STOP_WORDS = new Set([
'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from', 'has', 'i',
'in', 'is', 'it', 'its', 'of', 'on', 'or', 'that', 'the', 'their', 'this',
'to', 'was', 'with', 'you', 'your'
]);
function normalizeText(value: string): string {
return value.toLowerCase().replace(/[^a-z0-9\s]+/g, ' ').replace(/\s+/g, ' ').trim();
}
function tokenize(value: string | null | undefined): string[] {
if (!value) return [];
return normalizeText(value)
.split(' ')
.map((token) => token.trim())
.filter((token) => token.length >= 2 && !STOP_WORDS.has(token));
}
function uniqueTokens(values: Array<string | null | undefined>): string[] {
return Array.from(new Set(values.flatMap((value) => tokenize(value))));
}
function safeStringify(value: unknown): string {
try {
return JSON.stringify(value ?? {});
} catch {
return '';
}
}
function fetchContextCandidates(): ContextCandidate[] {
const sqlite = getSQLiteClient();
return sqlite.query<ContextCandidate>(`
WITH context_counts AS (
SELECT c.id, c.name, c.description, COUNT(n.id) AS count
FROM contexts c
LEFT JOIN nodes n ON n.context_id = c.id
GROUP BY c.id
),
ranked_anchors AS (
SELECT
c.id AS context_id,
n.title AS anchor_title,
n.description AS anchor_description,
ROW_NUMBER() OVER (
PARTITION BY c.id
ORDER BY COUNT(e.id) DESC, n.updated_at DESC, n.id ASC
) AS anchor_rank
FROM contexts c
LEFT JOIN nodes n ON n.context_id = c.id
LEFT JOIN edges e ON (e.from_node_id = n.id OR e.to_node_id = n.id)
GROUP BY c.id, n.id
)
SELECT
cc.id,
cc.name,
cc.description,
cc.count,
ra.anchor_title,
ra.anchor_description
FROM context_counts cc
LEFT JOIN ranked_anchors ra
ON ra.context_id = cc.id
AND ra.anchor_rank = 1
ORDER BY cc.name COLLATE NOCASE ASC
`).rows.map((row) => ({
id: Number(row.id),
name: row.name,
description: row.description ?? null,
count: Number(row.count ?? 0),
anchor_title: row.anchor_title ?? null,
anchor_description: row.anchor_description ?? null,
}));
}
function scoreContextCandidate(candidate: ContextCandidate, input: InferContextInput): number {
const titleText = normalizeText(input.title || '');
const descriptionText = normalizeText(input.description || '');
const sourceText = normalizeText((input.source || '').slice(0, 4000));
const metadataText = normalizeText(safeStringify(input.metadata));
const dimensionTokens = uniqueTokens(input.dimensions ?? []);
const contextName = normalizeText(candidate.name);
const contextNameTokens = tokenize(candidate.name);
const contextDescriptorTokens = uniqueTokens([
candidate.description,
candidate.anchor_title,
candidate.anchor_description,
]);
let score = 0;
if (contextName && (titleText.includes(contextName) || descriptionText.includes(contextName))) {
score += 80;
}
if (contextName && sourceText.includes(contextName)) {
score += 40;
}
for (const token of contextNameTokens) {
if (dimensionTokens.includes(token)) score += 30;
if (titleText.includes(token)) score += 16;
if (descriptionText.includes(token)) score += 12;
if (sourceText.includes(token)) score += 6;
if (metadataText.includes(token)) score += 4;
}
for (const token of contextDescriptorTokens) {
if (dimensionTokens.includes(token)) score += 8;
if (titleText.includes(token)) score += 4;
if (descriptionText.includes(token)) score += 3;
if (sourceText.includes(token)) score += 2;
}
return score;
}
export async function inferBestContextIdForNode(input: InferContextInput): Promise<number | null> {
const contexts = fetchContextCandidates();
if (contexts.length === 0) {
return null;
}
const ranked = contexts
.map((context) => ({
context,
score: scoreContextCandidate(context, input),
}))
.sort((a, b) =>
b.score - a.score ||
(b.context.count - a.context.count) ||
a.context.id - b.context.id
);
const best = ranked[0];
if (!best) {
return null;
}
if (best.score > 0) {
return best.context.id;
}
const research = contexts.find((context) => context.name.trim().toLowerCase() === 'research');
if (research) {
return research.id;
}
return ranked[0].context.id;
}
+225
View File
@@ -0,0 +1,225 @@
import { getSQLiteClient } from './sqlite-client';
import type { Context, ContextSummary, Node } from '@/types/database';
import { nodeService } from './nodes';
type ContextRow = Context;
function normalizeContextName(name: string): string {
return name.trim().replace(/\s+/g, ' ');
}
function assertContextName(name: unknown): string {
if (typeof name !== 'string') {
throw new Error('Context name is required.');
}
const normalized = normalizeContextName(name);
if (!normalized) {
throw new Error('Context name is required.');
}
return normalized;
}
function assertContextDescription(description: unknown): string {
if (typeof description !== 'string') {
throw new Error('Context description is required.');
}
const normalized = description.trim();
if (!normalized) {
throw new Error('Context description is required.');
}
return normalized;
}
function mapContextRow(row: ContextRow): Context {
return {
id: Number(row.id),
name: row.name,
description: row.description ?? null,
icon: row.icon ?? null,
created_at: row.created_at,
updated_at: row.updated_at,
};
}
export class ContextService {
async listContexts(): Promise<ContextSummary[]> {
const sqlite = getSQLiteClient();
const rows = sqlite.query<ContextSummary>(`
SELECT c.id, c.name, c.description, c.icon, COUNT(n.id) as count
FROM contexts c
LEFT JOIN nodes n ON n.context_id = c.id
GROUP BY c.id
ORDER BY c.name COLLATE NOCASE ASC
`).rows;
return rows.map((row) => ({
id: Number(row.id),
name: row.name,
description: row.description ?? null,
icon: row.icon ?? null,
count: Number(row.count ?? 0),
}));
}
async getContextById(id: number): Promise<ContextSummary | null> {
const sqlite = getSQLiteClient();
const row = sqlite.query<ContextSummary>(`
SELECT c.id, c.name, c.description, c.icon, COUNT(n.id) as count
FROM contexts c
LEFT JOIN nodes n ON n.context_id = c.id
WHERE c.id = ?
GROUP BY c.id
`, [id]).rows[0];
if (!row) return null;
return {
id: Number(row.id),
name: row.name,
description: row.description ?? null,
icon: row.icon ?? null,
count: Number(row.count ?? 0),
};
}
async getContextByName(name: string): Promise<Context | null> {
const normalized = assertContextName(name);
const sqlite = getSQLiteClient();
const row = sqlite.query<ContextRow>(`
SELECT id, name, description, icon, created_at, updated_at
FROM contexts
WHERE LOWER(TRIM(name)) = LOWER(TRIM(?))
LIMIT 1
`, [normalized]).rows[0];
return row ? mapContextRow(row) : null;
}
async createContext(input: { name: string; description: string; icon?: string | null }): Promise<Context> {
const name = assertContextName(input.name);
const description = assertContextDescription(input.description);
const icon = typeof input.icon === 'string' && input.icon.trim() ? input.icon.trim() : null;
const sqlite = getSQLiteClient();
const now = new Date().toISOString();
const existing = await this.getContextByName(name);
if (existing) {
throw new Error(`Context "${name}" already exists.`);
}
const result = sqlite.prepare(`
INSERT INTO contexts (name, description, icon, created_at, updated_at)
VALUES (?, ?, ?, ?, ?)
`).run(name, description, icon, now, now);
const created = await this.getContextById(Number(result.lastInsertRowid));
if (!created) {
throw new Error('Failed to create context.');
}
return {
...created,
created_at: now,
updated_at: now,
};
}
async updateContext(input: { id: number; name?: string; description?: string; icon?: string | null }): Promise<Context> {
const sqlite = getSQLiteClient();
const existing = sqlite.query<ContextRow>(`
SELECT id, name, description, icon, created_at, updated_at
FROM contexts
WHERE id = ?
`, [input.id]).rows[0];
if (!existing) {
throw new Error(`Context ${input.id} not found.`);
}
const nextName = input.name !== undefined ? assertContextName(input.name) : existing.name;
const nextDescription = input.description !== undefined
? assertContextDescription(input.description)
: existing.description;
const nextIcon = input.icon !== undefined
? (typeof input.icon === 'string' && input.icon.trim() ? input.icon.trim() : null)
: existing.icon;
const conflicting = sqlite.query<{ id: number }>(`
SELECT id FROM contexts
WHERE LOWER(TRIM(name)) = LOWER(TRIM(?))
AND id != ?
LIMIT 1
`, [nextName, input.id]).rows[0];
if (conflicting) {
throw new Error(`Context "${nextName}" already exists.`);
}
const now = new Date().toISOString();
sqlite.prepare(`
UPDATE contexts
SET name = ?, description = ?, icon = ?, updated_at = ?
WHERE id = ?
`).run(nextName, nextDescription, nextIcon, now, input.id);
return {
id: input.id,
name: nextName,
description: nextDescription ?? null,
icon: nextIcon ?? null,
created_at: existing.created_at,
updated_at: now,
};
}
async getNodesForContext(id: number): Promise<Node[]> {
return nodeService.getNodes({ contextId: id, limit: 500 });
}
async resolveContextId(input: { context_id?: number | null; context_name?: string | null }): Promise<number | null | undefined> {
const hasContextId = Object.prototype.hasOwnProperty.call(input, 'context_id');
const hasContextName = typeof input.context_name === 'string' && input.context_name.trim().length > 0;
if (!hasContextId && !hasContextName) {
return undefined;
}
if (hasContextId && input.context_id === null) {
if (hasContextName) {
throw new Error('context_name cannot be combined with context_id: null.');
}
return null;
}
let resolvedById: Context | null = null;
if (hasContextId) {
if (typeof input.context_id !== 'number' || !Number.isInteger(input.context_id) || input.context_id <= 0) {
throw new Error('context_id must be a positive integer or null.');
}
const byId = await this.getContextById(input.context_id);
if (!byId) {
throw new Error(`Context ${input.context_id} not found.`);
}
resolvedById = {
...byId,
created_at: '',
updated_at: '',
};
}
if (hasContextName) {
const byName = await this.getContextByName(input.context_name!);
if (!byName) {
throw new Error(`Context "${input.context_name}" not found.`);
}
if (resolvedById && resolvedById.id !== byName.id) {
throw new Error('context_id and context_name refer to different contexts.');
}
return byName.id;
}
return resolvedById?.id;
}
}
export const contextService = new ContextService();
+47 -76
View File
@@ -1,12 +1,13 @@
import { openai as openaiProvider } from '@ai-sdk/openai';
import { generateText } from 'ai';
import { hasValidOpenAiKey } from '../storage/apiKeys';
import type { CanonicalNodeMetadata } from '@/types/database';
export interface DescriptionInput {
title: string;
source?: string;
link?: string;
metadata?: {
metadata?: CanonicalNodeMetadata & {
source?: string;
channel_name?: string;
author?: string;
@@ -18,52 +19,12 @@ export interface DescriptionInput {
dimensions?: string[];
}
// Re-export for backwards compatibility — canonical source is ../storage/apiKeys
export { hasValidOpenAiKey } from '../storage/apiKeys';
/**
* Generate a simple fallback description without AI.
* Used when no API key is available or for simple inputs.
*/
export function generateFallbackDescription(input: DescriptionInput): string {
const { title, metadata, dimensions } = input;
// Build a contextual fallback
const parts: string[] = [];
if (metadata?.author || metadata?.channel_name) {
parts.push(`By ${metadata.author || metadata.channel_name}`);
}
if (dimensions?.length) {
parts.push(`in ${dimensions.slice(0, 2).join(', ')}`);
}
if (parts.length > 0) {
return `${parts.join(' — ')}: ${title.slice(0, 200)}`;
}
return title.slice(0, 280);
}
/**
* Generate a 280-character description for a knowledge node.
* Contextually grounded - adapts to node type (person, concept, article, etc.)
*
* IMPORTANT: Returns fallback immediately if no valid API key is configured.
* This prevents slow node creation (9-13s timeout) when OpenAI is unavailable.
*/
export async function generateDescription(input: DescriptionInput): Promise<string> {
// Fast path: skip AI if no valid API key
if (!hasValidOpenAiKey()) {
console.log(`[DescriptionService] No valid OpenAI key, using fallback for: "${input.title}"`);
return generateFallbackDescription(input);
}
// Fast path: skip AI for very short inputs (likely just notes)
if (!input.source && !input.link && input.title.length < 30) {
console.log(`[DescriptionService] Short input, using fallback for: "${input.title}"`);
return generateFallbackDescription(input);
return `${input.title}. Added via Quick Add with no further context yet, so the reason it belongs in the graph is not fully inferred. It has not been reviewed yet.`.slice(0, 500);
}
try {
@@ -78,31 +39,38 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
temperature: 0.3,
});
const finalDescription = sanitizeDescription(response.text, input);
const description = sanitizeDescription(response.text, input);
console.log(`[DescriptionService] Generated: "${finalDescription}"`);
console.log(`[DescriptionService] Generated: "${description}"`);
return finalDescription;
return description;
} catch (error) {
console.error('[DescriptionService] Error generating description:', error);
// Return a fallback description
return generateFallbackDescription(input);
return `${input.title}. Added via Quick Add with no further context yet, so the reason it belongs in the graph is not fully inferred. It has not been reviewed yet.`.slice(0, 500);
}
}
function buildDescriptionPrompt(input: DescriptionInput): string {
const normalizedSource = (input.metadata?.source || '').toLowerCase();
const sourceMetadata = input.metadata?.source_metadata as Record<string, unknown> | undefined;
const sourceType = typeof input.metadata?.type === 'string'
? input.metadata.type
: typeof input.metadata?.source === 'string'
? input.metadata.source
: '';
const normalizedSource = sourceType.toLowerCase();
const url = typeof input.link === 'string' ? input.link.trim() : '';
// Best-effort creator hint from structured metadata (when available),
// but never assume a particular extraction source (YouTube vs paper vs website vs note).
const creatorHint =
input.metadata?.author?.trim() ||
input.metadata?.channel_name?.trim() ||
(typeof sourceMetadata?.author === 'string' ? sourceMetadata.author.trim() : '') ||
(typeof sourceMetadata?.channel_name === 'string' ? sourceMetadata.channel_name.trim() : '') ||
(typeof input.metadata?.author === 'string' ? input.metadata.author.trim() : '') ||
(typeof input.metadata?.channel_name === 'string' ? input.metadata.channel_name.trim() : '') ||
'';
// Best-effort publisher / container hint (less ideal than a true author, but better than nothing).
const publisherHint = input.metadata?.site_name?.trim() || '';
const publisherHint =
(typeof sourceMetadata?.site_name === 'string' ? sourceMetadata.site_name.trim() : '') ||
(typeof input.metadata?.site_name === 'string' ? input.metadata.site_name.trim() : '') ||
'';
const likelyExternal =
Boolean(url) ||
@@ -123,38 +91,41 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
if (input.link) lines.push(`URL: ${input.link}`);
if (input.dimensions?.length) lines.push(`Dimensions: ${input.dimensions.join(', ')}`);
if (input.metadata?.channel_name) lines.push(`Channel: ${input.metadata.channel_name}`);
if (input.metadata?.author) lines.push(`Author: ${input.metadata.author}`);
if (input.metadata?.site_name) lines.push(`Site: ${input.metadata.site_name}`);
if (input.metadata?.source) lines.push(`Source type: ${input.metadata.source}`);
if (input.metadata?.original_filename) lines.push(`Original filename: ${input.metadata.original_filename}`);
if (typeof input.metadata?.pages === 'number') lines.push(`Pages: ${input.metadata.pages}`);
if (typeof input.metadata?.text_length === 'number') lines.push(`Text length: ${input.metadata.text_length}`);
if (sourceMetadata?.channel_name || input.metadata?.channel_name) lines.push(`Channel: ${sourceMetadata?.channel_name || input.metadata?.channel_name}`);
if (sourceMetadata?.author || input.metadata?.author) lines.push(`Author: ${sourceMetadata?.author || input.metadata?.author}`);
if (sourceMetadata?.site_name || input.metadata?.site_name) lines.push(`Site: ${sourceMetadata?.site_name || input.metadata?.site_name}`);
if (sourceType) lines.push(`Source type: ${sourceType}`);
if (sourceMetadata?.original_filename || input.metadata?.original_filename) lines.push(`Original filename: ${sourceMetadata?.original_filename || input.metadata?.original_filename}`);
if (typeof sourceMetadata?.pages === 'number' || typeof input.metadata?.pages === 'number') lines.push(`Pages: ${sourceMetadata?.pages || input.metadata?.pages}`);
if (typeof sourceMetadata?.text_length === 'number' || typeof input.metadata?.text_length === 'number') lines.push(`Text length: ${sourceMetadata?.text_length || input.metadata?.text_length}`);
if (creatorHint) lines.push(`Creator hint: ${creatorHint}`);
if (publisherHint) lines.push(`Publisher hint: ${publisherHint}`);
lines.push(`Likely user-authored: ${likelyUserAuthored ? 'yes' : 'no'}`);
const contentPreview = input.source?.slice(0, 800) || '';
if (contentPreview) lines.push(`Source excerpt: ${contentPreview}${input.source && input.source.length > 800 ? '...' : ''}`);
const sourcePreview = input.source?.slice(0, 800) || '';
if (sourcePreview) lines.push(`Source excerpt: ${sourcePreview}${input.source && input.source.length > 800 ? '...' : ''}`);
return `Write a description for this knowledge node. Max 280 characters.
return `Write a natural description for this knowledge node. Max 500 characters.
Say WHAT this literally is and WHY it matters. Be concrete and specific — like you're telling a friend what this thing is in one breath.
The description should read like normal prose, not a template or checklist. In one compact paragraph or a few natural sentences, make sure it clearly conveys:
1) what this literally is
2) why it is in Brad's graph
3) its current status in Brad's workflow
RULES:
1) Name the format only if the context clearly supports it: "Podcast episode where…", "Blog post arguing…", "Personal note capturing…", "Research paper showing…", "Resume/CV for…", "Document likely containing…", "Idea that…"
2) Name people by role — channel/host is the creator, title figures are guests/subjects. Use the Creator hint if available.
3) State the actual claim, finding, or insight from the content — not a vague summary of the topic.
4) End with why it's interesting or important — one concrete phrase.
5) ABSOLUTELY FORBIDDEN — these words will be rejected: "discusses", "explores", "examines", "talks about", "is about", "delves into", "emphasizing the need for". State things directly instead.
6) Do NOT start with "Your note —" or "This note —". Use a concrete opener tied to the actual artifact.
7) If the artifact type is unclear, say so explicitly using words like "likely", "appears to be", or "unclear" rather than guessing a confident format.
4) If the reason it belongs in the graph cannot be inferred from title, source excerpt, URL, metadata, or dimensions, say that naturally rather than inventing context.
5) If workflow status is unknown, say that naturally, for example by noting it has not been reviewed yet.
6) Do NOT use labels or headings like "WHAT:", "WHY:", or "STATUS:".
7) ABSOLUTELY FORBIDDEN — these words and phrases will be rejected: "discusses", "explores", "examines", "talks about", "is about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to", "important for", "useful for understanding". State things directly instead.
8) Do NOT start with "Your note —" or "This note —". Use a concrete opener tied to the actual artifact.
9) If the artifact type is unclear, say so explicitly using words like "likely", "appears to be", or "unclear" rather than guessing a confident format.
GOOD: "Karpathy blog post — AI agents make software fluid, ripping functionality from repos instead of taking dependencies. Signals the end of monolithic libraries."
GOOD: "Dwarkesh Patel interview with Anthropic CEO Dario Amodei — argues we're nearing the end of exponential AI scaling. Key signal for what comes next."
GOOD: "Personal note capturing a recurring pattern: morning optimism reverses to evening pessimism. Indicates a belief-level swing worth tracking."
GOOD: "Resume/CV for Brad Morris outlining work in AI systems, context engineering, and RA-H. Useful as a compact record of background, projects, and expertise."
GOOD: "Document likely related to Brad Morris's work history and AI consulting, but the exact artifact type is unclear from the available context. Still useful as a reference profile."
GOOD: "CS153 lecture by ElevenLabs co-founder Mati Staniszewski on production AI voice systems. Brad likely saved it as a follow-on to his interest in the ElevenLabs voice pipeline after CS153 ep.1, and it has not been reviewed yet."
GOOD: "YouTube talk by Lex Fridman with Sam Altman on AGI timelines and OpenAI strategy. It was added via Quick Add and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
GOOD: "Personal note capturing a recurring pattern: morning optimism reverses to evening pessimism. It belongs in the graph because it points to a belief-level pattern worth tracking against Brad's decision quality, and it has already been processed."
BAD: "By Dario Amodei — discusses reaching the limits of exponential growth in AI, emphasizing the need for a critical perspective on future advancements."
BAD: "This article explores ideas about how software is changing."
@@ -170,7 +141,7 @@ function sanitizeDescription(rawText: string, input: DescriptionInput): string {
.replace(/^["']|["']$/g, '');
if (!singleLine) {
return input.title.slice(0, 280);
return `${input.title}. Added via Quick Add with no further context yet, so the reason it belongs in the graph is not fully inferred. It has not been reviewed yet.`.slice(0, 500);
}
const noGenericPrefix = singleLine.replace(
@@ -178,7 +149,7 @@ function sanitizeDescription(rawText: string, input: DescriptionInput): string {
'Personal note capturing '
);
return noGenericPrefix.slice(0, 280);
return noGenericPrefix.slice(0, 500);
}
export const descriptionService = {
+1
View File
@@ -3,6 +3,7 @@ export { nodeService, NodeService } from './nodes';
export { chunkService, ChunkService } from './chunks';
export { edgeService, EdgeService } from './edges';
export { dimensionService, DimensionService } from './dimensionService';
export { contextService, ContextService } from './contextService';
// export { HelperService } from './helpers'; // Removed - migrated to JSON-based service
// Types
+43 -13
View File
@@ -3,8 +3,9 @@ import { Node, NodeFilters } from '@/types/database';
import { eventBroadcaster } from '../events';
import { EmbeddingService } from '@/services/embeddings';
import { scoreNodeSearchMatch } from './searchRanking';
import { buildCanonicalNodeMetadata, mergeNodeMetadata } from '@/services/nodes/metadata';
type NodeRow = Node & { dimensions_json: string };
type NodeRow = Node & { dimensions_json: string; context_json: string | null };
type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
function sanitizeFtsQuery(input: string): string {
@@ -81,7 +82,7 @@ export class NodeService {
async countNodes(filters: NodeFilters = {}): Promise<number> {
const { dimensions, search, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters;
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus, contextId } = filters;
if (search?.trim()) {
return this.countSearchNodesSQLite(filters);
@@ -120,6 +121,7 @@ export class NodeService {
if (eventAfter) { query += ` AND n.event_date >= ?`; params.push(eventAfter); }
if (eventBefore) { query += ` AND n.event_date < ?`; params.push(eventBefore); }
if (chunkStatus) { query += ` AND n.chunk_status = ?`; params.push(chunkStatus); }
if (contextId !== undefined) { query += ` AND n.context_id = ?`; params.push(contextId); }
const result = sqlite.query<{ total: number }>(query, params);
return result.rows[0]?.total ?? 0;
@@ -129,7 +131,7 @@ export class NodeService {
private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters;
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus, contextId } = filters;
if (search?.trim()) {
return this.searchNodesSQLite(filters);
@@ -141,11 +143,16 @@ export class NodeService {
let query = `
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
n.created_at, n.updated_at, n.context_id,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
CASE
WHEN c.id IS NULL THEN NULL
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
END as context_json,
(SELECT COUNT(*) FROM edges WHERE from_node_id = n.id OR to_node_id = n.id) as edge_count
FROM nodes n
LEFT JOIN contexts c ON c.id = n.context_id
WHERE 1=1
`;
const params: any[] = [];
@@ -198,6 +205,10 @@ export class NodeService {
query += ` AND n.chunk_status = ?`;
params.push(chunkStatus);
}
if (contextId !== undefined) {
query += ` AND n.context_id = ?`;
params.push(contextId);
}
// Sorting logic
if (search) {
@@ -255,10 +266,15 @@ export class NodeService {
const query = `
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
n.created_at, n.updated_at, n.context_id,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
CASE
WHEN c.id IS NULL THEN NULL
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
END as context_json
FROM nodes n
LEFT JOIN contexts c ON c.id = n.context_id
WHERE n.id = ?
`;
const result = sqlite.query<NodeRow>(query, [id]);
@@ -284,24 +300,27 @@ export class NodeService {
event_date,
dimensions = [],
chunk_status,
metadata = {}
metadata = {},
context_id,
} = nodeData;
const canonicalMetadata = buildCanonicalNodeMetadata({ metadata });
const now = new Date().toISOString();
const sqlite = getSQLiteClient();
const nodeId = sqlite.transaction(() => {
// Insert node using prepare/run for lastInsertRowid access
const nodeResult = sqlite.prepare(`
INSERT INTO nodes (title, description, source, link, event_date, metadata, chunk_status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO nodes (title, description, source, link, event_date, metadata, chunk_status, context_id, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
title,
description ?? null,
source ?? null,
link ?? null,
event_date ?? null,
JSON.stringify(metadata),
JSON.stringify(canonicalMetadata),
chunk_status ?? null,
context_id ?? null,
now,
now
);
@@ -349,13 +368,17 @@ export class NodeService {
const sqlite = getSQLiteClient();
const existingRow = sqlite
.query<{ id: number }>('SELECT id FROM nodes WHERE id = ?', [id])
.query<{ id: number; metadata: string | null }>('SELECT id, metadata FROM nodes WHERE id = ?', [id])
.rows[0];
if (!existingRow) {
throw new Error(`Node with ID ${id} not found`);
}
const mergedMetadata = metadata !== undefined
? mergeNodeMetadata(existingRow.metadata, metadata)
: undefined;
sqlite.transaction(() => {
// Update node columns (only update provided fields)
const setFields: string[] = [];
@@ -366,13 +389,17 @@ export class NodeService {
if (source !== undefined) { setFields.push('source = ?'); params.push(source); }
if (link !== undefined) { setFields.push('link = ?'); params.push(link); }
if (event_date !== undefined) { setFields.push('event_date = ?'); params.push(event_date); }
if (Object.prototype.hasOwnProperty.call(updates, 'context_id')) {
setFields.push('context_id = ?');
params.push(updates.context_id ?? null);
}
if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) {
setFields.push('chunk_status = ?');
params.push(updates.chunk_status ?? null);
}
if (metadata !== undefined) {
if (mergedMetadata !== undefined) {
setFields.push('metadata = ?');
params.push(JSON.stringify(metadata));
params.push(JSON.stringify(mergedMetadata));
}
// Always update timestamp
@@ -445,6 +472,7 @@ export class NodeService {
...row,
dimensions: JSON.parse(row.dimensions_json || '[]'),
metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null,
context: row.context_json ? JSON.parse(row.context_json) : null,
};
}
@@ -456,6 +484,7 @@ export class NodeService {
createdBefore,
eventAfter,
eventBefore,
contextId,
} = filters;
const clauses: string[] = [];
@@ -483,6 +512,7 @@ export class NodeService {
if (createdBefore) { clauses.push(`${alias}.created_at < ?`); params.push(createdBefore); }
if (eventAfter) { clauses.push(`${alias}.event_date >= ?`); params.push(eventAfter); }
if (eventBefore) { clauses.push(`${alias}.event_date < ?`); params.push(eventBefore); }
if (contextId !== undefined) { clauses.push(`${alias}.context_id = ?`); params.push(contextId); }
return { clauses, params };
}
+33
View File
@@ -1,6 +1,8 @@
const WEAK_DESCRIPTION_PATTERNS = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i;
const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|concept|conversation|dataset|decision|dimension|document|episode|essay|event|guide|idea|insight|interview|lesson|link|node|note|paper|person|plan|placeholder|podcast|post|presentation|project|question|record|research|resource|skill|source|status|summary|talk|target|test node|thread|tool|transcript|tweet|update|video|website|workflow)\b/i;
const UNCERTAINTY_PATTERNS = /\b(likely|probably|possibly|appears to be|seems to be|unclear|uncertain)\b/i;
const WHY_PATTERNS = /(why added:|added (?:after|as|for|because|to)|follow-?on|queued for|saved for|relevant because|connected to|not inferred|belongs in the graph because|belongs here because|captures .* idea|ties directly into|ties into)/i;
const STATUS_PATTERNS = /(status:|queued|not yet reviewed|in progress|processed|reviewed|saved for later|to review|to read|to watch|to listen|draft|not yet published|unpublished)/i;
const GENERIC_EDGE_PATTERNS = /^(related|related to|connected|connected to|association|associated with)$/i;
export function normalizeDimensionName(value: string): string {
@@ -29,6 +31,9 @@ export function normalizeDimensions(values: unknown, max = 5): string[] {
export function validateExplicitDescription(description: string): string | null {
const text = description.trim();
if (text.length > 500) {
return 'Description must be 500 characters or less.';
}
if (text.length < 24) {
return 'Description must be explicit and substantial (at least 24 characters).';
}
@@ -38,9 +43,37 @@ export function validateExplicitDescription(description: string): string | null
if (!EXPLICIT_ENTITY_PATTERNS.test(text) && !UNCERTAINTY_PATTERNS.test(text)) {
return 'Description must explicitly identify what this thing is, or state uncertainty explicitly.';
}
if (!WHY_PATTERNS.test(text)) {
return 'Description must clearly indicate why this belongs in the graph. If that reason is unknown, say so naturally instead of inventing it.';
}
if (!STATUS_PATTERNS.test(text)) {
return 'Description must make the workflow status clear. If status is unknown, say naturally that it has not been reviewed yet or is still in progress.';
}
return null;
}
interface DescriptionFallbackInput {
title: string;
description?: string | null;
}
export function coerceDescriptionForStorage(input: DescriptionFallbackInput): string {
const candidate = typeof input.description === 'string' ? input.description.trim() : '';
const clippedCandidate = candidate.slice(0, 500);
const validationError = clippedCandidate ? validateExplicitDescription(clippedCandidate) : 'missing';
if (!validationError && clippedCandidate) {
return clippedCandidate;
}
const title = input.title.trim() || 'Untitled node';
const opening = clippedCandidate || `${title}.`;
const normalizedOpening = opening.endsWith('.') ? opening : `${opening}.`;
const fallbackTail = ' It was added to the graph with incomplete context, so the exact reason it belongs here is not yet inferred, and it has not been reviewed yet.';
return `${normalizedOpening}${fallbackTail}`.replace(/\s+/g, ' ').trim().slice(0, 500);
}
export function validateEdgeExplanation(explanation: string): string | null {
const text = explanation.trim();
if (text.length < 8) {
+58
View File
@@ -69,6 +69,7 @@ class SQLiteClient {
// Ensure logging schema (rename memory->logs if needed, create triggers/views)
this.ensureLoggingAndMemorySchema();
this.ensureContextsSchema();
}
console.log('SQLite client initialized successfully');
@@ -219,6 +220,11 @@ class SQLiteClient {
return;
}
try {
const hasReadyLogs = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
if (hasReadyLogs) {
return;
}
// 1) If logs table missing but legacy memory table exists, migrate
const hasLogs = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
const hasLegacyMemory = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memory'").get();
@@ -748,6 +754,58 @@ class SQLiteClient {
}
}
private ensureContextsSchema(): void {
if (this.readOnly) {
return;
}
try {
this.db.exec(`
CREATE TABLE IF NOT EXISTS contexts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
icon TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
`);
const nodeCols = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
const nodeColNames = nodeCols.map((column) => column.name);
if (!nodeColNames.includes('context_id')) {
this.db.exec('ALTER TABLE nodes ADD COLUMN context_id INTEGER REFERENCES contexts(id) ON DELETE SET NULL;');
}
const contextCols = this.db.prepare('PRAGMA table_info(contexts)').all() as Array<{ name: string }>;
const contextColNames = contextCols.map((column) => column.name);
if (!contextColNames.includes('description')) {
this.db.exec("ALTER TABLE contexts ADD COLUMN description TEXT NOT NULL DEFAULT '';");
}
if (!contextColNames.includes('icon')) {
this.db.exec('ALTER TABLE contexts ADD COLUMN icon TEXT;');
}
if (!contextColNames.includes('created_at')) {
this.db.exec("ALTER TABLE contexts ADD COLUMN created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
}
if (!contextColNames.includes('updated_at')) {
this.db.exec("ALTER TABLE contexts ADD COLUMN updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
}
this.db.exec(`
UPDATE contexts
SET description = COALESCE(NULLIF(TRIM(description), ''), name)
WHERE description IS NULL OR LENGTH(TRIM(description)) = 0;
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized
ON contexts(LOWER(TRIM(name)));
CREATE INDEX IF NOT EXISTS idx_nodes_context_id ON nodes(context_id);
`);
} catch (error) {
console.warn('Failed to ensure contexts schema:', error);
}
}
private healVectorTablesIfCorrupt(): void {
if (this.readOnly || !this.vectorCapability.available) {
return;
+100
View File
@@ -0,0 +1,100 @@
import type {
CanonicalNodeMetadata,
NodeCapturedBy,
NodeMetadataState,
} from '@/types/database';
type MetadataLike = CanonicalNodeMetadata | Record<string, unknown> | string | null | undefined;
export interface BuildCanonicalMetadataInput {
existing?: MetadataLike;
metadata?: MetadataLike;
type?: string | null;
state?: NodeMetadataState | null;
captured_method?: string | null;
captured_by?: NodeCapturedBy | null;
source_metadata?: Record<string, unknown> | null;
}
function isObject(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
export function parseNodeMetadata(metadata: MetadataLike): CanonicalNodeMetadata {
if (!metadata) return {};
if (typeof metadata === 'string') {
try {
const parsed = JSON.parse(metadata);
return isObject(parsed) ? { ...parsed } : {};
} catch {
return {};
}
}
return isObject(metadata) ? { ...metadata } : {};
}
function normalizeSourceMetadata(sourceMetadata: unknown): Record<string, unknown> {
return isObject(sourceMetadata) ? { ...sourceMetadata } : {};
}
function normalizedString(value: unknown): string | undefined {
if (typeof value !== 'string') return undefined;
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : undefined;
}
export function buildCanonicalNodeMetadata(input: BuildCanonicalMetadataInput): CanonicalNodeMetadata {
const existing = parseNodeMetadata(input.existing);
const incoming = parseNodeMetadata(input.metadata);
const type = normalizedString(input.type) ?? normalizedString(incoming.type) ?? normalizedString(existing.type);
const state = input.state ?? (incoming.state as NodeMetadataState | undefined) ?? (existing.state as NodeMetadataState | undefined) ?? 'not_processed';
const capturedMethod =
normalizedString(input.captured_method)
?? normalizedString(incoming.captured_method)
?? normalizedString(existing.captured_method);
const capturedBy =
input.captured_by
?? (incoming.captured_by as NodeCapturedBy | undefined)
?? (existing.captured_by as NodeCapturedBy | undefined)
?? 'human';
const sourceMetadata = {
...normalizeSourceMetadata(existing.source_metadata),
...normalizeSourceMetadata(incoming.source_metadata),
...normalizeSourceMetadata(input.source_metadata),
};
const merged: CanonicalNodeMetadata = {
...existing,
...incoming,
source_metadata: sourceMetadata,
state,
captured_by: capturedBy,
};
if (type) {
merged.type = type;
} else {
delete merged.type;
}
if (capturedMethod) {
merged.captured_method = capturedMethod;
} else {
delete merged.captured_method;
}
return merged;
}
export function mergeNodeMetadata(existing: MetadataLike, incoming: MetadataLike): CanonicalNodeMetadata {
return buildCanonicalNodeMetadata({ existing, metadata: incoming });
}
export function getNodeProcessedState(metadata: MetadataLike): NodeMetadataState {
const parsed = parseNodeMetadata(metadata);
return parsed.state === 'processed' ? 'processed' : 'not_processed';
}
+1
View File
@@ -38,6 +38,7 @@ const SEEDED_SKILL_IDS = new Set([
'persona',
'calibration',
'connect',
'node-context-enrichment',
]);
const DEPRECATED_SKILL_IDS = new Set([