feat: port holistic node refinement contract
This commit is contained in:
“BeeRad”
@@ -12,9 +12,6 @@
|
||||
import { nodeService, edgeService } from '@/services/database';
|
||||
import { Node } from '@/types/database';
|
||||
|
||||
// Entity-like dimensions where we expect to find referenceable entities
|
||||
const ENTITY_DIMENSIONS = ['people', 'companies', 'organizations', 'books', 'papers', 'articles', 'podcasts', 'creators'];
|
||||
|
||||
/**
|
||||
* Clean up a candidate entity string by removing common prefixes/suffixes.
|
||||
*/
|
||||
@@ -113,7 +110,7 @@ function isGenericPhrase(phrase: string): boolean {
|
||||
|
||||
/**
|
||||
* Look up existing nodes that match candidate entity strings.
|
||||
* Uses exact title matching (case-insensitive) with optional dimension filtering.
|
||||
* Uses exact title matching (case-insensitive).
|
||||
*/
|
||||
async function findMatchingEntityNodes(candidates: string[]): Promise<Map<string, Node>> {
|
||||
const matches = new Map<string, Node>();
|
||||
@@ -131,16 +128,7 @@ async function findMatchingEntityNodes(candidates: string[]): Promise<Map<string
|
||||
const matchingNode = allNodes.find(node => {
|
||||
const normalizedTitle = (node.title || '').toLowerCase().trim();
|
||||
if (normalizedTitle !== normalizedCandidate) return false;
|
||||
|
||||
// Optionally filter to entity-like dimensions for higher confidence
|
||||
// But don't require it - some entities might not have dimensions yet
|
||||
const nodeDimensions = node.dimensions || [];
|
||||
const hasEntityDimension = nodeDimensions.some(dim =>
|
||||
ENTITY_DIMENSIONS.includes(dim.toLowerCase())
|
||||
);
|
||||
|
||||
// Accept if it has an entity dimension OR if it's a short title (likely a named entity)
|
||||
return hasEntityDimension || node.title.length < 50;
|
||||
return node.title.length < 80;
|
||||
});
|
||||
|
||||
if (matchingNode) {
|
||||
|
||||
@@ -15,6 +15,7 @@ export interface QuickAddInput {
|
||||
rawInput: string;
|
||||
mode?: QuickAddMode;
|
||||
description?: string;
|
||||
contextId?: number | null;
|
||||
}
|
||||
|
||||
export interface QuickAddResult {
|
||||
@@ -65,7 +66,7 @@ function buildTaskPrompt(type: QuickAddInputType, input: string): string {
|
||||
case 'pdf':
|
||||
return `Quick Add: extract PDF and create node → ${input}`;
|
||||
case 'note':
|
||||
return `Quick Add note: create a node from this text with no dimensions → ${input}`;
|
||||
return `Quick Add note: create a node from this text with optional context → ${input}`;
|
||||
case 'chat':
|
||||
return `Quick Add: import chat transcript and summarize → ${input.slice(0, 120)}${input.length > 120 ? '…' : ''}`;
|
||||
}
|
||||
@@ -199,12 +200,11 @@ function isCreateNodeResponse(value: unknown): value is CreateNodeResponse {
|
||||
return true;
|
||||
}
|
||||
|
||||
async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: string, task: string): Promise<string> {
|
||||
async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: string, task: string, contextId?: number | null): Promise<string> {
|
||||
const { toolName, execute } = EXTRACTION_TOOL_MAP[type];
|
||||
if (!execute) {
|
||||
throw new Error(`Tool ${toolName} does not have an execute function`);
|
||||
}
|
||||
|
||||
try {
|
||||
const rawResult = await execute({ url }, { toolCallId: 'quickadd-extract', messages: [] });
|
||||
|
||||
@@ -263,6 +263,7 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin
|
||||
refined_at: new Date().toISOString(),
|
||||
},
|
||||
},
|
||||
context_id: contextId,
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -289,7 +290,7 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin
|
||||
}
|
||||
}
|
||||
|
||||
async function handleNoteQuickAdd(rawInput: string, task: string, userDescription?: string): Promise<string> {
|
||||
async function handleNoteQuickAdd(rawInput: string, task: string, userDescription?: string, contextId?: number | null): Promise<string> {
|
||||
const content = rawInput.trim();
|
||||
if (!content) {
|
||||
throw new Error('Input is required to create a note');
|
||||
@@ -299,6 +300,7 @@ async function handleNoteQuickAdd(rawInput: string, task: string, userDescriptio
|
||||
const nodePayload: Record<string, unknown> = {
|
||||
title,
|
||||
source: content,
|
||||
context_id: contextId,
|
||||
metadata: {
|
||||
type: 'note',
|
||||
state: 'not_processed',
|
||||
@@ -310,6 +312,7 @@ async function handleNoteQuickAdd(rawInput: string, task: string, userDescriptio
|
||||
},
|
||||
};
|
||||
|
||||
// If user provided a description, use it instead of auto-generating
|
||||
if (userDescription && userDescription.trim()) {
|
||||
nodePayload.description = userDescription.trim();
|
||||
}
|
||||
@@ -342,7 +345,7 @@ async function handleNoteQuickAdd(rawInput: string, task: string, userDescriptio
|
||||
});
|
||||
}
|
||||
|
||||
async function handleChatTranscriptQuickAdd(rawInput: string, task: string): Promise<string> {
|
||||
async function handleChatTranscriptQuickAdd(rawInput: string, task: string, contextId?: number | null): Promise<string> {
|
||||
const transcript = rawInput.trim();
|
||||
if (!transcript) {
|
||||
throw new Error('Input is required to import a chat transcript');
|
||||
@@ -401,8 +404,9 @@ async function handleChatTranscriptQuickAdd(rawInput: string, task: string): Pro
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
title,
|
||||
source: transcript,
|
||||
description: nodeDescription,
|
||||
source: transcript,
|
||||
context_id: contextId,
|
||||
metadata,
|
||||
}),
|
||||
});
|
||||
@@ -429,7 +433,7 @@ async function handleChatTranscriptQuickAdd(rawInput: string, task: string): Pro
|
||||
});
|
||||
}
|
||||
|
||||
export async function enqueueQuickAdd({ rawInput, mode, description }: QuickAddInput): Promise<QuickAddResult> {
|
||||
export async function enqueueQuickAdd({ rawInput, mode, description, contextId }: QuickAddInput): Promise<QuickAddResult> {
|
||||
const inputType = detectInputType(rawInput, mode);
|
||||
const task = buildTaskPrompt(inputType, rawInput);
|
||||
const id = `qa_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
@@ -441,21 +445,25 @@ export async function enqueueQuickAdd({ rawInput, mode, description }: QuickAddI
|
||||
status: 'queued',
|
||||
};
|
||||
|
||||
// Run async - fire and forget
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
let summary: string;
|
||||
if (inputType === 'note') {
|
||||
await handleNoteQuickAdd(rawInput, task, description);
|
||||
summary = await handleNoteQuickAdd(rawInput, task, description, contextId);
|
||||
} else if (inputType === 'chat') {
|
||||
await handleChatTranscriptQuickAdd(rawInput, task);
|
||||
summary = await handleChatTranscriptQuickAdd(rawInput, task, contextId);
|
||||
} else {
|
||||
await handleExtractionQuickAdd(inputType as ExtractionQuickAddType, rawInput, task);
|
||||
summary = await handleExtractionQuickAdd(inputType as ExtractionQuickAddType, rawInput, task, contextId);
|
||||
}
|
||||
|
||||
console.log(`[QuickAdd] Completed: ${task}`);
|
||||
// Broadcast completion so ThreePanelLayout can remove the pending placeholder
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'QUICK_ADD_COMPLETED',
|
||||
data: { quickAddId: id, source: 'quick-add' }
|
||||
});
|
||||
// Also broadcast NODE_CREATED to refresh the feed
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'NODE_CREATED',
|
||||
data: { node: { title: task }, source: 'quick-add' }
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
import { getAutoContextSettings } from '@/services/settings/autoContextSettings';
|
||||
|
||||
export interface AutoContextSummary {
|
||||
id: number;
|
||||
@@ -34,28 +33,30 @@ function truncate(value: string | null | undefined, maxChars: number): string {
|
||||
|
||||
function fetchAutoContextRows(limit: number): AutoContextSummary[] {
|
||||
const db = getSQLiteClient();
|
||||
const rows = db.query<{
|
||||
id: number;
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
updated_at: string;
|
||||
edge_count: number | null;
|
||||
}>(
|
||||
`
|
||||
SELECT n.id,
|
||||
n.title,
|
||||
n.description,
|
||||
n.updated_at,
|
||||
COUNT(DISTINCT e.id) AS edge_count
|
||||
FROM nodes n
|
||||
LEFT JOIN edges e
|
||||
ON (e.from_node_id = n.id OR e.to_node_id = n.id)
|
||||
GROUP BY n.id
|
||||
ORDER BY edge_count DESC, n.updated_at DESC, n.id ASC
|
||||
LIMIT ?
|
||||
`,
|
||||
[limit]
|
||||
).rows;
|
||||
const rows = db
|
||||
.query<{
|
||||
id: number;
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
updated_at: string;
|
||||
edge_count: number | null;
|
||||
}>(
|
||||
`
|
||||
SELECT n.id,
|
||||
n.title,
|
||||
n.description,
|
||||
n.updated_at,
|
||||
COUNT(DISTINCT e.id) AS edge_count
|
||||
FROM nodes n
|
||||
LEFT JOIN edges e
|
||||
ON (e.from_node_id = n.id OR e.to_node_id = n.id)
|
||||
GROUP BY n.id
|
||||
ORDER BY edge_count DESC, n.updated_at DESC, n.id ASC
|
||||
LIMIT ?
|
||||
`,
|
||||
[limit]
|
||||
)
|
||||
.rows;
|
||||
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
@@ -150,7 +151,7 @@ export function buildContextsBlock(limit = 12): string | null {
|
||||
|
||||
const lines: string[] = [
|
||||
'User Contexts',
|
||||
'Use contexts as the primary scope layer. Dimensions are secondary metadata and filters.',
|
||||
'Contexts are optional soft hints. Use them when they are explicit and useful, but rely primarily on title, description, source, edges, and recency.',
|
||||
'',
|
||||
];
|
||||
|
||||
@@ -172,7 +173,7 @@ export function buildContextAnchorsBlock(limit = 12): string | null {
|
||||
|
||||
const lines: string[] = [
|
||||
'Context Anchors',
|
||||
'Each context anchor is the highest-edge node in that context. Use it as the starting graph waypoint for that scope.',
|
||||
'Each context anchor is the highest-edge node in that context. Use it only as an optional waypoint when that context is already clearly relevant.',
|
||||
'',
|
||||
];
|
||||
|
||||
@@ -195,7 +196,7 @@ export function buildHubNodesBlock(limit = 5): string | null {
|
||||
|
||||
const lines: string[] = [
|
||||
'Global Hub Diagnostics',
|
||||
'These are secondary graph diagnostics only. Do not use them as the primary scope layer when contexts are available.',
|
||||
'These are secondary graph diagnostics only. Do not treat them as the primary grounding mechanism.',
|
||||
'',
|
||||
];
|
||||
|
||||
@@ -210,24 +211,9 @@ export function buildHubNodesBlock(limit = 5): string | null {
|
||||
}
|
||||
|
||||
export function getAutoContextSummaries(limit = 5): AutoContextSummary[] {
|
||||
const settings = getAutoContextSettings();
|
||||
if (!settings.autoContextEnabled) {
|
||||
return [];
|
||||
}
|
||||
return getHubNodes(limit);
|
||||
}
|
||||
|
||||
export function buildAutoContextBlock(limit = 12): string | null {
|
||||
const settings = getAutoContextSettings();
|
||||
if (!settings.autoContextEnabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sections = [
|
||||
buildContextsBlock(limit),
|
||||
buildContextAnchorsBlock(limit),
|
||||
buildHubNodesBlock(5),
|
||||
].filter(Boolean);
|
||||
|
||||
return sections.length > 0 ? sections.join('\n\n') : null;
|
||||
export function buildAutoContextBlock(limit = 5): string | null {
|
||||
return buildContextsBlock(limit);
|
||||
}
|
||||
|
||||
@@ -1,168 +0,0 @@
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
|
||||
type ContextCandidate = {
|
||||
id: number;
|
||||
name: string;
|
||||
description: string | null;
|
||||
count: number;
|
||||
anchor_title: string | null;
|
||||
anchor_description: string | null;
|
||||
};
|
||||
|
||||
type InferContextInput = {
|
||||
title?: string | null;
|
||||
description?: string | null;
|
||||
source?: string | null;
|
||||
dimensions?: string[] | null;
|
||||
metadata?: unknown;
|
||||
};
|
||||
|
||||
const STOP_WORDS = new Set([
|
||||
'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from', 'has', 'i',
|
||||
'in', 'is', 'it', 'its', 'of', 'on', 'or', 'that', 'the', 'their', 'this',
|
||||
'to', 'was', 'with', 'you', 'your'
|
||||
]);
|
||||
|
||||
function normalizeText(value: string): string {
|
||||
return value.toLowerCase().replace(/[^a-z0-9\s]+/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function tokenize(value: string | null | undefined): string[] {
|
||||
if (!value) return [];
|
||||
return normalizeText(value)
|
||||
.split(' ')
|
||||
.map((token) => token.trim())
|
||||
.filter((token) => token.length >= 2 && !STOP_WORDS.has(token));
|
||||
}
|
||||
|
||||
function uniqueTokens(values: Array<string | null | undefined>): string[] {
|
||||
return Array.from(new Set(values.flatMap((value) => tokenize(value))));
|
||||
}
|
||||
|
||||
function safeStringify(value: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(value ?? {});
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function fetchContextCandidates(): ContextCandidate[] {
|
||||
const sqlite = getSQLiteClient();
|
||||
return sqlite.query<ContextCandidate>(`
|
||||
WITH context_counts AS (
|
||||
SELECT c.id, c.name, c.description, COUNT(n.id) AS count
|
||||
FROM contexts c
|
||||
LEFT JOIN nodes n ON n.context_id = c.id
|
||||
GROUP BY c.id
|
||||
),
|
||||
ranked_anchors AS (
|
||||
SELECT
|
||||
c.id AS context_id,
|
||||
n.title AS anchor_title,
|
||||
n.description AS anchor_description,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY c.id
|
||||
ORDER BY COUNT(e.id) DESC, n.updated_at DESC, n.id ASC
|
||||
) AS anchor_rank
|
||||
FROM contexts c
|
||||
LEFT JOIN nodes n ON n.context_id = c.id
|
||||
LEFT JOIN edges e ON (e.from_node_id = n.id OR e.to_node_id = n.id)
|
||||
GROUP BY c.id, n.id
|
||||
)
|
||||
SELECT
|
||||
cc.id,
|
||||
cc.name,
|
||||
cc.description,
|
||||
cc.count,
|
||||
ra.anchor_title,
|
||||
ra.anchor_description
|
||||
FROM context_counts cc
|
||||
LEFT JOIN ranked_anchors ra
|
||||
ON ra.context_id = cc.id
|
||||
AND ra.anchor_rank = 1
|
||||
ORDER BY cc.name COLLATE NOCASE ASC
|
||||
`).rows.map((row) => ({
|
||||
id: Number(row.id),
|
||||
name: row.name,
|
||||
description: row.description ?? null,
|
||||
count: Number(row.count ?? 0),
|
||||
anchor_title: row.anchor_title ?? null,
|
||||
anchor_description: row.anchor_description ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
function scoreContextCandidate(candidate: ContextCandidate, input: InferContextInput): number {
|
||||
const titleText = normalizeText(input.title || '');
|
||||
const descriptionText = normalizeText(input.description || '');
|
||||
const sourceText = normalizeText((input.source || '').slice(0, 4000));
|
||||
const metadataText = normalizeText(safeStringify(input.metadata));
|
||||
const dimensionTokens = uniqueTokens(input.dimensions ?? []);
|
||||
const contextName = normalizeText(candidate.name);
|
||||
const contextNameTokens = tokenize(candidate.name);
|
||||
const contextDescriptorTokens = uniqueTokens([
|
||||
candidate.description,
|
||||
candidate.anchor_title,
|
||||
candidate.anchor_description,
|
||||
]);
|
||||
|
||||
let score = 0;
|
||||
|
||||
if (contextName && (titleText.includes(contextName) || descriptionText.includes(contextName))) {
|
||||
score += 80;
|
||||
}
|
||||
if (contextName && sourceText.includes(contextName)) {
|
||||
score += 40;
|
||||
}
|
||||
|
||||
for (const token of contextNameTokens) {
|
||||
if (dimensionTokens.includes(token)) score += 30;
|
||||
if (titleText.includes(token)) score += 16;
|
||||
if (descriptionText.includes(token)) score += 12;
|
||||
if (sourceText.includes(token)) score += 6;
|
||||
if (metadataText.includes(token)) score += 4;
|
||||
}
|
||||
|
||||
for (const token of contextDescriptorTokens) {
|
||||
if (dimensionTokens.includes(token)) score += 8;
|
||||
if (titleText.includes(token)) score += 4;
|
||||
if (descriptionText.includes(token)) score += 3;
|
||||
if (sourceText.includes(token)) score += 2;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
export async function inferBestContextIdForNode(input: InferContextInput): Promise<number | null> {
|
||||
const contexts = fetchContextCandidates();
|
||||
if (contexts.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const ranked = contexts
|
||||
.map((context) => ({
|
||||
context,
|
||||
score: scoreContextCandidate(context, input),
|
||||
}))
|
||||
.sort((a, b) =>
|
||||
b.score - a.score ||
|
||||
(b.context.count - a.context.count) ||
|
||||
a.context.id - b.context.id
|
||||
);
|
||||
|
||||
const best = ranked[0];
|
||||
if (!best) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (best.score > 0) {
|
||||
return best.context.id;
|
||||
}
|
||||
|
||||
const research = contexts.find((context) => context.name.trim().toLowerCase() === 'research');
|
||||
if (research) {
|
||||
return research.id;
|
||||
}
|
||||
|
||||
return ranked[0].context.id;
|
||||
}
|
||||
@@ -177,7 +177,9 @@ export class ContextService {
|
||||
}
|
||||
|
||||
async resolveContextId(input: { context_id?: number | null; context_name?: string | null }): Promise<number | null | undefined> {
|
||||
const hasContextId = Object.prototype.hasOwnProperty.call(input, 'context_id');
|
||||
const hasContextId =
|
||||
Object.prototype.hasOwnProperty.call(input, 'context_id') &&
|
||||
input.context_id !== undefined;
|
||||
const hasContextName = typeof input.context_name === 'string' && input.context_name.trim().length > 0;
|
||||
|
||||
if (!hasContextId && !hasContextName) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { openai as openaiProvider } from '@ai-sdk/openai';
|
||||
import { generateText } from 'ai';
|
||||
import { openai as openaiProvider } from '@ai-sdk/openai';
|
||||
import { hasValidOpenAiKey } from '../storage/apiKeys';
|
||||
import type { CanonicalNodeMetadata } from '@/types/database';
|
||||
|
||||
@@ -16,11 +16,12 @@ export interface DescriptionInput {
|
||||
pages?: number;
|
||||
text_length?: number;
|
||||
};
|
||||
dimensions?: string[];
|
||||
}
|
||||
|
||||
export { hasValidOpenAiKey } from '../storage/apiKeys';
|
||||
|
||||
/**
|
||||
* Generate a context-rich description for a knowledge node.
|
||||
* The result must cover what the artifact is, why it is in the graph, and workflow status.
|
||||
*/
|
||||
export async function generateDescription(input: DescriptionInput): Promise<string> {
|
||||
if (!hasValidOpenAiKey()) {
|
||||
console.log(`[DescriptionService] No valid OpenAI key, using fallback for: "${input.title}"`);
|
||||
@@ -46,10 +47,13 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
|
||||
return description;
|
||||
} catch (error) {
|
||||
console.error('[DescriptionService] Error generating description:', error);
|
||||
// Fallback: just use the title — more useful than a vague template
|
||||
return `${input.title}. Added via Quick Add with no further context yet, so the reason it belongs in the graph is not fully inferred. It has not been reviewed yet.`.slice(0, 500);
|
||||
}
|
||||
}
|
||||
|
||||
export { hasValidOpenAiKey } from '../storage/apiKeys';
|
||||
|
||||
function buildDescriptionPrompt(input: DescriptionInput): string {
|
||||
const sourceMetadata = input.metadata?.source_metadata as Record<string, unknown> | undefined;
|
||||
const sourceType = typeof input.metadata?.type === 'string'
|
||||
@@ -60,6 +64,8 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
|
||||
const normalizedSource = sourceType.toLowerCase();
|
||||
const url = typeof input.link === 'string' ? input.link.trim() : '';
|
||||
|
||||
// Best-effort creator hint from structured metadata (when available),
|
||||
// but never assume a particular extraction source (YouTube vs paper vs website vs note).
|
||||
const creatorHint =
|
||||
(typeof sourceMetadata?.author === 'string' ? sourceMetadata.author.trim() : '') ||
|
||||
(typeof sourceMetadata?.channel_name === 'string' ? sourceMetadata.channel_name.trim() : '') ||
|
||||
@@ -67,6 +73,7 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
|
||||
(typeof input.metadata?.channel_name === 'string' ? input.metadata.channel_name.trim() : '') ||
|
||||
'';
|
||||
|
||||
// Best-effort publisher / container hint (less ideal than a true author, but better than nothing).
|
||||
const publisherHint =
|
||||
(typeof sourceMetadata?.site_name === 'string' ? sourceMetadata.site_name.trim() : '') ||
|
||||
(typeof input.metadata?.site_name === 'string' ? input.metadata.site_name.trim() : '') ||
|
||||
@@ -90,7 +97,6 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
|
||||
const lines: string[] = [`Title: ${input.title}`];
|
||||
|
||||
if (input.link) lines.push(`URL: ${input.link}`);
|
||||
if (input.dimensions?.length) lines.push(`Dimensions: ${input.dimensions.join(', ')}`);
|
||||
if (sourceMetadata?.channel_name || input.metadata?.channel_name) lines.push(`Channel: ${sourceMetadata?.channel_name || input.metadata?.channel_name}`);
|
||||
if (sourceMetadata?.author || input.metadata?.author) lines.push(`Author: ${sourceMetadata?.author || input.metadata?.author}`);
|
||||
if (sourceMetadata?.site_name || input.metadata?.site_name) lines.push(`Site: ${sourceMetadata?.site_name || input.metadata?.site_name}`);
|
||||
@@ -116,7 +122,7 @@ RULES:
|
||||
1) Name the format only if the context clearly supports it: "Podcast episode where…", "Blog post arguing…", "Personal note capturing…", "Research paper showing…", "Resume/CV for…", "Document likely containing…", "Idea that…"
|
||||
2) Name people by role — channel/host is the creator, title figures are guests/subjects. Use the Creator hint if available.
|
||||
3) State the actual claim, finding, or insight from the content — not a vague summary of the topic.
|
||||
4) If the reason it belongs in the graph cannot be inferred from title, source excerpt, URL, metadata, or dimensions, say that naturally rather than inventing context.
|
||||
4) If the reason it belongs in the graph cannot be inferred from title, source excerpt, URL, or metadata, say that naturally rather than inventing context.
|
||||
5) If workflow status is unknown, say that naturally, for example by noting it has not been reviewed yet.
|
||||
6) Do NOT use labels or headings like "WHAT:", "WHY:", or "STATUS:".
|
||||
7) ABSOLUTELY FORBIDDEN — these words and phrases will be rejected: "discusses", "explores", "examines", "talks about", "is about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to", "important for", "useful for understanding". State things directly instead.
|
||||
@@ -144,6 +150,7 @@ function sanitizeDescription(rawText: string, input: DescriptionInput): string {
|
||||
return `${input.title}. Added via Quick Add with no further context yet, so the reason it belongs in the graph is not fully inferred. It has not been reviewed yet.`.slice(0, 500);
|
||||
}
|
||||
|
||||
// Guard against weak generic openings from model drift.
|
||||
const noGenericPrefix = singleLine.replace(
|
||||
/^(your note|this note)\s*[—:-]\s*/i,
|
||||
'Personal note capturing '
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
|
||||
export interface Dimension {
|
||||
name: string;
|
||||
description: string | null;
|
||||
is_priority: boolean;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface LockedDimension {
|
||||
name: string;
|
||||
description: string | null;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export class DimensionService {
|
||||
/**
|
||||
* Legacy compatibility shim. Dimensions are now flat, so there is no locked subset.
|
||||
*/
|
||||
static async getLockedDimensions(): Promise<LockedDimension[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatic special-dimension assignment has been removed. Callers must provide dimensions explicitly.
|
||||
*/
|
||||
static async assignDimensions(nodeData: {
|
||||
title: string;
|
||||
content?: string;
|
||||
link?: string;
|
||||
description?: string;
|
||||
}): Promise<{ locked: string[]; keywords: string[] }> {
|
||||
console.log(`[DimensionAssignment] Skipped for "${nodeData.title}" — flat dimensions require explicit assignment.`);
|
||||
return { locked: [], keywords: [] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy method for backwards compatibility
|
||||
* @deprecated Use assignDimensions() instead
|
||||
*/
|
||||
static async assignLockedDimensions(nodeData: {
|
||||
title: string;
|
||||
content?: string;
|
||||
link?: string;
|
||||
}): Promise<string[]> {
|
||||
const result = await this.assignDimensions(nodeData);
|
||||
return result.locked;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update dimension description
|
||||
*/
|
||||
static async updateDimensionDescription(name: string, description: string): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
sqlite.query(`
|
||||
INSERT INTO dimensions(name, description, is_priority, updated_at)
|
||||
VALUES (?, ?, 0, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
description = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`, [name, description, description]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get dimension by name with description
|
||||
*/
|
||||
static async getDimensionByName(name: string): Promise<Dimension | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.query(`
|
||||
SELECT name, description, is_priority, updated_at
|
||||
FROM dimensions
|
||||
WHERE name = ?
|
||||
`, [name]);
|
||||
|
||||
if (result.rows.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const row = result.rows[0] as any;
|
||||
return {
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
is_priority: Boolean(row.is_priority),
|
||||
updated_at: row.updated_at
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy no-op prompt builder retained only for backward compatibility.
|
||||
*/
|
||||
private static buildAssignmentPrompt(
|
||||
nodeData: { title: string; notes?: string; link?: string; description?: string },
|
||||
lockedDimensions: LockedDimension[]
|
||||
): string {
|
||||
// Use description as primary context, content as fallback
|
||||
let nodeContextSection: string;
|
||||
if (nodeData.description) {
|
||||
const notesPreview = nodeData.notes?.slice(0, 500) || '';
|
||||
nodeContextSection = `DESCRIPTION: ${nodeData.description}
|
||||
|
||||
NOTES PREVIEW: ${notesPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`;
|
||||
} else {
|
||||
const notesPreview = nodeData.notes?.slice(0, 2000) || '';
|
||||
nodeContextSection = `NOTES: ${notesPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`;
|
||||
}
|
||||
|
||||
// Include ALL locked dimensions, using fallback text for those without descriptions
|
||||
const dimensionsList = lockedDimensions
|
||||
.map(d => {
|
||||
const description = d.description && d.description.trim().length > 0
|
||||
? d.description
|
||||
: '(none - infer from name)';
|
||||
return `DIMENSION: "${d.name}"\nDESCRIPTION: ${description}`;
|
||||
})
|
||||
.join('\n---\n');
|
||||
|
||||
return `Dimensions are now flat categories with no locked subset.
|
||||
|
||||
=== NODE TO CATEGORIZE ===
|
||||
Title: ${nodeData.title}
|
||||
${nodeContextSection}
|
||||
URL: ${nodeData.link || 'none'}
|
||||
|
||||
=== LOCKED DIMENSIONS ===
|
||||
CRITICAL: Read each dimension's DESCRIPTION carefully.
|
||||
The description defines what belongs in that dimension.
|
||||
Only assign if the content CLEARLY matches the description.
|
||||
If unsure, skip it — better to miss than assign incorrectly.
|
||||
|
||||
AVAILABLE DIMENSIONS:
|
||||
${dimensionsList}
|
||||
|
||||
=== RESPONSE FORMAT ===
|
||||
LOCKED:
|
||||
[dimension names from the list above, one per line, or "none"]`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy no-op parser retained only for backward compatibility.
|
||||
*/
|
||||
private static parseAssignmentResponse(
|
||||
response: string,
|
||||
availableDimensions: LockedDimension[]
|
||||
): { locked: string[]; keywords: string[] } {
|
||||
const lockedDimensions: string[] = [];
|
||||
|
||||
// Extract LOCKED section
|
||||
const lockedMatch = response.match(/LOCKED:\s*([\s\S]*?)$/i);
|
||||
|
||||
if (lockedMatch) {
|
||||
const lockedLines = lockedMatch[1].trim().split('\n');
|
||||
for (const line of lockedLines) {
|
||||
const dimensionName = line.trim().toLowerCase();
|
||||
|
||||
if (dimensionName === 'none' || dimensionName === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Find matching dimension (case-insensitive)
|
||||
const matchedDimension = availableDimensions.find(
|
||||
d => d.name.toLowerCase() === dimensionName
|
||||
);
|
||||
|
||||
if (matchedDimension && !lockedDimensions.includes(matchedDimension.name)) {
|
||||
lockedDimensions.push(matchedDimension.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { locked: lockedDimensions, keywords: [] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create or get a keyword dimension (unlocked)
|
||||
*/
|
||||
static async ensureKeywordDimension(keyword: string): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
// INSERT OR IGNORE - if dimension exists, do nothing
|
||||
sqlite.query(`
|
||||
INSERT OR IGNORE INTO dimensions(name, description, is_priority, updated_at)
|
||||
VALUES (?, ?, 0, CURRENT_TIMESTAMP)
|
||||
`, [keyword, null]);
|
||||
}
|
||||
}
|
||||
|
||||
export const dimensionService = new DimensionService();
|
||||
@@ -1,29 +0,0 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { normalizeDimensionName } from './quality';
|
||||
|
||||
export function getUnknownDimensions(values: string[]): string[] {
|
||||
if (values.length === 0) return [];
|
||||
|
||||
const sqlite = getSQLiteClient();
|
||||
const placeholders = values.map(() => '?').join(', ');
|
||||
const result = sqlite.query<{ name: string }>(
|
||||
`SELECT name FROM dimensions WHERE name IN (${placeholders})`,
|
||||
values
|
||||
);
|
||||
|
||||
const existing = new Set(
|
||||
result.rows
|
||||
.map(row => (typeof row.name === 'string' ? normalizeDimensionName(row.name) : ''))
|
||||
.filter(Boolean)
|
||||
);
|
||||
|
||||
return values.filter(value => !existing.has(normalizeDimensionName(value)));
|
||||
}
|
||||
|
||||
export function formatUnknownDimensionsError(values: string[]): string {
|
||||
if (values.length === 1) {
|
||||
return `Unknown dimension: "${values[0]}". Create it first or use an existing dimension.`;
|
||||
}
|
||||
|
||||
return `Unknown dimensions: ${values.map(value => `"${value}"`).join(', ')}. Create them first or use existing dimensions.`;
|
||||
}
|
||||
@@ -2,11 +2,11 @@ import { getSQLiteClient } from './sqlite-client';
|
||||
import { Edge, EdgeContext, EdgeData, EdgeCreatedVia, NodeConnection, Node } from '@/types/database';
|
||||
import { eventBroadcaster } from '../events';
|
||||
import { nodeService } from './nodes';
|
||||
import { getOpenAiKey } from '../storage/apiKeys';
|
||||
import { generateText } from 'ai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { z } from 'zod';
|
||||
import { validateEdgeExplanation } from './quality';
|
||||
import { hasValidOpenAiKey } from '../storage/apiKeys';
|
||||
|
||||
const inferredEdgeContextSchema = z.object({
|
||||
type: z.enum(['created_by', 'part_of', 'source_of', 'related_to']),
|
||||
@@ -53,14 +53,6 @@ async function inferEdgeContext(params: {
|
||||
return { type: 'related_to', confidence: 0.8, swap_direction: false };
|
||||
}
|
||||
|
||||
// If no API key is configured, degrade gracefully.
|
||||
// We still enforce explanation, but fall back to "related_to" classification.
|
||||
const apiKey = getOpenAiKey();
|
||||
if (!apiKey) {
|
||||
return { type: 'related_to', confidence: 0.0, swap_direction: false };
|
||||
}
|
||||
|
||||
const provider = createOpenAI({ apiKey });
|
||||
const prompt = [
|
||||
`Given two nodes and an explanation, determine the relationship type and direction.`,
|
||||
``,
|
||||
@@ -83,8 +75,12 @@ async function inferEdgeContext(params: {
|
||||
].join('\n');
|
||||
|
||||
try {
|
||||
if (!hasValidOpenAiKey()) {
|
||||
return { type: 'related_to', confidence: 0.2, swap_direction: false };
|
||||
}
|
||||
|
||||
const { text } = await generateText({
|
||||
model: provider('gpt-4o-mini'),
|
||||
model: openai('gpt-4o-mini'),
|
||||
prompt,
|
||||
temperature: 0.0,
|
||||
maxOutputTokens: 120,
|
||||
@@ -120,18 +116,6 @@ async function autoInferEdge(params: {
|
||||
}): Promise<{ explanation: string; type: EdgeContext['type']; confidence: number; swap_direction: boolean }> {
|
||||
const { fromNode, toNode } = params;
|
||||
|
||||
const apiKey = getOpenAiKey();
|
||||
if (!apiKey) {
|
||||
// Fallback without AI
|
||||
return {
|
||||
explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
|
||||
type: 'related_to',
|
||||
confidence: 0.0,
|
||||
swap_direction: false,
|
||||
};
|
||||
}
|
||||
|
||||
const provider = createOpenAI({ apiKey });
|
||||
const prompt = [
|
||||
`Given two knowledge base nodes, determine how they are related.`,
|
||||
``,
|
||||
@@ -157,8 +141,17 @@ async function autoInferEdge(params: {
|
||||
].join('\n');
|
||||
|
||||
try {
|
||||
if (!hasValidOpenAiKey()) {
|
||||
return {
|
||||
explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
|
||||
type: 'related_to',
|
||||
confidence: 0.2,
|
||||
swap_direction: false,
|
||||
};
|
||||
}
|
||||
|
||||
const { text } = await generateText({
|
||||
model: provider('gpt-4o-mini'),
|
||||
model: openai('gpt-4o-mini'),
|
||||
prompt,
|
||||
temperature: 0.0,
|
||||
maxOutputTokens: 150,
|
||||
@@ -477,15 +470,14 @@ export class EdgeService {
|
||||
WHEN e.from_node_id = ? THEN n_to.title
|
||||
ELSE n_from.title
|
||||
END as connected_node_title,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.link
|
||||
CASE WHEN e.from_node_id = ? THEN n_to.link
|
||||
ELSE n_from.link
|
||||
END as connected_node_link,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.source
|
||||
ELSE n_from.source
|
||||
END as connected_node_source,
|
||||
CASE
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.metadata
|
||||
ELSE n_from.metadata
|
||||
END as connected_node_metadata,
|
||||
@@ -496,17 +488,7 @@ export class EdgeService {
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.updated_at
|
||||
ELSE n_from.updated_at
|
||||
END as connected_node_updated_at,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN (
|
||||
SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n_to.id
|
||||
)
|
||||
ELSE (
|
||||
SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n_from.id
|
||||
)
|
||||
END as connected_node_dimensions_json
|
||||
END as connected_node_updated_at
|
||||
FROM edges e
|
||||
LEFT JOIN nodes n_from ON e.from_node_id = n_from.id
|
||||
LEFT JOIN nodes n_to ON e.to_node_id = n_to.id
|
||||
@@ -521,7 +503,6 @@ export class EdgeService {
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId
|
||||
]);
|
||||
|
||||
@@ -543,7 +524,6 @@ export class EdgeService {
|
||||
id: row.connected_node_id,
|
||||
title: row.connected_node_title,
|
||||
link: row.connected_node_link,
|
||||
dimensions: row.connected_node_dimensions,
|
||||
embedding: undefined, // Not needed for display
|
||||
source: row.connected_node_source,
|
||||
metadata: row.connected_node_metadata,
|
||||
@@ -587,7 +567,6 @@ export class EdgeService {
|
||||
id: row.connected_node_id,
|
||||
title: row.connected_node_title,
|
||||
link: row.connected_node_link,
|
||||
dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'),
|
||||
embedding: undefined, // Not needed for display
|
||||
source: row.connected_node_source,
|
||||
metadata: typeof row.connected_node_metadata === 'string' ? JSON.parse(row.connected_node_metadata) : row.connected_node_metadata,
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
export { nodeService, NodeService } from './nodes';
|
||||
export { chunkService, ChunkService } from './chunks';
|
||||
export { edgeService, EdgeService } from './edges';
|
||||
export { dimensionService, DimensionService } from './dimensionService';
|
||||
export { contextService, ContextService } from './contextService';
|
||||
// export { HelperService } from './helpers'; // Removed - migrated to JSON-based service
|
||||
|
||||
@@ -39,7 +38,6 @@ async function checkSQLiteDatabaseHealth(): Promise<{
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const connected = await sqlite.testConnection();
|
||||
const vectorCapability = sqlite.getVectorCapability();
|
||||
if (!connected) {
|
||||
return {
|
||||
connected: false,
|
||||
@@ -49,7 +47,7 @@ async function checkSQLiteDatabaseHealth(): Promise<{
|
||||
};
|
||||
}
|
||||
|
||||
const vectorExtension = vectorCapability.available;
|
||||
const vectorExtension = await sqlite.checkVectorExtension();
|
||||
|
||||
// Check if main tables exist
|
||||
const tables = await sqlite.checkTables();
|
||||
|
||||
+74
-152
@@ -5,7 +5,9 @@ import { EmbeddingService } from '@/services/embeddings';
|
||||
import { scoreNodeSearchMatch } from './searchRanking';
|
||||
import { buildCanonicalNodeMetadata, mergeNodeMetadata } from '@/services/nodes/metadata';
|
||||
|
||||
type NodeRow = Node & { dimensions_json: string; context_json: string | null };
|
||||
type NodeRow = Node & {
|
||||
context_json: string | null;
|
||||
};
|
||||
type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
|
||||
|
||||
function sanitizeFtsQuery(input: string): string {
|
||||
@@ -81,8 +83,15 @@ export class NodeService {
|
||||
}
|
||||
|
||||
async countNodes(filters: NodeFilters = {}): Promise<number> {
|
||||
const { dimensions, search, dimensionsMatch = 'any',
|
||||
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus, contextId } = filters;
|
||||
const {
|
||||
search,
|
||||
createdAfter,
|
||||
createdBefore,
|
||||
eventAfter,
|
||||
eventBefore,
|
||||
chunkStatus,
|
||||
contextId,
|
||||
} = filters;
|
||||
|
||||
if (search?.trim()) {
|
||||
return this.countSearchNodesSQLite(filters);
|
||||
@@ -93,24 +102,6 @@ export class NodeService {
|
||||
let query = `SELECT COUNT(*) as total FROM nodes n WHERE 1=1`;
|
||||
const params: any[] = [];
|
||||
|
||||
if (dimensions && dimensions.length > 0) {
|
||||
if (dimensionsMatch === 'all' && dimensions.length > 1) {
|
||||
query += ` AND (
|
||||
SELECT COUNT(DISTINCT nd.dimension) FROM node_dimensions nd
|
||||
WHERE nd.node_id = n.id
|
||||
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
|
||||
) = ?`;
|
||||
params.push(...dimensions, dimensions.length);
|
||||
} else {
|
||||
query += ` AND EXISTS (
|
||||
SELECT 1 FROM node_dimensions nd
|
||||
WHERE nd.node_id = n.id
|
||||
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
|
||||
)`;
|
||||
params.push(...dimensions);
|
||||
}
|
||||
}
|
||||
|
||||
if (search) {
|
||||
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
|
||||
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
|
||||
@@ -130,8 +121,18 @@ export class NodeService {
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
|
||||
const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any',
|
||||
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus, contextId } = filters;
|
||||
const {
|
||||
search,
|
||||
limit = 100,
|
||||
offset = 0,
|
||||
sortBy,
|
||||
createdAfter,
|
||||
createdBefore,
|
||||
eventAfter,
|
||||
eventBefore,
|
||||
chunkStatus,
|
||||
contextId,
|
||||
} = filters;
|
||||
|
||||
if (search?.trim()) {
|
||||
return this.searchNodesSQLite(filters);
|
||||
@@ -139,13 +140,10 @@ export class NodeService {
|
||||
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
|
||||
let query = `
|
||||
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at, n.context_id,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
CASE
|
||||
WHEN c.id IS NULL THEN NULL
|
||||
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
|
||||
@@ -157,27 +155,6 @@ export class NodeService {
|
||||
`;
|
||||
const params: any[] = [];
|
||||
|
||||
// Filter by dimensions (SQLite JOIN with node_dimensions)
|
||||
if (dimensions && dimensions.length > 0) {
|
||||
if (dimensionsMatch === 'all' && dimensions.length > 1) {
|
||||
// AND logic: node must have ALL specified dimensions
|
||||
query += ` AND (
|
||||
SELECT COUNT(DISTINCT nd.dimension) FROM node_dimensions nd
|
||||
WHERE nd.node_id = n.id
|
||||
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
|
||||
) = ?`;
|
||||
params.push(...dimensions, dimensions.length);
|
||||
} else {
|
||||
// OR logic: node must have at least one of the specified dimensions
|
||||
query += ` AND EXISTS (
|
||||
SELECT 1 FROM node_dimensions nd
|
||||
WHERE nd.node_id = n.id
|
||||
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
|
||||
)`;
|
||||
params.push(...dimensions);
|
||||
}
|
||||
}
|
||||
|
||||
// Text search in title, description, and source (SQLite LIKE with COLLATE NOCASE)
|
||||
if (search) {
|
||||
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
|
||||
@@ -251,7 +228,7 @@ export class NodeService {
|
||||
|
||||
const result = sqlite.query<NodeRow>(query, params);
|
||||
|
||||
// Parse dimensions_json and metadata back for compatibility
|
||||
// Parse metadata and normalize deprecated compatibility fields.
|
||||
return result.rows.map(row => this.mapNodeRow(row));
|
||||
}
|
||||
|
||||
@@ -267,8 +244,6 @@ export class NodeService {
|
||||
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at, n.context_id,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
CASE
|
||||
WHEN c.id IS NULL THEN NULL
|
||||
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
|
||||
@@ -298,7 +273,6 @@ export class NodeService {
|
||||
source,
|
||||
link,
|
||||
event_date,
|
||||
dimensions = [],
|
||||
chunk_status,
|
||||
metadata = {},
|
||||
context_id,
|
||||
@@ -327,20 +301,10 @@ export class NodeService {
|
||||
|
||||
const id = Number(nodeResult.lastInsertRowid);
|
||||
|
||||
// Insert dimensions separately with INSERT OR IGNORE for safety
|
||||
if (dimensions.length > 0) {
|
||||
const stmt = sqlite.prepare(
|
||||
"INSERT OR IGNORE INTO node_dimensions (node_id, dimension) VALUES (?, ?)"
|
||||
);
|
||||
for (const dimension of dimensions) {
|
||||
stmt.run(id, dimension);
|
||||
}
|
||||
}
|
||||
|
||||
return id; // Returns number directly
|
||||
});
|
||||
|
||||
// Get the created node with dimensions (outside transaction)
|
||||
// Re-read the created node outside the transaction so callers get the canonical shape.
|
||||
const createdNode = await this.getNodeByIdSQLite(nodeId);
|
||||
if (!createdNode) {
|
||||
throw new Error('Failed to create node');
|
||||
@@ -363,7 +327,7 @@ export class NodeService {
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async updateNodeSQLite(id: number, updates: Partial<Node>): Promise<Node> {
|
||||
const { title, description, source, link, event_date, dimensions, metadata } = updates;
|
||||
const { title, description, source, link, event_date, metadata } = updates;
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
@@ -411,14 +375,6 @@ export class NodeService {
|
||||
stmt.run(...params);
|
||||
}
|
||||
|
||||
// Handle dimensions separately
|
||||
if (Array.isArray(dimensions)) {
|
||||
sqlite.prepare('DELETE FROM node_dimensions WHERE node_id = ?').run(id);
|
||||
const dimStmt = sqlite.prepare('INSERT OR IGNORE INTO node_dimensions (node_id, dimension) VALUES (?, ?)');
|
||||
for (const dim of dimensions) {
|
||||
dimStmt.run(id, dim);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Get updated node
|
||||
@@ -458,28 +414,21 @@ export class NodeService {
|
||||
});
|
||||
}
|
||||
|
||||
// Dimension-based filtering methods
|
||||
async getNodesByDimension(dimension: string): Promise<Node[]> {
|
||||
return this.getNodes({ dimensions: [dimension] });
|
||||
}
|
||||
|
||||
async searchNodes(searchTerm: string, limit = 50): Promise<Node[]> {
|
||||
return this.getNodes({ search: searchTerm, limit });
|
||||
}
|
||||
|
||||
private mapNodeRow(row: NodeRow): Node {
|
||||
const { context_json, ...baseRow } = row;
|
||||
return {
|
||||
...row,
|
||||
dimensions: JSON.parse(row.dimensions_json || '[]'),
|
||||
metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null,
|
||||
context: row.context_json ? JSON.parse(row.context_json) : null,
|
||||
...baseRow,
|
||||
metadata: baseRow.metadata ? (typeof baseRow.metadata === 'string' ? JSON.parse(baseRow.metadata) : baseRow.metadata) : null,
|
||||
context: context_json ? JSON.parse(context_json) : null,
|
||||
};
|
||||
}
|
||||
|
||||
private buildNodeFilterClauses(filters: NodeFilters, alias = 'n'): { clauses: string[]; params: any[] } {
|
||||
const {
|
||||
dimensions,
|
||||
dimensionsMatch = 'any',
|
||||
createdAfter,
|
||||
createdBefore,
|
||||
eventAfter,
|
||||
@@ -490,24 +439,6 @@ export class NodeService {
|
||||
const clauses: string[] = [];
|
||||
const params: any[] = [];
|
||||
|
||||
if (dimensions && dimensions.length > 0) {
|
||||
if (dimensionsMatch === 'all' && dimensions.length > 1) {
|
||||
clauses.push(`(
|
||||
SELECT COUNT(DISTINCT nd.dimension) FROM node_dimensions nd
|
||||
WHERE nd.node_id = ${alias}.id
|
||||
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
|
||||
) = ?`);
|
||||
params.push(...dimensions, dimensions.length);
|
||||
} else {
|
||||
clauses.push(`EXISTS (
|
||||
SELECT 1 FROM node_dimensions nd
|
||||
WHERE nd.node_id = ${alias}.id
|
||||
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
|
||||
)`);
|
||||
params.push(...dimensions);
|
||||
}
|
||||
}
|
||||
|
||||
if (createdAfter) { clauses.push(`${alias}.created_at >= ?`); params.push(createdAfter); }
|
||||
if (createdBefore) { clauses.push(`${alias}.created_at < ?`); params.push(createdBefore); }
|
||||
if (eventAfter) { clauses.push(`${alias}.event_date >= ?`); params.push(eventAfter); }
|
||||
@@ -567,26 +498,30 @@ export class NodeService {
|
||||
if (!search) return 0;
|
||||
|
||||
const ftsQuery = sanitizeFtsQuery(search);
|
||||
const ftsExists = sqlite.prepare(
|
||||
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
|
||||
).get();
|
||||
const { clauses, params } = this.buildNodeFilterClauses(filters);
|
||||
|
||||
if (ftsExists && ftsQuery) {
|
||||
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
const result = sqlite.query<{ total: number }>(`
|
||||
WITH matched_nodes AS (
|
||||
SELECT rowid
|
||||
FROM nodes_fts
|
||||
WHERE nodes_fts MATCH ?
|
||||
)
|
||||
SELECT COUNT(*) as total
|
||||
FROM matched_nodes mn
|
||||
JOIN nodes n ON n.id = mn.rowid
|
||||
${whereClauses}
|
||||
`, [ftsQuery, ...params]);
|
||||
try {
|
||||
const result = sqlite.query<{ total: number }>(`
|
||||
WITH matched_nodes AS (
|
||||
SELECT rowid
|
||||
FROM nodes_fts
|
||||
WHERE nodes_fts MATCH ?
|
||||
)
|
||||
SELECT COUNT(*) as total
|
||||
FROM matched_nodes mn
|
||||
JOIN nodes n ON n.id = mn.rowid
|
||||
${whereClauses}
|
||||
`, [ftsQuery, ...params]);
|
||||
|
||||
return Number(result.rows[0]?.total ?? 0);
|
||||
return Number(result.rows[0]?.total ?? 0);
|
||||
} catch (error) {
|
||||
sqlite.disableNodesFts('nodes_fts query failed during count search', error);
|
||||
}
|
||||
}
|
||||
|
||||
const words = search.split(/\s+/).filter(Boolean);
|
||||
@@ -615,7 +550,7 @@ export class NodeService {
|
||||
const ftsQuery = sanitizeFtsQuery(search);
|
||||
if (!ftsQuery) return [];
|
||||
|
||||
const ftsExists = sqlite.prepare(
|
||||
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
|
||||
).get();
|
||||
if (!ftsExists) return [];
|
||||
@@ -633,12 +568,15 @@ export class NodeService {
|
||||
)
|
||||
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
n.created_at, n.updated_at, n.context_id,
|
||||
CASE
|
||||
WHEN c.id IS NULL THEN NULL
|
||||
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
|
||||
END as context_json,
|
||||
fm.rank
|
||||
FROM fts_matches fm
|
||||
JOIN nodes n ON n.id = fm.rowid
|
||||
LEFT JOIN contexts c ON c.id = n.context_id
|
||||
${whereClauses}
|
||||
ORDER BY fm.rank
|
||||
LIMIT ?
|
||||
@@ -646,7 +584,7 @@ export class NodeService {
|
||||
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
console.warn('[NodeSearch] FTS search failed, falling back to LIKE:', error);
|
||||
sqlite.disableNodesFts('nodes_fts query failed during node search', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -662,10 +600,13 @@ export class NodeService {
|
||||
let query = `
|
||||
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
n.created_at, n.updated_at, n.context_id,
|
||||
CASE
|
||||
WHEN c.id IS NULL THEN NULL
|
||||
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
|
||||
END as context_json
|
||||
FROM nodes n
|
||||
LEFT JOIN contexts c ON c.id = n.context_id
|
||||
WHERE 1=1
|
||||
`;
|
||||
const queryParams = [...params];
|
||||
@@ -707,10 +648,13 @@ export class NodeService {
|
||||
let query = `
|
||||
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
n.created_at, n.updated_at, n.context_id,
|
||||
CASE
|
||||
WHEN c.id IS NULL THEN NULL
|
||||
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
|
||||
END as context_json
|
||||
FROM nodes n
|
||||
LEFT JOIN contexts c ON c.id = n.context_id
|
||||
WHERE 1=1
|
||||
`;
|
||||
const queryParams = [...params];
|
||||
@@ -781,12 +725,15 @@ export class NodeService {
|
||||
)
|
||||
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
n.created_at, n.updated_at, n.context_id,
|
||||
CASE
|
||||
WHEN c.id IS NULL THEN NULL
|
||||
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
|
||||
END as context_json,
|
||||
(1.0 / (1.0 + vm.distance)) AS similarity
|
||||
FROM vector_matches vm
|
||||
JOIN nodes n ON n.id = vm.node_id
|
||||
LEFT JOIN contexts c ON c.id = n.context_id
|
||||
${whereClauses}
|
||||
ORDER BY vm.distance
|
||||
LIMIT ?
|
||||
@@ -829,31 +776,6 @@ export class NodeService {
|
||||
return updatedNodes;
|
||||
}
|
||||
|
||||
// Get all unique dimensions for UI filtering
|
||||
async getAllDimensions(): Promise<string[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const query = `
|
||||
SELECT DISTINCT dimension
|
||||
FROM node_dimensions
|
||||
ORDER BY dimension
|
||||
`;
|
||||
const result = sqlite.query<{dimension: string}>(query);
|
||||
return result.rows.map(row => row.dimension);
|
||||
}
|
||||
|
||||
// Get dimension usage statistics
|
||||
async getDimensionStats(): Promise<{dimension: string, count: number}[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const query = `
|
||||
SELECT dimension, COUNT(*) as count
|
||||
FROM node_dimensions
|
||||
GROUP BY dimension
|
||||
ORDER BY count DESC
|
||||
`;
|
||||
const result = sqlite.query<{dimension: string, count: number}>(query);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
|
||||
@@ -1,34 +1,10 @@
|
||||
const WEAK_DESCRIPTION_PATTERNS = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i;
|
||||
const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|concept|conversation|dataset|decision|dimension|document|episode|essay|event|guide|idea|insight|interview|lesson|link|node|note|paper|person|plan|placeholder|podcast|post|presentation|project|question|record|research|resource|skill|source|status|summary|talk|target|test node|thread|tool|transcript|tweet|update|video|website|workflow)\b/i;
|
||||
const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|concept|conversation|dataset|decision|document|episode|essay|event|guide|idea|insight|interview|lesson|link|node|note|paper|person|plan|placeholder|podcast|post|presentation|project|question|record|research|resource|skill|source|status|summary|talk|target|test node|thread|tool|transcript|tweet|update|video|website|workflow)\b/i;
|
||||
const UNCERTAINTY_PATTERNS = /\b(likely|probably|possibly|appears to be|seems to be|unclear|uncertain)\b/i;
|
||||
const WHY_PATTERNS = /(why added:|added (?:after|as|for|because|to)|follow-?on|queued for|saved for|relevant because|connected to|not inferred|belongs in the graph because|belongs here because|captures .* idea|ties directly into|ties into)/i;
|
||||
const STATUS_PATTERNS = /(status:|queued|not yet reviewed|in progress|processed|reviewed|saved for later|to review|to read|to watch|to listen|draft|not yet published|unpublished)/i;
|
||||
const GENERIC_EDGE_PATTERNS = /^(related|related to|connected|connected to|association|associated with)$/i;
|
||||
|
||||
export function normalizeDimensionName(value: string): string {
|
||||
return value.trim().replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
export function normalizeDimensions(values: unknown, max = 5): string[] {
|
||||
if (!Array.isArray(values)) return [];
|
||||
|
||||
const seen = new Set<string>();
|
||||
const normalized: string[] = [];
|
||||
|
||||
for (const value of values) {
|
||||
if (typeof value !== 'string') continue;
|
||||
const trimmed = normalizeDimensionName(value);
|
||||
if (!trimmed) continue;
|
||||
const key = trimmed.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
normalized.push(trimmed);
|
||||
if (normalized.length >= max) break;
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function validateExplicitDescription(description: string): string | null {
|
||||
const text = description.trim();
|
||||
if (text.length > 500) {
|
||||
@@ -84,14 +60,3 @@ export function validateEdgeExplanation(explanation: string): string | null {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function validateDimensionDescription(description: string): string | null {
|
||||
const text = description.trim();
|
||||
if (!text) {
|
||||
return 'Dimension description is required.';
|
||||
}
|
||||
if (text.length > 500) {
|
||||
return 'Description must be 500 characters or less.';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -25,6 +25,8 @@ class SQLiteClient {
|
||||
private config: SQLiteConfig;
|
||||
private readonly readOnly: boolean;
|
||||
private readonly vectorCapability: VectorCapability;
|
||||
private nodesFtsUsable = true;
|
||||
private nodesFtsDisabledReason: string | null = null;
|
||||
|
||||
private constructor() {
|
||||
this.config = this.getSQLiteConfig();
|
||||
@@ -61,6 +63,7 @@ class SQLiteClient {
|
||||
this.db.pragma('temp_store = memory');
|
||||
this.db.pragma('busy_timeout = 5000');
|
||||
|
||||
this.ensureCoreSchema();
|
||||
// Ensure vector virtual tables are present and healthy
|
||||
if (this.vectorCapability.available) {
|
||||
this.ensureVectorTables();
|
||||
@@ -162,6 +165,25 @@ class SQLiteClient {
|
||||
return this.vectorCapability;
|
||||
}
|
||||
|
||||
public isNodesFtsUsable(): boolean {
|
||||
return this.nodesFtsUsable;
|
||||
}
|
||||
|
||||
public disableNodesFts(reason: string, error?: unknown): void {
|
||||
this.nodesFtsUsable = false;
|
||||
if (this.nodesFtsDisabledReason === reason) {
|
||||
return;
|
||||
}
|
||||
this.nodesFtsDisabledReason = reason;
|
||||
|
||||
if (error && !this.isSqliteCorruptError(error)) {
|
||||
console.warn(`[SQLite] nodes_fts disabled: ${reason}`, error);
|
||||
return;
|
||||
}
|
||||
|
||||
console.warn(`[SQLite] nodes_fts disabled: ${reason}. Falling back to LIKE search for this database session.`);
|
||||
}
|
||||
|
||||
public async checkTables(): Promise<string[]> {
|
||||
try {
|
||||
const result = this.query(
|
||||
@@ -215,17 +237,283 @@ class SQLiteClient {
|
||||
this.ensureVectorExtensions();
|
||||
}
|
||||
|
||||
private ensureCoreSchema(): void {
|
||||
if (this.readOnly) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
source TEXT,
|
||||
link TEXT,
|
||||
event_date TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
metadata TEXT,
|
||||
embedding BLOB,
|
||||
embedding_updated_at TEXT,
|
||||
embedding_text TEXT,
|
||||
chunk_status TEXT DEFAULT 'not_chunked',
|
||||
context_id INTEGER,
|
||||
FOREIGN KEY (context_id) REFERENCES contexts(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS contexts (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
icon TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized
|
||||
ON contexts(LOWER(TRIM(name)));
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chunks (
|
||||
id INTEGER PRIMARY KEY,
|
||||
node_id INTEGER NOT NULL,
|
||||
chunk_idx INTEGER,
|
||||
text TEXT NOT NULL,
|
||||
embedding BLOB,
|
||||
embedding_type TEXT DEFAULT 'openai',
|
||||
metadata TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id INTEGER PRIMARY KEY,
|
||||
chat_type TEXT,
|
||||
helper_name TEXT,
|
||||
agent_type TEXT DEFAULT 'orchestrator',
|
||||
delegation_id INTEGER,
|
||||
user_message TEXT,
|
||||
assistant_message TEXT,
|
||||
thread_id TEXT,
|
||||
focused_node_id INTEGER,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
metadata TEXT,
|
||||
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_nodes_updated_at ON nodes(updated_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_node_id ON chunks(node_id);
|
||||
`);
|
||||
|
||||
this.ensureEdgesTableSchema();
|
||||
}
|
||||
|
||||
private ensureEdgesTableSchema(): void {
|
||||
const hasEdgesTable = this.db
|
||||
.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='edges'")
|
||||
.get();
|
||||
|
||||
if (!hasEdgesTable) {
|
||||
this.db.exec(`
|
||||
CREATE TABLE edges (
|
||||
id INTEGER PRIMARY KEY,
|
||||
from_node_id INTEGER NOT NULL,
|
||||
to_node_id INTEGER NOT NULL,
|
||||
source TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
context TEXT,
|
||||
explanation TEXT,
|
||||
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
} else {
|
||||
const edgeCols = this.db.prepare('PRAGMA table_info(edges)').all() as Array<{ name: string }>;
|
||||
const edgeColNames = new Set(edgeCols.map((col) => col.name));
|
||||
const needsLegacyRewrite =
|
||||
!edgeColNames.has('from_node_id') ||
|
||||
!edgeColNames.has('to_node_id') ||
|
||||
!edgeColNames.has('source') ||
|
||||
!edgeColNames.has('created_at') ||
|
||||
!edgeColNames.has('context') ||
|
||||
edgeColNames.has('from_id') ||
|
||||
edgeColNames.has('to_id') ||
|
||||
edgeColNames.has('description') ||
|
||||
edgeColNames.has('updated_at');
|
||||
|
||||
if (needsLegacyRewrite) {
|
||||
this.rebuildLegacyEdgesTable(edgeColNames);
|
||||
}
|
||||
}
|
||||
|
||||
this.db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node_id);
|
||||
`);
|
||||
}
|
||||
|
||||
private rebuildLegacyEdgesTable(edgeColNames: Set<string>): void {
|
||||
const fromExpr = edgeColNames.has('from_node_id')
|
||||
? 'from_node_id'
|
||||
: edgeColNames.has('from_id')
|
||||
? 'from_id'
|
||||
: 'NULL';
|
||||
const toExpr = edgeColNames.has('to_node_id')
|
||||
? 'to_node_id'
|
||||
: edgeColNames.has('to_id')
|
||||
? 'to_id'
|
||||
: 'NULL';
|
||||
const sourceExpr = edgeColNames.has('source') ? 'source' : "'legacy'";
|
||||
const createdAtExpr = edgeColNames.has('created_at') ? 'created_at' : 'CURRENT_TIMESTAMP';
|
||||
const contextExpr = edgeColNames.has('context') ? 'context' : 'NULL';
|
||||
const explanationExpr = edgeColNames.has('explanation')
|
||||
? 'explanation'
|
||||
: edgeColNames.has('description')
|
||||
? 'description'
|
||||
: edgeColNames.has('context')
|
||||
? "CASE WHEN json_valid(context) THEN json_extract(context, '$.explanation') ELSE NULL END"
|
||||
: 'NULL';
|
||||
|
||||
console.log('Migrating legacy edges table to canonical schema');
|
||||
|
||||
let flippedForeignKeys = false;
|
||||
try {
|
||||
this.db.exec('PRAGMA foreign_keys=OFF;');
|
||||
flippedForeignKeys = true;
|
||||
} catch {}
|
||||
|
||||
try {
|
||||
this.db.exec('BEGIN TRANSACTION;');
|
||||
this.db.exec(`
|
||||
DROP INDEX IF EXISTS idx_edges_from;
|
||||
DROP INDEX IF EXISTS idx_edges_to;
|
||||
ALTER TABLE edges RENAME TO edges_legacy_migration;
|
||||
CREATE TABLE edges (
|
||||
id INTEGER PRIMARY KEY,
|
||||
from_node_id INTEGER NOT NULL,
|
||||
to_node_id INTEGER NOT NULL,
|
||||
source TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
context TEXT,
|
||||
explanation TEXT,
|
||||
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
);
|
||||
INSERT INTO edges (id, from_node_id, to_node_id, source, created_at, context, explanation)
|
||||
SELECT
|
||||
id,
|
||||
${fromExpr},
|
||||
${toExpr},
|
||||
${sourceExpr},
|
||||
COALESCE(${createdAtExpr}, CURRENT_TIMESTAMP),
|
||||
${contextExpr},
|
||||
${explanationExpr}
|
||||
FROM edges_legacy_migration
|
||||
WHERE ${fromExpr} IS NOT NULL
|
||||
AND ${toExpr} IS NOT NULL;
|
||||
DROP TABLE edges_legacy_migration;
|
||||
COMMIT;
|
||||
`);
|
||||
} catch (error) {
|
||||
try {
|
||||
this.db.exec('ROLLBACK;');
|
||||
} catch {}
|
||||
throw error;
|
||||
} finally {
|
||||
if (flippedForeignKeys) {
|
||||
try {
|
||||
this.db.exec('PRAGMA foreign_keys=ON;');
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private rebuildLegacyChatsTable(chatColNames: Set<string>): void {
|
||||
const chatTypeExpr = chatColNames.has('chat_type') ? 'chat_type' : 'NULL';
|
||||
const helperNameExpr = chatColNames.has('helper_name')
|
||||
? 'helper_name'
|
||||
: chatColNames.has('title')
|
||||
? 'title'
|
||||
: 'NULL';
|
||||
const agentTypeExpr = chatColNames.has('agent_type')
|
||||
? "COALESCE(agent_type, 'orchestrator')"
|
||||
: "'orchestrator'";
|
||||
const delegationIdExpr = chatColNames.has('delegation_id') ? 'delegation_id' : 'NULL';
|
||||
const userMessageExpr = chatColNames.has('user_message') ? 'user_message' : 'NULL';
|
||||
const assistantMessageExpr = chatColNames.has('assistant_message') ? 'assistant_message' : 'NULL';
|
||||
const threadIdExpr = chatColNames.has('thread_id') ? 'thread_id' : 'NULL';
|
||||
const focusedNodeIdExpr = chatColNames.has('focused_node_id') ? 'focused_node_id' : 'NULL';
|
||||
const createdAtExpr = chatColNames.has('created_at') ? 'created_at' : 'CURRENT_TIMESTAMP';
|
||||
const metadataExpr = chatColNames.has('metadata') ? 'metadata' : 'NULL';
|
||||
|
||||
console.log('Migrating legacy chats table to canonical schema');
|
||||
|
||||
let flippedForeignKeys = false;
|
||||
try {
|
||||
this.db.exec('PRAGMA foreign_keys=OFF;');
|
||||
flippedForeignKeys = true;
|
||||
} catch {}
|
||||
|
||||
try {
|
||||
this.db.exec('BEGIN TRANSACTION;');
|
||||
this.db.exec(`
|
||||
DROP INDEX IF EXISTS idx_chats_thread;
|
||||
ALTER TABLE chats RENAME TO chats_legacy_cleanup;
|
||||
CREATE TABLE chats (
|
||||
id INTEGER PRIMARY KEY,
|
||||
chat_type TEXT,
|
||||
helper_name TEXT,
|
||||
agent_type TEXT DEFAULT 'orchestrator',
|
||||
delegation_id INTEGER,
|
||||
user_message TEXT,
|
||||
assistant_message TEXT,
|
||||
thread_id TEXT,
|
||||
focused_node_id INTEGER,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
metadata TEXT,
|
||||
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
|
||||
);
|
||||
INSERT INTO chats (
|
||||
id, chat_type, helper_name, agent_type, delegation_id,
|
||||
user_message, assistant_message, thread_id, focused_node_id,
|
||||
created_at, metadata
|
||||
)
|
||||
SELECT
|
||||
id,
|
||||
${chatTypeExpr},
|
||||
${helperNameExpr},
|
||||
${agentTypeExpr},
|
||||
${delegationIdExpr},
|
||||
${userMessageExpr},
|
||||
${assistantMessageExpr},
|
||||
${threadIdExpr},
|
||||
${focusedNodeIdExpr},
|
||||
COALESCE(${createdAtExpr}, CURRENT_TIMESTAMP),
|
||||
${metadataExpr}
|
||||
FROM chats_legacy_cleanup;
|
||||
DROP TABLE chats_legacy_cleanup;
|
||||
COMMIT;
|
||||
`);
|
||||
} catch (error) {
|
||||
try {
|
||||
this.db.exec('ROLLBACK;');
|
||||
} catch {}
|
||||
throw error;
|
||||
} finally {
|
||||
if (flippedForeignKeys) {
|
||||
try {
|
||||
this.db.exec('PRAGMA foreign_keys=ON;');
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ensureLoggingAndMemorySchema(): void {
|
||||
if (this.readOnly) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const hasReadyLogs = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
|
||||
if (hasReadyLogs) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 1) If logs table missing but legacy memory table exists, migrate
|
||||
// Existing installs may already have logs but still need the idempotent schema pass below.
|
||||
// Only skip the legacy memory rename step when logs already exists.
|
||||
const hasLogs = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
|
||||
const hasLegacyMemory = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memory'").get();
|
||||
if (!hasLogs && hasLegacyMemory) {
|
||||
@@ -263,6 +551,11 @@ class SQLiteClient {
|
||||
}
|
||||
};
|
||||
ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;");
|
||||
ensureNodeCol('link', 'ALTER TABLE nodes ADD COLUMN link TEXT;');
|
||||
ensureNodeCol('source', 'ALTER TABLE nodes ADD COLUMN source TEXT;');
|
||||
ensureNodeCol('metadata', 'ALTER TABLE nodes ADD COLUMN metadata TEXT;');
|
||||
ensureNodeCol('created_at', "ALTER TABLE nodes ADD COLUMN created_at TEXT DEFAULT CURRENT_TIMESTAMP;");
|
||||
ensureNodeCol('updated_at', "ALTER TABLE nodes ADD COLUMN updated_at TEXT DEFAULT CURRENT_TIMESTAMP;");
|
||||
} catch (nodeErr) {
|
||||
console.warn('Failed to ensure nodes columns:', nodeErr);
|
||||
}
|
||||
@@ -279,6 +572,25 @@ class SQLiteClient {
|
||||
console.warn('Failed to ensure chats.created_at column:', chatErr);
|
||||
}
|
||||
|
||||
// Normalize legacy chats table before creating chat triggers or views that reference modern columns.
|
||||
try {
|
||||
const hasChatsTable = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='chats'").get();
|
||||
if (hasChatsTable) {
|
||||
const chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as Array<{ name: string }>;
|
||||
const chatColNames = new Set(chatCols.map((col) => col.name));
|
||||
const needsChatRewrite =
|
||||
chatColNames.has('focused_memory_id') ||
|
||||
['chat_type', 'helper_name', 'agent_type', 'delegation_id', 'user_message', 'assistant_message', 'thread_id', 'focused_node_id', 'created_at', 'metadata']
|
||||
.some((name) => !chatColNames.has(name));
|
||||
|
||||
if (needsChatRewrite) {
|
||||
this.rebuildLegacyChatsTable(chatColNames);
|
||||
}
|
||||
}
|
||||
} catch (chatSchemaErr) {
|
||||
console.warn('Failed to normalize chats schema before log setup:', chatSchemaErr);
|
||||
}
|
||||
|
||||
// 3) Helpful indexes on logs (clean up old names first)
|
||||
this.db.exec(`
|
||||
DROP INDEX IF EXISTS idx_memory_ts;
|
||||
@@ -494,55 +806,20 @@ class SQLiteClient {
|
||||
if (hasChats) {
|
||||
try {
|
||||
let chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as any[];
|
||||
const hasFocusedMemoryId = chatCols.some((c: any) => c.name === 'focused_memory_id');
|
||||
if (hasFocusedMemoryId) {
|
||||
console.log('Removing legacy chats.focused_memory_id column');
|
||||
let flippedForeignKeys = false;
|
||||
try {
|
||||
this.db.exec('PRAGMA foreign_keys=OFF;');
|
||||
flippedForeignKeys = true;
|
||||
this.db.exec(`
|
||||
BEGIN TRANSACTION;
|
||||
ALTER TABLE chats RENAME TO chats_legacy_cleanup;
|
||||
CREATE TABLE chats (
|
||||
id INTEGER PRIMARY KEY,
|
||||
chat_type TEXT,
|
||||
helper_name TEXT,
|
||||
agent_type TEXT DEFAULT 'orchestrator',
|
||||
delegation_id INTEGER,
|
||||
user_message TEXT,
|
||||
assistant_message TEXT,
|
||||
thread_id TEXT,
|
||||
focused_node_id INTEGER,
|
||||
created_at TEXT DEFAULT (CURRENT_TIMESTAMP),
|
||||
metadata TEXT,
|
||||
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
|
||||
);
|
||||
INSERT INTO chats (
|
||||
id, chat_type, helper_name, agent_type, delegation_id,
|
||||
user_message, assistant_message, thread_id, focused_node_id,
|
||||
created_at, metadata
|
||||
)
|
||||
SELECT id, chat_type, helper_name, agent_type, delegation_id,
|
||||
user_message, assistant_message, thread_id, focused_node_id,
|
||||
created_at, metadata
|
||||
FROM chats_legacy_cleanup;
|
||||
DROP TABLE chats_legacy_cleanup;
|
||||
CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);
|
||||
COMMIT;
|
||||
`);
|
||||
} catch (migrationErr) {
|
||||
console.warn('Failed to migrate chats table (focused_memory_id removal):', migrationErr);
|
||||
try { this.db.exec('ROLLBACK;'); } catch {}
|
||||
} finally {
|
||||
if (flippedForeignKeys) {
|
||||
try { this.db.exec('PRAGMA foreign_keys=ON;'); } catch {}
|
||||
}
|
||||
}
|
||||
const chatColNames = new Set(chatCols.map((c: any) => c.name));
|
||||
const needsChatRewrite =
|
||||
chatColNames.has('focused_memory_id') ||
|
||||
['chat_type', 'helper_name', 'agent_type', 'delegation_id', 'user_message', 'assistant_message', 'thread_id', 'focused_node_id', 'created_at', 'metadata']
|
||||
.some((name) => !chatColNames.has(name));
|
||||
|
||||
if (needsChatRewrite) {
|
||||
this.rebuildLegacyChatsTable(chatColNames);
|
||||
chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as any[];
|
||||
}
|
||||
|
||||
this.db.exec("CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);");
|
||||
if (chatCols.some((c: any) => c.name === 'thread_id')) {
|
||||
this.db.exec("CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);");
|
||||
}
|
||||
|
||||
const ensureCol = (name: string, ddl: string) => {
|
||||
if (!chatCols.some((c: any) => c.name === name)) {
|
||||
@@ -578,51 +855,7 @@ class SQLiteClient {
|
||||
console.warn('Failed to drop legacy memory pipeline tables:', dropLegacyErr);
|
||||
}
|
||||
|
||||
// 9) Ensure dimensions table exists (v0.1.16+ schema migration)
|
||||
const hasDimensions = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='dimensions'").get();
|
||||
if (!hasDimensions) {
|
||||
console.log('Creating dimensions table for v0.1.16+ features...');
|
||||
this.db.exec(`
|
||||
CREATE TABLE dimensions (
|
||||
name TEXT PRIMARY KEY,
|
||||
description TEXT,
|
||||
is_priority INTEGER DEFAULT 0,
|
||||
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
// Seed default locked dimensions
|
||||
const defaultDimensions = ['research', 'ideas', 'projects', 'memory', 'preferences'];
|
||||
const insertDimension = this.db.prepare(`
|
||||
INSERT INTO dimensions (name, is_priority, updated_at)
|
||||
VALUES (?, 1, datetime('now'))
|
||||
ON CONFLICT(name) DO UPDATE SET is_priority = 1, updated_at = datetime('now')
|
||||
`);
|
||||
|
||||
for (const dimension of defaultDimensions) {
|
||||
try {
|
||||
insertDimension.run(dimension);
|
||||
} catch (e) {
|
||||
console.warn(`Failed to seed dimension '${dimension}':`, e);
|
||||
}
|
||||
}
|
||||
console.log('Dimensions table created and seeded with default locked dimensions');
|
||||
} else {
|
||||
// Check if existing dimensions table has description column
|
||||
const dimensionCols = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
|
||||
const hasDescription = dimensionCols.some(col => col.name === 'description');
|
||||
if (!hasDescription) {
|
||||
console.log('Adding description column to existing dimensions table...');
|
||||
try {
|
||||
this.db.exec('ALTER TABLE dimensions ADD COLUMN description TEXT;');
|
||||
console.log('Description column added to dimensions table');
|
||||
} catch (e) {
|
||||
console.warn('Failed to add description column to dimensions table:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 10) Final schema pass migrations (source canonicalization, event_date, dimensions.icon, drop dead columns)
|
||||
// 9) Final schema pass migrations (source-first backfill, event_date, soft contexts, drop dimensions)
|
||||
try {
|
||||
let nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
|
||||
let nodeColNames = nodeCols2.map(c => c.name);
|
||||
@@ -634,6 +867,12 @@ class SQLiteClient {
|
||||
nodeColNames = nodeCols2.map(c => c.name);
|
||||
}
|
||||
|
||||
if (!nodeColNames.includes('chunk_status')) {
|
||||
this.db.exec("ALTER TABLE nodes ADD COLUMN chunk_status TEXT DEFAULT 'not_chunked';");
|
||||
nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
|
||||
nodeColNames = nodeCols2.map(c => c.name);
|
||||
}
|
||||
|
||||
if (nodeColNames.includes('source')) {
|
||||
if (nodeColNames.includes('content')) {
|
||||
this.db.exec(`
|
||||
@@ -691,19 +930,121 @@ class SQLiteClient {
|
||||
// Add event_date
|
||||
if (!nodeColNames.includes('event_date')) {
|
||||
this.db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;');
|
||||
// Backfill from metadata.published_date where available
|
||||
// Backfill from metadata.published_date or metadata.source_metadata.published_date where available
|
||||
try {
|
||||
this.db.exec(`
|
||||
UPDATE nodes SET event_date = json_extract(metadata, '$.published_date')
|
||||
WHERE event_date IS NULL AND json_extract(metadata, '$.published_date') IS NOT NULL;
|
||||
UPDATE nodes
|
||||
SET event_date = COALESCE(
|
||||
json_extract(metadata, '$.source_metadata.published_date'),
|
||||
json_extract(metadata, '$.published_date')
|
||||
)
|
||||
WHERE event_date IS NULL
|
||||
AND COALESCE(
|
||||
json_extract(metadata, '$.source_metadata.published_date'),
|
||||
json_extract(metadata, '$.published_date')
|
||||
) IS NOT NULL;
|
||||
`);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Add dimensions.icon
|
||||
const dimCols2 = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
|
||||
if (!dimCols2.some(c => c.name === 'icon')) {
|
||||
this.db.exec('ALTER TABLE dimensions ADD COLUMN icon TEXT;');
|
||||
if (!nodeColNames.includes('context_id')) {
|
||||
this.db.exec('ALTER TABLE nodes ADD COLUMN context_id INTEGER REFERENCES contexts(id) ON DELETE SET NULL;');
|
||||
}
|
||||
|
||||
const hasContexts = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='contexts'").get();
|
||||
if (!hasContexts) {
|
||||
this.db.exec(`
|
||||
CREATE TABLE contexts (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
icon TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
const contextCols = this.db.prepare('PRAGMA table_info(contexts)').all() as Array<{ name: string }>;
|
||||
const ensureContextCol = (name: string, ddl: string) => {
|
||||
if (!contextCols.some(col => col.name === name)) {
|
||||
this.db.exec(ddl);
|
||||
}
|
||||
};
|
||||
ensureContextCol('description', "ALTER TABLE contexts ADD COLUMN description TEXT NOT NULL DEFAULT '';");
|
||||
ensureContextCol('icon', 'ALTER TABLE contexts ADD COLUMN icon TEXT;');
|
||||
ensureContextCol('created_at', "ALTER TABLE contexts ADD COLUMN created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
|
||||
ensureContextCol('updated_at', "ALTER TABLE contexts ADD COLUMN updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
|
||||
|
||||
this.db.exec(`
|
||||
UPDATE contexts
|
||||
SET description = COALESCE(NULLIF(TRIM(description), ''), name)
|
||||
WHERE description IS NULL OR LENGTH(TRIM(description)) = 0;
|
||||
`);
|
||||
|
||||
this.db.exec(`
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized
|
||||
ON contexts(LOWER(TRIM(name)));
|
||||
CREATE INDEX IF NOT EXISTS idx_nodes_context_id ON nodes(context_id);
|
||||
`);
|
||||
|
||||
const hasLegacyDimensions = this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='dimensions'").get();
|
||||
const hasLegacyNodeDimensions = this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='node_dimensions'").get();
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS dimension_migration_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
migrated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
dimension_count INTEGER NOT NULL,
|
||||
assignment_count INTEGER NOT NULL,
|
||||
payload TEXT
|
||||
);
|
||||
`);
|
||||
|
||||
if (hasLegacyDimensions || hasLegacyNodeDimensions) {
|
||||
const existingSnapshotCount = Number(
|
||||
(this.db.prepare('SELECT COUNT(*) as count FROM dimension_migration_snapshots').get() as { count?: number } | undefined)?.count ?? 0
|
||||
);
|
||||
|
||||
if (existingSnapshotCount === 0) {
|
||||
const dimensionCount = hasLegacyDimensions
|
||||
? Number((this.db.prepare('SELECT COUNT(*) as count FROM dimensions').get() as { count?: number } | undefined)?.count ?? 0)
|
||||
: 0;
|
||||
const assignmentCount = hasLegacyNodeDimensions
|
||||
? Number((this.db.prepare('SELECT COUNT(*) as count FROM node_dimensions').get() as { count?: number } | undefined)?.count ?? 0)
|
||||
: 0;
|
||||
const payload = hasLegacyNodeDimensions
|
||||
? (
|
||||
this.db.prepare(`
|
||||
SELECT COALESCE(
|
||||
json_group_array(
|
||||
json_object(
|
||||
'node_id', nd.node_id,
|
||||
'dimension', nd.dimension,
|
||||
'description', d.description,
|
||||
'icon', d.icon,
|
||||
'is_priority', d.is_priority
|
||||
)
|
||||
),
|
||||
'[]'
|
||||
) AS payload
|
||||
FROM node_dimensions nd
|
||||
LEFT JOIN dimensions d ON d.name = nd.dimension
|
||||
`).get() as { payload?: string } | undefined
|
||||
)?.payload ?? '[]'
|
||||
: '[]';
|
||||
|
||||
this.db.prepare(`
|
||||
INSERT INTO dimension_migration_snapshots (dimension_count, assignment_count, payload)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(dimensionCount, assignmentCount, payload);
|
||||
}
|
||||
|
||||
this.db.exec(`
|
||||
DROP INDEX IF EXISTS idx_dim_by_dimension;
|
||||
DROP INDEX IF EXISTS idx_dim_by_node;
|
||||
DROP TABLE IF EXISTS node_dimensions;
|
||||
DROP TABLE IF EXISTS dimensions;
|
||||
`);
|
||||
}
|
||||
|
||||
// Drop dead columns (requires SQLite 3.35+)
|
||||
@@ -742,7 +1083,11 @@ class SQLiteClient {
|
||||
this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
|
||||
}
|
||||
} catch (ftsErr) {
|
||||
console.warn('Failed to rebuild nodes_fts:', ftsErr);
|
||||
if (this.isSqliteCorruptError(ftsErr)) {
|
||||
this.disableNodesFts('existing nodes_fts is corrupt and could not be rebuilt', ftsErr);
|
||||
} else {
|
||||
console.warn('Failed to rebuild nodes_fts:', ftsErr);
|
||||
}
|
||||
}
|
||||
} catch (schemaErr) {
|
||||
console.warn('Final schema pass migration error:', schemaErr);
|
||||
@@ -854,6 +1199,15 @@ class SQLiteClient {
|
||||
public close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
private isSqliteCorruptError(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const sqliteError = error as Error & { code?: string };
|
||||
return sqliteError.code === 'SQLITE_CORRUPT' || /database disk image is malformed/i.test(sqliteError.message || '');
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance (similar to PostgreSQL client interface)
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
|
||||
export interface AutoContextSettings {
|
||||
autoContextEnabled: boolean;
|
||||
lastPinnedMigration?: string;
|
||||
}
|
||||
|
||||
const SETTINGS_FILE = 'settings.json';
|
||||
const DEFAULT_SETTINGS: AutoContextSettings = {
|
||||
autoContextEnabled: false,
|
||||
};
|
||||
|
||||
let bootstrapAttempted = false;
|
||||
|
||||
function resolveBaseConfigDir(): string {
|
||||
const override = process.env.RAH_CONFIG_DIR;
|
||||
if (override && override.trim().length > 0) {
|
||||
return override;
|
||||
}
|
||||
|
||||
const home = os.homedir();
|
||||
if (process.platform === 'darwin') {
|
||||
return path.join(home, 'Library', 'Application Support', 'RA-H');
|
||||
}
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
const roaming = process.env.APPDATA || path.join(home, 'AppData', 'Roaming');
|
||||
return path.join(roaming, 'RA-H');
|
||||
}
|
||||
|
||||
const xdgConfig = process.env.XDG_CONFIG_HOME || path.join(home, '.config');
|
||||
return path.join(xdgConfig, 'ra-h');
|
||||
}
|
||||
|
||||
function getSettingsDir(): string {
|
||||
return path.join(resolveBaseConfigDir(), 'config');
|
||||
}
|
||||
|
||||
function getSettingsPath(): string {
|
||||
return path.join(getSettingsDir(), SETTINGS_FILE);
|
||||
}
|
||||
|
||||
function ensureSettingsDirExists(): void {
|
||||
const dir = getSettingsDir();
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
function writeSettingsFile(settings: AutoContextSettings): AutoContextSettings {
|
||||
ensureSettingsDirExists();
|
||||
fs.writeFileSync(getSettingsPath(), JSON.stringify(settings, null, 2), 'utf-8');
|
||||
return settings;
|
||||
}
|
||||
|
||||
function bootstrapFromLegacyPins(): void {
|
||||
if (bootstrapAttempted) return;
|
||||
bootstrapAttempted = true;
|
||||
|
||||
const settingsPath = getSettingsPath();
|
||||
if (fs.existsSync(settingsPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const db = getSQLiteClient();
|
||||
const countRow = db
|
||||
.query<{ count: number }>('SELECT COUNT(*) as count FROM nodes WHERE 1=0 /* /* is_pinned removed */ removed */')
|
||||
.rows[0];
|
||||
const pinnedCount = Number(countRow?.count ?? 0);
|
||||
if (pinnedCount > 0) {
|
||||
writeSettingsFile({
|
||||
autoContextEnabled: true,
|
||||
lastPinnedMigration: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Auto-context pin bootstrap failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
export function getAutoContextSettings(): AutoContextSettings {
|
||||
bootstrapFromLegacyPins();
|
||||
const settingsPath = getSettingsPath();
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(settingsPath)) {
|
||||
return { ...DEFAULT_SETTINGS };
|
||||
}
|
||||
|
||||
const raw = fs.readFileSync(settingsPath, 'utf-8');
|
||||
const parsed = JSON.parse(raw);
|
||||
return {
|
||||
...DEFAULT_SETTINGS,
|
||||
...parsed,
|
||||
autoContextEnabled: Boolean(parsed?.autoContextEnabled),
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn('Failed to read auto-context settings, using defaults:', error);
|
||||
return { ...DEFAULT_SETTINGS };
|
||||
}
|
||||
}
|
||||
|
||||
export function updateAutoContextSettings(
|
||||
partial: Partial<AutoContextSettings>
|
||||
): AutoContextSettings {
|
||||
const current = getAutoContextSettings();
|
||||
const next: AutoContextSettings = {
|
||||
...current,
|
||||
...partial,
|
||||
autoContextEnabled:
|
||||
typeof partial.autoContextEnabled === 'boolean'
|
||||
? partial.autoContextEnabled
|
||||
: current.autoContextEnabled,
|
||||
};
|
||||
return writeSettingsFile(next);
|
||||
}
|
||||
|
||||
export function setAutoContextEnabled(enabled: boolean): AutoContextSettings {
|
||||
return updateAutoContextSettings({ autoContextEnabled: enabled });
|
||||
}
|
||||
@@ -1,26 +1,24 @@
|
||||
/**
|
||||
* Node metadata embedding service for RA-H knowledge management system
|
||||
* Embeds node metadata (title, content, dimensions, AI analysis) into nodes.embedding field
|
||||
* Embeds node metadata (title, source, context, AI analysis) into nodes.embedding field
|
||||
*/
|
||||
|
||||
import OpenAI from 'openai';
|
||||
import { generateText } from 'ai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import {
|
||||
createDatabaseConnection,
|
||||
getDbVectorCapability,
|
||||
import {
|
||||
createDatabaseConnection,
|
||||
serializeFloat32Vector,
|
||||
formatEmbeddingText,
|
||||
batchProcess
|
||||
batchProcess
|
||||
} from './sqlite-vec';
|
||||
import type { VectorCapability } from '@/services/database/sqlite-runtime';
|
||||
|
||||
interface NodeRecord {
|
||||
id: number;
|
||||
title: string;
|
||||
source: string | null;
|
||||
description: string | null;
|
||||
dimensions_json: string;
|
||||
context_name: string | null;
|
||||
embedding?: Buffer | null;
|
||||
embedding_updated_at?: string | null;
|
||||
embedding_text?: string | null;
|
||||
@@ -36,7 +34,6 @@ export class NodeEmbedder {
|
||||
private openaiClient: OpenAI;
|
||||
private openaiProvider: ReturnType<typeof createOpenAI>;
|
||||
private db: ReturnType<typeof createDatabaseConnection>;
|
||||
private readonly vectorCapability: VectorCapability;
|
||||
private processedCount: number = 0;
|
||||
private failedCount: number = 0;
|
||||
|
||||
@@ -45,25 +42,21 @@ export class NodeEmbedder {
|
||||
if (!apiKey) {
|
||||
throw new Error('OPENAI_API_KEY environment variable is not set');
|
||||
}
|
||||
|
||||
|
||||
this.openaiClient = new OpenAI({ apiKey });
|
||||
this.openaiProvider = createOpenAI({ apiKey });
|
||||
this.db = createDatabaseConnection();
|
||||
this.vectorCapability = getDbVectorCapability(this.db);
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze node content with AI to extract insights
|
||||
*/
|
||||
private async analyzeNodeWithAI(node: NodeRecord): Promise<string> {
|
||||
const dimensions = node.dimensions_json ? JSON.parse(node.dimensions_json) : [];
|
||||
const dimensionsText = Array.isArray(dimensions) && dimensions.length ? dimensions.join(', ') : 'none';
|
||||
|
||||
const prompt = `Analyze this content and provide 2-3 key insights or themes in a concise paragraph (max 100 words):
|
||||
|
||||
Title: ${node.title}
|
||||
Source: ${node.source || 'No source'}
|
||||
Dimensions: ${dimensionsText}
|
||||
Context: ${node.context_name || 'none'}
|
||||
|
||||
Focus on the main concepts, key relationships, and practical implications.`;
|
||||
|
||||
@@ -90,7 +83,7 @@ Focus on the main concepts, key relationships, and practical implications.`;
|
||||
model: 'text-embedding-3-small',
|
||||
input: text,
|
||||
});
|
||||
|
||||
|
||||
return response.data[0].embedding;
|
||||
}
|
||||
|
||||
@@ -104,15 +97,12 @@ Focus on the main concepts, key relationships, and practical implications.`;
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse dimensions from JSON string
|
||||
const dimensions = node.dimensions_json ? JSON.parse(node.dimensions_json) : [];
|
||||
|
||||
// Create base embedding text
|
||||
let embeddingText = formatEmbeddingText(
|
||||
node.title,
|
||||
node.source || '',
|
||||
dimensions,
|
||||
node.description
|
||||
node.description,
|
||||
node.context_name
|
||||
);
|
||||
|
||||
// Add AI analysis if source exists
|
||||
@@ -128,36 +118,41 @@ Focus on the main concepts, key relationships, and practical implications.`;
|
||||
// Generate embedding
|
||||
const embedding = await this.generateEmbedding(embeddingText);
|
||||
const embeddingBlob = serializeFloat32Vector(embedding);
|
||||
|
||||
|
||||
// Update database
|
||||
const updateStmt = this.db.prepare(`
|
||||
UPDATE nodes
|
||||
SET embedding = ?,
|
||||
embedding_updated_at = ?,
|
||||
UPDATE nodes
|
||||
SET embedding = ?,
|
||||
embedding_updated_at = ?,
|
||||
embedding_text = ?
|
||||
WHERE id = ?
|
||||
`);
|
||||
|
||||
|
||||
const now = new Date().toISOString();
|
||||
updateStmt.run(embeddingBlob, now, embeddingText, node.id);
|
||||
|
||||
if (this.vectorCapability.available) {
|
||||
try {
|
||||
const pkCol = 'node_id';
|
||||
const deleteStmt = this.db.prepare(`DELETE FROM vec_nodes WHERE ${pkCol} = ?`);
|
||||
deleteStmt.run(BigInt(node.id));
|
||||
|
||||
const vectorString = `[${embedding.join(',')}]`;
|
||||
const insertStmt = this.db.prepare(`INSERT INTO vec_nodes (${pkCol}, embedding) VALUES (?, ?)`);
|
||||
insertStmt.run(BigInt(node.id), vectorString);
|
||||
} catch (vecError) {
|
||||
console.warn(`Could not update vec_nodes for node ${node.id}:`, vecError);
|
||||
}
|
||||
// Update vec_nodes virtual table
|
||||
try {
|
||||
// Determine correct column name for primary key (node_id vs id)
|
||||
// Use declared PK column from your DB schema (confirmed: node_id)
|
||||
const pkCol = 'node_id';
|
||||
|
||||
// Delete existing entry if any
|
||||
const deleteStmt = this.db.prepare(`DELETE FROM vec_nodes WHERE ${pkCol} = ?`);
|
||||
deleteStmt.run(BigInt(node.id));
|
||||
|
||||
// Insert new entry (use bracketed string format compatible with sqlite-vec)
|
||||
const vectorString = `[${embedding.join(',')}]`;
|
||||
const insertStmt = this.db.prepare(`INSERT INTO vec_nodes (${pkCol}, embedding) VALUES (?, ?)`);
|
||||
insertStmt.run(BigInt(node.id), vectorString);
|
||||
} catch (vecError) {
|
||||
console.warn(`Could not update vec_nodes for node ${node.id}:`, vecError);
|
||||
// Continue - main embedding is still saved
|
||||
}
|
||||
|
||||
|
||||
this.processedCount++;
|
||||
console.log(`✓ Embedded node ${node.id}: "${node.title}"`);
|
||||
|
||||
|
||||
} catch (error) {
|
||||
this.failedCount++;
|
||||
console.error(`✗ Failed to embed node ${node.id}:`, error);
|
||||
@@ -170,54 +165,51 @@ Focus on the main concepts, key relationships, and practical implications.`;
|
||||
*/
|
||||
async embedNodes(options: EmbedNodeOptions = {}): Promise<{ processed: number; failed: number }> {
|
||||
const { nodeId, forceReEmbed = false, verbose = false } = options;
|
||||
|
||||
|
||||
let query: string;
|
||||
let params: any[] = [];
|
||||
|
||||
|
||||
if (nodeId) {
|
||||
// Single node
|
||||
query = `
|
||||
SELECT n.id, n.title, n.source, n.description,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
SELECT n.id, n.title, n.source, n.description, c.name as context_name,
|
||||
n.embedding, n.embedding_updated_at
|
||||
FROM nodes n
|
||||
LEFT JOIN contexts c ON c.id = n.context_id
|
||||
WHERE n.id = ?
|
||||
`;
|
||||
params = [nodeId];
|
||||
} else if (forceReEmbed) {
|
||||
// All nodes
|
||||
query = `
|
||||
SELECT n.id, n.title, n.source, n.description,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
SELECT n.id, n.title, n.source, n.description, c.name as context_name,
|
||||
n.embedding, n.embedding_updated_at
|
||||
FROM nodes n
|
||||
LEFT JOIN contexts c ON c.id = n.context_id
|
||||
ORDER BY n.id
|
||||
`;
|
||||
} else {
|
||||
// Only nodes without embeddings
|
||||
query = `
|
||||
SELECT n.id, n.title, n.source, n.description,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
SELECT n.id, n.title, n.source, n.description, c.name as context_name,
|
||||
n.embedding, n.embedding_updated_at
|
||||
FROM nodes n
|
||||
LEFT JOIN contexts c ON c.id = n.context_id
|
||||
WHERE n.embedding IS NULL OR n.embedding_updated_at IS NULL
|
||||
ORDER BY n.id
|
||||
`;
|
||||
}
|
||||
|
||||
|
||||
const stmt = this.db.prepare(query);
|
||||
const nodes = stmt.all(...params) as NodeRecord[];
|
||||
|
||||
|
||||
if (nodes.length === 0) {
|
||||
console.log('No nodes to process');
|
||||
return { processed: 0, failed: 0 };
|
||||
}
|
||||
|
||||
|
||||
console.log(`Processing ${nodes.length} nodes...`);
|
||||
|
||||
|
||||
// Process in batches
|
||||
await batchProcess(
|
||||
nodes,
|
||||
@@ -233,9 +225,9 @@ Focus on the main concepts, key relationships, and practical implications.`;
|
||||
console.log(`Progress: ${processed}/${total} nodes`);
|
||||
} : undefined
|
||||
);
|
||||
|
||||
|
||||
console.log(`\nComplete! Processed: ${this.processedCount}, Failed: ${this.failedCount}`);
|
||||
|
||||
|
||||
return {
|
||||
processed: this.processedCount,
|
||||
failed: this.failedCount
|
||||
@@ -254,15 +246,15 @@ Focus on the main concepts, key relationships, and practical implications.`;
|
||||
* CLI interface for direct execution
|
||||
*/
|
||||
export async function runCLI(args: string[]): Promise<void> {
|
||||
const nodeId = args.includes('--node-id')
|
||||
const nodeId = args.includes('--node-id')
|
||||
? parseInt(args[args.indexOf('--node-id') + 1])
|
||||
: undefined;
|
||||
|
||||
|
||||
const forceReEmbed = args.includes('--force');
|
||||
const verbose = args.includes('--verbose');
|
||||
|
||||
|
||||
const embedder = new NodeEmbedder();
|
||||
|
||||
|
||||
try {
|
||||
await embedder.embedNodes({ nodeId, forceReEmbed, verbose });
|
||||
} finally {
|
||||
|
||||
@@ -4,13 +4,9 @@
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import {
|
||||
ensureDatabaseDirectory,
|
||||
getDatabasePath,
|
||||
getDbVectorCapability,
|
||||
getVecExtensionPath,
|
||||
loadVecExtension,
|
||||
} from '@/services/database/sqlite-runtime';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { getDbVectorCapability as getVectorCapability } from '@/services/database/sqlite-runtime';
|
||||
|
||||
/**
|
||||
* Serialize a float array to binary format for vec0 storage
|
||||
@@ -35,24 +31,56 @@ export function deserializeFloat32Vector(blob: Buffer): number[] {
|
||||
return vector;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get SQLite database path from environment or default location
|
||||
*/
|
||||
export function getDatabasePath(): string {
|
||||
const envPath = process.env.SQLITE_DB_PATH;
|
||||
if (envPath) {
|
||||
return envPath;
|
||||
}
|
||||
|
||||
// Default path: ~/Library/Application Support/RA-H/db/rah.sqlite
|
||||
const homeDir = os.homedir();
|
||||
return path.join(homeDir, 'Library', 'Application Support', 'RA-H', 'db', 'rah.sqlite');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get vec extension path from environment or default location
|
||||
*/
|
||||
export function getVecExtensionPath(): string {
|
||||
const envPath = process.env.SQLITE_VEC_EXTENSION_PATH;
|
||||
if (envPath) {
|
||||
return envPath;
|
||||
}
|
||||
|
||||
// Default path relative to project root
|
||||
return path.join(process.cwd(), 'vendor', 'sqlite-extensions', 'vec0.dylib');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create database connection with vec0 extension loaded
|
||||
*/
|
||||
export function createDatabaseConnection(): Database.Database {
|
||||
const dbPath = getDatabasePath();
|
||||
const vecPath = getVecExtensionPath();
|
||||
ensureDatabaseDirectory(dbPath);
|
||||
|
||||
const db = new Database(dbPath);
|
||||
|
||||
const capability = loadVecExtension(db, vecPath);
|
||||
if (!capability.available) {
|
||||
console.warn(`Warning: ${capability.reason}`);
|
||||
// Load vec0 extension
|
||||
try {
|
||||
db.loadExtension(vecPath);
|
||||
} catch (error) {
|
||||
console.error('Warning: Could not load vec0 extension:', error);
|
||||
// Continue without vector support for non-vector operations
|
||||
}
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
export { getDatabasePath, getDbVectorCapability, getVecExtensionPath };
|
||||
export function getDbVectorCapability(db: Database.Database) {
|
||||
return getVectorCapability(db);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format embedding text for node metadata
|
||||
@@ -60,12 +88,12 @@ export { getDatabasePath, getDbVectorCapability, getVecExtensionPath };
|
||||
export function formatEmbeddingText(
|
||||
title: string,
|
||||
content: string,
|
||||
dimensions: string[],
|
||||
description?: string | null
|
||||
description?: string | null,
|
||||
contextName?: string | null
|
||||
): string {
|
||||
const descriptionText = description && description.trim() ? description.trim() : 'none';
|
||||
const dimensionsText = dimensions.length > 0 ? dimensions.join(', ') : 'none';
|
||||
return `Title: ${title}\n\nDescription: ${descriptionText}\n\nContent: ${content}\n\nDimensions: ${dimensionsText}`;
|
||||
const contextText = contextName && contextName.trim() ? contextName.trim() : 'none';
|
||||
return `Title: ${title}\n\nDescription: ${descriptionText}\n\nContent: ${content}\n\nContext: ${contextText}`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -78,16 +106,16 @@ export async function batchProcess<T, R>(
|
||||
onProgress?: (processed: number, total: number) => void
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
|
||||
|
||||
for (let i = 0; i < items.length; i += batchSize) {
|
||||
const batch = items.slice(i, Math.min(i + batchSize, items.length));
|
||||
const batchResults = await Promise.all(batch.map(processor));
|
||||
results.push(...batchResults);
|
||||
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(Math.min(i + batchSize, items.length), items.length);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user