SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port holistic node refinement contract

Browse Source
This commit is contained in:
“BeeRad”
2026-04-11 21:37:52 +10:00
parent 35f9ecf89c
commit 3ae46245ec
119 changed files with 6596 additions and 10982 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/database/contextService.ts
+3 -1
View File
@@ -177,7 +177,9 @@ export class ContextService {
}
async resolveContextId(input: { context_id?: number | null; context_name?: string | null }): Promise<number | null | undefined> {
const hasContextId = Object.prototype.hasOwnProperty.call(input, 'context_id');
const hasContextId =
Object.prototype.hasOwnProperty.call(input, 'context_id') &&
input.context_id !== undefined;
const hasContextName = typeof input.context_name === 'string' && input.context_name.trim().length > 0;
if (!hasContextId && !hasContextName) {
src/services/database/descriptionService.ts
+13 -6
View File
@@ -1,5 +1,5 @@
import { openai as openaiProvider } from '@ai-sdk/openai';
import { generateText } from 'ai';
import { openai as openaiProvider } from '@ai-sdk/openai';
import { hasValidOpenAiKey } from '../storage/apiKeys';
import type { CanonicalNodeMetadata } from '@/types/database';
@@ -16,11 +16,12 @@ export interface DescriptionInput {
pages?: number;
text_length?: number;
};
dimensions?: string[];
}
export { hasValidOpenAiKey } from '../storage/apiKeys';
/**
* Generate a context-rich description for a knowledge node.
* The result must cover what the artifact is, why it is in the graph, and workflow status.
*/
export async function generateDescription(input: DescriptionInput): Promise<string> {
if (!hasValidOpenAiKey()) {
console.log(`[DescriptionService] No valid OpenAI key, using fallback for: "${input.title}"`);
@@ -46,10 +47,13 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
return description;
} catch (error) {
console.error('[DescriptionService] Error generating description:', error);
// Fallback: just use the title — more useful than a vague template
return `${input.title}. Added via Quick Add with no further context yet, so the reason it belongs in the graph is not fully inferred. It has not been reviewed yet.`.slice(0, 500);
}
}
export { hasValidOpenAiKey } from '../storage/apiKeys';
function buildDescriptionPrompt(input: DescriptionInput): string {
const sourceMetadata = input.metadata?.source_metadata as Record<string, unknown> | undefined;
const sourceType = typeof input.metadata?.type === 'string'
@@ -60,6 +64,8 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
const normalizedSource = sourceType.toLowerCase();
const url = typeof input.link === 'string' ? input.link.trim() : '';
// Best-effort creator hint from structured metadata (when available),
// but never assume a particular extraction source (YouTube vs paper vs website vs note).
const creatorHint =
(typeof sourceMetadata?.author === 'string' ? sourceMetadata.author.trim() : '') ||
(typeof sourceMetadata?.channel_name === 'string' ? sourceMetadata.channel_name.trim() : '') ||
@@ -67,6 +73,7 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
(typeof input.metadata?.channel_name === 'string' ? input.metadata.channel_name.trim() : '') ||
'';
// Best-effort publisher / container hint (less ideal than a true author, but better than nothing).
const publisherHint =
(typeof sourceMetadata?.site_name === 'string' ? sourceMetadata.site_name.trim() : '') ||
(typeof input.metadata?.site_name === 'string' ? input.metadata.site_name.trim() : '') ||
@@ -90,7 +97,6 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
const lines: string[] = [`Title: ${input.title}`];
if (input.link) lines.push(`URL: ${input.link}`);
if (input.dimensions?.length) lines.push(`Dimensions: ${input.dimensions.join(', ')}`);
if (sourceMetadata?.channel_name || input.metadata?.channel_name) lines.push(`Channel: ${sourceMetadata?.channel_name || input.metadata?.channel_name}`);
if (sourceMetadata?.author || input.metadata?.author) lines.push(`Author: ${sourceMetadata?.author || input.metadata?.author}`);
if (sourceMetadata?.site_name || input.metadata?.site_name) lines.push(`Site: ${sourceMetadata?.site_name || input.metadata?.site_name}`);
@@ -116,7 +122,7 @@ RULES:
1) Name the format only if the context clearly supports it: "Podcast episode where…", "Blog post arguing…", "Personal note capturing…", "Research paper showing…", "Resume/CV for…", "Document likely containing…", "Idea that…"
2) Name people by role — channel/host is the creator, title figures are guests/subjects. Use the Creator hint if available.
3) State the actual claim, finding, or insight from the content — not a vague summary of the topic.
4) If the reason it belongs in the graph cannot be inferred from title, source excerpt, URL, metadata, or dimensions, say that naturally rather than inventing context.
4) If the reason it belongs in the graph cannot be inferred from title, source excerpt, URL, or metadata, say that naturally rather than inventing context.
5) If workflow status is unknown, say that naturally, for example by noting it has not been reviewed yet.
6) Do NOT use labels or headings like "WHAT:", "WHY:", or "STATUS:".
7) ABSOLUTELY FORBIDDEN — these words and phrases will be rejected: "discusses", "explores", "examines", "talks about", "is about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to", "important for", "useful for understanding". State things directly instead.
@@ -144,6 +150,7 @@ function sanitizeDescription(rawText: string, input: DescriptionInput): string {
return `${input.title}. Added via Quick Add with no further context yet, so the reason it belongs in the graph is not fully inferred. It has not been reviewed yet.`.slice(0, 500);
}
// Guard against weak generic openings from model drift.
const noGenericPrefix = singleLine.replace(
/^(your note|this note)\s*[—:-]\s*/i,
'Personal note capturing '
src/services/database/dimensionService.ts
-189
View File
@@ -1,189 +0,0 @@
import { getSQLiteClient } from './sqlite-client';
export interface Dimension {
name: string;
description: string | null;
is_priority: boolean;
updated_at: string;
}
export interface LockedDimension {
name: string;
description: string | null;
count: number;
}
export class DimensionService {
/**
* Legacy compatibility shim. Dimensions are now flat, so there is no locked subset.
*/
static async getLockedDimensions(): Promise<LockedDimension[]> {
return [];
}
/**
* Automatic special-dimension assignment has been removed. Callers must provide dimensions explicitly.
*/
static async assignDimensions(nodeData: {
title: string;
content?: string;
link?: string;
description?: string;
}): Promise<{ locked: string[]; keywords: string[] }> {
console.log(`[DimensionAssignment] Skipped for "${nodeData.title}" — flat dimensions require explicit assignment.`);
return { locked: [], keywords: [] };
}
/**
* Legacy method for backwards compatibility
* @deprecated Use assignDimensions() instead
*/
static async assignLockedDimensions(nodeData: {
title: string;
content?: string;
link?: string;
}): Promise<string[]> {
const result = await this.assignDimensions(nodeData);
return result.locked;
}
/**
* Update dimension description
*/
static async updateDimensionDescription(name: string, description: string): Promise<void> {
const sqlite = getSQLiteClient();
sqlite.query(`
INSERT INTO dimensions(name, description, is_priority, updated_at)
VALUES (?, ?, 0, CURRENT_TIMESTAMP)
ON CONFLICT(name) DO UPDATE SET
description = ?,
updated_at = CURRENT_TIMESTAMP
`, [name, description, description]);
}
/**
* Get dimension by name with description
*/
static async getDimensionByName(name: string): Promise<Dimension | null> {
const sqlite = getSQLiteClient();
const result = sqlite.query(`
SELECT name, description, is_priority, updated_at
FROM dimensions
WHERE name = ?
`, [name]);
if (result.rows.length === 0) {
return null;
}
const row = result.rows[0] as any;
return {
name: row.name,
description: row.description,
is_priority: Boolean(row.is_priority),
updated_at: row.updated_at
};
}
/**
* Legacy no-op prompt builder retained only for backward compatibility.
*/
private static buildAssignmentPrompt(
nodeData: { title: string; notes?: string; link?: string; description?: string },
lockedDimensions: LockedDimension[]
): string {
// Use description as primary context, content as fallback
let nodeContextSection: string;
if (nodeData.description) {
const notesPreview = nodeData.notes?.slice(0, 500) || '';
nodeContextSection = `DESCRIPTION: ${nodeData.description}
NOTES PREVIEW: ${notesPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`;
} else {
const notesPreview = nodeData.notes?.slice(0, 2000) || '';
nodeContextSection = `NOTES: ${notesPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`;
}
// Include ALL locked dimensions, using fallback text for those without descriptions
const dimensionsList = lockedDimensions
.map(d => {
const description = d.description && d.description.trim().length > 0
? d.description
: '(none - infer from name)';
return `DIMENSION: "${d.name}"\nDESCRIPTION: ${description}`;
})
.join('\n---\n');
return `Dimensions are now flat categories with no locked subset.
=== NODE TO CATEGORIZE ===
Title: ${nodeData.title}
${nodeContextSection}
URL: ${nodeData.link || 'none'}
=== LOCKED DIMENSIONS ===
CRITICAL: Read each dimension's DESCRIPTION carefully.
The description defines what belongs in that dimension.
Only assign if the content CLEARLY matches the description.
If unsure, skip it — better to miss than assign incorrectly.
AVAILABLE DIMENSIONS:
${dimensionsList}
=== RESPONSE FORMAT ===
LOCKED:
[dimension names from the list above, one per line, or "none"]`;
}
/**
* Legacy no-op parser retained only for backward compatibility.
*/
private static parseAssignmentResponse(
response: string,
availableDimensions: LockedDimension[]
): { locked: string[]; keywords: string[] } {
const lockedDimensions: string[] = [];
// Extract LOCKED section
const lockedMatch = response.match(/LOCKED:\s*([\s\S]*?)$/i);
if (lockedMatch) {
const lockedLines = lockedMatch[1].trim().split('\n');
for (const line of lockedLines) {
const dimensionName = line.trim().toLowerCase();
if (dimensionName === 'none' || dimensionName === '') {
continue;
}
// Find matching dimension (case-insensitive)
const matchedDimension = availableDimensions.find(
d => d.name.toLowerCase() === dimensionName
);
if (matchedDimension && !lockedDimensions.includes(matchedDimension.name)) {
lockedDimensions.push(matchedDimension.name);
}
}
}
return { locked: lockedDimensions, keywords: [] };
}
/**
* Create or get a keyword dimension (unlocked)
*/
static async ensureKeywordDimension(keyword: string): Promise<void> {
const sqlite = getSQLiteClient();
// INSERT OR IGNORE - if dimension exists, do nothing
sqlite.query(`
INSERT OR IGNORE INTO dimensions(name, description, is_priority, updated_at)
VALUES (?, ?, 0, CURRENT_TIMESTAMP)
`, [keyword, null]);
}
}
export const dimensionService = new DimensionService();
src/services/database/dimensionValidation.ts
-29
View File
@@ -1,29 +0,0 @@
import { getSQLiteClient } from './sqlite-client';
import { normalizeDimensionName } from './quality';
export function getUnknownDimensions(values: string[]): string[] {
if (values.length === 0) return [];
const sqlite = getSQLiteClient();
const placeholders = values.map(() => '?').join(', ');
const result = sqlite.query<{ name: string }>(
`SELECT name FROM dimensions WHERE name IN (${placeholders})`,
values
);
const existing = new Set(
result.rows
.map(row => (typeof row.name === 'string' ? normalizeDimensionName(row.name) : ''))
.filter(Boolean)
);
return values.filter(value => !existing.has(normalizeDimensionName(value)));
}
export function formatUnknownDimensionsError(values: string[]): string {
if (values.length === 1) {
return `Unknown dimension: "${values[0]}". Create it first or use an existing dimension.`;
}
return `Unknown dimensions: ${values.map(value => `"${value}"`).join(', ')}. Create them first or use existing dimensions.`;
}
src/services/database/edges.ts
+20 -41
View File
@@ -2,11 +2,11 @@ import { getSQLiteClient } from './sqlite-client';
import { Edge, EdgeContext, EdgeData, EdgeCreatedVia, NodeConnection, Node } from '@/types/database';
import { eventBroadcaster } from '../events';
import { nodeService } from './nodes';
import { getOpenAiKey } from '../storage/apiKeys';
import { generateText } from 'ai';
import { createOpenAI } from '@ai-sdk/openai';
import { openai } from '@ai-sdk/openai';
import { z } from 'zod';
import { validateEdgeExplanation } from './quality';
import { hasValidOpenAiKey } from '../storage/apiKeys';
const inferredEdgeContextSchema = z.object({
type: z.enum(['created_by', 'part_of', 'source_of', 'related_to']),
@@ -53,14 +53,6 @@ async function inferEdgeContext(params: {
return { type: 'related_to', confidence: 0.8, swap_direction: false };
}
// If no API key is configured, degrade gracefully.
// We still enforce explanation, but fall back to "related_to" classification.
const apiKey = getOpenAiKey();
if (!apiKey) {
return { type: 'related_to', confidence: 0.0, swap_direction: false };
}
const provider = createOpenAI({ apiKey });
const prompt = [
`Given two nodes and an explanation, determine the relationship type and direction.`,
``,
@@ -83,8 +75,12 @@ async function inferEdgeContext(params: {
].join('\n');
try {
if (!hasValidOpenAiKey()) {
return { type: 'related_to', confidence: 0.2, swap_direction: false };
}
const { text } = await generateText({
model: provider('gpt-4o-mini'),
model: openai('gpt-4o-mini'),
prompt,
temperature: 0.0,
maxOutputTokens: 120,
@@ -120,18 +116,6 @@ async function autoInferEdge(params: {
}): Promise<{ explanation: string; type: EdgeContext['type']; confidence: number; swap_direction: boolean }> {
const { fromNode, toNode } = params;
const apiKey = getOpenAiKey();
if (!apiKey) {
// Fallback without AI
return {
explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
type: 'related_to',
confidence: 0.0,
swap_direction: false,
};
}
const provider = createOpenAI({ apiKey });
const prompt = [
`Given two knowledge base nodes, determine how they are related.`,
``,
@@ -157,8 +141,17 @@ async function autoInferEdge(params: {
].join('\n');
try {
if (!hasValidOpenAiKey()) {
return {
explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
type: 'related_to',
confidence: 0.2,
swap_direction: false,
};
}
const { text } = await generateText({
model: provider('gpt-4o-mini'),
model: openai('gpt-4o-mini'),
prompt,
temperature: 0.0,
maxOutputTokens: 150,
@@ -477,15 +470,14 @@ export class EdgeService {
WHEN e.from_node_id = ? THEN n_to.title
ELSE n_from.title
END as connected_node_title,
CASE
WHEN e.from_node_id = ? THEN n_to.link
CASE WHEN e.from_node_id = ? THEN n_to.link
ELSE n_from.link
END as connected_node_link,
CASE
WHEN e.from_node_id = ? THEN n_to.source
ELSE n_from.source
END as connected_node_source,
CASE
CASE
WHEN e.from_node_id = ? THEN n_to.metadata
ELSE n_from.metadata
END as connected_node_metadata,
@@ -496,17 +488,7 @@ export class EdgeService {
CASE
WHEN e.from_node_id = ? THEN n_to.updated_at
ELSE n_from.updated_at
END as connected_node_updated_at,
CASE
WHEN e.from_node_id = ? THEN (
SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n_to.id
)
ELSE (
SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n_from.id
)
END as connected_node_dimensions_json
END as connected_node_updated_at
FROM edges e
LEFT JOIN nodes n_from ON e.from_node_id = n_from.id
LEFT JOIN nodes n_to ON e.to_node_id = n_to.id
@@ -521,7 +503,6 @@ export class EdgeService {
nodeId,
nodeId,
nodeId,
nodeId,
nodeId
]);
@@ -543,7 +524,6 @@ export class EdgeService {
id: row.connected_node_id,
title: row.connected_node_title,
link: row.connected_node_link,
dimensions: row.connected_node_dimensions,
embedding: undefined, // Not needed for display
source: row.connected_node_source,
metadata: row.connected_node_metadata,
@@ -587,7 +567,6 @@ export class EdgeService {
id: row.connected_node_id,
title: row.connected_node_title,
link: row.connected_node_link,
dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'),
embedding: undefined, // Not needed for display
source: row.connected_node_source,
metadata: typeof row.connected_node_metadata === 'string' ? JSON.parse(row.connected_node_metadata) : row.connected_node_metadata,
src/services/database/index.ts
+1 -3
View File
@@ -2,7 +2,6 @@
export { nodeService, NodeService } from './nodes';
export { chunkService, ChunkService } from './chunks';
export { edgeService, EdgeService } from './edges';
export { dimensionService, DimensionService } from './dimensionService';
export { contextService, ContextService } from './contextService';
// export { HelperService } from './helpers'; // Removed - migrated to JSON-based service
@@ -39,7 +38,6 @@ async function checkSQLiteDatabaseHealth(): Promise<{
const sqlite = getSQLiteClient();
const connected = await sqlite.testConnection();
const vectorCapability = sqlite.getVectorCapability();
if (!connected) {
return {
connected: false,
@@ -49,7 +47,7 @@ async function checkSQLiteDatabaseHealth(): Promise<{
};
}
const vectorExtension = vectorCapability.available;
const vectorExtension = await sqlite.checkVectorExtension();
// Check if main tables exist
const tables = await sqlite.checkTables();
src/services/database/nodes.ts
+74 -152
View File
@@ -5,7 +5,9 @@ import { EmbeddingService } from '@/services/embeddings';
import { scoreNodeSearchMatch } from './searchRanking';
import { buildCanonicalNodeMetadata, mergeNodeMetadata } from '@/services/nodes/metadata';
type NodeRow = Node & { dimensions_json: string; context_json: string | null };
type NodeRow = Node & {
context_json: string | null;
};
type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
function sanitizeFtsQuery(input: string): string {
@@ -81,8 +83,15 @@ export class NodeService {
}
async countNodes(filters: NodeFilters = {}): Promise<number> {
const { dimensions, search, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus, contextId } = filters;
const {
search,
createdAfter,
createdBefore,
eventAfter,
eventBefore,
chunkStatus,
contextId,
} = filters;
if (search?.trim()) {
return this.countSearchNodesSQLite(filters);
@@ -93,24 +102,6 @@ export class NodeService {
let query = `SELECT COUNT(*) as total FROM nodes n WHERE 1=1`;
const params: any[] = [];
if (dimensions && dimensions.length > 0) {
if (dimensionsMatch === 'all' && dimensions.length > 1) {
query += ` AND (
SELECT COUNT(DISTINCT nd.dimension) FROM node_dimensions nd
WHERE nd.node_id = n.id
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
) = ?`;
params.push(...dimensions, dimensions.length);
} else {
query += ` AND EXISTS (
SELECT 1 FROM node_dimensions nd
WHERE nd.node_id = n.id
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
)`;
params.push(...dimensions);
}
}
if (search) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
@@ -130,8 +121,18 @@ export class NodeService {
// PostgreSQL path removed in SQLite-only consolidation
private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus, contextId } = filters;
const {
search,
limit = 100,
offset = 0,
sortBy,
createdAfter,
createdBefore,
eventAfter,
eventBefore,
chunkStatus,
contextId,
} = filters;
if (search?.trim()) {
return this.searchNodesSQLite(filters);
@@ -139,13 +140,10 @@ export class NodeService {
const sqlite = getSQLiteClient();
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
let query = `
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at, n.context_id,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
CASE
WHEN c.id IS NULL THEN NULL
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
@@ -157,27 +155,6 @@ export class NodeService {
`;
const params: any[] = [];
// Filter by dimensions (SQLite JOIN with node_dimensions)
if (dimensions && dimensions.length > 0) {
if (dimensionsMatch === 'all' && dimensions.length > 1) {
// AND logic: node must have ALL specified dimensions
query += ` AND (
SELECT COUNT(DISTINCT nd.dimension) FROM node_dimensions nd
WHERE nd.node_id = n.id
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
) = ?`;
params.push(...dimensions, dimensions.length);
} else {
// OR logic: node must have at least one of the specified dimensions
query += ` AND EXISTS (
SELECT 1 FROM node_dimensions nd
WHERE nd.node_id = n.id
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
)`;
params.push(...dimensions);
}
}
// Text search in title, description, and source (SQLite LIKE with COLLATE NOCASE)
if (search) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
@@ -251,7 +228,7 @@ export class NodeService {
const result = sqlite.query<NodeRow>(query, params);
// Parse dimensions_json and metadata back for compatibility
// Parse metadata and normalize deprecated compatibility fields.
return result.rows.map(row => this.mapNodeRow(row));
}
@@ -267,8 +244,6 @@ export class NodeService {
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at, n.context_id,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
CASE
WHEN c.id IS NULL THEN NULL
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
@@ -298,7 +273,6 @@ export class NodeService {
source,
link,
event_date,
dimensions = [],
chunk_status,
metadata = {},
context_id,
@@ -327,20 +301,10 @@ export class NodeService {
const id = Number(nodeResult.lastInsertRowid);
// Insert dimensions separately with INSERT OR IGNORE for safety
if (dimensions.length > 0) {
const stmt = sqlite.prepare(
"INSERT OR IGNORE INTO node_dimensions (node_id, dimension) VALUES (?, ?)"
);
for (const dimension of dimensions) {
stmt.run(id, dimension);
}
}
return id; // Returns number directly
});
// Get the created node with dimensions (outside transaction)
// Re-read the created node outside the transaction so callers get the canonical shape.
const createdNode = await this.getNodeByIdSQLite(nodeId);
if (!createdNode) {
throw new Error('Failed to create node');
@@ -363,7 +327,7 @@ export class NodeService {
// PostgreSQL path removed in SQLite-only consolidation
private async updateNodeSQLite(id: number, updates: Partial<Node>): Promise<Node> {
const { title, description, source, link, event_date, dimensions, metadata } = updates;
const { title, description, source, link, event_date, metadata } = updates;
const now = new Date().toISOString();
const sqlite = getSQLiteClient();
@@ -411,14 +375,6 @@ export class NodeService {
stmt.run(...params);
}
// Handle dimensions separately
if (Array.isArray(dimensions)) {
sqlite.prepare('DELETE FROM node_dimensions WHERE node_id = ?').run(id);
const dimStmt = sqlite.prepare('INSERT OR IGNORE INTO node_dimensions (node_id, dimension) VALUES (?, ?)');
for (const dim of dimensions) {
dimStmt.run(id, dim);
}
}
});
// Get updated node
@@ -458,28 +414,21 @@ export class NodeService {
});
}
// Dimension-based filtering methods
async getNodesByDimension(dimension: string): Promise<Node[]> {
return this.getNodes({ dimensions: [dimension] });
}
async searchNodes(searchTerm: string, limit = 50): Promise<Node[]> {
return this.getNodes({ search: searchTerm, limit });
}
private mapNodeRow(row: NodeRow): Node {
const { context_json, ...baseRow } = row;
return {
...row,
dimensions: JSON.parse(row.dimensions_json || '[]'),
metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null,
context: row.context_json ? JSON.parse(row.context_json) : null,
...baseRow,
metadata: baseRow.metadata ? (typeof baseRow.metadata === 'string' ? JSON.parse(baseRow.metadata) : baseRow.metadata) : null,
context: context_json ? JSON.parse(context_json) : null,
};
}
private buildNodeFilterClauses(filters: NodeFilters, alias = 'n'): { clauses: string[]; params: any[] } {
const {
dimensions,
dimensionsMatch = 'any',
createdAfter,
createdBefore,
eventAfter,
@@ -490,24 +439,6 @@ export class NodeService {
const clauses: string[] = [];
const params: any[] = [];
if (dimensions && dimensions.length > 0) {
if (dimensionsMatch === 'all' && dimensions.length > 1) {
clauses.push(`(
SELECT COUNT(DISTINCT nd.dimension) FROM node_dimensions nd
WHERE nd.node_id = ${alias}.id
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
) = ?`);
params.push(...dimensions, dimensions.length);
} else {
clauses.push(`EXISTS (
SELECT 1 FROM node_dimensions nd
WHERE nd.node_id = ${alias}.id
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
)`);
params.push(...dimensions);
}
}
if (createdAfter) { clauses.push(`${alias}.created_at >= ?`); params.push(createdAfter); }
if (createdBefore) { clauses.push(`${alias}.created_at < ?`); params.push(createdBefore); }
if (eventAfter) { clauses.push(`${alias}.event_date >= ?`); params.push(eventAfter); }
@@ -567,26 +498,30 @@ export class NodeService {
if (!search) return 0;
const ftsQuery = sanitizeFtsQuery(search);
const ftsExists = sqlite.prepare(
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
).get();
const { clauses, params } = this.buildNodeFilterClauses(filters);
if (ftsExists && ftsQuery) {
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
const result = sqlite.query<{ total: number }>(`
WITH matched_nodes AS (
SELECT rowid
FROM nodes_fts
WHERE nodes_fts MATCH ?
)
SELECT COUNT(*) as total
FROM matched_nodes mn
JOIN nodes n ON n.id = mn.rowid
${whereClauses}
`, [ftsQuery, ...params]);
try {
const result = sqlite.query<{ total: number }>(`
WITH matched_nodes AS (
SELECT rowid
FROM nodes_fts
WHERE nodes_fts MATCH ?
)
SELECT COUNT(*) as total
FROM matched_nodes mn
JOIN nodes n ON n.id = mn.rowid
${whereClauses}
`, [ftsQuery, ...params]);
return Number(result.rows[0]?.total ?? 0);
return Number(result.rows[0]?.total ?? 0);
} catch (error) {
sqlite.disableNodesFts('nodes_fts query failed during count search', error);
}
}
const words = search.split(/\s+/).filter(Boolean);
@@ -615,7 +550,7 @@ export class NodeService {
const ftsQuery = sanitizeFtsQuery(search);
if (!ftsQuery) return [];
const ftsExists = sqlite.prepare(
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
).get();
if (!ftsExists) return [];
@@ -633,12 +568,15 @@ export class NodeService {
)
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
n.created_at, n.updated_at, n.context_id,
CASE
WHEN c.id IS NULL THEN NULL
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
END as context_json,
fm.rank
FROM fts_matches fm
JOIN nodes n ON n.id = fm.rowid
LEFT JOIN contexts c ON c.id = n.context_id
${whereClauses}
ORDER BY fm.rank
LIMIT ?
@@ -646,7 +584,7 @@ export class NodeService {
return result.rows;
} catch (error) {
console.warn('[NodeSearch] FTS search failed, falling back to LIKE:', error);
sqlite.disableNodesFts('nodes_fts query failed during node search', error);
return [];
}
}
@@ -662,10 +600,13 @@ export class NodeService {
let query = `
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
n.created_at, n.updated_at, n.context_id,
CASE
WHEN c.id IS NULL THEN NULL
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
END as context_json
FROM nodes n
LEFT JOIN contexts c ON c.id = n.context_id
WHERE 1=1
`;
const queryParams = [...params];
@@ -707,10 +648,13 @@ export class NodeService {
let query = `
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
n.created_at, n.updated_at, n.context_id,
CASE
WHEN c.id IS NULL THEN NULL
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
END as context_json
FROM nodes n
LEFT JOIN contexts c ON c.id = n.context_id
WHERE 1=1
`;
const queryParams = [...params];
@@ -781,12 +725,15 @@ export class NodeService {
)
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
n.created_at, n.updated_at, n.context_id,
CASE
WHEN c.id IS NULL THEN NULL
ELSE json_object('id', c.id, 'name', c.name, 'description', c.description, 'icon', c.icon)
END as context_json,
(1.0 / (1.0 + vm.distance)) AS similarity
FROM vector_matches vm
JOIN nodes n ON n.id = vm.node_id
LEFT JOIN contexts c ON c.id = n.context_id
${whereClauses}
ORDER BY vm.distance
LIMIT ?
@@ -829,31 +776,6 @@ export class NodeService {
return updatedNodes;
}
// Get all unique dimensions for UI filtering
async getAllDimensions(): Promise<string[]> {
const sqlite = getSQLiteClient();
const query = `
SELECT DISTINCT dimension
FROM node_dimensions
ORDER BY dimension
`;
const result = sqlite.query<{dimension: string}>(query);
return result.rows.map(row => row.dimension);
}
// Get dimension usage statistics
async getDimensionStats(): Promise<{dimension: string, count: number}[]> {
const sqlite = getSQLiteClient();
const query = `
SELECT dimension, COUNT(*) as count
FROM node_dimensions
GROUP BY dimension
ORDER BY count DESC
`;
const result = sqlite.query<{dimension: string, count: number}>(query);
return result.rows;
}
}
// Export singleton instance
src/services/database/quality.ts
+1 -36
View File
@@ -1,34 +1,10 @@
const WEAK_DESCRIPTION_PATTERNS = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i;
const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|concept|conversation|dataset|decision|dimension|document|episode|essay|event|guide|idea|insight|interview|lesson|link|node|note|paper|person|plan|placeholder|podcast|post|presentation|project|question|record|research|resource|skill|source|status|summary|talk|target|test node|thread|tool|transcript|tweet|update|video|website|workflow)\b/i;
const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|concept|conversation|dataset|decision|document|episode|essay|event|guide|idea|insight|interview|lesson|link|node|note|paper|person|plan|placeholder|podcast|post|presentation|project|question|record|research|resource|skill|source|status|summary|talk|target|test node|thread|tool|transcript|tweet|update|video|website|workflow)\b/i;
const UNCERTAINTY_PATTERNS = /\b(likely|probably|possibly|appears to be|seems to be|unclear|uncertain)\b/i;
const WHY_PATTERNS = /(why added:|added (?:after|as|for|because|to)|follow-?on|queued for|saved for|relevant because|connected to|not inferred|belongs in the graph because|belongs here because|captures .* idea|ties directly into|ties into)/i;
const STATUS_PATTERNS = /(status:|queued|not yet reviewed|in progress|processed|reviewed|saved for later|to review|to read|to watch|to listen|draft|not yet published|unpublished)/i;
const GENERIC_EDGE_PATTERNS = /^(related|related to|connected|connected to|association|associated with)$/i;
export function normalizeDimensionName(value: string): string {
return value.trim().replace(/\s+/g, ' ');
}
export function normalizeDimensions(values: unknown, max = 5): string[] {
if (!Array.isArray(values)) return [];
const seen = new Set<string>();
const normalized: string[] = [];
for (const value of values) {
if (typeof value !== 'string') continue;
const trimmed = normalizeDimensionName(value);
if (!trimmed) continue;
const key = trimmed.toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
normalized.push(trimmed);
if (normalized.length >= max) break;
}
return normalized;
}
export function validateExplicitDescription(description: string): string | null {
const text = description.trim();
if (text.length > 500) {
@@ -84,14 +60,3 @@ export function validateEdgeExplanation(explanation: string): string | null {
}
return null;
}
export function validateDimensionDescription(description: string): string | null {
const text = description.trim();
if (!text) {
return 'Dimension description is required.';
}
if (text.length > 500) {
return 'Description must be 500 characters or less.';
}
return null;
}
src/services/database/sqlite-client.ts
+459 -105
View File
@@ -25,6 +25,8 @@ class SQLiteClient {
private config: SQLiteConfig;
private readonly readOnly: boolean;
private readonly vectorCapability: VectorCapability;
private nodesFtsUsable = true;
private nodesFtsDisabledReason: string | null = null;
private constructor() {
this.config = this.getSQLiteConfig();
@@ -61,6 +63,7 @@ class SQLiteClient {
this.db.pragma('temp_store = memory');
this.db.pragma('busy_timeout = 5000');
this.ensureCoreSchema();
// Ensure vector virtual tables are present and healthy
if (this.vectorCapability.available) {
this.ensureVectorTables();
@@ -162,6 +165,25 @@ class SQLiteClient {
return this.vectorCapability;
}
public isNodesFtsUsable(): boolean {
return this.nodesFtsUsable;
}
public disableNodesFts(reason: string, error?: unknown): void {
this.nodesFtsUsable = false;
if (this.nodesFtsDisabledReason === reason) {
return;
}
this.nodesFtsDisabledReason = reason;
if (error && !this.isSqliteCorruptError(error)) {
console.warn(`[SQLite] nodes_fts disabled: ${reason}`, error);
return;
}
console.warn(`[SQLite] nodes_fts disabled: ${reason}. Falling back to LIKE search for this database session.`);
}
public async checkTables(): Promise<string[]> {
try {
const result = this.query(
@@ -215,17 +237,283 @@ class SQLiteClient {
this.ensureVectorExtensions();
}
private ensureCoreSchema(): void {
if (this.readOnly) {
return;
}
this.db.exec(`
CREATE TABLE IF NOT EXISTS nodes (
id INTEGER PRIMARY KEY,
title TEXT,
description TEXT,
source TEXT,
link TEXT,
event_date TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
embedding BLOB,
embedding_updated_at TEXT,
embedding_text TEXT,
chunk_status TEXT DEFAULT 'not_chunked',
context_id INTEGER,
FOREIGN KEY (context_id) REFERENCES contexts(id) ON DELETE SET NULL
);
CREATE TABLE IF NOT EXISTS contexts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL,
icon TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized
ON contexts(LOWER(TRIM(name)));
CREATE TABLE IF NOT EXISTS chunks (
id INTEGER PRIMARY KEY,
node_id INTEGER NOT NULL,
chunk_idx INTEGER,
text TEXT NOT NULL,
embedding BLOB,
embedding_type TEXT DEFAULT 'openai',
metadata TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS chats (
id INTEGER PRIMARY KEY,
chat_type TEXT,
helper_name TEXT,
agent_type TEXT DEFAULT 'orchestrator',
delegation_id INTEGER,
user_message TEXT,
assistant_message TEXT,
thread_id TEXT,
focused_node_id INTEGER,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_nodes_updated_at ON nodes(updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_chunks_node_id ON chunks(node_id);
`);
this.ensureEdgesTableSchema();
}
private ensureEdgesTableSchema(): void {
const hasEdgesTable = this.db
.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='edges'")
.get();
if (!hasEdgesTable) {
this.db.exec(`
CREATE TABLE edges (
id INTEGER PRIMARY KEY,
from_node_id INTEGER NOT NULL,
to_node_id INTEGER NOT NULL,
source TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
`);
} else {
const edgeCols = this.db.prepare('PRAGMA table_info(edges)').all() as Array<{ name: string }>;
const edgeColNames = new Set(edgeCols.map((col) => col.name));
const needsLegacyRewrite =
!edgeColNames.has('from_node_id') ||
!edgeColNames.has('to_node_id') ||
!edgeColNames.has('source') ||
!edgeColNames.has('created_at') ||
!edgeColNames.has('context') ||
edgeColNames.has('from_id') ||
edgeColNames.has('to_id') ||
edgeColNames.has('description') ||
edgeColNames.has('updated_at');
if (needsLegacyRewrite) {
this.rebuildLegacyEdgesTable(edgeColNames);
}
}
this.db.exec(`
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node_id);
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node_id);
`);
}
private rebuildLegacyEdgesTable(edgeColNames: Set<string>): void {
const fromExpr = edgeColNames.has('from_node_id')
? 'from_node_id'
: edgeColNames.has('from_id')
? 'from_id'
: 'NULL';
const toExpr = edgeColNames.has('to_node_id')
? 'to_node_id'
: edgeColNames.has('to_id')
? 'to_id'
: 'NULL';
const sourceExpr = edgeColNames.has('source') ? 'source' : "'legacy'";
const createdAtExpr = edgeColNames.has('created_at') ? 'created_at' : 'CURRENT_TIMESTAMP';
const contextExpr = edgeColNames.has('context') ? 'context' : 'NULL';
const explanationExpr = edgeColNames.has('explanation')
? 'explanation'
: edgeColNames.has('description')
? 'description'
: edgeColNames.has('context')
? "CASE WHEN json_valid(context) THEN json_extract(context, '$.explanation') ELSE NULL END"
: 'NULL';
console.log('Migrating legacy edges table to canonical schema');
let flippedForeignKeys = false;
try {
this.db.exec('PRAGMA foreign_keys=OFF;');
flippedForeignKeys = true;
} catch {}
try {
this.db.exec('BEGIN TRANSACTION;');
this.db.exec(`
DROP INDEX IF EXISTS idx_edges_from;
DROP INDEX IF EXISTS idx_edges_to;
ALTER TABLE edges RENAME TO edges_legacy_migration;
CREATE TABLE edges (
id INTEGER PRIMARY KEY,
from_node_id INTEGER NOT NULL,
to_node_id INTEGER NOT NULL,
source TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
INSERT INTO edges (id, from_node_id, to_node_id, source, created_at, context, explanation)
SELECT
id,
${fromExpr},
${toExpr},
${sourceExpr},
COALESCE(${createdAtExpr}, CURRENT_TIMESTAMP),
${contextExpr},
${explanationExpr}
FROM edges_legacy_migration
WHERE ${fromExpr} IS NOT NULL
AND ${toExpr} IS NOT NULL;
DROP TABLE edges_legacy_migration;
COMMIT;
`);
} catch (error) {
try {
this.db.exec('ROLLBACK;');
} catch {}
throw error;
} finally {
if (flippedForeignKeys) {
try {
this.db.exec('PRAGMA foreign_keys=ON;');
} catch {}
}
}
}
private rebuildLegacyChatsTable(chatColNames: Set<string>): void {
const chatTypeExpr = chatColNames.has('chat_type') ? 'chat_type' : 'NULL';
const helperNameExpr = chatColNames.has('helper_name')
? 'helper_name'
: chatColNames.has('title')
? 'title'
: 'NULL';
const agentTypeExpr = chatColNames.has('agent_type')
? "COALESCE(agent_type, 'orchestrator')"
: "'orchestrator'";
const delegationIdExpr = chatColNames.has('delegation_id') ? 'delegation_id' : 'NULL';
const userMessageExpr = chatColNames.has('user_message') ? 'user_message' : 'NULL';
const assistantMessageExpr = chatColNames.has('assistant_message') ? 'assistant_message' : 'NULL';
const threadIdExpr = chatColNames.has('thread_id') ? 'thread_id' : 'NULL';
const focusedNodeIdExpr = chatColNames.has('focused_node_id') ? 'focused_node_id' : 'NULL';
const createdAtExpr = chatColNames.has('created_at') ? 'created_at' : 'CURRENT_TIMESTAMP';
const metadataExpr = chatColNames.has('metadata') ? 'metadata' : 'NULL';
console.log('Migrating legacy chats table to canonical schema');
let flippedForeignKeys = false;
try {
this.db.exec('PRAGMA foreign_keys=OFF;');
flippedForeignKeys = true;
} catch {}
try {
this.db.exec('BEGIN TRANSACTION;');
this.db.exec(`
DROP INDEX IF EXISTS idx_chats_thread;
ALTER TABLE chats RENAME TO chats_legacy_cleanup;
CREATE TABLE chats (
id INTEGER PRIMARY KEY,
chat_type TEXT,
helper_name TEXT,
agent_type TEXT DEFAULT 'orchestrator',
delegation_id INTEGER,
user_message TEXT,
assistant_message TEXT,
thread_id TEXT,
focused_node_id INTEGER,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
);
INSERT INTO chats (
id, chat_type, helper_name, agent_type, delegation_id,
user_message, assistant_message, thread_id, focused_node_id,
created_at, metadata
)
SELECT
id,
${chatTypeExpr},
${helperNameExpr},
${agentTypeExpr},
${delegationIdExpr},
${userMessageExpr},
${assistantMessageExpr},
${threadIdExpr},
${focusedNodeIdExpr},
COALESCE(${createdAtExpr}, CURRENT_TIMESTAMP),
${metadataExpr}
FROM chats_legacy_cleanup;
DROP TABLE chats_legacy_cleanup;
COMMIT;
`);
} catch (error) {
try {
this.db.exec('ROLLBACK;');
} catch {}
throw error;
} finally {
if (flippedForeignKeys) {
try {
this.db.exec('PRAGMA foreign_keys=ON;');
} catch {}
}
}
}
private ensureLoggingAndMemorySchema(): void {
if (this.readOnly) {
return;
}
try {
const hasReadyLogs = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
if (hasReadyLogs) {
return;
}
// 1) If logs table missing but legacy memory table exists, migrate
// Existing installs may already have logs but still need the idempotent schema pass below.
// Only skip the legacy memory rename step when logs already exists.
const hasLogs = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
const hasLegacyMemory = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memory'").get();
if (!hasLogs && hasLegacyMemory) {
@@ -263,6 +551,11 @@ class SQLiteClient {
}
};
ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;");
ensureNodeCol('link', 'ALTER TABLE nodes ADD COLUMN link TEXT;');
ensureNodeCol('source', 'ALTER TABLE nodes ADD COLUMN source TEXT;');
ensureNodeCol('metadata', 'ALTER TABLE nodes ADD COLUMN metadata TEXT;');
ensureNodeCol('created_at', "ALTER TABLE nodes ADD COLUMN created_at TEXT DEFAULT CURRENT_TIMESTAMP;");
ensureNodeCol('updated_at', "ALTER TABLE nodes ADD COLUMN updated_at TEXT DEFAULT CURRENT_TIMESTAMP;");
} catch (nodeErr) {
console.warn('Failed to ensure nodes columns:', nodeErr);
}
@@ -279,6 +572,25 @@ class SQLiteClient {
console.warn('Failed to ensure chats.created_at column:', chatErr);
}
// Normalize legacy chats table before creating chat triggers or views that reference modern columns.
try {
const hasChatsTable = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='chats'").get();
if (hasChatsTable) {
const chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as Array<{ name: string }>;
const chatColNames = new Set(chatCols.map((col) => col.name));
const needsChatRewrite =
chatColNames.has('focused_memory_id') ||
['chat_type', 'helper_name', 'agent_type', 'delegation_id', 'user_message', 'assistant_message', 'thread_id', 'focused_node_id', 'created_at', 'metadata']
.some((name) => !chatColNames.has(name));
if (needsChatRewrite) {
this.rebuildLegacyChatsTable(chatColNames);
}
}
} catch (chatSchemaErr) {
console.warn('Failed to normalize chats schema before log setup:', chatSchemaErr);
}
// 3) Helpful indexes on logs (clean up old names first)
this.db.exec(`
DROP INDEX IF EXISTS idx_memory_ts;
@@ -494,55 +806,20 @@ class SQLiteClient {
if (hasChats) {
try {
let chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as any[];
const hasFocusedMemoryId = chatCols.some((c: any) => c.name === 'focused_memory_id');
if (hasFocusedMemoryId) {
console.log('Removing legacy chats.focused_memory_id column');
let flippedForeignKeys = false;
try {
this.db.exec('PRAGMA foreign_keys=OFF;');
flippedForeignKeys = true;
this.db.exec(`
BEGIN TRANSACTION;
ALTER TABLE chats RENAME TO chats_legacy_cleanup;
CREATE TABLE chats (
id INTEGER PRIMARY KEY,
chat_type TEXT,
helper_name TEXT,
agent_type TEXT DEFAULT 'orchestrator',
delegation_id INTEGER,
user_message TEXT,
assistant_message TEXT,
thread_id TEXT,
focused_node_id INTEGER,
created_at TEXT DEFAULT (CURRENT_TIMESTAMP),
metadata TEXT,
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
);
INSERT INTO chats (
id, chat_type, helper_name, agent_type, delegation_id,
user_message, assistant_message, thread_id, focused_node_id,
created_at, metadata
)
SELECT id, chat_type, helper_name, agent_type, delegation_id,
user_message, assistant_message, thread_id, focused_node_id,
created_at, metadata
FROM chats_legacy_cleanup;
DROP TABLE chats_legacy_cleanup;
CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);
COMMIT;
`);
} catch (migrationErr) {
console.warn('Failed to migrate chats table (focused_memory_id removal):', migrationErr);
try { this.db.exec('ROLLBACK;'); } catch {}
} finally {
if (flippedForeignKeys) {
try { this.db.exec('PRAGMA foreign_keys=ON;'); } catch {}
}
}
const chatColNames = new Set(chatCols.map((c: any) => c.name));
const needsChatRewrite =
chatColNames.has('focused_memory_id') ||
['chat_type', 'helper_name', 'agent_type', 'delegation_id', 'user_message', 'assistant_message', 'thread_id', 'focused_node_id', 'created_at', 'metadata']
.some((name) => !chatColNames.has(name));
if (needsChatRewrite) {
this.rebuildLegacyChatsTable(chatColNames);
chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as any[];
}
this.db.exec("CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);");
if (chatCols.some((c: any) => c.name === 'thread_id')) {
this.db.exec("CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);");
}
const ensureCol = (name: string, ddl: string) => {
if (!chatCols.some((c: any) => c.name === name)) {
@@ -578,51 +855,7 @@ class SQLiteClient {
console.warn('Failed to drop legacy memory pipeline tables:', dropLegacyErr);
}
// 9) Ensure dimensions table exists (v0.1.16+ schema migration)
const hasDimensions = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='dimensions'").get();
if (!hasDimensions) {
console.log('Creating dimensions table for v0.1.16+ features...');
this.db.exec(`
CREATE TABLE dimensions (
name TEXT PRIMARY KEY,
description TEXT,
is_priority INTEGER DEFAULT 0,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
`);
// Seed default locked dimensions
const defaultDimensions = ['research', 'ideas', 'projects', 'memory', 'preferences'];
const insertDimension = this.db.prepare(`
INSERT INTO dimensions (name, is_priority, updated_at)
VALUES (?, 1, datetime('now'))
ON CONFLICT(name) DO UPDATE SET is_priority = 1, updated_at = datetime('now')
`);
for (const dimension of defaultDimensions) {
try {
insertDimension.run(dimension);
} catch (e) {
console.warn(`Failed to seed dimension '${dimension}':`, e);
}
}
console.log('Dimensions table created and seeded with default locked dimensions');
} else {
// Check if existing dimensions table has description column
const dimensionCols = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
const hasDescription = dimensionCols.some(col => col.name === 'description');
if (!hasDescription) {
console.log('Adding description column to existing dimensions table...');
try {
this.db.exec('ALTER TABLE dimensions ADD COLUMN description TEXT;');
console.log('Description column added to dimensions table');
} catch (e) {
console.warn('Failed to add description column to dimensions table:', e);
}
}
}
// 10) Final schema pass migrations (source canonicalization, event_date, dimensions.icon, drop dead columns)
// 9) Final schema pass migrations (source-first backfill, event_date, soft contexts, drop dimensions)
try {
let nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
let nodeColNames = nodeCols2.map(c => c.name);
@@ -634,6 +867,12 @@ class SQLiteClient {
nodeColNames = nodeCols2.map(c => c.name);
}
if (!nodeColNames.includes('chunk_status')) {
this.db.exec("ALTER TABLE nodes ADD COLUMN chunk_status TEXT DEFAULT 'not_chunked';");
nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
nodeColNames = nodeCols2.map(c => c.name);
}
if (nodeColNames.includes('source')) {
if (nodeColNames.includes('content')) {
this.db.exec(`
@@ -691,19 +930,121 @@ class SQLiteClient {
// Add event_date
if (!nodeColNames.includes('event_date')) {
this.db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;');
// Backfill from metadata.published_date where available
// Backfill from metadata.published_date or metadata.source_metadata.published_date where available
try {
this.db.exec(`
UPDATE nodes SET event_date = json_extract(metadata, '$.published_date')
WHERE event_date IS NULL AND json_extract(metadata, '$.published_date') IS NOT NULL;
UPDATE nodes
SET event_date = COALESCE(
json_extract(metadata, '$.source_metadata.published_date'),
json_extract(metadata, '$.published_date')
)
WHERE event_date IS NULL
AND COALESCE(
json_extract(metadata, '$.source_metadata.published_date'),
json_extract(metadata, '$.published_date')
) IS NOT NULL;
`);
} catch {}
}
// Add dimensions.icon
const dimCols2 = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
if (!dimCols2.some(c => c.name === 'icon')) {
this.db.exec('ALTER TABLE dimensions ADD COLUMN icon TEXT;');
if (!nodeColNames.includes('context_id')) {
this.db.exec('ALTER TABLE nodes ADD COLUMN context_id INTEGER REFERENCES contexts(id) ON DELETE SET NULL;');
}
const hasContexts = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='contexts'").get();
if (!hasContexts) {
this.db.exec(`
CREATE TABLE contexts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL,
icon TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
`);
}
const contextCols = this.db.prepare('PRAGMA table_info(contexts)').all() as Array<{ name: string }>;
const ensureContextCol = (name: string, ddl: string) => {
if (!contextCols.some(col => col.name === name)) {
this.db.exec(ddl);
}
};
ensureContextCol('description', "ALTER TABLE contexts ADD COLUMN description TEXT NOT NULL DEFAULT '';");
ensureContextCol('icon', 'ALTER TABLE contexts ADD COLUMN icon TEXT;');
ensureContextCol('created_at', "ALTER TABLE contexts ADD COLUMN created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
ensureContextCol('updated_at', "ALTER TABLE contexts ADD COLUMN updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
this.db.exec(`
UPDATE contexts
SET description = COALESCE(NULLIF(TRIM(description), ''), name)
WHERE description IS NULL OR LENGTH(TRIM(description)) = 0;
`);
this.db.exec(`
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized
ON contexts(LOWER(TRIM(name)));
CREATE INDEX IF NOT EXISTS idx_nodes_context_id ON nodes(context_id);
`);
const hasLegacyDimensions = this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='dimensions'").get();
const hasLegacyNodeDimensions = this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='node_dimensions'").get();
this.db.exec(`
CREATE TABLE IF NOT EXISTS dimension_migration_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
migrated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
dimension_count INTEGER NOT NULL,
assignment_count INTEGER NOT NULL,
payload TEXT
);
`);
if (hasLegacyDimensions || hasLegacyNodeDimensions) {
const existingSnapshotCount = Number(
(this.db.prepare('SELECT COUNT(*) as count FROM dimension_migration_snapshots').get() as { count?: number } | undefined)?.count ?? 0
);
if (existingSnapshotCount === 0) {
const dimensionCount = hasLegacyDimensions
? Number((this.db.prepare('SELECT COUNT(*) as count FROM dimensions').get() as { count?: number } | undefined)?.count ?? 0)
: 0;
const assignmentCount = hasLegacyNodeDimensions
? Number((this.db.prepare('SELECT COUNT(*) as count FROM node_dimensions').get() as { count?: number } | undefined)?.count ?? 0)
: 0;
const payload = hasLegacyNodeDimensions
? (
this.db.prepare(`
SELECT COALESCE(
json_group_array(
json_object(
'node_id', nd.node_id,
'dimension', nd.dimension,
'description', d.description,
'icon', d.icon,
'is_priority', d.is_priority
)
),
'[]'
) AS payload
FROM node_dimensions nd
LEFT JOIN dimensions d ON d.name = nd.dimension
`).get() as { payload?: string } | undefined
)?.payload ?? '[]'
: '[]';
this.db.prepare(`
INSERT INTO dimension_migration_snapshots (dimension_count, assignment_count, payload)
VALUES (?, ?, ?)
`).run(dimensionCount, assignmentCount, payload);
}
this.db.exec(`
DROP INDEX IF EXISTS idx_dim_by_dimension;
DROP INDEX IF EXISTS idx_dim_by_node;
DROP TABLE IF EXISTS node_dimensions;
DROP TABLE IF EXISTS dimensions;
`);
}
// Drop dead columns (requires SQLite 3.35+)
@@ -742,7 +1083,11 @@ class SQLiteClient {
this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
}
} catch (ftsErr) {
console.warn('Failed to rebuild nodes_fts:', ftsErr);
if (this.isSqliteCorruptError(ftsErr)) {
this.disableNodesFts('existing nodes_fts is corrupt and could not be rebuilt', ftsErr);
} else {
console.warn('Failed to rebuild nodes_fts:', ftsErr);
}
}
} catch (schemaErr) {
console.warn('Final schema pass migration error:', schemaErr);
@@ -854,6 +1199,15 @@ class SQLiteClient {
public close(): void {
this.db.close();
}
private isSqliteCorruptError(error: unknown): boolean {
if (!(error instanceof Error)) {
return false;
}
const sqliteError = error as Error & { code?: string };
return sqliteError.code === 'SQLITE_CORRUPT' || /database disk image is malformed/i.test(sqliteError.message || '');
}
}
// Export singleton instance (similar to PostgreSQL client interface)