refactor: align source-first schema surfaces

This commit is contained in:
“BeeRad”
2026-03-22 20:17:07 +11:00
parent 255377658b
commit b9af8dc385
21 changed files with 267 additions and 93 deletions
+4 -4
View File
@@ -4,7 +4,7 @@ import { hasValidOpenAiKey } from '../storage/apiKeys';
export interface DescriptionInput {
title: string;
notes?: string;
source?: string;
link?: string;
metadata?: {
source?: string;
@@ -61,7 +61,7 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
}
// Fast path: skip AI for very short inputs (likely just notes)
if (!input.notes && !input.link && input.title.length < 30) {
if (!input.source && !input.link && input.title.length < 30) {
console.log(`[DescriptionService] Short input, using fallback for: "${input.title}"`);
return generateFallbackDescription(input);
}
@@ -134,8 +134,8 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
if (publisherHint) lines.push(`Publisher hint: ${publisherHint}`);
lines.push(`Likely user-authored: ${likelyUserAuthored ? 'yes' : 'no'}`);
const contentPreview = input.notes?.slice(0, 800) || '';
if (contentPreview) lines.push(`Notes: ${contentPreview}${input.notes && input.notes.length > 800 ? '...' : ''}`);
const contentPreview = input.source?.slice(0, 800) || '';
if (contentPreview) lines.push(`Source excerpt: ${contentPreview}${input.source && input.source.length > 800 ? '...' : ''}`);
return `Write a description for this knowledge node. Max 280 characters.
+6 -6
View File
@@ -138,7 +138,7 @@ export class NodeService {
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
let query = `
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -252,7 +252,7 @@ export class NodeService {
private async getNodeByIdSQLite(id: number): Promise<Node | null> {
const sqlite = getSQLiteClient();
const query = `
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -589,7 +589,7 @@ export class NodeService {
WHERE nodes_fts MATCH ?
LIMIT ?
)
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -618,7 +618,7 @@ export class NodeService {
const words = search.split(/\s+/).filter(Boolean);
const { clauses, params } = this.buildNodeFilterClauses(filters);
let query = `
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -663,7 +663,7 @@ export class NodeService {
const { clauses, params } = this.buildNodeFilterClauses(filters);
let query = `
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -737,7 +737,7 @@ export class NodeService {
ORDER BY distance
LIMIT ?
)
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
+32 -24
View File
@@ -616,19 +616,11 @@ class SQLiteClient {
}
}
// 10) Final schema pass migrations (content→notes, event_date, dimensions.icon, drop dead columns)
// 10) Final schema pass migrations (source canonicalization, event_date, dimensions.icon, drop dead columns)
try {
let nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
let nodeColNames = nodeCols2.map(c => c.name);
// Rename content → notes (additive first)
if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) {
console.log('Migrating nodes.content → nodes.notes...');
this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;');
nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
nodeColNames = nodeCols2.map(c => c.name);
}
if (!nodeColNames.includes('source')) {
console.log('Adding nodes.source column...');
this.db.exec('ALTER TABLE nodes ADD COLUMN source TEXT;');
@@ -637,22 +629,38 @@ class SQLiteClient {
}
if (nodeColNames.includes('source')) {
this.db.exec(`
UPDATE nodes
SET source = chunk
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND chunk IS NOT NULL
AND LENGTH(TRIM(chunk)) > 0;
`);
if (nodeColNames.includes('content')) {
this.db.exec(`
UPDATE nodes
SET source = content,
chunk_status = 'not_chunked'
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND content IS NOT NULL
AND LENGTH(TRIM(content)) > 0;
`);
}
this.db.exec(`
UPDATE nodes
SET source = notes,
chunk_status = 'not_chunked'
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND notes IS NOT NULL
AND LENGTH(TRIM(notes)) > 0;
`);
if (nodeColNames.includes('notes')) {
this.db.exec(`
UPDATE nodes
SET source = notes,
chunk_status = 'not_chunked'
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND notes IS NOT NULL
AND LENGTH(TRIM(notes)) > 0;
`);
}
if (nodeColNames.includes('chunk')) {
this.db.exec(`
UPDATE nodes
SET source = chunk,
chunk_status = 'not_chunked'
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND chunk IS NOT NULL
AND LENGTH(TRIM(chunk)) > 0;
`);
}
this.db.exec(`
UPDATE nodes