feat: port source-first ingestion to os repo

- make source the canonical field across os routes, tools, ui, and mcp
- align fresh-install schema, search, and embedding flows with source-first ingestion

Generated with Claude Code

src/services/agents/quickAdd.ts

@@ -236,7 +236,7 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin
     const title = deriveFallbackLinkTitle(url);
     const description =
       `Link record for this source. RA-H could not correctly process the URL during ingestion because ${message}. Stored so the source is not lost and can be revisited later.`;
-    const notes = [
+    const source = [
       `Original URL: ${url}`,
       `Ingestion failure: ${message}`,
       `Attempted pipeline: ${type}`,
@@ -248,7 +248,7 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin
       body: JSON.stringify({
         title,
         description,
-        notes,
+        source,
         link: url,
         metadata: {
           source: 'quick-add-link-fallback',
@@ -292,7 +292,7 @@ async function handleNoteQuickAdd(rawInput: string, task: string, userDescriptio
   const title = deriveNoteTitle(content);
   const nodePayload: Record<string, unknown> = {
     title,
-    notes: content,
+    source: content,
     metadata: {
       source: 'quick-add-note',
       refined_at: new Date().toISOString(),
@@ -396,8 +396,8 @@ async function handleChatTranscriptQuickAdd(rawInput: string, task: string): Pro
     headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({
       title,
-      notes: content,
-      chunk: transcript,
+      source: transcript,
+      description: content,
       metadata,
     }),
   });

src/services/database/edges.ts

@@ -478,17 +478,13 @@ export class EdgeService {
           ELSE n_from.title
         END as connected_node_title,
         CASE
-          WHEN e.from_node_id = ? THEN n_to.notes
-          ELSE n_from.notes
-        END as connected_node_notes,
-        CASE 
           WHEN e.from_node_id = ? THEN n_to.link
           ELSE n_from.link
         END as connected_node_link,
         CASE 
-          WHEN e.from_node_id = ? THEN n_to.chunk
-          ELSE n_from.chunk
-        END as connected_node_chunk,
+          WHEN e.from_node_id = ? THEN n_to.source
+          ELSE n_from.source
+        END as connected_node_source,
         CASE 
           WHEN e.from_node_id = ? THEN n_to.metadata
           ELSE n_from.metadata
@@ -547,11 +543,10 @@ export class EdgeService {
       const connected_node: Node = {
         id: row.connected_node_id,
         title: row.connected_node_title,
-        notes: row.connected_node_notes,
         link: row.connected_node_link,
         dimensions: row.connected_node_dimensions,
         embedding: undefined, // Not needed for display
-        chunk: row.connected_node_chunk,
+        source: row.connected_node_source,
         metadata: row.connected_node_metadata,
         created_at: row.connected_node_created_at,
         updated_at: row.connected_node_updated_at
@@ -592,11 +587,10 @@ export class EdgeService {
       const connected_node: Node = {
         id: row.connected_node_id,
         title: row.connected_node_title,
-        notes: row.connected_node_notes,
         link: row.connected_node_link,
         dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'),
         embedding: undefined, // Not needed for display
-        chunk: row.connected_node_chunk,
+        source: row.connected_node_source,
         metadata: typeof row.connected_node_metadata === 'string' ? JSON.parse(row.connected_node_metadata) : row.connected_node_metadata,
         created_at: row.connected_node_created_at,
         updated_at: row.connected_node_updated_at

src/services/database/nodes.ts

@@ -16,6 +16,38 @@ function sanitizeFtsQuery(input: string): string {
     .join(' ');
 }
 
+function extractRelaxedSearchTerms(query: string): string[] {
+  const stopWords = new Set([
+    'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'can', 'do', 'find',
+    'for', 'from', 'hello', 'i', 'in', 'is', 'it', 'me', 'my', 'of', 'on',
+    'or', 'recent', 'stuff', 'term', 'that', 'the', 'this', 'to', 'with', 'you'
+  ]);
+
+  const rawTerms = query
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]+/g, ' ')
+    .split(/\s+/)
+    .map(term => term.trim())
+    .filter(Boolean);
+
+  const expanded = new Set<string>();
+
+  for (const term of rawTerms) {
+    if (!stopWords.has(term) && term.length >= 3) {
+      expanded.add(term);
+    }
+
+    const alphaParts = term.replace(/\d+/g, ' ').split(/\s+/).filter(Boolean);
+    for (const part of alphaParts) {
+      if (!stopWords.has(part) && part.length >= 3) {
+        expanded.add(part);
+      }
+    }
+  }
+
+  return Array.from(expanded).slice(0, 8);
+}
+
 function reciprocalRankFuse<T extends { id: number }>(
   rankedLists: T[][],
   limit: number,
@@ -48,7 +80,7 @@ export class NodeService {
 
   async countNodes(filters: NodeFilters = {}): Promise<number> {
     const { dimensions, search, dimensionsMatch = 'any',
-            createdAfter, createdBefore, eventAfter, eventBefore } = filters;
+            createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters;
 
     if (search?.trim()) {
       return this.countSearchNodesSQLite(filters);
@@ -78,7 +110,7 @@ export class NodeService {
     }
 
     if (search) {
-      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
+      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
       params.push(`%${search}%`, `%${search}%`, `%${search}%`);
     }
 
@@ -86,6 +118,7 @@ export class NodeService {
     if (createdBefore) { query += ` AND n.created_at < ?`; params.push(createdBefore); }
     if (eventAfter) { query += ` AND n.event_date >= ?`; params.push(eventAfter); }
     if (eventBefore) { query += ` AND n.event_date < ?`; params.push(eventBefore); }
+    if (chunkStatus) { query += ` AND n.chunk_status = ?`; params.push(chunkStatus); }
 
     const result = sqlite.query<{ total: number }>(query, params);
     return result.rows[0]?.total ?? 0;
@@ -95,7 +128,7 @@ export class NodeService {
 
   private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
     const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any',
-            createdAfter, createdBefore, eventAfter, eventBefore } = filters;
+            createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters;
 
     if (search?.trim()) {
       return this.searchNodesSQLite(filters);
@@ -105,7 +138,7 @@ export class NodeService {
     
     // Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
     let query = `
-      SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
+      SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
              n.chunk_status, n.embedding_updated_at, n.embedding_text,
              n.created_at, n.updated_at,
              COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -137,9 +170,9 @@ export class NodeService {
       }
     }
 
-    // Text search in title, description, and notes (SQLite LIKE with COLLATE NOCASE)
+    // Text search in title, description, and source (SQLite LIKE with COLLATE NOCASE)
     if (search) {
-      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
+      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
       params.push(`%${search}%`, `%${search}%`, `%${search}%`);
     }
 
@@ -160,23 +193,27 @@ export class NodeService {
       query += ` AND n.event_date < ?`;
       params.push(eventBefore);
     }
+    if (chunkStatus) {
+      query += ` AND n.chunk_status = ?`;
+      params.push(chunkStatus);
+    }
 
     // Sorting logic
     if (search) {
-      // For search queries, prioritize by relevance: exact title → starts with → contains in title → description → notes
+      // For search queries, prioritize by relevance: exact title → starts with → contains in title → description → source
       query += ` ORDER BY
         CASE WHEN LOWER(n.title) = LOWER(?) THEN 1 ELSE 6 END,
         CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
         CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
         CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
-        CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
+        CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
         n.updated_at DESC`;
       params.push(
         search,           // Exact match (case-insensitive)
         `${search}%`,     // Starts with search term
         `%${search}%`,    // Contains in title
         `%${search}%`,    // Contains in description
-        `%${search}%`     // Contains in notes
+        `%${search}%`     // Contains in source
       );
     } else if (sortBy === 'edges') {
       // Sort by edge count (most connected first)
@@ -215,7 +252,7 @@ export class NodeService {
   private async getNodeByIdSQLite(id: number): Promise<Node | null> {
     const sqlite = getSQLiteClient();
     const query = `
-      SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
+      SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
              n.chunk_status, n.embedding_updated_at, n.embedding_text,
              n.created_at, n.updated_at,
              COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) 
@@ -241,11 +278,10 @@ export class NodeService {
     const {
       title,
       description,
-      notes,
+      source,
       link,
       event_date,
       dimensions = [],
-      chunk,
       chunk_status,
       metadata = {}
     } = nodeData;
@@ -255,16 +291,15 @@ export class NodeService {
     const nodeId = sqlite.transaction(() => {
       // Insert node using prepare/run for lastInsertRowid access
       const nodeResult = sqlite.prepare(`
-        INSERT INTO nodes (title, description, notes, link, event_date, metadata, chunk, chunk_status, created_at, updated_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        INSERT INTO nodes (title, description, source, link, event_date, metadata, chunk_status, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
       `).run(
         title,
         description ?? null,
-        notes ?? null,
+        source ?? null,
         link ?? null,
         event_date ?? null,
         JSON.stringify(metadata),
-        chunk ?? null,
         chunk_status ?? null,
         now,
         now
@@ -308,7 +343,7 @@ export class NodeService {
   // PostgreSQL path removed in SQLite-only consolidation
 
   private async updateNodeSQLite(id: number, updates: Partial<Node>): Promise<Node> {
-    const { title, description, notes, link, event_date, dimensions, chunk, metadata } = updates;
+    const { title, description, source, link, event_date, dimensions, metadata } = updates;
     const now = new Date().toISOString();
     const sqlite = getSQLiteClient();
 
@@ -327,10 +362,9 @@ export class NodeService {
       
       if (title !== undefined) { setFields.push('title = ?'); params.push(title); }
       if (description !== undefined) { setFields.push('description = ?'); params.push(description); }
-      if (notes !== undefined) { setFields.push('notes = ?'); params.push(notes); }
+      if (source !== undefined) { setFields.push('source = ?'); params.push(source); }
       if (link !== undefined) { setFields.push('link = ?'); params.push(link); }
       if (event_date !== undefined) { setFields.push('event_date = ?'); params.push(event_date); }
-      if (chunk !== undefined) { setFields.push('chunk = ?'); params.push(chunk); }
       if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) {
         setFields.push('chunk_status = ?');
         params.push(updates.chunk_status ?? null);
@@ -469,6 +503,10 @@ export class NodeService {
       rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit);
     }
 
+    if (rankedRows.length === 0) {
+      rankedRows = this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit);
+    }
+
     if ((filters.searchMode ?? 'standard') === 'hybrid') {
       const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit);
       if (vectorRows.length > 0) {
@@ -518,7 +556,7 @@ export class NodeService {
     }
 
     for (const word of words) {
-      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
+      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
       queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
     }
 
@@ -551,7 +589,7 @@ export class NodeService {
           WHERE nodes_fts MATCH ?
           LIMIT ?
         )
-        SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
+        SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
                n.chunk_status, n.embedding_updated_at, n.embedding_text,
                n.created_at, n.updated_at,
                COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -580,7 +618,7 @@ export class NodeService {
     const words = search.split(/\s+/).filter(Boolean);
     const { clauses, params } = this.buildNodeFilterClauses(filters);
     let query = `
-      SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
+      SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
              n.chunk_status, n.embedding_updated_at, n.embedding_text,
              n.created_at, n.updated_at,
              COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -595,7 +633,7 @@ export class NodeService {
     }
 
     for (const word of words) {
-      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
+      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
       queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
     }
 
@@ -604,7 +642,7 @@ export class NodeService {
       CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
       CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
       CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
-      CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
+      CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
       n.updated_at DESC
       LIMIT ?`;
 
@@ -614,6 +652,62 @@ export class NodeService {
     return result.rows;
   }
 
+  private searchNodesLikeRelaxed(
+    sqlite: ReturnType<typeof getSQLiteClient>,
+    search: string,
+    filters: NodeFilters,
+    limit: number,
+  ): NodeSearchRow[] {
+    const terms = extractRelaxedSearchTerms(search);
+    if (terms.length === 0) return [];
+
+    const { clauses, params } = this.buildNodeFilterClauses(filters);
+    let query = `
+      SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
+             n.chunk_status, n.embedding_updated_at, n.embedding_text,
+             n.created_at, n.updated_at,
+             COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
+                       FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
+      FROM nodes n
+      WHERE 1=1
+    `;
+    const queryParams = [...params];
+
+    if (clauses.length > 0) {
+      query += ` AND ${clauses.join(' AND ')}`;
+    }
+
+    const termClauses: string[] = [];
+    for (const term of terms) {
+      termClauses.push(`n.title LIKE ? COLLATE NOCASE`);
+      termClauses.push(`n.description LIKE ? COLLATE NOCASE`);
+      termClauses.push(`n.source LIKE ? COLLATE NOCASE`);
+      queryParams.push(`%${term}%`, `%${term}%`, `%${term}%`);
+    }
+
+    query += ` AND (${termClauses.join(' OR ')})`;
+
+    const scoreClauses: string[] = [];
+    const scoreParams: string[] = [];
+    for (const term of terms) {
+      scoreClauses.push(`CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 0 END`);
+      scoreClauses.push(`CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 2 ELSE 0 END`);
+      scoreClauses.push(`CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 1 ELSE 0 END`);
+      scoreParams.push(`%${term}%`, `%${term}%`, `%${term}%`);
+    }
+
+    query += ` ORDER BY
+      (${scoreClauses.join(' + ')}) DESC,
+      CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 0 ELSE 1 END,
+      n.updated_at DESC
+      LIMIT ?`;
+
+    queryParams.push(...scoreParams, `%${search}%`, limit);
+
+    const result = sqlite.query<NodeSearchRow>(query, queryParams);
+    return result.rows;
+  }
+
   private async searchNodesVector(
     sqlite: ReturnType<typeof getSQLiteClient>,
     search: string,
@@ -643,7 +737,7 @@ export class NodeService {
           ORDER BY distance
           LIMIT ?
         )
-        SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
+        SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
                n.chunk_status, n.embedding_updated_at, n.embedding_text,
                n.created_at, n.updated_at,
                COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)

src/services/database/sqlite-client.ts

@@ -616,13 +616,60 @@ class SQLiteClient {
 
       // 10) Final schema pass migrations (content→notes, event_date, dimensions.icon, drop dead columns)
       try {
-        const nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
-        const nodeColNames = nodeCols2.map(c => c.name);
+        let nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
+        let nodeColNames = nodeCols2.map(c => c.name);
 
         // Rename content → notes (additive first)
         if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) {
           console.log('Migrating nodes.content → nodes.notes...');
           this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;');
+          nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
+          nodeColNames = nodeCols2.map(c => c.name);
+        }
+
+        if (!nodeColNames.includes('source')) {
+          console.log('Adding nodes.source column...');
+          this.db.exec('ALTER TABLE nodes ADD COLUMN source TEXT;');
+          nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
+          nodeColNames = nodeCols2.map(c => c.name);
+        }
+
+        if (nodeColNames.includes('source')) {
+          this.db.exec(`
+            UPDATE nodes
+            SET source = chunk
+            WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
+              AND chunk IS NOT NULL
+              AND LENGTH(TRIM(chunk)) > 0;
+          `);
+
+          this.db.exec(`
+            UPDATE nodes
+            SET source = notes,
+                chunk_status = 'not_chunked'
+            WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
+              AND notes IS NOT NULL
+              AND LENGTH(TRIM(notes)) > 0;
+          `);
+
+          this.db.exec(`
+            UPDATE nodes
+            SET source = title || CASE
+              WHEN description IS NOT NULL AND LENGTH(TRIM(description)) > 0
+                THEN char(10) || char(10) || description
+              ELSE ''
+            END,
+            chunk_status = 'not_chunked'
+            WHERE source IS NULL OR LENGTH(TRIM(source)) = 0;
+          `);
+
+          this.db.exec(`
+            UPDATE nodes
+            SET chunk_status = 'not_chunked'
+            WHERE source IS NOT NULL
+              AND LENGTH(TRIM(source)) > 0
+              AND (chunk_status IS NULL OR chunk_status != 'chunked');
+          `);
         }
 
         // Add event_date
@@ -657,8 +704,10 @@ class SQLiteClient {
         if (edgeCols.some(c => c.name === 'user_feedback')) {
           try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch {}
         }
+        // edges.explanation (top-level column added alongside context JSON)
         if (!edgeCols.some(c => c.name === 'explanation')) {
           this.db.exec('ALTER TABLE edges ADD COLUMN explanation TEXT;');
+          // Backfill from context JSON where available
           try {
             this.db.exec(`
               UPDATE edges SET explanation = json_extract(context, '$.explanation')
@@ -667,13 +716,13 @@ class SQLiteClient {
           } catch {}
         }
 
-        // Recreate nodes_fts to index title + description + notes
+        // Recreate nodes_fts to index title + source + description
         try {
           const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined;
-          if (ftsCheck?.sql && (!ftsCheck.sql.includes('description') || ftsCheck.sql.includes('content'))) {
+          const needsRebuild = !ftsCheck?.sql || !ftsCheck.sql.includes('source') || ftsCheck.sql.includes('notes') || ftsCheck.sql.includes('content');
+          if (needsRebuild) {
             this.db.exec('DROP TABLE IF EXISTS nodes_fts;');
-            this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content='nodes', content_rowid='id');");
-            // Rebuild FTS index
+            this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');");
             this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
           }
         } catch (ftsErr) {

src/services/embedding/autoEmbedQueue.ts

@@ -16,12 +16,15 @@ export class AutoEmbedQueue {
   private readonly lastRunAt = new Map<number, number>();
   private readonly maxConcurrent = 1;
   private readonly cooldownMs = DEFAULT_COOLDOWN_MS;
-  private readonly embeddingsDisabled = process.env.DISABLE_EMBEDDINGS === 'true';
+
+  async recoverStuckNodes(): Promise<void> {
+    const stuckNodes = await nodeService.getNodes({ chunkStatus: 'not_chunked', limit: 1000 });
+    for (const node of stuckNodes) {
+      this.enqueue(node.id, { reason: 'startup_recovery' });
+    }
+  }
 
   enqueue(nodeId: number, task: Omit<AutoEmbedTask, 'nodeId'> = {}): boolean {
-    if (this.embeddingsDisabled && !task.force) {
-      return false;
-    }
     const existing = this.pendingTasks.get(nodeId);
     if (!existing) {
       this.pendingTasks.set(nodeId, { nodeId, ...task });
@@ -77,21 +80,12 @@ export class AutoEmbedQueue {
   }
 
   private async executeTask(task: AutoEmbedTask) {
-    if (this.embeddingsDisabled && !task.force) {
-      return;
-    }
     const node = await nodeService.getNodeById(task.nodeId);
     if (!node) {
       console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId);
       return;
     }
 
-    const chunkText = node.chunk?.trim();
-    if (!chunkText) {
-      console.warn('[AutoEmbedQueue] Node has no chunk content, skipping', task.nodeId);
-      return;
-    }
-
     if (!task.force && node.chunk_status === 'chunked') {
       return;
     }
@@ -117,4 +111,7 @@ declare global {
 export const autoEmbedQueue = globalThis.autoEmbedQueue ?? new AutoEmbedQueue();
 if (!globalThis.autoEmbedQueue) {
   globalThis.autoEmbedQueue = autoEmbedQueue;
+  autoEmbedQueue.recoverStuckNodes().catch(error => {
+    console.error('[AutoEmbedQueue] Startup recovery failed', error);
+  });
 }

src/services/embedding/ingestion.ts

@@ -103,10 +103,10 @@ export async function embedNodeContent(nodeId: number): Promise<EmbeddingPipelin
     };
   }
 
-  if (!node.chunk || !node.chunk.trim()) {
+  if (!node.source || !node.source.trim()) {
     results.chunk_embeddings = {
       status: 'skipped',
-      message: 'No chunk content to embed',
+      message: 'No source content to embed',
       chunks_created: 0
     };
   } else {

src/services/typescript/embed-nodes.ts

@@ -16,6 +16,7 @@ import {
 interface NodeRecord {
   id: number;
   title: string;
+  source: string | null;
   notes: string | null;
   description: string | null;
   dimensions_json: string;
@@ -58,7 +59,7 @@ export class NodeEmbedder {
     const prompt = `Analyze this content and provide 2-3 key insights or themes in a concise paragraph (max 100 words):
 
 Title: ${node.title}
-Content: ${node.notes || 'No content'}
+Source: ${node.source || node.notes || 'No source'}
 Dimensions: ${dimensionsText}
 
 Focus on the main concepts, key relationships, and practical implications.`;
@@ -106,13 +107,14 @@ Focus on the main concepts, key relationships, and practical implications.`;
     // Create base embedding text
     let embeddingText = formatEmbeddingText(
       node.title,
-      node.notes || '',
+      node.source || node.notes || '',
       dimensions,
       node.description
     );
 
-    // Add AI analysis if content exists
-    if (node.notes && node.notes.trim().length > 0) {
+    // Add AI analysis if source exists
+    const sourceText = node.source || node.notes || '';
+    if (sourceText.trim().length > 0) {
       const analysis = await this.analyzeNodeWithAI(node);
       if (analysis) {
         embeddingText += `\n\nAI Analysis: ${analysis}`;
@@ -176,8 +178,8 @@ Focus on the main concepts, key relationships, and practical implications.`;
     
     if (nodeId) {
       // Single node
-      query = `
-        SELECT n.id, n.title, n.content, n.description,
+        query = `
+        SELECT n.id, n.title, n.source, n.notes, n.description,
                COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
                         FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
                n.embedding, n.embedding_updated_at
@@ -188,7 +190,7 @@ Focus on the main concepts, key relationships, and practical implications.`;
     } else if (forceReEmbed) {
       // All nodes
       query = `
-        SELECT n.id, n.title, n.content, n.description,
+        SELECT n.id, n.title, n.source, n.notes, n.description,
                COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
                         FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
                n.embedding, n.embedding_updated_at
@@ -198,7 +200,7 @@ Focus on the main concepts, key relationships, and practical implications.`;
     } else {
       // Only nodes without embeddings
       query = `
-        SELECT n.id, n.title, n.content, n.description,
+        SELECT n.id, n.title, n.source, n.notes, n.description,
                COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
                         FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
                n.embedding, n.embedding_updated_at

src/services/typescript/embed-universal.ts

@@ -1,6 +1,6 @@
 /**
  * Universal chunking and embedding service for RA-H knowledge management system
- * Takes a node_id, reads chunk content from nodes table, chunks it, and stores in chunks table
+ * Takes a node_id, reads source content from nodes table, chunks it, and stores in chunks table
  */
 
 import OpenAI from 'openai';
@@ -14,7 +14,7 @@ import {
 interface Node {
   id: number;
   title: string;
-  chunk: string | null;
+  source: string | null;
   chunk_status?: string | null;
 }
 
@@ -157,7 +157,7 @@ export class UniversalEmbedder {
     
     // Get node data
     const stmt = this.db.prepare(`
-      SELECT id, title, chunk, chunk_status
+      SELECT id, title, source, chunk_status
       FROM nodes
       WHERE id = ?
     `);
@@ -168,8 +168,8 @@ export class UniversalEmbedder {
       throw new Error(`Node ${nodeId} not found`);
     }
     
-    if (!node.chunk || node.chunk.trim().length === 0) {
-      console.log(`Node ${nodeId} has no chunk content to process`);
+    if (!node.source || node.source.trim().length === 0) {
+      console.log(`Node ${nodeId} has no source content to process`);
       return { chunks: 0 };
     }
     
@@ -179,7 +179,7 @@ export class UniversalEmbedder {
     this.deleteExistingChunks(nodeId);
     
     // Split text into chunks
-    const chunks = await this.textSplitter.splitText(node.chunk);
+    const chunks = await this.textSplitter.splitText(node.source);
     
     if (verbose) {
       console.log(`Split into ${chunks.length} chunks`);