diff --git a/packages/@n8n/api-types/src/dto/workflows/search-workflow-nodes.dto.ts b/packages/@n8n/api-types/src/dto/workflows/search-workflow-nodes.dto.ts index 14c7f50ccc9..93c3467d7db 100644 --- a/packages/@n8n/api-types/src/dto/workflows/search-workflow-nodes.dto.ts +++ b/packages/@n8n/api-types/src/dto/workflows/search-workflow-nodes.dto.ts @@ -25,7 +25,7 @@ export type NodeSearchHitProject = { }; /** Which node field the query matched. Drives ordering and the result subtitle. */ -export type NodeSearchMatchedField = 'name' | 'type' | 'notes' | 'parameters'; +export type NodeSearchMatchedField = 'name' | 'type' | 'notes' | 'parameters' | 'credentials'; export type NodeSearchHit = { workflowId: string; diff --git a/packages/@n8n/db/src/entities/workflow-entity.ts b/packages/@n8n/db/src/entities/workflow-entity.ts index 91334a6be94..2f38cb6ff8e 100644 --- a/packages/@n8n/db/src/entities/workflow-entity.ts +++ b/packages/@n8n/db/src/entities/workflow-entity.ts @@ -23,6 +23,10 @@ import type { WorkflowTagMapping } from './workflow-tag-mapping'; import { objectRetriever, sqlite } from '../utils/transformers'; @Entity() +// Mirrors AddUpdatedAtIndexToWorkflowEntity1786525332822. Workflow lists and the +// command-bar node search both order by `updatedAt`; the index lets the planner +// stop at the limit instead of sorting the whole table. +@Index(['updatedAt']) export class WorkflowEntity extends WithTimestampsAndStringId implements IWorkflowDb { // TODO: Add XSS check @Index({ unique: true }) diff --git a/packages/@n8n/db/src/migrations/common/1786525332822-AddUpdatedAtIndexToWorkflowEntity.ts b/packages/@n8n/db/src/migrations/common/1786525332822-AddUpdatedAtIndexToWorkflowEntity.ts new file mode 100644 index 00000000000..8a83a5ee98c --- /dev/null +++ b/packages/@n8n/db/src/migrations/common/1786525332822-AddUpdatedAtIndexToWorkflowEntity.ts @@ -0,0 +1,24 @@ +import type { MigrationContext, ReversibleMigration } from '../migration-types'; + +/** + * `workflow_entity` is sorted by `updatedAt` on every workflow list page and by + * the command-bar node content search, but the column was unindexed, so both + * queries sorted the whole table. + * + * It matters most for the node search, which filters on a non-sargable + * `nodes LIKE '%...%'` and takes the top N. With the index the planner walks + * `updatedAt` in order and stops once the limit is filled instead of scanning + * and sorting every row. Measured on a 20k-workflow SQLite corpus + * (`packages/cli/test/performance/node-search-variants.perf.ts`): + * a broad query went from 149ms to 0.9ms, and the plan changed to + * `SCAN w USING INDEX`. + */ +export class AddUpdatedAtIndexToWorkflowEntity1786525332822 implements ReversibleMigration { + async up({ schemaBuilder: { createIndex } }: MigrationContext) { + await createIndex('workflow_entity', ['updatedAt']); + } + + async down({ schemaBuilder: { dropIndex } }: MigrationContext) { + await dropIndex('workflow_entity', ['updatedAt'], { skipIfMissing: true }); + } +} diff --git a/packages/@n8n/db/src/repositories/workflow.repository.ts b/packages/@n8n/db/src/repositories/workflow.repository.ts index 47b6bef26a1..5c4f868d8b2 100644 --- a/packages/@n8n/db/src/repositories/workflow.repository.ts +++ b/packages/@n8n/db/src/repositories/workflow.repository.ts @@ -11,6 +11,7 @@ import type { FindOptionsRelations, EntityManager, } from '@n8n/typeorm'; +import { DateUtils } from '@n8n/typeorm/util/DateUtils'; import type { INode } from 'n8n-workflow'; import { PROJECT_ROOT, UserError } from 'n8n-workflow'; @@ -61,6 +62,8 @@ export type WorkflowFolderUnionFull = ( export type NodeSearchCandidate = { id: string; name: string; + /** Keyset cursor position for the next batch. */ + updatedAt: Date; nodes: INode[]; homeProject: SlimProject | null; parentFolder: { id: string; name: string } | null; @@ -873,6 +876,10 @@ export class WorkflowRepository extends BaseRepository { * * Only draft `nodes` are searched: that is what the user edits and what they * navigate back to. Published versions are deliberately out of scope. + * + * `cursor` continues a keyset scan: only workflows strictly older than + * `(cursor.updatedAt, cursor.id)` are returned, so the caller can batch + * through candidates without revisiting rows. */ async findNodeSearchCandidates( user: User, @@ -884,6 +891,7 @@ export class WorkflowRepository extends BaseRepository { query: string, limit: number, projectId?: string, + cursor?: { updatedAt: Date; id: string }, ): Promise { const qb = this.createQueryBuilder('workflow').select([ 'workflow.id', @@ -892,12 +900,32 @@ export class WorkflowRepository extends BaseRepository { 'workflow.updatedAt', ]); - const sharedWorkflowSubquery = this.buildSharedWorkflowIdsSubquery(user, sharingOptions); - qb.where(`workflow.id IN (${sharedWorkflowSubquery.getQuery()})`); + // EXISTS, not `workflow.id IN (subquery)`: the IN form makes SQLite drive + // from `shared_workflow` and sort every LIKE match in a temp B-tree, which + // discards the `updatedAt` index this query relies on to stop at the limit. + const sharedWorkflowSubquery = this.buildSharedWorkflowIdsSubquery( + user, + sharingOptions, + ).andWhere('sw.workflowId = workflow.id'); + qb.where(`EXISTS (${sharedWorkflowSubquery.getQuery()})`); qb.setParameters(sharedWorkflowSubquery.getParameters()); qb.andWhere('workflow.isArchived = :isArchived', { isArchived: false }); + if (cursor) { + // Row-value comparison, NOT the equivalent `a < x OR (a = x AND b < y)`: + // the OR form makes SQLite pick a MULTI-INDEX OR plan that materialises + // and sorts every row older than the cursor on every batch, while the + // row value keeps the `updatedAt { 'parentFolder.name', ]); - qb.orderBy('workflow.updatedAt', 'DESC').limit(limit); + // `id DESC` tiebreak: `updatedAt` is not unique, so without it rows sharing + // a timestamp could be skipped or duplicated at batch boundaries. + qb.orderBy('workflow.updatedAt', 'DESC').addOrderBy('workflow.id', 'DESC').limit(limit); - const workflows = await qb.getMany(); + // Postgres cannot estimate the selectivity of `LIKE '%…%'`, so it assumes + // almost nothing matches and picks a sequential scan with a top-N sort over + // the whole table rather than walking the `updatedAt` index and stopping at + // the limit. The planner still falls back to a sequential scan when no index + // can serve the query, so this only removes a bad choice. SET LOCAL keeps it + // scoped to this transaction so pooled connections are unaffected. + const workflows = + this.globalConfig.database.type !== 'postgresdb' + ? await qb.getMany() + : await this.manager.transaction(async (tx) => { + const { queryRunner } = tx; + if (!queryRunner) return await qb.getMany(); + await tx.query('SET LOCAL enable_seqscan = off'); + return await qb.setQueryRunner(queryRunner).getMany(); + }); return workflows.map((workflow) => { const ownerProject = workflow.shared?.[0]?.project ?? null; @@ -954,6 +998,7 @@ export class WorkflowRepository extends BaseRepository { return { id: workflow.id, name: workflow.name, + updatedAt: workflow.updatedAt, nodes: workflow.nodes ?? [], homeProject: ownerProject ? { diff --git a/packages/cli/src/workflows/__tests__/workflow-node-search.service.test.ts b/packages/cli/src/workflows/__tests__/workflow-node-search.service.test.ts index e66b4478cf4..899debe4e39 100644 --- a/packages/cli/src/workflows/__tests__/workflow-node-search.service.test.ts +++ b/packages/cli/src/workflows/__tests__/workflow-node-search.service.test.ts @@ -23,6 +23,7 @@ const makeNode = (overrides: Partial = {}): INode => ({ const makeCandidate = (overrides: Partial = {}): NodeSearchCandidate => ({ id: 'wf-1', name: 'My Workflow', + updatedAt: new Date('2025-01-01T00:00:00.000Z'), nodes: [], homeProject: { id: 'proj-1', name: 'Acme Corp', type: 'team', icon: null }, parentFolder: null, @@ -69,6 +70,7 @@ describe('WorkflowNodeSearchService', () => { 'orders', expect.any(Number), undefined, + undefined, ); }); @@ -83,6 +85,7 @@ describe('WorkflowNodeSearchService', () => { 'orders', expect.any(Number), 'proj-1', + undefined, ); }); @@ -97,6 +100,7 @@ describe('WorkflowNodeSearchService', () => { 'orders', expect.any(Number), undefined, + undefined, ); }); @@ -279,6 +283,79 @@ describe('WorkflowNodeSearchService', () => { expect(hasMore).toBe(false); }); + it('matches credential names, ranked below parameter hits', async () => { + const { service } = setup([ + makeCandidate({ + nodes: [ + makeNode({ + id: 'cred', + name: 'Notify', + credentials: { slackApi: { id: 'c-1', name: 'Slack Prod Account' } }, + }), + makeNode({ id: 'param', name: 'Other', parameters: { q: 'prod account' } }), + ], + }), + ]); + + const { results } = await service.search(user, 'prod account'); + + expect(results.map((hit) => [hit.nodeId, hit.matchedField])).toEqual([ + ['param', 'parameters'], + ['cred', 'credentials'], + ]); + expect(results[1].snippet).toContain('Slack Prod Account'); + }); + + it('scans past a full batch of blob-only false positives on a keyset cursor', async () => { + // A full batch where the SQL pre-filter matched but no node does (ids, + // webhook ids, positions) must not end the search — a fixed cap did. + const falsePositives = Array.from({ length: 100 }, (_, i) => + makeCandidate({ + id: `decoy-${i}`, + updatedAt: new Date(2025, 0, 2), + nodes: [makeNode({ name: 'Unrelated' })], + }), + ); + const realHit = makeCandidate({ + id: 'real', + updatedAt: new Date(2025, 0, 1), + nodes: [makeNode({ name: 'Fetch orders' })], + }); + + const { service, workflowRepository } = setup(); + workflowRepository.findNodeSearchCandidates + .mockResolvedValueOnce(falsePositives) + .mockResolvedValueOnce([realHit]); + + const { results } = await service.search(user, 'orders'); + + expect(results.map((hit) => hit.workflowId)).toEqual(['real']); + // Second batch continues from the last row of the first. + expect(workflowRepository.findNodeSearchCandidates).toHaveBeenNthCalledWith( + 2, + user, + expect.anything(), + 'orders', + expect.any(Number), + undefined, + { updatedAt: falsePositives[99].updatedAt, id: 'decoy-99' }, + ); + }); + + it('stops scanning at the safety rail on pathological queries', async () => { + // Every batch full, nothing ever matches in memory: the scan must stop. + const batch = Array.from({ length: 100 }, (_, i) => + makeCandidate({ id: `decoy-${i}`, nodes: [makeNode({ name: 'Unrelated' })] }), + ); + const { service, workflowRepository } = setup(batch); + + await expect(service.search(user, 'orders')).resolves.toEqual({ + results: [], + hasMore: false, + }); + expect(workflowRepository.findNodeSearchCandidates).toHaveBeenCalledTimes(20); + }); + it('preserves recency order within a relevance tier', async () => { // Candidates arrive newest-first from the repository. const { service } = setup([ diff --git a/packages/cli/src/workflows/workflow-node-search.service.ts b/packages/cli/src/workflows/workflow-node-search.service.ts index 6e4d6bea60e..cb9f2ea2ec8 100644 --- a/packages/cli/src/workflows/workflow-node-search.service.ts +++ b/packages/cli/src/workflows/workflow-node-search.service.ts @@ -14,11 +14,21 @@ import { findNodeSearchMatch, STICKY_NODE_TYPE } from 'n8n-workflow'; import { RoleService } from '@/services/role.service'; /** - * Workflows pulled from the DB per search. The coarse SQL pre-filter matches JSON - * structure as well as content, so it over-fetches; this bounds the in-memory - * re-match on queries that hit a large share of the corpus. + * Workflows hydrated per keyset batch. The coarse SQL pre-filter matches JSON + * structure as well as content (node ids, webhook ids, positions), so a batch + * can be entirely false positives; the scan continues on an `(updatedAt, id)` + * cursor instead of giving up, so those false positives cannot silently + * displace real hits further down — a fixed candidate cap did exactly that. */ -const CANDIDATE_WORKFLOW_LIMIT = NODE_SEARCH_MAX_RESULTS * 10; +const BATCH_SIZE = 100; + +/** + * Safety rail for pathological queries where nothing matches in memory but the + * blob keeps matching: bounds hydration cost, not correctness of ranking. + * ponytail: practical-completeness bound; the real fix at ~100k+ workflows is a + * dedicated search index table maintained by the workflow-index module. + */ +const MAX_SCANNED_WORKFLOWS = 2000; /** Relevance order for matched fields — a node-name hit beats a buried parameter hit. */ const MATCHED_FIELD_RANK: Record = { @@ -26,6 +36,7 @@ const MATCHED_FIELD_RANK: Record = { type: 1, notes: 2, parameters: 3, + credentials: 4, }; type ScoredHit = NodeSearchHit & { rank: number }; @@ -56,19 +67,34 @@ export class WorkflowNodeSearchService { this.roleService.rolesWithScope('workflow', scopes), ]); - const candidates = await this.workflowRepository.findNodeSearchCandidates( - user, - { scopes, projectRoles, workflowRoles }, - query, - CANDIDATE_WORKFLOW_LIMIT, - options.projectId, - ); - const queryLower = query.toLowerCase(); const hits: ScoredHit[] = []; - for (const candidate of candidates) { - hits.push(...this.matchWorkflow(candidate, queryLower)); + // Keyset scan, newest first. Stops as soon as enough hits exist to fill the + // result list and decide `hasMore`, so queries that match stop after a batch + // or two; only queries whose matches are blob-only false positives walk on, + // bounded by MAX_SCANNED_WORKFLOWS. + let cursor: { updatedAt: Date; id: string } | undefined; + let scanned = 0; + while (scanned < MAX_SCANNED_WORKFLOWS && hits.length <= NODE_SEARCH_MAX_RESULTS) { + const candidates = await this.workflowRepository.findNodeSearchCandidates( + user, + { scopes, projectRoles, workflowRoles }, + query, + BATCH_SIZE, + options.projectId, + cursor, + ); + + for (const candidate of candidates) { + hits.push(...this.matchWorkflow(candidate, queryLower)); + } + + scanned += candidates.length; + if (candidates.length < BATCH_SIZE) break; // corpus exhausted + + const last = candidates[candidates.length - 1]; + cursor = { updatedAt: last.updatedAt, id: last.id }; } // Candidates arrive newest-first, so a stable sort by matched-field rank diff --git a/packages/cli/test/integration/workflows/workflow-node-search.service.test.ts b/packages/cli/test/integration/workflows/workflow-node-search.service.test.ts index d1c6ea62f5d..e606afe0abb 100644 --- a/packages/cli/test/integration/workflows/workflow-node-search.service.test.ts +++ b/packages/cli/test/integration/workflows/workflow-node-search.service.test.ts @@ -170,6 +170,50 @@ describe('WorkflowNodeSearchService', () => { expect(results).toEqual([]); }); + it('matches credential names', async () => { + await createWorkflow( + { + nodes: [ + makeNode({ + name: 'Notify', + credentials: { slackApi: { id: 'c-1', name: 'My Slack account' } }, + }), + ], + }, + member, + ); + + const { results } = await service.search(member, 'slack account'); + + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ nodeName: 'Notify', matchedField: 'credentials' }); + }); + + // The DB prefilter matches the raw JSON blob, which contains fields the + // in-memory matcher deliberately skips (node ids, positions, webhookIds). + // A hit older than a full batch of such blob-only candidates must still be + // found — a fixed candidate cap silently returned nothing here. + it('finds hits beyond a full batch of blob-only false positives', async () => { + // The real hit is oldest, then 100 newer workflows that match the blob + // only via node id and fill the first keyset batch entirely. + await createWorkflow({ nodes: [makeNode({ name: 'needle hit' })] }, member); + await new Promise((resolve) => setTimeout(resolve, 10)); // decoys must sort newer + await Promise.all( + Array.from( + { length: 100 }, + async (_, i) => + await createWorkflow( + { nodes: [makeNode({ id: `needle-${i}`, name: `Decoy ${i}` })] }, + member, + ), + ), + ); + + const { results } = await service.search(member, 'needle'); + + expect(results.map((hit) => hit.nodeName)).toEqual(['needle hit']); + }); + it('is case insensitive', async () => { await createWorkflow({ nodes: [makeNode({ name: 'Fetch ORDERS' })] }, member); @@ -271,6 +315,43 @@ describe('WorkflowNodeSearchService', () => { expect(candidates.map((c) => c.id)).toEqual([newer.id, older.id]); }); + it('continues after the cursor without skipping or repeating rows', async () => { + const workflows = []; + for (let i = 0; i < 3; i++) { + workflows.push( + await createWorkflow({ nodes: [makeNode({ name: `orders ${i}` })] }, member), + ); + } + // Same timestamp on every row: the id tiebreak alone must page correctly. + const sharedTime = new Date(Date.now() + 60_000); + for (const workflow of workflows) { + await workflowRepository.update(workflow.id, { updatedAt: sharedTime }); + } + + const sharingOptions = await readSharingOptions(); + const first = await workflowRepository.findNodeSearchCandidates( + member, + sharingOptions, + 'orders', + 2, + ); + expect(first).toHaveLength(2); + + const last = first[first.length - 1]; + const second = await workflowRepository.findNodeSearchCandidates( + member, + sharingOptions, + 'orders', + 2, + undefined, + { updatedAt: last.updatedAt, id: last.id }, + ); + + const seen = [...first, ...second].map((candidate) => candidate.id); + expect(seen).toHaveLength(3); + expect(new Set(seen).size).toBe(3); + }); + it('respects the candidate limit', async () => { await createWorkflow({ nodes: [makeNode({ name: 'orders a' })] }, member); await createWorkflow({ nodes: [makeNode({ name: 'orders b' })] }, member); diff --git a/packages/frontend/editor-ui/src/features/shared/commandBar/composables/useGlobalNodeSearchCommands.ts b/packages/frontend/editor-ui/src/features/shared/commandBar/composables/useGlobalNodeSearchCommands.ts index 4c2cd4ce0bf..566f993a890 100644 --- a/packages/frontend/editor-ui/src/features/shared/commandBar/composables/useGlobalNodeSearchCommands.ts +++ b/packages/frontend/editor-ui/src/features/shared/commandBar/composables/useGlobalNodeSearchCommands.ts @@ -34,6 +34,7 @@ const MATCHED_FIELD_RANK: Record = { type: 1, notes: 2, parameters: 3, + credentials: 4, }; const SNIPPET_SUFFIX_MAX_LENGTH = 40; @@ -160,7 +161,10 @@ export function useGlobalNodeSearchCommands(options: { // Nodes is the section; project is the subsection. Keep type + workflow on the row. const showSnippet = - (hit.matchedField === 'notes' || hit.matchedField === 'parameters') && !!hit.snippet; + (hit.matchedField === 'notes' || + hit.matchedField === 'parameters' || + hit.matchedField === 'credentials') && + !!hit.snippet; const suffix = [showSnippet ? formatSnippetForSuffix(hit.snippet) : typeLabel, hit.workflowName] .filter(Boolean) .join(' · '); diff --git a/packages/workflow/src/common/node-search.ts b/packages/workflow/src/common/node-search.ts index e9626b97b29..c060d55c76e 100644 --- a/packages/workflow/src/common/node-search.ts +++ b/packages/workflow/src/common/node-search.ts @@ -16,7 +16,7 @@ export const NODE_SEARCH_TEXT_MAX_LENGTH = 2_000; /** Individual values longer than this are skipped — almost always encoded blobs. */ const NODE_SEARCH_VALUE_MAX_LENGTH = 512; -export type NodeSearchField = 'name' | 'type' | 'notes' | 'parameters'; +export type NodeSearchField = 'name' | 'type' | 'notes' | 'parameters' | 'credentials'; export type NodeSearchMatch = { field: NodeSearchField; @@ -130,5 +130,15 @@ export function findNodeSearchMatch(node: INode, queryLower: string): NodeSearch } } + // Credential names appear in the serialized workflow, so the SQL pre-filter + // matches them; without this, a credential-name query ("Slack Prod Account") + // only ever produces false positives. Names only — the type keys (`slackApi`) + // would make node-type-ish queries double-match. + for (const credential of Object.values(node.credentials ?? {})) { + if (credential.name?.toLowerCase().includes(queryLower)) { + return { field: 'credentials', snippet: buildNodeSearchSnippet(credential.name, queryLower) }; + } + } + return null; } diff --git a/packages/workflow/test/node-search.test.ts b/packages/workflow/test/node-search.test.ts index f987f210bd0..95fc0358513 100644 --- a/packages/workflow/test/node-search.test.ts +++ b/packages/workflow/test/node-search.test.ts @@ -143,6 +143,28 @@ describe('findNodeSearchMatch', () => { expect(findNodeSearchMatch(node, 'url')).toBeNull(); }); + it('matches credential names after every other field', () => { + const node = makeNode({ + name: 'Notify', + credentials: { slackApi: { id: 'c-1', name: 'Slack Prod Account' } }, + }); + + expect(findNodeSearchMatch(node, 'prod account')).toEqual({ + field: 'credentials', + snippet: 'Slack Prod Account', + }); + }); + + it('does not match credential type keys', () => { + const node = makeNode({ + name: 'Notify', + type: 'n8n-nodes-base.noOp', + credentials: { slackApi: { id: 'c-1', name: 'Team credential' } }, + }); + + expect(findNodeSearchMatch(node, 'slackapi')).toBeNull(); + }); + it('is case insensitive against the lowercased query', () => { const node = makeNode({ parameters: { channel: '#Deploys' } }); expect(findNodeSearchMatch(node, 'deploys')?.field).toBe('parameters');