perf(editor): harden node search data layer with index, EXISTS sharing and keyset scan

Ports the measured DB-layer hardening from #30294 onto the node content
search, and closes two correctness gaps:

1. Sharing as EXISTS, not `workflow.id IN (subquery)`. The IN form makes
   SQLite drive from shared_workflow and sort every LIKE match in a temp
   B-tree, discarding the updatedAt index (measured on a 20k corpus in
   #30294; plan-asserted there in node-search-variants.perf.ts).

2. Index on workflow_entity.updatedAt (migration
   AddUpdatedAtIndexToWorkflowEntity1786525332822 + entity mirror). The
   query orders by updatedAt and stops at the limit; without the index
   every keystroke sorts the whole table (149ms -> 0.9ms at 20k). The
   migration was already registered in the generated migration index on
   this branch but the file itself was never committed.

3. Keyset scan instead of a fixed 500-candidate cap. The SQL pre-filter
   matches the raw JSON blob - node ids, webhook ids, positions - which
   the in-memory matcher deliberately skips, so a fixed cap let blob-only
   false positives displace real hits and silently return nothing. The
   service now batches 100 candidates on an (updatedAt, id) cursor (id
   tiebreak: updatedAt is not unique) until the result list fills,
   candidates run out, or a 2000-workflow safety rail stops pathological
   queries. The cursor is a row-value comparison, NOT the equivalent
   'a < x OR (a = x AND b < y)': the OR form makes SQLite pick a
   MULTI-INDEX OR plan that re-materialises per batch (measured 3.5s vs
   96ms per search at 20k in #30294).

   Credential names are matched too (new 'credentials' field, ranked
   last): they appear in the serialized workflow, so the pre-filter
   matches them, and without an in-memory counterpart a credential-name
   query only ever produced false positives - the most realistic trigger
   for the empty-results bug.

Also ports the Postgres planner hint (SET LOCAL enable_seqscan = off)
from #30294: Postgres cannot estimate '%...%' selectivity and otherwise
seq-scans + top-N sorts instead of walking the index (75ms -> 1ms).

Deliberately not ported from #30294 (yet): the pLimit(1) concurrency
gate + per-user rate limiter, and the performance benchmark suite.

Verified: matcher 40/40, service unit 19/19, integration 22/22 on both
SQLite and Postgres 16; typecheck and lint clean across db, cli,
workflow, api-types and editor-ui; editor-ui composable tests 24/24.
This commit is contained in:
Garrit Franke 2026-08-12 16:24:01 +02:00
parent 9474b63609
commit d24fa54996
No known key found for this signature in database
10 changed files with 314 additions and 21 deletions

View File

@ -25,7 +25,7 @@ export type NodeSearchHitProject = {
};
/** Which node field the query matched. Drives ordering and the result subtitle. */
export type NodeSearchMatchedField = 'name' | 'type' | 'notes' | 'parameters';
export type NodeSearchMatchedField = 'name' | 'type' | 'notes' | 'parameters' | 'credentials';
export type NodeSearchHit = {
workflowId: string;

View File

@ -23,6 +23,10 @@ import type { WorkflowTagMapping } from './workflow-tag-mapping';
import { objectRetriever, sqlite } from '../utils/transformers';
@Entity()
// Mirrors AddUpdatedAtIndexToWorkflowEntity1786525332822. Workflow lists and the
// command-bar node search both order by `updatedAt`; the index lets the planner
// stop at the limit instead of sorting the whole table.
@Index(['updatedAt'])
export class WorkflowEntity extends WithTimestampsAndStringId implements IWorkflowDb {
// TODO: Add XSS check
@Index({ unique: true })

View File

@ -0,0 +1,24 @@
import type { MigrationContext, ReversibleMigration } from '../migration-types';
/**
* `workflow_entity` is sorted by `updatedAt` on every workflow list page and by
* the command-bar node content search, but the column was unindexed, so both
* queries sorted the whole table.
*
* It matters most for the node search, which filters on a non-sargable
* `nodes LIKE '%...%'` and takes the top N. With the index the planner walks
* `updatedAt` in order and stops once the limit is filled instead of scanning
* and sorting every row. Measured on a 20k-workflow SQLite corpus
* (`packages/cli/test/performance/node-search-variants.perf.ts`):
* a broad query went from 149ms to 0.9ms, and the plan changed to
* `SCAN w USING INDEX`.
*/
export class AddUpdatedAtIndexToWorkflowEntity1786525332822 implements ReversibleMigration {
async up({ schemaBuilder: { createIndex } }: MigrationContext) {
await createIndex('workflow_entity', ['updatedAt']);
}
async down({ schemaBuilder: { dropIndex } }: MigrationContext) {
await dropIndex('workflow_entity', ['updatedAt'], { skipIfMissing: true });
}
}

View File

@ -11,6 +11,7 @@ import type {
FindOptionsRelations,
EntityManager,
} from '@n8n/typeorm';
import { DateUtils } from '@n8n/typeorm/util/DateUtils';
import type { INode } from 'n8n-workflow';
import { PROJECT_ROOT, UserError } from 'n8n-workflow';
@ -61,6 +62,8 @@ export type WorkflowFolderUnionFull = (
export type NodeSearchCandidate = {
id: string;
name: string;
/** Keyset cursor position for the next batch. */
updatedAt: Date;
nodes: INode[];
homeProject: SlimProject | null;
parentFolder: { id: string; name: string } | null;
@ -873,6 +876,10 @@ export class WorkflowRepository extends BaseRepository<WorkflowEntity> {
*
* Only draft `nodes` are searched: that is what the user edits and what they
* navigate back to. Published versions are deliberately out of scope.
*
* `cursor` continues a keyset scan: only workflows strictly older than
* `(cursor.updatedAt, cursor.id)` are returned, so the caller can batch
* through candidates without revisiting rows.
*/
async findNodeSearchCandidates(
user: User,
@ -884,6 +891,7 @@ export class WorkflowRepository extends BaseRepository<WorkflowEntity> {
query: string,
limit: number,
projectId?: string,
cursor?: { updatedAt: Date; id: string },
): Promise<NodeSearchCandidate[]> {
const qb = this.createQueryBuilder('workflow').select([
'workflow.id',
@ -892,12 +900,32 @@ export class WorkflowRepository extends BaseRepository<WorkflowEntity> {
'workflow.updatedAt',
]);
const sharedWorkflowSubquery = this.buildSharedWorkflowIdsSubquery(user, sharingOptions);
qb.where(`workflow.id IN (${sharedWorkflowSubquery.getQuery()})`);
// EXISTS, not `workflow.id IN (subquery)`: the IN form makes SQLite drive
// from `shared_workflow` and sort every LIKE match in a temp B-tree, which
// discards the `updatedAt` index this query relies on to stop at the limit.
const sharedWorkflowSubquery = this.buildSharedWorkflowIdsSubquery(
user,
sharingOptions,
).andWhere('sw.workflowId = workflow.id');
qb.where(`EXISTS (${sharedWorkflowSubquery.getQuery()})`);
qb.setParameters(sharedWorkflowSubquery.getParameters());
qb.andWhere('workflow.isArchived = :isArchived', { isArchived: false });
if (cursor) {
// Row-value comparison, NOT the equivalent `a < x OR (a = x AND b < y)`:
// the OR form makes SQLite pick a MULTI-INDEX OR plan that materialises
// and sorts every row older than the cursor on every batch, while the
// row value keeps the `updatedAt<?` index seek and short-circuits at
// the limit (measured ~175ms vs ~3ms per batch on a 20k corpus).
qb.andWhere('(workflow.updatedAt, workflow.id) < (:cursorUpdatedAt, :cursorId)', {
// Stringified for the raw comparison; same pattern as execution.repository.
// String() narrows DateUtils' `any` return — always a string for a Date input.
cursorUpdatedAt: String(DateUtils.mixedDateToUtcDatetimeString(cursor.updatedAt)),
cursorId: cursor.id,
});
}
// Postgres stores `nodes` as json and needs an explicit text cast for LIKE;
// sqlite already stores it as text. Mirrors `applyTriggerNodeTypesFilter`.
const nodesAsText =
@ -944,9 +972,25 @@ export class WorkflowRepository extends BaseRepository<WorkflowEntity> {
'parentFolder.name',
]);
qb.orderBy('workflow.updatedAt', 'DESC').limit(limit);
// `id DESC` tiebreak: `updatedAt` is not unique, so without it rows sharing
// a timestamp could be skipped or duplicated at batch boundaries.
qb.orderBy('workflow.updatedAt', 'DESC').addOrderBy('workflow.id', 'DESC').limit(limit);
const workflows = await qb.getMany();
// Postgres cannot estimate the selectivity of `LIKE '%…%'`, so it assumes
// almost nothing matches and picks a sequential scan with a top-N sort over
// the whole table rather than walking the `updatedAt` index and stopping at
// the limit. The planner still falls back to a sequential scan when no index
// can serve the query, so this only removes a bad choice. SET LOCAL keeps it
// scoped to this transaction so pooled connections are unaffected.
const workflows =
this.globalConfig.database.type !== 'postgresdb'
? await qb.getMany()
: await this.manager.transaction(async (tx) => {
const { queryRunner } = tx;
if (!queryRunner) return await qb.getMany();
await tx.query('SET LOCAL enable_seqscan = off');
return await qb.setQueryRunner(queryRunner).getMany();
});
return workflows.map((workflow) => {
const ownerProject = workflow.shared?.[0]?.project ?? null;
@ -954,6 +998,7 @@ export class WorkflowRepository extends BaseRepository<WorkflowEntity> {
return {
id: workflow.id,
name: workflow.name,
updatedAt: workflow.updatedAt,
nodes: workflow.nodes ?? [],
homeProject: ownerProject
? {

View File

@ -23,6 +23,7 @@ const makeNode = (overrides: Partial<INode> = {}): INode => ({
const makeCandidate = (overrides: Partial<NodeSearchCandidate> = {}): NodeSearchCandidate => ({
id: 'wf-1',
name: 'My Workflow',
updatedAt: new Date('2025-01-01T00:00:00.000Z'),
nodes: [],
homeProject: { id: 'proj-1', name: 'Acme Corp', type: 'team', icon: null },
parentFolder: null,
@ -69,6 +70,7 @@ describe('WorkflowNodeSearchService', () => {
'orders',
expect.any(Number),
undefined,
undefined,
);
});
@ -83,6 +85,7 @@ describe('WorkflowNodeSearchService', () => {
'orders',
expect.any(Number),
'proj-1',
undefined,
);
});
@ -97,6 +100,7 @@ describe('WorkflowNodeSearchService', () => {
'orders',
expect.any(Number),
undefined,
undefined,
);
});
@ -279,6 +283,79 @@ describe('WorkflowNodeSearchService', () => {
expect(hasMore).toBe(false);
});
it('matches credential names, ranked below parameter hits', async () => {
const { service } = setup([
makeCandidate({
nodes: [
makeNode({
id: 'cred',
name: 'Notify',
credentials: { slackApi: { id: 'c-1', name: 'Slack Prod Account' } },
}),
makeNode({ id: 'param', name: 'Other', parameters: { q: 'prod account' } }),
],
}),
]);
const { results } = await service.search(user, 'prod account');
expect(results.map((hit) => [hit.nodeId, hit.matchedField])).toEqual([
['param', 'parameters'],
['cred', 'credentials'],
]);
expect(results[1].snippet).toContain('Slack Prod Account');
});
it('scans past a full batch of blob-only false positives on a keyset cursor', async () => {
// A full batch where the SQL pre-filter matched but no node does (ids,
// webhook ids, positions) must not end the search — a fixed cap did.
const falsePositives = Array.from({ length: 100 }, (_, i) =>
makeCandidate({
id: `decoy-${i}`,
updatedAt: new Date(2025, 0, 2),
nodes: [makeNode({ name: 'Unrelated' })],
}),
);
const realHit = makeCandidate({
id: 'real',
updatedAt: new Date(2025, 0, 1),
nodes: [makeNode({ name: 'Fetch orders' })],
});
const { service, workflowRepository } = setup();
workflowRepository.findNodeSearchCandidates
.mockResolvedValueOnce(falsePositives)
.mockResolvedValueOnce([realHit]);
const { results } = await service.search(user, 'orders');
expect(results.map((hit) => hit.workflowId)).toEqual(['real']);
// Second batch continues from the last row of the first.
expect(workflowRepository.findNodeSearchCandidates).toHaveBeenNthCalledWith(
2,
user,
expect.anything(),
'orders',
expect.any(Number),
undefined,
{ updatedAt: falsePositives[99].updatedAt, id: 'decoy-99' },
);
});
it('stops scanning at the safety rail on pathological queries', async () => {
// Every batch full, nothing ever matches in memory: the scan must stop.
const batch = Array.from({ length: 100 }, (_, i) =>
makeCandidate({ id: `decoy-${i}`, nodes: [makeNode({ name: 'Unrelated' })] }),
);
const { service, workflowRepository } = setup(batch);
await expect(service.search(user, 'orders')).resolves.toEqual({
results: [],
hasMore: false,
});
expect(workflowRepository.findNodeSearchCandidates).toHaveBeenCalledTimes(20);
});
it('preserves recency order within a relevance tier', async () => {
// Candidates arrive newest-first from the repository.
const { service } = setup([

View File

@ -14,11 +14,21 @@ import { findNodeSearchMatch, STICKY_NODE_TYPE } from 'n8n-workflow';
import { RoleService } from '@/services/role.service';
/**
* Workflows pulled from the DB per search. The coarse SQL pre-filter matches JSON
* structure as well as content, so it over-fetches; this bounds the in-memory
* re-match on queries that hit a large share of the corpus.
* Workflows hydrated per keyset batch. The coarse SQL pre-filter matches JSON
* structure as well as content (node ids, webhook ids, positions), so a batch
* can be entirely false positives; the scan continues on an `(updatedAt, id)`
* cursor instead of giving up, so those false positives cannot silently
* displace real hits further down — a fixed candidate cap did exactly that.
*/
const CANDIDATE_WORKFLOW_LIMIT = NODE_SEARCH_MAX_RESULTS * 10;
const BATCH_SIZE = 100;
/**
* Safety rail for pathological queries where nothing matches in memory but the
* blob keeps matching: bounds hydration cost, not correctness of ranking.
* ponytail: practical-completeness bound; the real fix at ~100k+ workflows is a
* dedicated search index table maintained by the workflow-index module.
*/
const MAX_SCANNED_WORKFLOWS = 2000;
/** Relevance order for matched fields — a node-name hit beats a buried parameter hit. */
const MATCHED_FIELD_RANK: Record<NodeSearchMatchedField, number> = {
@ -26,6 +36,7 @@ const MATCHED_FIELD_RANK: Record<NodeSearchMatchedField, number> = {
type: 1,
notes: 2,
parameters: 3,
credentials: 4,
};
type ScoredHit = NodeSearchHit & { rank: number };
@ -56,19 +67,34 @@ export class WorkflowNodeSearchService {
this.roleService.rolesWithScope('workflow', scopes),
]);
const candidates = await this.workflowRepository.findNodeSearchCandidates(
user,
{ scopes, projectRoles, workflowRoles },
query,
CANDIDATE_WORKFLOW_LIMIT,
options.projectId,
);
const queryLower = query.toLowerCase();
const hits: ScoredHit[] = [];
for (const candidate of candidates) {
hits.push(...this.matchWorkflow(candidate, queryLower));
// Keyset scan, newest first. Stops as soon as enough hits exist to fill the
// result list and decide `hasMore`, so queries that match stop after a batch
// or two; only queries whose matches are blob-only false positives walk on,
// bounded by MAX_SCANNED_WORKFLOWS.
let cursor: { updatedAt: Date; id: string } | undefined;
let scanned = 0;
while (scanned < MAX_SCANNED_WORKFLOWS && hits.length <= NODE_SEARCH_MAX_RESULTS) {
const candidates = await this.workflowRepository.findNodeSearchCandidates(
user,
{ scopes, projectRoles, workflowRoles },
query,
BATCH_SIZE,
options.projectId,
cursor,
);
for (const candidate of candidates) {
hits.push(...this.matchWorkflow(candidate, queryLower));
}
scanned += candidates.length;
if (candidates.length < BATCH_SIZE) break; // corpus exhausted
const last = candidates[candidates.length - 1];
cursor = { updatedAt: last.updatedAt, id: last.id };
}
// Candidates arrive newest-first, so a stable sort by matched-field rank

View File

@ -170,6 +170,50 @@ describe('WorkflowNodeSearchService', () => {
expect(results).toEqual([]);
});
it('matches credential names', async () => {
await createWorkflow(
{
nodes: [
makeNode({
name: 'Notify',
credentials: { slackApi: { id: 'c-1', name: 'My Slack account' } },
}),
],
},
member,
);
const { results } = await service.search(member, 'slack account');
expect(results).toHaveLength(1);
expect(results[0]).toMatchObject({ nodeName: 'Notify', matchedField: 'credentials' });
});
// The DB prefilter matches the raw JSON blob, which contains fields the
// in-memory matcher deliberately skips (node ids, positions, webhookIds).
// A hit older than a full batch of such blob-only candidates must still be
// found — a fixed candidate cap silently returned nothing here.
it('finds hits beyond a full batch of blob-only false positives', async () => {
// The real hit is oldest, then 100 newer workflows that match the blob
// only via node id and fill the first keyset batch entirely.
await createWorkflow({ nodes: [makeNode({ name: 'needle hit' })] }, member);
await new Promise((resolve) => setTimeout(resolve, 10)); // decoys must sort newer
await Promise.all(
Array.from(
{ length: 100 },
async (_, i) =>
await createWorkflow(
{ nodes: [makeNode({ id: `needle-${i}`, name: `Decoy ${i}` })] },
member,
),
),
);
const { results } = await service.search(member, 'needle');
expect(results.map((hit) => hit.nodeName)).toEqual(['needle hit']);
});
it('is case insensitive', async () => {
await createWorkflow({ nodes: [makeNode({ name: 'Fetch ORDERS' })] }, member);
@ -271,6 +315,43 @@ describe('WorkflowNodeSearchService', () => {
expect(candidates.map((c) => c.id)).toEqual([newer.id, older.id]);
});
it('continues after the cursor without skipping or repeating rows', async () => {
const workflows = [];
for (let i = 0; i < 3; i++) {
workflows.push(
await createWorkflow({ nodes: [makeNode({ name: `orders ${i}` })] }, member),
);
}
// Same timestamp on every row: the id tiebreak alone must page correctly.
const sharedTime = new Date(Date.now() + 60_000);
for (const workflow of workflows) {
await workflowRepository.update(workflow.id, { updatedAt: sharedTime });
}
const sharingOptions = await readSharingOptions();
const first = await workflowRepository.findNodeSearchCandidates(
member,
sharingOptions,
'orders',
2,
);
expect(first).toHaveLength(2);
const last = first[first.length - 1];
const second = await workflowRepository.findNodeSearchCandidates(
member,
sharingOptions,
'orders',
2,
undefined,
{ updatedAt: last.updatedAt, id: last.id },
);
const seen = [...first, ...second].map((candidate) => candidate.id);
expect(seen).toHaveLength(3);
expect(new Set(seen).size).toBe(3);
});
it('respects the candidate limit', async () => {
await createWorkflow({ nodes: [makeNode({ name: 'orders a' })] }, member);
await createWorkflow({ nodes: [makeNode({ name: 'orders b' })] }, member);

View File

@ -34,6 +34,7 @@ const MATCHED_FIELD_RANK: Record<NodeSearchMatchedField, number> = {
type: 1,
notes: 2,
parameters: 3,
credentials: 4,
};
const SNIPPET_SUFFIX_MAX_LENGTH = 40;
@ -160,7 +161,10 @@ export function useGlobalNodeSearchCommands(options: {
// Nodes is the section; project is the subsection. Keep type + workflow on the row.
const showSnippet =
(hit.matchedField === 'notes' || hit.matchedField === 'parameters') && !!hit.snippet;
(hit.matchedField === 'notes' ||
hit.matchedField === 'parameters' ||
hit.matchedField === 'credentials') &&
!!hit.snippet;
const suffix = [showSnippet ? formatSnippetForSuffix(hit.snippet) : typeLabel, hit.workflowName]
.filter(Boolean)
.join(' · ');

View File

@ -16,7 +16,7 @@ export const NODE_SEARCH_TEXT_MAX_LENGTH = 2_000;
/** Individual values longer than this are skipped — almost always encoded blobs. */
const NODE_SEARCH_VALUE_MAX_LENGTH = 512;
export type NodeSearchField = 'name' | 'type' | 'notes' | 'parameters';
export type NodeSearchField = 'name' | 'type' | 'notes' | 'parameters' | 'credentials';
export type NodeSearchMatch = {
field: NodeSearchField;
@ -130,5 +130,15 @@ export function findNodeSearchMatch(node: INode, queryLower: string): NodeSearch
}
}
// Credential names appear in the serialized workflow, so the SQL pre-filter
// matches them; without this, a credential-name query ("Slack Prod Account")
// only ever produces false positives. Names only — the type keys (`slackApi`)
// would make node-type-ish queries double-match.
for (const credential of Object.values(node.credentials ?? {})) {
if (credential.name?.toLowerCase().includes(queryLower)) {
return { field: 'credentials', snippet: buildNodeSearchSnippet(credential.name, queryLower) };
}
}
return null;
}

View File

@ -143,6 +143,28 @@ describe('findNodeSearchMatch', () => {
expect(findNodeSearchMatch(node, 'url')).toBeNull();
});
it('matches credential names after every other field', () => {
const node = makeNode({
name: 'Notify',
credentials: { slackApi: { id: 'c-1', name: 'Slack Prod Account' } },
});
expect(findNodeSearchMatch(node, 'prod account')).toEqual({
field: 'credentials',
snippet: 'Slack Prod Account',
});
});
it('does not match credential type keys', () => {
const node = makeNode({
name: 'Notify',
type: 'n8n-nodes-base.noOp',
credentials: { slackApi: { id: 'c-1', name: 'Team credential' } },
});
expect(findNodeSearchMatch(node, 'slackapi')).toBeNull();
});
it('is case insensitive against the lowercased query', () => {
const node = makeNode({ parameters: { channel: '#Deploys' } });
expect(findNodeSearchMatch(node, 'deploys')?.field).toBe('parameters');