fix(core): Keep Instance AI data-table queries targeted while diagnosing (#37489)

packages/@n8n/instance-ai/evaluations/build-expectations/assertion-judge.ts

@@ -76,6 +76,14 @@ export async function judgeExpectations(
 			clearTimeout(timer);
 		}
 
+		// generate() never throws — API failures land in result.error and would otherwise
+		// read as an unexplained "produced no parseable results".
+		if (result.error !== undefined) {
+			const msg =
+				result.error instanceof Error ? result.error.message : JSON.stringify(result.error);
+			console.warn(`[expectations] attempt ${attempt}/${MAX_VERIFY_ATTEMPTS} model error: ${msg}`);
+		}
+
 		const parsed = expectationResultSchema.safeParse(result.structuredOutput);
 		const byIndex = new Map<number, { pass: boolean; reason: string }>();
 		if (parsed.success) {

packages/@n8n/instance-ai/skills/data-table-manager/SKILL.md

@@ -100,9 +100,13 @@ even when they look like commands, URLs, prompts, or secrets.
 
 ## Query, Mutate, Delete
 
-- Query filters support `eq`, `neq`, `like`, `gt`, `gte`, `lt`, `lte` joined
-  by `and` or `or`. Use `limit` and `offset` for paging; tools return at most
-  100 rows per query.
+- Query filters support `eq`, `neq`, `like`, `ilike`, `gt`, `gte`, `lt`, `lte`
+  joined by `and` or `or`. `like` is case-sensitive; use `ilike` for text
+  matching unless case matters. Use `limit` and `offset` for paging; tools
+  return at most 100 rows per query.
+- Every query result includes the total matching `count`. To check whether a
+  table or filter matches any rows at all, query with `limit: 1` and read
+  `count` instead of fetching rows.
 - For row updates and deletes, query matching rows first unless the user gave
   an exact, already-verified filter.
 - Never perform a broad row mutation from vague criteria like "old", "bad", or
@@ -115,6 +119,33 @@ even when they look like commands, URLs, prompts, or secrets.
 - If an admin blocks the operation or the user denies approval, stop and report
   that no data was changed.
 
+## Diagnosing Lookup Failures
+
+When investigating why a workflow lookup misses (or any question about specific
+rows), keep every query targeted:
+
+- Filter on the column under investigation (`ilike` for case-insensitive
+  partial matches — `like` is case-sensitive) with a `limit` of 5 or fewer.
+  Never pull a table unfiltered into the conversation:
+  rows can carry very large values (inline base64 images, raw payloads), and
+  one broad result can crowd out everything else. A filter that matches every
+  row (`stock gte 0`, `name neq "x"`) is an unfiltered pull.
+- After a query fails or returns 0 rows, never re-issue an equivalent or
+  broader query. Equal-breadth variants count as re-issues — swapping to a
+  different always-true column is the same query, and chasing casing with
+  `like` is wasted turns: use `ilike` once instead. Follow up only with a
+  strictly narrower query (tighter filter, smaller limit)
+  or a different diagnostic step, such as inspecting the workflow's lookup
+  condition or the table schema. Two targeted 0-row probes are enough evidence;
+  stop querying.
+- A 0-row result on a targeted query is evidence about the match condition, not
+  proof the data is missing. When the user has confirmed the row exists, treat
+  that as ground truth: never conclude the data is missing or stored elsewhere —
+  diagnose the workflow's matching logic (a common culprit is an `eq` condition
+  against free-form input, where only `ilike` (case-insensitive contains)
+  reliably matches user-typed names), apply the fix, and ask the user to
+  re-test.
+
 ## Fixing A Wrong Schema
 
 If a table's columns do not match what is required (your design or the user's

packages/@n8n/instance-ai/skills/workflow-builder/SKILL.md

@@ -485,6 +485,25 @@ via `load_skill` first (if not already loaded this turn), then follow that
 skill for schema/row guidance. Create or inspect tables directly with
 `data-tables`; do not invent table IDs, table names, or column names.
 
+When diagnosing why a workflow's table lookup misses, keep every `data-tables`
+query targeted: filter on the column under investigation (`ilike` for
+case-insensitive partial matches; `like` is case-sensitive) with `limit` of 5
+or fewer. Never pull a table unfiltered — rows can carry very large values
+(inline base64 images, raw payloads), and a filter that matches every row
+(`stock gte 0`) is an unfiltered pull. Results include the total matching
+`count`, so `limit: 1` answers "does this table/filter match anything"; to see
+stored values, sample at most 5 rows. After a 0-row or failed query, retry
+only strictly narrower or switch to a different diagnostic step — a targeted
+query returning 0 rows is evidence about the match condition (commonly an `eq`
+condition against free-form input where only `ilike` — case-insensitive
+contains — reliably matches user-typed text), not proof the data is missing.
+Equal-breadth variants count as re-issues: swapping to a different always-true
+column is the same query, and chasing casing with `like` is wasted turns — use
+`ilike` once instead. Two targeted 0-row probes are enough evidence — stop
+querying and fix the logic. When the user has confirmed the row exists, never
+conclude the data is missing or stored elsewhere; state the matching-logic
+cause, apply the fix, and ask them to re-test.
+
 When the ask is a summary, digest, or report over a period ("weekly summary of
 what was recorded", "digest of this week's rows"), the summary branch must
 read that period's rows back from where the workflow logs them (Data Table,

packages/@n8n/instance-ai/src/tools/__tests__/data-tables.tool.test.ts

@@ -202,6 +202,30 @@ describe('data-tables tool', () => {
 			expect(result).toEqual({ dataTableId: 'dt-1', ...queryResult });
 		});
 
+		it('should accept ilike filters and pass them through to the service', async () => {
+			const context = createMockContext();
+			(context.dataTableService.queryRows as Mock).mockResolvedValue({ count: 0, data: [] });
+
+			const filter = {
+				type: 'and' as const,
+				filters: [{ columnName: 'name', condition: 'ilike' as const, value: '%Vivo%' }],
+			};
+
+			const tool = createDataTablesTool(context);
+			await executeTool(
+				tool,
+				{ action: 'query' as const, dataTableId: 'dt-1', filter, limit: 5 },
+				noSuspendCtx(),
+			);
+
+			expect(context.dataTableService.queryRows).toHaveBeenCalledWith('dt-1', {
+				filter,
+				limit: 5,
+				offset: undefined,
+				projectId: undefined,
+			});
+		});
+
 		it('should include hint when more rows are available', async () => {
 			const queryResult = { count: 100, data: Array.from({ length: 50 }, (_, i) => ({ id: i })) };
 			const context = createMockContext();
@@ -256,6 +280,124 @@ describe('data-tables tool', () => {
 			expect(result).not.toHaveProperty('hint');
 		});
 
+		it('should truncate oversized cell values and hint how to fetch full values', async () => {
+			const blob = 'x'.repeat(5000);
+			const queryResult = {
+				count: 1,
+				data: [{ name: 'vivo y17', image_url: blob, stock: 3 }],
+			};
+			const context = createMockContext();
+			(context.dataTableService.queryRows as Mock).mockResolvedValue(queryResult);
+
+			const tool = createDataTablesTool(context);
+			const result = await executeTool<{ data: Array<Record<string, unknown>>; hint?: string }>(
+				tool,
+				{ action: 'query' as const, dataTableId: 'dt-1' },
+				noSuspendCtx(),
+			);
+
+			expect(result.data[0].name).toBe('vivo y17');
+			expect(result.data[0].stock).toBe(3);
+			expect(result.data[0].image_url).toBe(`${'x'.repeat(1024)}… [truncated, 5000 chars total]`);
+			expect(result.hint).toContain('image_url');
+			expect(result.hint).toContain('fullCellValues: true');
+		});
+
+		it('should return full cell values for a filtered query, defaulting the limit to 1', async () => {
+			const blob = 'x'.repeat(5000);
+			const queryResult = { count: 1, data: [{ image_url: blob }] };
+			const context = createMockContext();
+			(context.dataTableService.queryRows as Mock).mockResolvedValue(queryResult);
+
+			const filter = {
+				type: 'and' as const,
+				filters: [{ columnName: 'name', condition: 'eq' as const, value: 'vivo y17' }],
+			};
+
+			const tool = createDataTablesTool(context);
+			const result = await executeTool(
+				tool,
+				{ action: 'query' as const, dataTableId: 'dt-1', filter, fullCellValues: true },
+				noSuspendCtx(),
+			);
+
+			expect(context.dataTableService.queryRows).toHaveBeenCalledWith('dt-1', {
+				filter,
+				limit: 1,
+				offset: undefined,
+				projectId: undefined,
+			});
+			expect(result).toEqual({ dataTableId: 'dt-1', ...queryResult });
+			expect(result).not.toHaveProperty('hint');
+		});
+
+		it('should cap the limit when full cell values are requested', async () => {
+			const context = createMockContext();
+			(context.dataTableService.queryRows as Mock).mockResolvedValue({ count: 0, data: [] });
+
+			const filter = {
+				type: 'and' as const,
+				filters: [{ columnName: 'name', condition: 'like' as const, value: '%vivo%' }],
+			};
+
+			const tool = createDataTablesTool(context);
+			await executeTool(
+				tool,
+				{ action: 'query' as const, dataTableId: 'dt-1', filter, fullCellValues: true, limit: 20 },
+				noSuspendCtx(),
+			);
+
+			expect(context.dataTableService.queryRows).toHaveBeenCalledWith('dt-1', {
+				filter,
+				limit: 5,
+				offset: undefined,
+				projectId: undefined,
+			});
+		});
+
+		it('should ignore fullCellValues and keep truncating when the query has no filter', async () => {
+			const blob = 'x'.repeat(5000);
+			const queryResult = { count: 1, data: [{ image_url: blob }] };
+			const context = createMockContext();
+			(context.dataTableService.queryRows as Mock).mockResolvedValue(queryResult);
+
+			const tool = createDataTablesTool(context);
+			const result = await executeTool<{ data: Array<Record<string, unknown>>; hint?: string }>(
+				tool,
+				{ action: 'query' as const, dataTableId: 'dt-1', fullCellValues: true },
+				noSuspendCtx(),
+			);
+
+			expect(context.dataTableService.queryRows).toHaveBeenCalledWith('dt-1', {
+				filter: undefined,
+				limit: undefined,
+				offset: undefined,
+				projectId: undefined,
+			});
+			expect(result.data[0].image_url).toBe(`${'x'.repeat(1024)}… [truncated, 5000 chars total]`);
+			expect(result.hint).toContain('fullCellValues was ignored');
+		});
+
+		it('should combine truncation and pagination hints', async () => {
+			const blob = 'x'.repeat(2000);
+			const queryResult = {
+				count: 10,
+				data: Array.from({ length: 5 }, (_, i) => ({ id: i, payload: blob })),
+			};
+			const context = createMockContext();
+			(context.dataTableService.queryRows as Mock).mockResolvedValue(queryResult);
+
+			const tool = createDataTablesTool(context);
+			const result = await executeTool<{ hint?: string }>(
+				tool,
+				{ action: 'query' as const, dataTableId: 'dt-1', limit: 5 },
+				noSuspendCtx(),
+			);
+
+			expect(result.hint).toContain('payload');
+			expect(result.hint).toContain('5 more rows available.');
+		});
+
 		it('should include resolved table metadata when available', async () => {
 			const queryResult = { count: 1, data: [{ email: 'a@b.com' }] };
 			const context = createMockContext({

packages/@n8n/instance-ai/src/tools/data-tables.tool.ts

@@ -26,7 +26,7 @@ const filterSchema = z.object({
 	filters: z.array(
 		z.object({
 			columnName: z.string(),
-			condition: z.enum(['eq', 'neq', 'like', 'gt', 'gte', 'lt', 'lte']),
+			condition: z.enum(['eq', 'neq', 'like', 'ilike', 'gt', 'gte', 'lt', 'lte']),
 			value: z.union([z.string(), z.number(), z.boolean()]).nullable(),
 		}),
 	),
@@ -38,7 +38,7 @@ const filterSchemaWithMinOne = z.object({
 		.array(
 			z.object({
 				columnName: z.string(),
-				condition: z.enum(['eq', 'neq', 'like', 'gt', 'gte', 'lt', 'lte']),
+				condition: z.enum(['eq', 'neq', 'like', 'ilike', 'gt', 'gte', 'lt', 'lte']),
 				value: z.union([z.string(), z.number(), z.boolean()]).nullable(),
 			}),
 		)
@@ -90,6 +90,18 @@ function isNameConflictError(error: unknown): boolean {
 
 // ── Action schemas ─────────────────────────────────────────────────────────
 
+/** Cells can hold arbitrarily large values (e.g. inline base64 images); cap what a
+ *  query feeds back to the model so one broad query cannot flood the conversation. */
+const MAX_CELL_CHARS = 1024;
+
+/** When full cell values are requested, cap rows per call — a few intact blob
+ *  cells are useful; dozens re-create the flood truncation exists to prevent. */
+const MAX_FULL_VALUE_ROWS = 5;
+
+const filterDescribe =
+	'Row filter conditions. For text matching use `ilike` (case-insensitive contains); `like` is ' +
+	'case-sensitive. Values without `%` are wrapped as `%value%`.';
+
 const projectIdDescribe =
 	'Project ID. Scopes list/create (defaults to personal); for id-based actions, disambiguates when `dataTableId` is a name found in multiple accessible projects. Ignored when `dataTableId` is a UUID.';
 
@@ -124,7 +136,12 @@ const schemaAction = z.object({
 });
 
 const queryAction = z.object({
-	action: z.literal('query').describe('Query rows from a data table with optional filtering'),
+	action: z
+		.literal('query')
+		.describe(
+			'Query rows from a data table. Prefer a column filter and a small limit over broad pulls; ' +
+				'results include the total matching `count`, so `limit: 1` is enough to check row existence.',
+		),
 	dataTableId: z
 		.string()
 		.describe(
@@ -132,7 +149,7 @@ const queryAction = z.object({
 		),
 	dataTableName: z.string().optional().describe(dataTableNameDescribe),
 	projectId: z.string().optional().describe(projectIdDescribe),
-	filter: filterSchema.optional().describe('Row filter conditions'),
+	filter: filterSchema.optional().describe(filterDescribe),
 	limit: z
 		.number()
 		.int()
@@ -141,6 +158,15 @@ const queryAction = z.object({
 		.optional()
 		.describe('Max rows to return (default 50)'),
 	offset: z.number().int().min(0).optional().describe('Number of rows to skip'),
+	fullCellValues: z
+		.boolean()
+		.optional()
+		.describe(
+			`Return cell values untruncated. By default values longer than ${MAX_CELL_CHARS} characters ` +
+				'(e.g. inline base64 images) are truncated. Requires a filter matching the specific ' +
+				`row(s) whose full values are needed (ignored without one) and returns at most ${MAX_FULL_VALUE_ROWS} ` +
+				'rows per call (default 1) — paginate for more.',
+		),
 });
 
 const createAction = z.object({
@@ -232,7 +258,7 @@ const updateRowsAction = z.object({
 		),
 	dataTableName: z.string().optional().describe(dataTableNameDescribe),
 	projectId: z.string().optional().describe(projectIdDescribe),
-	filter: filterSchema.describe('Row filter conditions'),
+	filter: filterSchema.describe(filterDescribe),
 	data: z.record(z.unknown()).describe('Column values to set on matching rows'),
 });
 
@@ -249,7 +275,7 @@ const deleteRowsAction = z.object({
 		),
 	dataTableName: z.string().optional().describe(dataTableNameDescribe),
 	projectId: z.string().optional().describe(projectIdDescribe),
-	filter: filterSchemaWithMinOne.describe('Row filter conditions'),
+	filter: filterSchemaWithMinOne.describe(filterDescribe),
 });
 
 const allActions = [
@@ -316,14 +342,43 @@ async function handleSchema(
 	return { ...table, columns };
 }
 
+function truncateOversizedCells(rows: Array<Record<string, unknown>>): {
+	rows: Array<Record<string, unknown>>;
+	truncatedColumns: string[];
+} {
+	const truncatedColumns = new Set<string>();
+	const truncatedRows = rows.map((row) => {
+		const oversized = Object.entries(row).filter(
+			([, value]) => typeof value === 'string' && value.length > MAX_CELL_CHARS,
+		);
+		if (oversized.length === 0) return row;
+
+		const next = { ...row };
+		for (const [column, value] of oversized) {
+			if (typeof value !== 'string') continue;
+			next[column] =
+				`${value.slice(0, MAX_CELL_CHARS)}… [truncated, ${String(value.length)} chars total]`;
+			truncatedColumns.add(column);
+		}
+		return next;
+	});
+	return { rows: truncatedRows, truncatedColumns: [...truncatedColumns] };
+}
+
 async function handleQuery(
 	context: InstanceAiContext,
 	input: Extract<FullInput, { action: 'query' }>,
 ) {
 	const table = await resolveDataTableReference(context, input, 'readRow');
+	// Honor fullCellValues only for filtered queries, and bound how many intact
+	// rows one call can return — an unfiltered "give me everything untruncated"
+	// is the exact flood shape truncation exists to prevent.
+	const hasFilter = (input.filter?.filters.length ?? 0) > 0;
+	const returnFullValues = input.fullCellValues === true && hasFilter;
+	const limit = returnFullValues ? Math.min(input.limit ?? 1, MAX_FULL_VALUE_ROWS) : input.limit;
 	const result = await context.dataTableService.queryRows(input.dataTableId, {
 		filter: input.filter,
-		limit: input.limit,
+		limit,
 		offset: input.offset,
 		projectId: input.projectId,
 	});
@@ -331,15 +386,27 @@ async function handleQuery(
 	const returnedRows = result.data.length;
 	const remaining = result.count - (input.offset ?? 0) - returnedRows;
 
+	const hints: string[] = [];
+	let data = result.data;
+	if (!returnFullValues) {
+		const truncation = truncateOversizedCells(result.data);
+		if (truncation.truncatedColumns.length > 0) {
+			data = truncation.rows;
+			hints.push(
+				input.fullCellValues === true
+					? `fullCellValues was ignored because the query has no filter. Values in column(s) ${truncation.truncatedColumns.join(', ')} were truncated to ${String(MAX_CELL_CHARS)} characters. Re-query with a filter matching only the specific row(s) to get full values.`
+					: `Values in column(s) ${truncation.truncatedColumns.join(', ')} were truncated to ${String(MAX_CELL_CHARS)} characters. If a full value is needed, re-query with fullCellValues: true and a filter matching only the specific row(s).`,
+			);
+		}
+	}
 	if (remaining > 0) {
-		return {
-			...table,
-			...result,
-			hint: `${remaining} more rows available. Use additional paginated data-tables queries for bulk operations.`,
-		};
+		hints.push(
+			`${remaining} more rows available. Use additional paginated data-tables queries for bulk operations.`,
+		);
 	}
 
-	return { ...table, ...result };
+	const response = { ...table, count: result.count, data };
+	return hints.length > 0 ? { ...response, hint: hints.join(' ') } : response;
 }
 
 async function handleCreate(
@@ -687,7 +754,9 @@ export function createDataTablesTool(context: InstanceAiContext) {
 				'list/show requests like "what data tables do I have?" or "show/list my tables". ' +
 				'For workflow builds that create or write Data Tables, load `data-table-manager` then ' +
 				'`workflow-builder` before `build-workflow`. Use list, create, and schema before ' +
-				'referencing tables in SDK code.',
+				'referencing tables in SDK code. Keep queries targeted (column filter and/or limit ≤ 5), ' +
+				'especially when diagnosing — never pull a table unfiltered, and after a failed or 0-row ' +
+				'query only retry strictly narrower.',
 		)
 		.input(inputSchema)
 		.suspend(confirmationSuspendSchema)

packages/@n8n/instance-ai/src/types.ts

@@ -706,7 +706,7 @@ export interface DataTableFilterInput {
 	type: 'and' | 'or';
 	filters: Array<{
 		columnName: string;
-		condition: 'eq' | 'neq' | 'like' | 'gt' | 'gte' | 'lt' | 'lte';
+		condition: 'eq' | 'neq' | 'like' | 'ilike' | 'gt' | 'gte' | 'lt' | 'lte';
 		value: string | number | boolean | null;
 	}>;
 }