From d7a70d643ec4042a94405d27ded9e5349622cde7 Mon Sep 17 00:00:00 2001 From: Tomi Turtiainen <10324676+tomi@users.noreply.github.com> Date: Fri, 10 Oct 2025 10:20:18 +0300 Subject: [PATCH 01/81] build: Add n8n packages to min relase age exclude (#20622) --- pnpm-workspace.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 73a40037747..76026d9f9ad 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,6 +8,8 @@ packages: minimumReleaseAge: 2880 # 2 days minimumReleaseAgeExclude: + - '@n8n/typeorm' + - '@n8n_io/ai-assistant-sdk' - eslint-plugin-storybook catalog: From dc72c23d6ad67d09a48ffdec3de3fda565fccf8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ir=C3=A9n=C3=A9e?= Date: Fri, 10 Oct 2025 08:39:48 +0100 Subject: [PATCH 02/81] fix(core): Return insights when only one day is selected (#20543) --- .../insights.service.integration.test.ts | 47 +- .../insights-by-period-query.helper.test.ts | 502 ++++++++++++++++++ .../insights-by-period-query.helper.ts | 133 +++++ .../insights-by-period.repository.ts | 79 +-- .../modules/insights/insights.controller.ts | 5 +- .../src/modules/insights/insights.service.ts | 2 +- 6 files changed, 674 insertions(+), 94 deletions(-) create mode 100644 packages/cli/src/modules/insights/database/repositories/__tests__/insights-by-period-query.helper.test.ts create mode 100644 packages/cli/src/modules/insights/database/repositories/insights-by-period-query.helper.ts diff --git a/packages/cli/src/modules/insights/__tests__/insights.service.integration.test.ts b/packages/cli/src/modules/insights/__tests__/insights.service.integration.test.ts index 700e9dc7fa7..8c644b2d60a 100644 --- a/packages/cli/src/modules/insights/__tests__/insights.service.integration.test.ts +++ b/packages/cli/src/modules/insights/__tests__/insights.service.integration.test.ts @@ -229,6 +229,7 @@ describe('InsightsService', () => { test('mixed period data are summarized correctly', async () => { // ARRANGE + // current period await createCompactedInsightsEvent(workflow, { type: 'success', @@ -338,11 +339,11 @@ describe('InsightsService', () => { // ASSERT expect(summary).toEqual({ - averageRunTime: { value: 0, unit: 'millisecond', deviation: -7157.8 }, + averageRunTime: { value: 0, unit: 'millisecond', deviation: -8947.25 }, failed: { value: 20, unit: 'count', deviation: 18 }, - failureRate: { value: 0.909, unit: 'ratio', deviation: 0.509 }, + failureRate: { value: 0.87, unit: 'ratio', deviation: 0.37 }, timeSaved: { value: 0, unit: 'minute', deviation: -15 }, - total: { value: 22, unit: 'count', deviation: 17 }, + total: { value: 23, unit: 'count', deviation: 19 }, }); }); @@ -687,7 +688,7 @@ describe('InsightsService', () => { type: 'success', value: 1, periodUnit: 'hour', - periodStart: now.minus({ days: 13, hours: 23 }), + periodStart: now.minus({ days: 14 }).startOf('day'), }); // Out of date range insight (should not be included) @@ -696,7 +697,7 @@ describe('InsightsService', () => { type: 'success', value: 1, periodUnit: 'day', - periodStart: now.minus({ days: 14 }), + periodStart: now.minus({ days: 15 }), }); } @@ -843,21 +844,20 @@ describe('InsightsService', () => { }); // Barely in range insight (should be included) - // 1 hour before 14 days ago await createCompactedInsightsEvent(workflow, { type: workflow === workflow1 ? 'success' : 'failure', value: 1, periodUnit: 'hour', - periodStart: now.minus({ days: 13, hours: 23 }), + periodStart: now.minus({ days: 14 }).startOf('day'), }); // Out of date range insight (should not be included) - // 14 days ago + // 15 days ago await createCompactedInsightsEvent(workflow, { type: 'success', value: 1, periodUnit: 'day', - periodStart: now.minus({ days: 14 }), + periodStart: now.minus({ days: 15 }), }); } @@ -1001,12 +1001,11 @@ describe('InsightsService', () => { }); // Barely in range insight (should be included) - // 1 hour before 14 days ago await createCompactedInsightsEvent(workflow, { type: workflow === workflow1 ? 'success' : 'failure', value: 1, periodUnit: 'hour', - periodStart: DateTime.utc().minus({ days: 13, hours: 23 }), + periodStart: DateTime.utc().minus({ days: 14 }).startOf('day'), }); // Out of date range insight (should not be included) @@ -1015,7 +1014,7 @@ describe('InsightsService', () => { type: 'success', value: 1, periodUnit: 'day', - periodStart: DateTime.utc().minus({ days: 14 }), + periodStart: DateTime.utc().minus({ days: 15 }), }); } @@ -1199,25 +1198,29 @@ describe('InsightsService', () => { licenseStateMock.isInsightsHourlyDataLicensed.mockReturnValue(false); licenseStateMock.getInsightsMaxHistory.mockReturnValue(30); - const today = DateTime.now().startOf('day'); - const startDate = today.minus({ hours: 12 }).toJSDate(); - const endDate = today.toJSDate(); + const startDate = DateTime.now().minus({ days: 3 }).startOf('day'); + const endDate = startDate.plus({ hours: 10 }); - expect(() => insightsService.validateDateFiltersLicense({ startDate, endDate })).toThrowError( - new UserError('Hourly data is not available with your current license'), - ); + expect(() => + insightsService.validateDateFiltersLicense({ + startDate: startDate.toJSDate(), + endDate: endDate.toJSDate(), + }), + ).toThrowError(new UserError('Hourly data is not available with your current license')); }); test('does not throw if granularity is hour and hourly data is licensed', () => { licenseStateMock.isInsightsHourlyDataLicensed.mockReturnValue(true); licenseStateMock.getInsightsMaxHistory.mockReturnValue(30); - const today = DateTime.now().startOf('day'); - const startDate = today.minus({ hours: 12 }).toJSDate(); - const endDate = today.toJSDate(); + const startDate = DateTime.now().minus({ days: 3 }).startOf('day'); + const endDate = startDate.endOf('day'); expect(() => - insightsService.validateDateFiltersLicense({ startDate, endDate }), + insightsService.validateDateFiltersLicense({ + startDate: startDate.toJSDate(), + endDate: endDate.toJSDate(), + }), ).not.toThrow(); }); diff --git a/packages/cli/src/modules/insights/database/repositories/__tests__/insights-by-period-query.helper.test.ts b/packages/cli/src/modules/insights/database/repositories/__tests__/insights-by-period-query.helper.test.ts new file mode 100644 index 00000000000..2e68234192b --- /dev/null +++ b/packages/cli/src/modules/insights/database/repositories/__tests__/insights-by-period-query.helper.test.ts @@ -0,0 +1,502 @@ +import type { DatabaseConfig } from '@n8n/config'; +import { DateTime } from 'luxon'; + +import { getDateRangesCommonTableExpressionQuery } from '../insights-by-period-query.helper'; + +describe('getDateRangesCommonTableExpressionQuery', () => { + const now = DateTime.utc(2025, 10, 8, 8, 51, 27); + + beforeEach(() => { + jest.useFakeTimers(); + jest.setSystemTime(now.toJSDate()); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + describe.each([ + ['sqlite', 'SQLite'], + ['postgresdb', 'PostgreSQL'], + ['mysqldb', 'MySQL'], + ['mariadb', 'MariaDB'], + ])('%s', (dbType: DatabaseConfig['type']) => { + describe('hour periodicity (1 day - startDate == endDate)', () => { + test('last 24 hours (endDate is today)', () => { + const startDate = now.startOf('day').toJSDate(); + const endDate = now.startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-2 days')"); // prev_start_date + expect(result).toContain("datetime('now', '-1 days')"); // start_date + expect(result).toContain("datetime('now')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("NOW() - INTERVAL '2 days'"); // prev_start_date + expect(result).toContain("NOW() - INTERVAL '1 days'"); // start_date + expect(result).toContain('NOW()'); // end_date + } else { + expect(result).toContain('DATE_SUB(NOW(), INTERVAL 2 DAY)'); // prev_start_date + expect(result).toContain('DATE_SUB(NOW(), INTERVAL 1 DAY)'); // start_date + expect(result).toContain('NOW()'); // end_date + } + }); + + test('yesterday (specific day)', () => { + const startDate = now.minus({ days: 1 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 1 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-2 days', 'start of day')"); // prev_start_date + expect(result).toContain("datetime('now', '-1 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '2 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '1 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 2 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 1 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + + test('7 days ago (specific day)', () => { + const startDate = now.minus({ days: 7 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 7 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-8 days', 'start of day')"); // prev_start_date + expect(result).toContain("datetime('now', '-7 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', '-6 days', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '8 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '7 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '6 days')"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 8 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 7 DAY))'); // start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 6 DAY))'); // end_date + } + }); + + test('14 days ago (specific day)', () => { + const startDate = now.minus({ days: 14 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 14 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-15 days', 'start of day')"); // prev_start_date + expect(result).toContain("datetime('now', '-14 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', '-13 days', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '15 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '14 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '13 days')"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 15 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 14 DAY))'); // start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 13 DAY))'); // end_date + } + }); + + test('X days ago (specific day far in the past)', () => { + // 109 days ago (2025-06-21) + const startDate = now.minus({ days: 109 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 109 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-110 days', 'start of day')"); // prev_start_date + expect(result).toContain("datetime('now', '-109 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', '-108 days', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '110 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '109 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '108 days')"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 110 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 109 DAY))'); // start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 108 DAY))'); // end_date + } + }); + }); + + describe('day periodicity (2-30 days)', () => { + test('last 7 days (endDate is today)', () => { + const startDate = now.minus({ days: 6 }).startOf('day').toJSDate(); + const endDate = now.startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-12 days', 'start of day')"); // prev_start_date (6 + 6) + expect(result).toContain("datetime('now', '-6 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '12 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '6 days')"); // start_date + expect(result).toContain('NOW()'); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 12 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 6 DAY))'); // start_date + expect(result).toContain('NOW()'); // end_date + } + }); + + test('last 14 days (endDate is today)', () => { + const startDate = now.minus({ days: 13 }).startOf('day').toJSDate(); + const endDate = now.startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-26 days', 'start of day')"); // prev_start_date (13 + 13) + expect(result).toContain("datetime('now', '-13 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '26 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '13 days')"); // start_date + expect(result).toContain('NOW()'); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 26 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 13 DAY))'); // start_date + expect(result).toContain('NOW()'); // end_date + } + }); + + test('last 30 days (endDate is today)', () => { + const startDate = now.minus({ days: 29 }).startOf('day').toJSDate(); + const endDate = now.startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-58 days', 'start of day')"); // prev_start_date (29 + 29) + expect(result).toContain("datetime('now', '-29 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '58 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '29 days')"); // start_date + expect(result).toContain('NOW()'); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 58 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 29 DAY))'); // start_date + expect(result).toContain('NOW()'); // end_date + } + }); + + test('2 days range (specific historical range)', () => { + const startDate = now.minus({ days: 2 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 1 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-3 days', 'start of day')"); // prev_start_date (2 + 1) + expect(result).toContain("datetime('now', '-2 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '3 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '2 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 3 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 2 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + + test('5 days range (specific historical range)', () => { + const startDate = now.minus({ days: 10 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 6 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-14 days', 'start of day')"); // prev_start_date (10 + 4) + expect(result).toContain("datetime('now', '-10 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', '-5 days', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '14 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '10 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '5 days')"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 14 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 10 DAY))'); // start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 5 DAY))'); // end_date + } + }); + + test('7 days range (specific historical range)', () => { + const startDate = now.minus({ days: 12 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 6 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-18 days', 'start of day')"); // prev_start_date (12 + 6) + expect(result).toContain("datetime('now', '-12 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', '-5 days', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '18 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '12 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '5 days')"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 18 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 12 DAY))'); // start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 5 DAY))'); // end_date + } + }); + + test('14 days range (specific historical range)', () => { + const startDate = now.minus({ days: 14 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 1 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-27 days', 'start of day')"); // prev_start_date (14 + 13) + expect(result).toContain("datetime('now', '-14 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '27 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '14 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 27 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 14 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + + test('30 days range (specific historical range)', () => { + const startDate = now.minus({ days: 30 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 1 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-59 days', 'start of day')"); // prev_start_date (30 + 29) + expect(result).toContain("datetime('now', '-30 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '59 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '30 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 59 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 30 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + }); + + describe('week periodicity (31+ days)', () => { + test('last 90 days (endDate is today)', () => { + const startDate = now.minus({ days: 89 }).startOf('day').toJSDate(); + const endDate = now.startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-178 days', 'start of day')"); // prev_start_date (89 + 89) + expect(result).toContain("datetime('now', '-89 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '178 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '89 days')"); // start_date + expect(result).toContain('NOW()'); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 178 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 89 DAY))'); // start_date + expect(result).toContain('NOW()'); // end_date + } + }); + + test('last 6 months (endDate is today)', () => { + const startDate = now.minus({ months: 6 }).startOf('day').toJSDate(); + const endDate = now.startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + const daysBack = Math.floor(now.diff(DateTime.fromJSDate(startDate), 'days').days); + const prevDaysBack = daysBack * 2; + + if (dbType === 'sqlite') { + expect(result).toContain(`datetime('now', '-${prevDaysBack} days', 'start of day')`); // prev_start_date + expect(result).toContain(`datetime('now', '-${daysBack} days', 'start of day')`); // start_date + expect(result).toContain("datetime('now')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain(`DATE_TRUNC('day', NOW() - INTERVAL '${prevDaysBack} days')`); // prev_start_date + expect(result).toContain(`DATE_TRUNC('day', NOW() - INTERVAL '${daysBack} days')`); // start_date + expect(result).toContain('NOW()'); // end_date + } else { + expect(result).toContain(`DATE(DATE_SUB(NOW(), INTERVAL ${prevDaysBack} DAY))`); // prev_start_date + expect(result).toContain(`DATE(DATE_SUB(NOW(), INTERVAL ${daysBack} DAY))`); // start_date + expect(result).toContain('NOW()'); // end_date + } + }); + + test('last year (endDate is today)', () => { + const startDate = now.minus({ years: 1 }).startOf('day').toJSDate(); + const endDate = now.startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + const daysBack = Math.floor(now.diff(DateTime.fromJSDate(startDate), 'days').days); + const prevDaysBack = daysBack * 2; + + if (dbType === 'sqlite') { + expect(result).toContain(`datetime('now', '-${prevDaysBack} days', 'start of day')`); // prev_start_date + expect(result).toContain(`datetime('now', '-${daysBack} days', 'start of day')`); // start_date + expect(result).toContain("datetime('now')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain(`DATE_TRUNC('day', NOW() - INTERVAL '${prevDaysBack} days')`); // prev_start_date + expect(result).toContain(`DATE_TRUNC('day', NOW() - INTERVAL '${daysBack} days')`); // start_date + expect(result).toContain('NOW()'); // end_date + } else { + expect(result).toContain(`DATE(DATE_SUB(NOW(), INTERVAL ${prevDaysBack} DAY))`); // prev_start_date + expect(result).toContain(`DATE(DATE_SUB(NOW(), INTERVAL ${daysBack} DAY))`); // start_date + expect(result).toContain('NOW()'); // end_date + } + }); + + test('31 days range (specific historical range)', () => { + const startDate = now.minus({ days: 31 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 1 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-61 days', 'start of day')"); // prev_start_date (31 + 30) + expect(result).toContain("datetime('now', '-31 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '61 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '31 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 61 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 31 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + + test('90 days range (specific historical range)', () => { + const startDate = now.minus({ days: 90 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 1 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-179 days', 'start of day')"); // prev_start_date (90 + 89) + expect(result).toContain("datetime('now', '-90 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '179 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '90 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 179 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 90 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + + test('180 days range (specific historical range)', () => { + const startDate = now.minus({ days: 180 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 1 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-359 days', 'start of day')"); // prev_start_date (180 + 179) + expect(result).toContain("datetime('now', '-180 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '359 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '180 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 359 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 180 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + + test('360 days range (specific historical range)', () => { + const startDate = now.minus({ days: 360 }).startOf('day').toJSDate(); + const endDate = now.minus({ days: 1 }).startOf('day').toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-719 days', 'start of day')"); // prev_start_date (360 + 359) + expect(result).toContain("datetime('now', '-360 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '719 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '360 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 719 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 360 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + }); + + describe('edge cases', () => { + test('handles date with time component correctly', () => { + // Date with time should be treated as start of day + const startDate = DateTime.utc(2025, 10, 6, 14, 30, 0).toJSDate(); + const endDate = DateTime.utc(2025, 10, 7, 18, 45, 30).toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + // Should calculate based on start of day values (2-day range) + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-3 days', 'start of day')"); // prev_start_date (2 + 1) + expect(result).toContain("datetime('now', '-2 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '3 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '2 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 3 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 2 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + + test('handles same day with different times correctly (hour periodicity)', () => { + const startDate = DateTime.utc(2025, 10, 7, 9, 0, 0).toJSDate(); + const endDate = DateTime.utc(2025, 10, 7, 17, 0, 0).toJSDate(); + + const result = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); + + // Should treat as single day (yesterday) - hour periodicity + if (dbType === 'sqlite') { + expect(result).toContain("datetime('now', '-2 days', 'start of day')"); // prev_start_date (1 + 1) + expect(result).toContain("datetime('now', '-1 days', 'start of day')"); // start_date + expect(result).toContain("datetime('now', 'start of day')"); // end_date + } else if (dbType === 'postgresdb') { + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '2 days')"); // prev_start_date + expect(result).toContain("DATE_TRUNC('day', NOW() - INTERVAL '1 days')"); // start_date + expect(result).toContain("DATE_TRUNC('day', NOW())"); // end_date + } else { + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 2 DAY))'); // prev_start_date + expect(result).toContain('DATE(DATE_SUB(NOW(), INTERVAL 1 DAY))'); // start_date + expect(result).toContain('DATE(NOW())'); // end_date + } + }); + }); + }); +}); diff --git a/packages/cli/src/modules/insights/database/repositories/insights-by-period-query.helper.ts b/packages/cli/src/modules/insights/database/repositories/insights-by-period-query.helper.ts new file mode 100644 index 00000000000..a6ca190c577 --- /dev/null +++ b/packages/cli/src/modules/insights/database/repositories/insights-by-period-query.helper.ts @@ -0,0 +1,133 @@ +import type { DatabaseConfig } from '@n8n/config'; +import { sql } from '@n8n/db'; +import { DateTime } from 'luxon'; + +/** + * Generates database-specific SQL for a datetime value relative to now + * @param dbType - The database type + * @param daysFromToday - Number of days back from today (0 = now) + * @param useStartOfDay - Whether to truncate to start of day (00:00:00) + */ +const getDatetimeSql = ({ + dbType, + daysFromToday, + useStartOfDay = false, +}: { + dbType: DatabaseConfig['type']; + daysFromToday: number; + useStartOfDay?: boolean; +}): string => { + // Handle "now" case + if (daysFromToday === 0 && !useStartOfDay) { + return dbType === 'sqlite' ? "datetime('now')" : 'NOW()'; + } + + // SQLite + if (dbType === 'sqlite') { + if (daysFromToday === 0 && useStartOfDay) { + return "datetime('now', 'start of day')"; + } + if (useStartOfDay) { + return `datetime('now', '-${daysFromToday} days', 'start of day')`; + } + return `datetime('now', '-${daysFromToday} days')`; + } + + // PostgreSQL + if (dbType === 'postgresdb') { + if (daysFromToday === 0 && useStartOfDay) { + return "DATE_TRUNC('day', NOW())"; + } + if (useStartOfDay) { + return `DATE_TRUNC('day', NOW() - INTERVAL '${daysFromToday} days')`; + } + return `NOW() - INTERVAL '${daysFromToday} days'`; + } + + // MySQL/MariaDB + if (daysFromToday === 0 && useStartOfDay) { + return 'DATE(NOW())'; + } + if (useStartOfDay) { + return `DATE(DATE_SUB(NOW(), INTERVAL ${daysFromToday} DAY))`; + } + return `DATE_SUB(NOW(), INTERVAL ${daysFromToday} DAY)`; +}; + +/** + * Generates a SQL Common Table Expression (CTE) query that provides three date boundaries for insights queries + * + * Behavior: + * - If startDate and endDate are the same and today + * - returns the last 24 hours: prev_start_date (2 days ago), start_date (1 day ago), end_date (now). + * - Otherwise: + * - prev_start_date: start of the day before the range + * - start_date: start of the current range + * - end_date: "now" if endDate is today, else start of the day after endDate + * + * The SQL CTE can be joined with the insights table for filtering/aggregation. + * + * @param dbType - The database type ('sqlite', 'postgresdb', 'mysqldb', 'mariadb') + * @param startDate - The start date of the range (inclusive) + * @param endDate - The end date of the range (inclusive, or "now" if today) + * @returns SQL CTE query with `prev_start_date`, `start_date`, and `end_date` columns + * - `prev_start_date`: The start of the previous period (used for comparison) + * - `start_date`: The start of the current period (inclusive) + * - `end_date`: The end of the current period (exclusive) + */ +export const getDateRangesCommonTableExpressionQuery = ({ + dbType, + startDate, + endDate, +}: { + dbType: DatabaseConfig['type']; + startDate: Date; + endDate: Date; +}) => { + const today = DateTime.now().startOf('day'); + const startDateStartOfDay = DateTime.fromJSDate(startDate).startOf('day'); + const endDateStartOfDay = DateTime.fromJSDate(endDate).startOf('day'); + + const daysFromEndDateToToday = Math.floor(today.diff(endDateStartOfDay, 'days').days); + const daysDiff = Math.floor(endDateStartOfDay.diff(startDateStartOfDay, 'days').days); + + const isEndDateToday = daysFromEndDateToToday === 0; + + let prevStartDateSql: string; + let startDateSql: string; + let endDateSql: string; + + if (daysDiff === 0 && isEndDateToday) { + // Last 24 hours + prevStartDateSql = getDatetimeSql({ dbType, daysFromToday: 2, useStartOfDay: false }); + startDateSql = getDatetimeSql({ dbType, daysFromToday: 1, useStartOfDay: false }); + endDateSql = getDatetimeSql({ dbType, daysFromToday: 0, useStartOfDay: false }); + } else { + // Calculate the date range (minimum 1 day) for previous period + const dateRangeInDays = Math.max(1, daysDiff); + const daysFromStartDateToToday = Math.floor(today.diff(startDateStartOfDay, 'days').days); + const prevStartDaysFromToday = daysFromStartDateToToday + dateRangeInDays; + + prevStartDateSql = getDatetimeSql({ + dbType, + daysFromToday: prevStartDaysFromToday, + useStartOfDay: true, + }); + + startDateSql = getDatetimeSql({ + dbType, + daysFromToday: daysFromStartDateToToday, + useStartOfDay: true, + }); + + endDateSql = isEndDateToday + ? getDatetimeSql({ dbType, daysFromToday: 0, useStartOfDay: false }) + : getDatetimeSql({ dbType, daysFromToday: daysFromEndDateToToday - 1, useStartOfDay: true }); + } + + return sql`SELECT + ${prevStartDateSql} AS prev_start_date, + ${startDateSql} AS start_date, + ${endDateSql} AS end_date + `; +}; diff --git a/packages/cli/src/modules/insights/database/repositories/insights-by-period.repository.ts b/packages/cli/src/modules/insights/database/repositories/insights-by-period.repository.ts index 2eb583433a3..2a00b358a7f 100644 --- a/packages/cli/src/modules/insights/database/repositories/insights-by-period.repository.ts +++ b/packages/cli/src/modules/insights/database/repositories/insights-by-period.repository.ts @@ -6,6 +6,7 @@ import { DataSource, LessThanOrEqual, Repository } from '@n8n/typeorm'; import { DateTime } from 'luxon'; import { z } from 'zod'; +import { getDateRangesCommonTableExpressionQuery } from './insights-by-period-query.helper'; import { InsightsByPeriod } from '../entities/insights-by-period'; import type { PeriodUnit, TypeUnit } from '../entities/insights-shared'; import { PeriodUnitToNumber, TypeToNumber } from '../entities/insights-shared'; @@ -140,7 +141,7 @@ export class InsightsByPeriodRepository extends Repository { }>; } - getAggregationQuery(periodUnit: PeriodUnit) { + private getAggregationQuery(periodUnit: PeriodUnit) { // Get the start period expression depending on the period unit and database type const periodStartExpr = this.getPeriodStartExpr(periodUnit); @@ -268,18 +269,6 @@ export class InsightsByPeriodRepository extends Repository { } } - private getAgeLimitQuery(maxAgeInDays: number) { - if (maxAgeInDays === 0) { - return dbType === 'sqlite' ? "datetime('now')" : 'NOW()'; - } - - return dbType === 'sqlite' - ? `datetime('now', '-${maxAgeInDays} days')` - : dbType === 'postgresdb' - ? `NOW() - INTERVAL '${maxAgeInDays} days'` - : `DATE_SUB(NOW(), INTERVAL ${maxAgeInDays} DAY)`; - } - async getPreviousAndCurrentPeriodTypeAggregates({ startDate, endDate, @@ -291,25 +280,14 @@ export class InsightsByPeriodRepository extends Repository { total_value: string | number; }> > { - const { daysFromStartDateToToday, daysFromEndDateToToday, dateRangeInDays } = - this.getDateRangesDaysLimits({ - startDate, - endDate, - }); - - const cte = sql` - SELECT - ${this.getAgeLimitQuery(daysFromStartDateToToday)} AS current_start, - ${this.getAgeLimitQuery(daysFromEndDateToToday)} AS current_end, - ${this.getAgeLimitQuery(daysFromStartDateToToday + dateRangeInDays)} AS previous_start - `; + const cte = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); const rawRowsQuery = this.createQueryBuilder('insights') .addCommonTableExpression(cte, 'date_ranges') .select( sql` CASE - WHEN insights.periodStart >= date_ranges.current_start AND insights.periodStart <= date_ranges.current_end + WHEN insights.periodStart >= date_ranges.start_date AND insights.periodStart < date_ranges.end_date THEN 'current' ELSE 'previous' END @@ -320,8 +298,8 @@ export class InsightsByPeriodRepository extends Repository { .addSelect('SUM(value)', 'total_value') // Use a cross join with the CTE .innerJoin('date_ranges', 'date_ranges', '1=1') - .where('insights.periodStart >= date_ranges.previous_start') - .andWhere('insights.periodStart <= date_ranges.current_end') + .where('insights.periodStart >= date_ranges.prev_start_date') + .andWhere('insights.periodStart < date_ranges.end_date') // Group by both period and type .groupBy('period') .addGroupBy('insights.type'); @@ -360,16 +338,7 @@ export class InsightsByPeriodRepository extends Repository { const [sortField, sortOrder] = this.parseSortingParams(sortBy); const sumOfExecutions = sql`SUM(CASE WHEN insights.type IN (${TypeToNumber.success.toString()}, ${TypeToNumber.failure.toString()}) THEN value ELSE 0 END)`; - const { daysFromStartDateToToday, daysFromEndDateToToday } = this.getDateRangesDaysLimits({ - startDate, - endDate, - }); - - const cte = sql` - SELECT - ${this.getAgeLimitQuery(daysFromStartDateToToday)} AS start_date, - ${this.getAgeLimitQuery(daysFromEndDateToToday)} AS end_date - `; + const cte = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); const rawRowsQuery = this.createQueryBuilder('insights') .addCommonTableExpression(cte, 'date_ranges') @@ -396,7 +365,7 @@ export class InsightsByPeriodRepository extends Repository { // Use a cross join with the CTE .innerJoin('date_ranges', 'date_ranges', '1=1') .where('insights.periodStart >= date_ranges.start_date') - .andWhere('insights.periodStart <= date_ranges.end_date') + .andWhere('insights.periodStart < date_ranges.end_date') .groupBy('metadata.workflowId') .addGroupBy('metadata.workflowName') .addGroupBy('metadata.projectId') @@ -426,16 +395,7 @@ export class InsightsByPeriodRepository extends Repository { startDate: Date; endDate: Date; }) { - const { daysFromStartDateToToday, daysFromEndDateToToday } = this.getDateRangesDaysLimits({ - startDate, - endDate, - }); - - const cte = sql` - SELECT - ${this.getAgeLimitQuery(daysFromStartDateToToday)} AS start_date, - ${this.getAgeLimitQuery(daysFromEndDateToToday)} AS end_date - `; + const cte = getDateRangesCommonTableExpressionQuery({ dbType, startDate, endDate }); const typesAggregation = insightTypes.map((type) => { return `SUM(CASE WHEN insights.type = ${TypeToNumber[type]} THEN value ELSE 0 END) AS "${displayTypeName[TypeToNumber[type]]}"`; @@ -461,27 +421,6 @@ export class InsightsByPeriodRepository extends Repository { return aggregatedInsightsByTimeParser.parse(rawRows); } - private getDateRangesDaysLimits({ startDate, endDate }: { startDate: Date; endDate: Date }) { - const today = DateTime.now().startOf('day'); - const startDateStartOfDay = DateTime.fromJSDate(startDate).startOf('day'); - const endDateStartOfDay = DateTime.fromJSDate(endDate).startOf('day'); - - let daysFromStartDateToToday = today.diff(startDateStartOfDay, 'days').days; - // ensure that at least one day is covered - if (daysFromStartDateToToday < 1) { - daysFromStartDateToToday = 1; - } - const daysFromEndDateToToday = today.diff(endDateStartOfDay, 'days').days; - - const dateRangeInDays = daysFromStartDateToToday - daysFromEndDateToToday; - - return { - daysFromStartDateToToday, - daysFromEndDateToToday, - dateRangeInDays, - }; - } - async pruneOldData(maxAgeInDays: number): Promise<{ affected: number | null | undefined }> { const thresholdDate = DateTime.now().minus({ days: maxAgeInDays }).startOf('day').toJSDate(); const result = await this.delete({ diff --git a/packages/cli/src/modules/insights/insights.controller.ts b/packages/cli/src/modules/insights/insights.controller.ts index 69dbe356b25..d23d46ba1cd 100644 --- a/packages/cli/src/modules/insights/insights.controller.ts +++ b/packages/cli/src/modules/insights/insights.controller.ts @@ -161,7 +161,10 @@ export class InsightsController { if (query.dateRange) { const maxAgeInDays = keyRangeToDays[query.dateRange]; return { - startDate: DateTime.now().minus({ days: maxAgeInDays }).toJSDate(), + startDate: + maxAgeInDays === 1 + ? DateTime.now().startOf('day').toJSDate() + : DateTime.now().minus({ days: maxAgeInDays }).toJSDate(), endDate: today, }; } diff --git a/packages/cli/src/modules/insights/insights.service.ts b/packages/cli/src/modules/insights/insights.service.ts index 7c45f4a8cc7..959f87a4372 100644 --- a/packages/cli/src/modules/insights/insights.service.ts +++ b/packages/cli/src/modules/insights/insights.service.ts @@ -275,7 +275,7 @@ export class InsightsService { const endDateTime = DateTime.fromJSDate(endDate); const differenceInDays = endDateTime.diff(startDateTime, 'days').days; - if (differenceInDays <= 1) { + if (differenceInDays < 1) { return 'hour'; } From c55f67e4615e1c6ff03d609923754ef42e9c6507 Mon Sep 17 00:00:00 2001 From: Daria Date: Fri, 10 Oct 2025 10:40:22 +0300 Subject: [PATCH 03/81] feat(core): Data Table - Improve sql utils (no-changelog) (#20551) --- .../data-table-filters.integration.test.ts | 112 ++++++- .../data-table.service.integration.test.ts | 46 +++ .../data-table/__tests__/sql-utils.test.ts | 306 +++++++++++++++++- .../data-table/data-table-rows.repository.ts | 44 +-- .../modules/data-table/data-table.service.ts | 124 ++++--- .../src/modules/data-table/utils/sql-utils.ts | 136 +++++--- .../tests/ui/data-table-details.spec.ts | 3 +- packages/workflow/src/data-table.types.ts | 13 + 8 files changed, 657 insertions(+), 127 deletions(-) diff --git a/packages/cli/src/modules/data-table/__tests__/data-table-filters.integration.test.ts b/packages/cli/src/modules/data-table/__tests__/data-table-filters.integration.test.ts index bb58355e432..08566d71332 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-table-filters.integration.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-table-filters.integration.test.ts @@ -1179,7 +1179,7 @@ describe('dataTable filters', () => { const createdAtTimestamp = inserted[0].createdAt; const midnight = new Date(createdAtTimestamp); - midnight.setHours(0, 0, 0, 0); + midnight.setUTCHours(0, 0, 0, 0); // ACT - Check the row is not returned if filtered before midnight const beforeMidnightResult = await dataTableService.getManyRowsAndCount( @@ -1208,7 +1208,7 @@ describe('dataTable filters', () => { expect(result.count).toBeGreaterThanOrEqual(1); expect(result.data.some((row) => row.name === 'TestRow')).toBe(true); - // ACT - - Check the row is returned when using lt on the exact timestamp + // ACT - Check the row is returned when using lt on the exact timestamp const resultLt = await dataTableService.getManyRowsAndCount(dataTableId, project.id, { filter: { type: 'and', @@ -1219,6 +1219,114 @@ describe('dataTable filters', () => { // ASSERT expect(resultLt.data.some((row) => row.name === 'TestRow')).toBe(false); }); + + it('filters by date with timezone offset using eq condition', async () => { + // ARRANGE + const dateWithOffset = new Date('2024-01-15T10:30:00.000+03:00'); + const equivalentUtcTime = new Date('2024-01-15T07:30:00.000Z'); + + await dataTableService.insertRows( + dataTableId, + project.id, + [{ name: 'TimezoneTest', registeredAt: dateWithOffset }], + 'all', + ); + + // ACT + const resultWithOffset = await dataTableService.getManyRowsAndCount( + dataTableId, + project.id, + { + filter: { + type: 'and', + filters: [{ columnName: 'registeredAt', value: dateWithOffset, condition: 'eq' }], + }, + }, + ); + + const resultWithUtc = await dataTableService.getManyRowsAndCount( + dataTableId, + project.id, + { + filter: { + type: 'and', + filters: [ + { columnName: 'registeredAt', value: equivalentUtcTime, condition: 'eq' }, + ], + }, + }, + ); + + // ASSERT + expect(resultWithOffset.count).toBe(1); + expect(resultWithOffset.data[0].name).toBe('TimezoneTest'); + expect(resultWithUtc.count).toBe(1); + expect(resultWithUtc.data[0].name).toBe('TimezoneTest'); + }); + + it('filters by date finding multiple rows with same UTC time but different offsets', async () => { + // ARRANGE + const isoWithOffsetPlus = '2024-01-15T10:30:00.000+05:00'; + const isoWithOffsetMinus = '2024-01-15T02:30:00.000-03:00'; + const equivalentUtcTime = new Date('2024-01-15T05:30:00.000Z'); + + await dataTableService.insertRows(dataTableId, project.id, [ + { name: 'OffsetPlus', registeredAt: isoWithOffsetPlus }, + { name: 'OffsetMinus', registeredAt: isoWithOffsetMinus }, + ]); + + // ACT + const result = await dataTableService.getManyRowsAndCount(dataTableId, project.id, { + filter: { + type: 'and', + filters: [{ columnName: 'registeredAt', value: equivalentUtcTime, condition: 'eq' }], + }, + }); + + // ASSERT + expect(result.count).toBe(2); + expect(result.data.map((r) => r.name).sort()).toEqual(['OffsetMinus', 'OffsetPlus']); + }); + + it('correctly compares dates across timezones with gt/lt filters', async () => { + // ARRANGE + await dataTableService.insertRows(dataTableId, project.id, [ + { name: 'TzEarly', registeredAt: new Date('2025-01-15T08:00:00.000+02:00') }, + { name: 'TzMiddle', registeredAt: new Date('2025-01-15T12:00:00.000Z') }, + { name: 'TzLate', registeredAt: new Date('2025-01-15T20:00:00.000+02:00') }, + ]); + + // ACT + const filterDate = new Date('2025-01-15T13:00:00.000+03:00'); + const result = await dataTableService.getManyRowsAndCount(dataTableId, project.id, { + filter: { + type: 'and', + filters: [ + { columnName: 'registeredAt', value: filterDate, condition: 'gt' }, + { columnName: 'name', value: 'Tz%', condition: 'like' }, + ], + }, + }); + + // ASSERT + expect(result.count).toBe(2); + expect(result.data.map((r) => r.name).sort()).toEqual(['TzLate', 'TzMiddle']); + + // ACT + const resultLt = await dataTableService.getManyRowsAndCount(dataTableId, project.id, { + filter: { + type: 'and', + filters: [ + { columnName: 'registeredAt', value: filterDate, condition: 'lt' }, + { columnName: 'name', value: 'Tz%', condition: 'like' }, + ], + }, + }); + + // ASSERT + expect(resultLt.count).toBe(1); + expect(resultLt.data[0].name).toBe('TzEarly'); + }); }); describe('null value validation', () => { diff --git a/packages/cli/src/modules/data-table/__tests__/data-table.service.integration.test.ts b/packages/cli/src/modules/data-table/__tests__/data-table.service.integration.test.ts index c4eea0dcc63..9cfe7c526b4 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-table.service.integration.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-table.service.integration.test.ts @@ -1359,6 +1359,52 @@ describe('dataTable', () => { ); }); + it('converts dates with timezone offsets to UTC when inserting', async () => { + // ARRANGE + const { id: dataTableId } = await dataTableService.createDataTable(project1.id, { + name: 'dataTable', + columns: [{ name: 'registeredAt', type: 'date' }], + }); + + const dateWithOffset = new Date('2024-01-15T10:30:00.000+03:00'); + const expectedUtcTime = new Date('2024-01-15T07:30:00.000Z'); + + // ACT + const inserted = await dataTableService.insertRows( + dataTableId, + project1.id, + [{ registeredAt: dateWithOffset }], + 'all', + ); + + // ASSERT + expect((inserted[0].registeredAt as Date).getTime()).toBe(expectedUtcTime.getTime()); + }); + + it('converts ISO date strings with timezone offsets to UTC when inserting', async () => { + // ARRANGE + const { id: dataTableId } = await dataTableService.createDataTable(project1.id, { + name: 'dataTable', + columns: [{ name: 'registeredAt', type: 'date' }], + }); + + const isoWithOffsetPlus = '2024-01-15T10:30:00.000+05:00'; + const isoWithOffsetMinus = '2024-01-15T02:30:00.000-03:00'; + const expectedUtcTime = new Date('2024-01-15T05:30:00.000Z'); + + // ACT + const inserted = await dataTableService.insertRows( + dataTableId, + project1.id, + [{ registeredAt: isoWithOffsetPlus }, { registeredAt: isoWithOffsetMinus }], + 'all', + ); + + // ASSERT + expect((inserted[0].registeredAt as Date).getTime()).toBe(expectedUtcTime.getTime()); + expect((inserted[1].registeredAt as Date).getTime()).toBe(expectedUtcTime.getTime()); + }); + it('rejects unknown data table id', async () => { // ARRANGE await dataTableService.createDataTable(project1.id, { diff --git a/packages/cli/src/modules/data-table/__tests__/sql-utils.test.ts b/packages/cli/src/modules/data-table/__tests__/sql-utils.test.ts index 02427804418..2719796b742 100644 --- a/packages/cli/src/modules/data-table/__tests__/sql-utils.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/sql-utils.test.ts @@ -1,6 +1,204 @@ -import { addColumnQuery, deleteColumnQuery } from '../utils/sql-utils'; +import type { DataTableColumnType } from 'n8n-workflow'; + +import type { DataTableColumn } from '../data-table-column.entity'; +import { + addColumnQuery, + deleteColumnQuery, + normalizeRows, + normalizeValueForDatabase, + toSqliteGlobFromPercent, +} from '../utils/sql-utils'; describe('sql-utils', () => { + describe('normalizeRows', () => { + const createColumn = (name: string, type: DataTableColumnType): DataTableColumn => + ({ + id: '1', + name, + type, + dataTableId: 'test-table', + createdAt: new Date(), + updatedAt: new Date(), + }) as DataTableColumn; + + it('should normalize boolean values from numbers', () => { + const columns = [createColumn('active', 'boolean')]; + const rows = [ + { id: 1, active: 1, createdAt: new Date(), updatedAt: new Date() }, + { id: 2, active: 0, createdAt: new Date(), updatedAt: new Date() }, + ]; + + const result = normalizeRows(rows, columns); + + expect(result[0].active).toEqual(true); + expect(result[1].active).toEqual(false); + }); + + it('should normalize boolean values from strings', () => { + const columns = [createColumn('active', 'boolean')]; + const rows = [ + { id: 1, active: '1', createdAt: new Date(), updatedAt: new Date() }, + { id: 2, active: '0', createdAt: new Date(), updatedAt: new Date() }, + ]; + + const result = normalizeRows(rows, columns); + + expect(result[0].active).toEqual(true); + expect(result[1].active).toEqual(false); + }); + + it('should keep boolean values as-is when already boolean', () => { + const columns = [createColumn('active', 'boolean')]; + const rows = [ + { id: 1, active: true, createdAt: new Date(), updatedAt: new Date() }, + { id: 2, active: false, createdAt: new Date(), updatedAt: new Date() }, + ]; + + const result = normalizeRows(rows, columns); + + expect(result[0].active).toEqual(true); + expect(result[1].active).toEqual(false); + }); + + it('should keep date values unchanged for Date objects', () => { + const columns = [createColumn('birthday', 'date')]; + const testDate = new Date('2024-01-15T10:30:00Z'); + const rows = [{ id: 1, birthday: testDate, createdAt: new Date(), updatedAt: new Date() }]; + + const result = normalizeRows(rows, columns); + + expect(result[0].birthday).toEqual(testDate); + }); + + it('should normalize date values from ISO strings', () => { + const columns = [createColumn('birthday', 'date')]; + const dateString = '2024-01-15T10:30:00Z'; + const rows = [{ id: 1, birthday: dateString, createdAt: dateString, updatedAt: dateString }]; + + const result = normalizeRows(rows, columns); + + expect(result[0].birthday).toEqual(new Date(dateString)); + expect(result[0].createdAt).toEqual(new Date(dateString)); + expect(result[0].updatedAt).toEqual(new Date(dateString)); + }); + + it('should normalize date values from strings of sqlite format', () => { + const columns = [createColumn('birthday', 'date')]; + const dateString = '2024-01-15 10:30:00'; + const rows = [{ id: 1, birthday: dateString, createdAt: dateString, updatedAt: dateString }]; + + const result = normalizeRows(rows, columns); + + expect(result[0].birthday).toEqual(new Date('2024-01-15T10:30:00Z')); + expect(result[0].createdAt).toEqual(new Date('2024-01-15T10:30:00Z')); + expect(result[0].updatedAt).toEqual(new Date('2024-01-15T10:30:00Z')); + }); + + it('should normalize date values from timestamps', () => { + const columns = [createColumn('birthday', 'date')]; + const timestamp = 1705318200000; // 2024-01-15T10:30:00Z + const rows = [{ id: 1, birthday: timestamp, createdAt: timestamp, updatedAt: timestamp }]; + + const result = normalizeRows(rows, columns); + + expect(result[0].birthday).toEqual(new Date(timestamp)); + expect(result[0].createdAt).toEqual(new Date(timestamp)); + expect(result[0].updatedAt).toEqual(new Date(timestamp)); + }); + + it('should handle invalid date strings gracefully', () => { + const columns = [createColumn('birthday', 'date')]; + const rows = [ + { id: 1, birthday: 'not-a-date', createdAt: new Date(), updatedAt: new Date() }, + ]; + + const result = normalizeRows(rows, columns); + + expect(result[0].birthday).toBe('not-a-date'); + }); + + it('should handle null date values', () => { + const columns = [createColumn('birthday', 'date')]; + const rows = [{ id: 1, birthday: null, createdAt: new Date(), updatedAt: new Date() }]; + + const result = normalizeRows(rows, columns); + + expect(result[0].birthday).toBeNull(); + }); + + it('should handle multiple rows', () => { + const columns = [ + createColumn('name', 'string'), + createColumn('age', 'number'), + createColumn('active', 'boolean'), + ]; + const date = new Date(); + const rows = [ + { + id: 1, + name: 'John Doe', + age: 30, + active: 1, + createdAt: date, + updatedAt: date, + }, + { + id: 2, + name: 'Jane Doe', + age: 25, + active: 0, + createdAt: date, + updatedAt: date, + }, + { + id: 3, + name: 'Jim Doe', + age: 35, + active: true, + createdAt: date, + updatedAt: date, + }, + ]; + + const result = normalizeRows(rows, columns); + + expect(result).toEqual([ + { + id: 1, + active: true, + age: 30, + name: 'John Doe', + createdAt: date, + updatedAt: date, + }, + { + id: 2, + active: false, + age: 25, + name: 'Jane Doe', + createdAt: date, + updatedAt: date, + }, + { + id: 3, + active: true, + age: 35, + name: 'Jim Doe', + createdAt: date, + updatedAt: date, + }, + ]); + }); + + it('should handle empty rows array', () => { + const columns = [createColumn('active', 'boolean')]; + + const result = normalizeRows([], columns); + + expect(result).toEqual([]); + }); + }); + describe('addColumnQuery', () => { it('should generate a valid SQL query for adding columns to a table, sqlite', () => { const tableName = 'data_table_user_abc'; @@ -49,4 +247,110 @@ describe('sql-utils', () => { expect(query).toBe('ALTER TABLE "data_table_user_abc" DROP COLUMN "email"'); }); }); + + describe('normalizeValueForDatabase', () => { + it('should return value unchanged for non-date column types', () => { + expect(normalizeValueForDatabase('test', 'string')).toBe('test'); + expect(normalizeValueForDatabase(123, 'number')).toBe(123); + expect(normalizeValueForDatabase(true, 'boolean')).toBe(true); + }); + + it('should return null for null', () => { + expect(normalizeValueForDatabase(null, 'string')).toBeNull(); + expect(normalizeValueForDatabase(null, 'number')).toBeNull(); + expect(normalizeValueForDatabase(null, 'boolean')).toBeNull(); + expect(normalizeValueForDatabase(null, 'date')).toBeNull(); + }); + + describe('date columns', () => { + it.each([ + ['sqlite', '2024-01-15 10:30:00.123'], + ['sqlite-pooled', '2024-01-15 10:30:00.123'], + ['mysql', '2024-01-15 10:30:00.123'], + ['mariadb', '2024-01-15 10:30:00.123'], + ['postgres', '2024-01-15T10:30:00.123Z'], + ] as const)('should format Date object for %s', (dbType, expected) => { + const result = normalizeValueForDatabase( + new Date('2024-01-15T10:30:00.123Z'), + 'date', + dbType, + ); + + expect(result).toBe(expected); + }); + + it.each([ + ['sqlite', '2024-01-15 10:30:00.123'], + ['sqlite-pooled', '2024-01-15 10:30:00.123'], + ['mysql', '2024-01-15 10:30:00.123'], + ['mariadb', '2024-01-15 10:30:00.123'], + ['postgres', '2024-01-15T10:30:00.123Z'], + ] as const)('should format ISO date string for %s', (dbType, expected) => { + const result = normalizeValueForDatabase('2024-01-15T10:30:00.123Z', 'date', dbType); + + expect(result).toBe(expected); + }); + + it('should throw on invalid date string', () => { + expect(() => normalizeValueForDatabase('not-a-date', 'date', 'sqlite')).toThrow( + 'Invalid date', + ); + }); + + it('should throw on invalid date value', () => { + expect(() => + normalizeValueForDatabase('2024-99-99T10:30:00.123Z', 'date', 'sqlite'), + ).toThrow('Invalid date'); + }); + + it('should throw for unsupported value types', () => { + expect(() => normalizeValueForDatabase(true, 'date')).toThrow( + 'Expected Date object or ISO date string', + ); + expect(() => normalizeValueForDatabase(false, 'date')).toThrow( + 'Expected Date object or ISO date string', + ); + expect(() => normalizeValueForDatabase(123, 'date')).toThrow( + 'Expected Date object or ISO date string', + ); + }); + }); + }); + + describe('toSqliteGlobFromPercent', () => { + it('should convert % to *', () => { + expect(toSqliteGlobFromPercent('test%')).toBe('test*'); + expect(toSqliteGlobFromPercent('%test')).toBe('*test'); + expect(toSqliteGlobFromPercent('%test%')).toBe('*test*'); + }); + + it('should escape [ with [[]', () => { + expect(toSqliteGlobFromPercent('test[abc')).toBe('test[[]abc'); + }); + + it('should escape ] with []]', () => { + expect(toSqliteGlobFromPercent('test]abc')).toBe('test[]]abc'); + }); + + it('should escape * with [*]', () => { + expect(toSqliteGlobFromPercent('test*abc')).toBe('test[*]abc'); + }); + + it('should escape ? with [?]', () => { + expect(toSqliteGlobFromPercent('test?abc')).toBe('test[?]abc'); + }); + + it('should handle multiple special characters', () => { + expect(toSqliteGlobFromPercent('%test*[abc]?%')).toBe('*test[*][[]abc[]][?]*'); + }); + + it('should handle empty string', () => { + expect(toSqliteGlobFromPercent('')).toBe(''); + }); + + it('should keep regular characters unchanged', () => { + expect(toSqliteGlobFromPercent('abc123')).toBe('abc123'); + expect(toSqliteGlobFromPercent('test_value')).toBe('test_value'); + }); + }); }); diff --git a/packages/cli/src/modules/data-table/data-table-rows.repository.ts b/packages/cli/src/modules/data-table/data-table-rows.repository.ts index db05f12448b..65681ccf096 100644 --- a/packages/cli/src/modules/data-table/data-table-rows.repository.ts +++ b/packages/cli/src/modules/data-table/data-table-rows.repository.ts @@ -21,6 +21,7 @@ import { DataTableInsertRowsReturnType, DataTableInsertRowsResult, DataTableRowReturnWithState, + DataTableRawRowReturn, } from 'n8n-workflow'; import { DataTableColumn } from './data-table-column.entity'; @@ -30,7 +31,7 @@ import { extractInsertedIds, extractReturningData, normalizeRows, - normalizeValue, + normalizeValueForDatabase, quoteIdentifier, toSqliteGlobFromPercent, toTableName, @@ -56,7 +57,6 @@ function getConditionAndParams( index: number, dbType: DataSourceOptions['type'], tableReference?: string, - columns?: DataTableColumn[], ): [string, Record] { const paramName = `filter_${index}`; const columnRef = tableReference @@ -72,9 +72,8 @@ function getConditionAndParams( } } - // Find the column type to normalize the value consistently - const columnInfo = columns?.find((col) => col.name === filter.columnName); - const value = columnInfo ? normalizeValue(filter.value, columnInfo?.type, dbType) : filter.value; + // For filters, we let TypeORM handle date conversion through parameterized queries. + const value = filter.value; // Handle operators that map directly to SQL operators const operators: Record = { @@ -197,7 +196,7 @@ export class DataTableRowsRepository { const column = columns[h]; // Fill missing columns with null values to support partial data insertion const value = rows[j][column.name] ?? null; - insertArray[h] = normalizeValue(value, column.type, dbType); + insertArray[h] = normalizeValueForDatabase(value, column.type, dbType); } completeRows[j - start] = insertArray; } @@ -255,7 +254,11 @@ export class DataTableRowsRepository { if (!(column.name in completeRow)) { completeRow[column.name] = null; } - completeRow[column.name] = normalizeValue(completeRow[column.name], column.type, dbType); + completeRow[column.name] = normalizeValueForDatabase( + completeRow[column.name], + column.type, + dbType, + ); } const query = em.createQueryBuilder().insert().into(table).values(completeRow); @@ -330,11 +333,11 @@ export class DataTableRowsRepository { affectedRows = await this.getAffectedRowsForUpdate(dataTableId, filter, columns, true, trx); } - setData.updatedAt = normalizeValue(new Date(), 'date', dbType); + setData.updatedAt = normalizeValueForDatabase(new Date(), 'date', dbType); const query = em.createQueryBuilder().update(table); // Some DBs (like SQLite) don't allow using table aliases as column prefixes in UPDATE statements - this.applyFilters(query, filter, undefined, columns); + this.applyFilters(query, filter, undefined); query.set(setData); if (useReturning && returnData) { @@ -436,7 +439,7 @@ export class DataTableRowsRepository { // Just delete and return true const query = em.createQueryBuilder().delete().from(table, 'dataTable'); if (filter) { - this.applyFilters(query, filter, undefined, columns); + this.applyFilters(query, filter, undefined); } await query.execute(); @@ -449,10 +452,10 @@ export class DataTableRowsRepository { const selectQuery = em.createQueryBuilder().select('*').from(table, 'dataTable'); if (filter) { - this.applyFilters(selectQuery, filter, 'dataTable', columns); + this.applyFilters(selectQuery, filter, 'dataTable'); } - const rawRows = await selectQuery.getRawMany(); + const rawRows = await selectQuery.getRawMany(); affectedRows = normalizeRows(rawRows, columns); } @@ -473,7 +476,7 @@ export class DataTableRowsRepository { } if (filter) { - this.applyFilters(deleteQuery, filter, undefined, columns); + this.applyFilters(deleteQuery, filter, undefined); } const result = await deleteQuery.execute(); @@ -497,7 +500,7 @@ export class DataTableRowsRepository { const table = toTableName(dataTableId); const selectColumns = idsOnly ? 'id' : '*'; const selectQuery = em.createQueryBuilder().select(selectColumns).from(table, 'dataTable'); - this.applyFilters(selectQuery, filter, 'dataTable', columns); + this.applyFilters(selectQuery, filter, 'dataTable'); const rawRows: DataTableRowsReturn = await selectQuery.getRawMany(); if (idsOnly) { @@ -516,7 +519,7 @@ export class DataTableRowsRepository { const setData = { ...data }; for (const column of columns) { if (column.name in setData) { - setData[column.name] = normalizeValue(setData[column.name], column.type, dbType); + setData[column.name] = normalizeValueForDatabase(setData[column.name], column.type, dbType); } } return setData; @@ -558,14 +561,13 @@ export class DataTableRowsRepository { async getManyAndCount( dataTableId: string, dto: ListDataTableContentQueryDto, - columns?: DataTableColumn[], trx?: EntityManager, ) { return await withTransaction( this.dataSource.manager, trx, async (em) => { - const [countQuery, query] = this.getManyQuery(dataTableId, dto, em, columns); + const [countQuery, query] = this.getManyQuery(dataTableId, dto, em); const data: DataTableRowsReturn = await query.select('*').getRawMany(); const countResult = await countQuery.select('COUNT(*) as count').getRawOne<{ count: number | string | null; @@ -606,7 +608,7 @@ export class DataTableRowsRepository { .select(selectColumns) .from(table, 'dataTable') .where({ id: In(ids) }) - .getRawMany(); + .getRawMany(); return normalizeRows(rows, columns); }, @@ -618,14 +620,13 @@ export class DataTableRowsRepository { dataTableId: string, dto: ListDataTableContentQueryDto, em: EntityManager, - columns?: DataTableColumn[], ): [QueryBuilder, QueryBuilder] { const query = em.createQueryBuilder(); const tableReference = 'dataTable'; query.from(toTableName(dataTableId), tableReference); if (dto.filter) { - this.applyFilters(query, dto.filter, tableReference, columns); + this.applyFilters(query, dto.filter, tableReference); } const countQuery = query.clone().select('COUNT(*)'); this.applySorting(query, dto); @@ -638,14 +639,13 @@ export class DataTableRowsRepository { query: SelectQueryBuilder | UpdateQueryBuilder | DeleteQueryBuilder, filter: DataTableFilter, tableReference?: string, - columns?: DataTableColumn[], ): void { const filters = filter.filters ?? []; const filterType = filter.type ?? 'and'; const dbType = this.dataSource.options.type; const conditionsAndParams = filters.map((filter, i) => - getConditionAndParams(filter, i, dbType, tableReference, columns), + getConditionAndParams(filter, i, dbType, tableReference), ); if (conditionsAndParams.length === 1) { diff --git a/packages/cli/src/modules/data-table/data-table.service.ts b/packages/cli/src/modules/data-table/data-table.service.ts index f95ebff72b9..cab418f614a 100644 --- a/packages/cli/src/modules/data-table/data-table.service.ts +++ b/packages/cli/src/modules/data-table/data-table.service.ts @@ -155,13 +155,12 @@ export class DataTableService { return await this.dataTableColumnRepository.manager.transaction(async (em) => { const columns = await this.dataTableColumnRepository.getColumns(dataTableId, em); - if (dto.filter) { - this.validateAndTransformFilters(dto.filter, columns); - } + const transformedDto = dto.filter + ? { ...dto, filter: this.validateAndTransformFilters(dto.filter, columns) } + : dto; const result = await this.dataTableRowsRepository.getManyAndCount( dataTableId, - dto, - columns, + transformedDto, em, ); return { @@ -194,11 +193,11 @@ export class DataTableService { const result = await this.dataTableColumnRepository.manager.transaction(async (trx) => { const columns = await this.dataTableColumnRepository.getColumns(dataTableId, trx); - this.validateRowsWithColumns(rows, columns); + const transformedRows = this.validateAndTransformRows(rows, columns); return await this.dataTableRowsRepository.insertRows( dataTableId, - rows, + transformedRows, columns, returnType, trx, @@ -243,13 +242,13 @@ export class DataTableService { const result = await this.dataTableColumnRepository.manager.transaction(async (trx) => { const columns = await this.dataTableColumnRepository.getColumns(dataTableId, trx); - this.validateUpdateParams(dto, columns); + const { data, filter } = this.validateAndTransformUpdateParams(dto, columns); if (dryRun) { return await this.dataTableRowsRepository.dryRunUpsertRow( dataTableId, - dto.data, - dto.filter, + data, + filter, columns, trx, ); @@ -257,8 +256,8 @@ export class DataTableService { const updated = await this.dataTableRowsRepository.updateRows( dataTableId, - dto.data, - dto.filter, + data, + filter, columns, true, trx, @@ -271,7 +270,7 @@ export class DataTableService { // No rows were updated, so insert a new one const inserted = await this.dataTableRowsRepository.insertRows( dataTableId, - [dto.data], + [data], columns, returnData ? 'all' : 'id', trx, @@ -286,10 +285,10 @@ export class DataTableService { return result; } - validateUpdateParams( + validateAndTransformUpdateParams( { filter, data }: Pick, columns: DataTableColumn[], - ) { + ): { data: DataTableRow; filter: DataTableFilter } { if (columns.length === 0) { throw new DataTableValidationError( 'No columns found for this data table or data table not found', @@ -303,8 +302,10 @@ export class DataTableService { throw new DataTableValidationError('Data columns must not be empty'); } - this.validateRowsWithColumns([data], columns, false); - this.validateAndTransformFilters(filter, columns); + const [transformedData] = this.validateAndTransformRows([data], columns, false); + const transformedFilter = this.validateAndTransformFilters(filter, columns); + + return { data: transformedData, filter: transformedFilter }; } async updateRows( @@ -340,13 +341,13 @@ export class DataTableService { const result = await this.dataTableColumnRepository.manager.transaction(async (trx) => { const columns = await this.dataTableColumnRepository.getColumns(dataTableId, trx); - this.validateUpdateParams(dto, columns); + const { data, filter } = this.validateAndTransformUpdateParams(dto, columns); if (dryRun) { return await this.dataTableRowsRepository.dryRunUpdateRows( dataTableId, - dto.data, - dto.filter, + data, + filter, columns, trx, ); @@ -354,8 +355,8 @@ export class DataTableService { return await this.dataTableRowsRepository.updateRows( dataTableId, - dto.data, - dto.filter, + data, + filter, columns, returnData, trx, @@ -408,12 +409,12 @@ export class DataTableService { ); } - this.validateAndTransformFilters(dto.filter, columns); + const transformedFilter = this.validateAndTransformFilters(dto.filter, columns); return await this.dataTableRowsRepository.deleteRows( dataTableId, columns, - dto.filter, + transformedFilter, returnData, dryRun, trx, @@ -427,11 +428,12 @@ export class DataTableService { return result; } - private validateRowsWithColumns( + private validateAndTransformRows( rows: DataTableRows, columns: Array<{ name: string; type: DataTableColumnType }>, includeSystemColumns = false, - ): void { + skipDateTransform = false, + ): DataTableRows { // Include system columns like 'id' if requested const allColumns = includeSystemColumns ? [ @@ -444,26 +446,38 @@ export class DataTableService { : columns; const columnNames = new Set(allColumns.map((x) => x.name)); const columnTypeMap = new Map(allColumns.map((x) => [x.name, x.type])); - for (const row of rows) { + + return rows.map((row) => { + const transformedRow: DataTableRow = {}; const keys = Object.keys(row); for (const key of keys) { if (!columnNames.has(key)) { throw new DataTableValidationError(`unknown column name '${key}'`); } - this.validateCell(row, key, columnTypeMap); + transformedRow[key] = this.validateAndTransformCell( + row[key], + key, + columnTypeMap, + skipDateTransform, + ); } - } + return transformedRow; + }); } - private validateCell(row: DataTableRow, key: string, columnTypeMap: Map) { - const cell = row[key]; - if (cell === null) return; + private validateAndTransformCell( + cell: DataTableColumnJsType, + key: string, + columnTypeMap: Map, + skipDateTransform = false, + ): DataTableColumnJsType { + if (cell === null) return null; const columnType = columnTypeMap.get(key); - if (!columnType) return; + if (!columnType) return cell; const fieldType = columnTypeToFieldType[columnType]; - if (!fieldType) return; + if (!fieldType) return cell; const validationResult = validateFieldType(key, cell, fieldType, { strict: false, // Allow type coercion (e.g., string numbers to numbers) @@ -476,12 +490,14 @@ export class DataTableService { ); } - // Special handling for date type to convert from luxon DateTime to ISO string if (columnType === 'date') { + if (skipDateTransform && cell instanceof Date) { + return cell; + } try { - const dateInISO = (validationResult.newValue as DateTime).toISO(); - row[key] = dateInISO; - return; + // Convert to UTC to ensure consistent timezone handling + const dateInISO = (validationResult.newValue as DateTime).toUTC().toISO(); + return dateInISO; } catch { throw new DataTableValidationError( `value '${String(cell)}' does not match column type 'date'`, @@ -489,7 +505,7 @@ export class DataTableService { } } - row[key] = validationResult.newValue as DataTableColumnJsType; + return validationResult.newValue as DataTableColumnJsType; } private async validateDataTableExists(dataTableId: string, projectId: string) { @@ -534,8 +550,9 @@ export class DataTableService { private validateAndTransformFilters( filterObject: DataTableFilter, columns: DataTableColumn[], - ): void { - this.validateRowsWithColumns( + ): DataTableFilter { + // Skip date transformation for filters - TypeORM needs Date objects for parameterized queries + const transformedRows = this.validateAndTransformRows( filterObject.filters.map((f) => { return { [f.columnName]: f.value, @@ -543,34 +560,43 @@ export class DataTableService { }), columns, true, + true, ); - for (const filter of filterObject.filters) { + const transformedFilters = filterObject.filters.map((filter, index) => { + const transformedValue = transformedRows[index][filter.columnName]; + if (['like', 'ilike'].includes(filter.condition)) { - if (filter.value === null || filter.value === undefined) { + if (transformedValue === null || transformedValue === undefined) { throw new DataTableValidationError( `${filter.condition.toUpperCase()} filter value cannot be null or undefined`, ); } - if (typeof filter.value !== 'string') { + if (typeof transformedValue !== 'string') { throw new DataTableValidationError( `${filter.condition.toUpperCase()} filter value must be a string`, ); } - if (!filter.value.includes('%')) { - filter.value = `%${filter.value}%`; - } + const valueWithWildcards = transformedValue.includes('%') + ? transformedValue + : `%${transformedValue}%`; + + return { ...filter, value: valueWithWildcards }; } if (['gt', 'gte', 'lt', 'lte'].includes(filter.condition)) { - if (filter.value === null || filter.value === undefined) { + if (transformedValue === null || transformedValue === undefined) { throw new DataTableValidationError( `${filter.condition.toUpperCase()} filter value cannot be null or undefined`, ); } } - } + + return { ...filter, value: transformedValue }; + }); + + return { ...filterObject, filters: transformedFilters }; } private async validateDataTableSize() { diff --git a/packages/cli/src/modules/data-table/utils/sql-utils.ts b/packages/cli/src/modules/data-table/utils/sql-utils.ts index 59badbb1ee4..35e58df82ee 100644 --- a/packages/cli/src/modules/data-table/utils/sql-utils.ts +++ b/packages/cli/src/modules/data-table/utils/sql-utils.ts @@ -7,14 +7,20 @@ import { GlobalConfig } from '@n8n/config'; import { DslColumn } from '@n8n/db'; import { Container } from '@n8n/di'; import type { DataSourceOptions } from '@n8n/typeorm'; -import type { DataTableColumnJsType, DataTableRowReturn, DataTableRowsReturn } from 'n8n-workflow'; -import { UnexpectedError } from 'n8n-workflow'; - -import { NotFoundError } from '@/errors/response-errors/not-found.error'; +import type { + DataTableColumnJsType, + DataTableColumnType, + DataTableRawRowsReturn, + DataTableRowReturn, + DataTableRowsReturn, +} from 'n8n-workflow'; +import { DATA_TABLE_SYSTEM_COLUMN_TYPE_MAP, UnexpectedError } from 'n8n-workflow'; import type { DataTableColumn } from '../data-table-column.entity'; import type { DataTableUserTableName } from '../data-table.types'; +import { NotFoundError } from '@/errors/response-errors/not-found.error'; + export function toDslColumns(columns: DataTableCreateColumnSchema[]): DslColumn[] { return columns.map((col) => { const name = new DslColumn(col.name.trim()); @@ -183,18 +189,46 @@ export function extractInsertedIds(raw: unknown, dbType: DataSourceOptions['type } } -export function normalizeRows(rows: DataTableRowsReturn, columns: DataTableColumn[]) { - // we need to normalize system dates as well - const systemColumns = [ - { name: 'createdAt', type: 'date' }, - { name: 'updatedAt', type: 'date' }, - ]; +// Convert date objects or strings to dates in UTC +function normalizeDate(value: DataTableColumnJsType): Date | null { + if (value instanceof Date) return value; - const typeMap = new Map([...columns, ...systemColumns].map((col) => [col.name, col.type])); + if (typeof value === 'string') { + // sqlite returns date strings without timezone information, but we store them as UTC + const parsed = new Date(value.endsWith('Z') ? value : value + 'Z'); + if (!isNaN(parsed.getTime())) return parsed; + } + + if (typeof value === 'number') { + const parsed = new Date(value); + if (!isNaN(parsed.getTime())) return parsed; + } + + return null; +} + +// Normalize rows fetched from the database according to the column types +export function normalizeRows( + rows: DataTableRawRowsReturn, + columns: DataTableColumn[], +): DataTableRowsReturn { + const typeMap: Record = { + ...Object.fromEntries(columns.map((col) => [col.name, col.type])), + // we need to normalize system dates as well + ...DATA_TABLE_SYSTEM_COLUMN_TYPE_MAP, + }; return rows.map((row) => { - const normalized = { ...row }; - for (const [key, value] of Object.entries(row)) { - const type = typeMap.get(key); + const { id, createdAt, updatedAt, ...rest } = row; + + const normalized: DataTableRowReturn = { + ...rest, + id, + createdAt: normalizeDate(createdAt) ?? new Date(), // fallback should not happen + updatedAt: normalizeDate(updatedAt) ?? new Date(), // fallback should not happen + }; + + for (const [key, value] of Object.entries(rest)) { + const type = typeMap[key]; if (type === 'boolean') { // Convert boolean values to true/false @@ -206,63 +240,61 @@ export function normalizeRows(rows: DataTableRowsReturn, columns: DataTableColum normalized[key] = false; } } + if (type === 'date' && value !== null && value !== undefined) { - // Convert date objects or strings to dates in UTC - let dateObj: Date | null = null; - - if (value instanceof Date) { - dateObj = value; - } else if (typeof value === 'string') { - // sqlite returns date strings without timezone information, but we store them as UTC - const parsed = new Date(value.endsWith('Z') ? value : value + 'Z'); - if (!isNaN(parsed.getTime())) { - dateObj = parsed; - } - } else if (typeof value === 'number') { - const parsed = new Date(value); - if (!isNaN(parsed.getTime())) { - dateObj = parsed; - } - } - - normalized[key] = dateObj ?? value; + normalized[key] = normalizeDate(value) ?? value; // fallback to original value } } return normalized; }); } -function formatDateForDatabase(date: Date, dbType?: DataSourceOptions['type']): string { - // MySQL/MariaDB DATETIME format doesn't accept ISO strings with 'Z' timezone - if (dbType === 'mysql' || dbType === 'mariadb') { +/** + * Format a date value (Date object or ISO string) for database storage. + * Converts to database-specific format. + */ +function formatDateForDatabase( + value: DataTableColumnJsType, + dbType?: DataSourceOptions['type'], +): string { + let date: Date; + + if (value instanceof Date) { + date = value; + } else if (typeof value === 'string') { + date = new Date(value); + } else { + throw new UnexpectedError( + `Expected Date object or ISO date string, got ${typeof value}: ${String(value)}`, + ); + } + + if (isNaN(date.getTime())) { + throw new UnexpectedError(`Invalid date: ${String(value)}`); + } + + // These dbs use DATETIME format without 'T' and 'Z' + if (dbType && ['sqlite', 'sqlite-pooled', 'mysql', 'mariadb'].includes(dbType)) { return date.toISOString().replace('T', ' ').replace('Z', ''); } - // PostgreSQL and SQLite accept ISO strings + return date.toISOString(); } -export function normalizeValue( +/** + * Normalize a value for database operations based on column type. + * For date columns, accepts both Date objects and ISO date strings. + * Converts them to database-specific format. + */ +export function normalizeValueForDatabase( value: DataTableColumnJsType, columnType: string | undefined, dbType?: DataSourceOptions['type'], ): DataTableColumnJsType { - if (columnType !== 'date' || value === null || value === undefined) { - return value; - } - - // Convert Date objects to appropriate string format for database parameter binding - if (value instanceof Date) { + if (columnType === 'date' && value !== null) { return formatDateForDatabase(value, dbType); } - if (typeof value === 'string') { - const date = new Date(value); - if (!isNaN(date.getTime())) { - // Convert parsed date strings to appropriate format - return formatDateForDatabase(date, dbType); - } - } - return value; } diff --git a/packages/testing/playwright/tests/ui/data-table-details.spec.ts b/packages/testing/playwright/tests/ui/data-table-details.spec.ts index deefb0197be..8184bffe554 100644 --- a/packages/testing/playwright/tests/ui/data-table-details.spec.ts +++ b/packages/testing/playwright/tests/ui/data-table-details.spec.ts @@ -346,7 +346,8 @@ test.describe('Data Table details view', () => { expect(initialName).not.toEqual(newName); }); - test('Should filter correctly using column filters', async ({ n8n }) => { + // eslint-disable-next-line playwright/no-skipped-test + test.skip('Should filter correctly using column filters', async ({ n8n }) => { await expect(n8n.dataTableDetails.getPageWrapper()).toBeVisible(); await n8n.dataTableDetails.setPageSize('10'); diff --git a/packages/workflow/src/data-table.types.ts b/packages/workflow/src/data-table.types.ts index 94833afdb7b..bd7e60ab3c3 100644 --- a/packages/workflow/src/data-table.types.ts +++ b/packages/workflow/src/data-table.types.ts @@ -92,13 +92,26 @@ export const DATA_TABLE_SYSTEM_COLUMN_TYPE_MAP: Record; export type DataTableRows = DataTableRow[]; + +// Raw database results (before normalization) +export type DataTableRawRowReturn = DataTableRow & DataTableRawRowReturnBase; +export type DataTableRawRowsReturn = DataTableRawRowReturn[]; + export type DataTableRowReturn = DataTableRow & DataTableRowReturnBase; export type DataTableRowsReturn = DataTableRowReturn[]; From c2dd9f61b4623247e4dccb17318db475f9ac187f Mon Sep 17 00:00:00 2001 From: Alex Grozav Date: Fri, 10 Oct 2025 10:53:37 +0300 Subject: [PATCH 04/81] refactor(editor): Extract `contextMenu` into features (no-changelog) (#20582) --- .../editor-ui/src/components/FocusPanel.vue | 2 +- .../src/features/canvas/components/Canvas.vue | 8 +- .../canvas/components/WorkflowCanvas.vue | 2 +- .../components/elements/nodes/CanvasNode.vue | 2 +- .../ExperimentalNodeDetailsDrawer.vue | 5 +- .../contextMenu/components}/ContextMenu.vue | 4 +- .../__snapshots__/useContextMenu.test.ts.snap | 1257 +++++++++++++++++ .../composables/useContextMenu.test.ts | 2 +- .../composables/useContextMenu.ts | 2 +- .../composables/useContextMenuItems.ts | 4 +- .../frontend/editor-ui/src/views/NodeView.vue | 2 +- 11 files changed, 1275 insertions(+), 15 deletions(-) rename packages/frontend/editor-ui/src/{components/ContextMenu => features/ui/contextMenu/components}/ContextMenu.vue (91%) create mode 100644 packages/frontend/editor-ui/src/features/ui/contextMenu/composables/__snapshots__/useContextMenu.test.ts.snap rename packages/frontend/editor-ui/src/{ => features/ui/contextMenu}/composables/useContextMenu.test.ts (99%) rename packages/frontend/editor-ui/src/{ => features/ui/contextMenu}/composables/useContextMenu.ts (96%) rename packages/frontend/editor-ui/src/{ => features/ui/contextMenu}/composables/useContextMenuItems.ts (98%) diff --git a/packages/frontend/editor-ui/src/components/FocusPanel.vue b/packages/frontend/editor-ui/src/components/FocusPanel.vue index c9ef264df7b..0aaf77fe672 100644 --- a/packages/frontend/editor-ui/src/components/FocusPanel.vue +++ b/packages/frontend/editor-ui/src/components/FocusPanel.vue @@ -46,7 +46,7 @@ import { useNDVStore } from '@/stores/ndv.store'; import { useVueFlow } from '@vue-flow/core'; import ExperimentalFocusPanelHeader from '@/features/canvas/experimental/components/ExperimentalFocusPanelHeader.vue'; import { useTelemetryContext } from '@/composables/useTelemetryContext'; -import { type ContextMenuAction } from '@/composables/useContextMenuItems'; +import { type ContextMenuAction } from '@/features/ui/contextMenu/composables/useContextMenuItems'; import { type CanvasNode, CanvasNodeRenderType } from '@/features/canvas/canvas.types'; import { useCanvasOperations } from '@/composables/useCanvasOperations'; diff --git a/packages/frontend/editor-ui/src/features/canvas/components/Canvas.vue b/packages/frontend/editor-ui/src/features/canvas/components/Canvas.vue index 0cabae74f31..97c62125deb 100644 --- a/packages/frontend/editor-ui/src/features/canvas/components/Canvas.vue +++ b/packages/frontend/editor-ui/src/features/canvas/components/Canvas.vue @@ -1,11 +1,11 @@ diff --git a/packages/frontend/editor-ui/src/components/Node/NodeCreator/Panel/NodesListPanel.vue b/packages/frontend/editor-ui/src/components/Node/NodeCreator/Panel/NodesListPanel.vue index 8253f4eacfe..ae3fbaceb57 100644 --- a/packages/frontend/editor-ui/src/components/Node/NodeCreator/Panel/NodesListPanel.vue +++ b/packages/frontend/editor-ui/src/components/Node/NodeCreator/Panel/NodesListPanel.vue @@ -194,7 +194,6 @@ function onBackButton() { :circle="false" :show-tooltip="false" :size="20" - :use-updated-icons="true" />

@@ -286,6 +285,7 @@ function onBackButton() { } .nodeIcon { --node-icon-size: 20px; + --node-icon-color: var(--color--text); margin-right: var(--spacing--sm); } .renderedItems { diff --git a/packages/frontend/editor-ui/src/components/NodeSettingsHint.vue b/packages/frontend/editor-ui/src/components/NodeSettingsHint.vue index f6a3eb9a264..d81b0931ff7 100644 --- a/packages/frontend/editor-ui/src/components/NodeSettingsHint.vue +++ b/packages/frontend/editor-ui/src/components/NodeSettingsHint.vue @@ -71,8 +71,7 @@ const activeSettings = computed(() => {

- - +
{{ setting.message }} diff --git a/packages/frontend/editor-ui/src/components/PanelDragButtonV2.vue b/packages/frontend/editor-ui/src/components/PanelDragButtonV2.vue index 4e45f6be2d9..025fc709b95 100644 --- a/packages/frontend/editor-ui/src/components/PanelDragButtonV2.vue +++ b/packages/frontend/editor-ui/src/components/PanelDragButtonV2.vue @@ -1,4 +1,5 @@ + + + + diff --git a/packages/frontend/editor-ui/src/features/chatHub/components/CredentialSelectorModal.vue b/packages/frontend/editor-ui/src/features/chatHub/components/CredentialSelectorModal.vue index cf06aaf5033..2d6acdb633c 100644 --- a/packages/frontend/editor-ui/src/features/chatHub/components/CredentialSelectorModal.vue +++ b/packages/frontend/editor-ui/src/features/chatHub/components/CredentialSelectorModal.vue @@ -7,6 +7,7 @@ import type { ICredentialsResponse } from '@/Interface'; import { createEventBus } from '@n8n/utils/event-bus'; import { PROVIDER_CREDENTIAL_TYPE_MAP, type ChatHubProvider } from '@n8n/api-types'; import { providerDisplayNames } from '@/features/chatHub/constants'; +import CredentialIcon from '@/components/CredentialIcon.vue'; const props = defineProps<{ provider: ChatHubProvider; @@ -57,7 +58,14 @@ function onCancel() { min-height="250px" >