diff --git a/packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts b/packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts new file mode 100644 index 00000000000..4e3f370cd20 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts @@ -0,0 +1,94 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema'; + +const FilterConditionSchema = z.union([z.literal('eq'), z.literal('neq')]); +export type ListDataStoreContentFilterConditionType = z.infer; + +const filterRecord = z.object({ + columnName: dataStoreColumnNameSchema, + condition: FilterConditionSchema.default('eq'), + value: z.union([z.string(), z.number(), z.boolean(), z.date()]), +}); + +const chainedFilterSchema = z.union([z.literal('and'), z.literal('or')]); + +export type ListDataStoreContentFilter = z.infer; + +// --------------------- +// Parameter Validators +// --------------------- + +// Filter parameter validation +const filterValidator = z.object({ + type: chainedFilterSchema.default('and'), + filters: z.array(filterRecord).default([]), +}); + +// Skip parameter validation +const skipValidator = z + .string() + .optional() + .transform((val) => (val ? parseInt(val, 10) : 0)) + .refine((val) => !isNaN(val), { + message: 'Skip must be a valid number', + }); + +// Take parameter validation +const takeValidator = z + .string() + .optional() + .transform((val) => (val ? parseInt(val, 10) : 10)) + .refine((val) => !isNaN(val), { + message: 'Take must be a valid number', + }); + +// SortBy parameter validation +const sortByValidator = z + .string() + .optional() + .transform((val, ctx) => { + if (val === undefined) return val; + + if (!val.includes(':')) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid sort format, expected :', + path: ['sort'], + }); + return z.NEVER; + } + + let [column, direction] = val.split(':'); + + try { + column = dataStoreColumnNameSchema.parse(column); + } catch { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid sort columnName', + path: ['sort'], + }); + return z.NEVER; + } + + direction = direction?.toUpperCase(); + if (direction !== 'ASC' && direction !== 'DESC') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid sort direction', + path: ['sort'], + }); + + return z.NEVER; + } + return [column, direction] as const; + }); + +export class ListDataStoreContentQueryDto extends Z.class({ + filter: filterValidator, + skip: skipValidator, + take: takeValidator, + sortBy: sortByValidator, +}) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts b/packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts index a5efd930375..cdd4ebc9831 100644 --- a/packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts +++ b/packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts @@ -76,41 +76,6 @@ const takeValidator = z message: 'Take must be a valid number', }); -// Select parameter validation -const selectFieldsValidator = z.array(z.enum(VALID_SELECT_FIELDS)); -const selectValidator = z - .string() - .optional() - .transform((val, ctx) => { - if (!val) return undefined; - try { - const parsed: unknown = JSON.parse(val); - try { - const selectFields = selectFieldsValidator.parse(parsed); - if (selectFields.length === 0) return undefined; - type SelectField = (typeof VALID_SELECT_FIELDS)[number]; - return selectFields.reduce>( - (acc, field) => ({ ...acc, [field]: true }), - {} as Record, - ); - } catch (e) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `Invalid select fields. Valid fields are: ${VALID_SELECT_FIELDS.join(', ')}`, - path: ['select'], - }); - return z.NEVER; - } - } catch (e) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: 'Invalid select format', - path: ['select'], - }); - return z.NEVER; - } - }); - // SortBy parameter validation const sortByValidator = z .enum(VALID_SORT_OPTIONS, { message: `sortBy must be one of: ${VALID_SORT_OPTIONS.join(', ')}` }) @@ -120,6 +85,5 @@ export class ListDataStoreQueryDto extends Z.class({ filter: filterValidator, skip: skipValidator, take: takeValidator, - select: selectValidator, sortBy: sortByValidator, }) {} diff --git a/packages/@n8n/api-types/src/dto/index.ts b/packages/@n8n/api-types/src/dto/index.ts index 239872376c1..209480aecf6 100644 --- a/packages/@n8n/api-types/src/dto/index.ts +++ b/packages/@n8n/api-types/src/dto/index.ts @@ -83,6 +83,7 @@ export { OidcConfigDto } from './oidc/config.dto'; export { CreateDataStoreDto } from './data-store/create-data-store.dto'; export { RenameDataStoreDto } from './data-store/rename-data-store.dto'; export { ListDataStoreQueryDto } from './data-store/list-data-store-query.dto'; +export { ListDataStoreContentQueryDto } from './data-store/list-data-store-content-query.dto'; export { CreateDataStoreColumnDto } from './data-store/create-data-store-column.dto'; export { AddDatastoreRecordsDto } from './data-store/add-data-store-records.dto'; export { AddDataStoreColumnDto } from './data-store/add-data-store-column.dto'; diff --git a/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts b/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts index c30b255e86d..90676f23249 100644 --- a/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts +++ b/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts @@ -1,4 +1,4 @@ -import type { AddDataStoreColumnDto, DataStore } from '@n8n/api-types'; +import type { AddDataStoreColumnDto } from '@n8n/api-types'; import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils'; import { Project } from '@n8n/db'; import { Container } from '@n8n/di'; @@ -429,4 +429,126 @@ describe('dataStore', () => { expect(sizeBytesDesc[0].map((x) => x.name)).toEqual(['ds0', 'ds1', 'ds2']); }); }); + describe('appendRows', () => { + it('appends a row to an existing table', async () => { + // ARRANGE + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } }); + + // ACT + const result = await dataStoreService.appendRows(dataStore1.id, [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { c1: 4, c2: false, c3: new Date(), c4: 'hello!' }, + { c1: 5, c2: true, c3: new Date(), c4: 'hello.' }, + ]); + + // ASSERT + expect(result).toBe(true); + }); + + it('rejects a mismatched row with extra column', async () => { + // ARRANGE + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } }); + + // ACT + const result = await dataStoreService.appendRows(dataStore1.id, [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { cWrong: 3, c1: 4, c2: true, c3: new Date(), c4: 'hello?' }, + ]); + + // ASSERT + expect(result).toBe('mismatched key count'); + }); + it('rejects a mismatched row with missing column', async () => { + // ARRANGE + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } }); + + // ACT + const result = await dataStoreService.appendRows(dataStore1.id, [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { c2: true, c3: new Date(), c4: 'hello?' }, + ]); + + // ASSERT + expect(result).toBe('mismatched key count'); + }); + it('rejects a mismatched row with replaced column', async () => { + // ARRANGE + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } }); + + // ACT + const result = await dataStoreService.appendRows(dataStore1.id, [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, + ]); + + // ASSERT + expect(result).toBe('unknown column name'); + }); + it('rejects unknown data store id', async () => { + // ARRANGE + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } }); + + // ACT + const result = await dataStoreService.appendRows('this is not an id', [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, + ]); + + // ASSERT + expect(result).toBe('no columns found for id'); + }); + it('fails on type mismatch', async () => { + // ARRANGE + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } }); + + // ACT + const result = await dataStoreService.appendRows(dataStore1.id, [{ c1: 3 }, { c1: true }]); + + // ASSERT + expect(result).toBe('type mismatch'); + }); + }); + describe('getManyRowsAndCount', () => { + it('retrieves rows correctly', async () => { + // ARRANGE + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } }); + await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } }); + + await dataStoreService.appendRows(dataStore1.id, [ + { c1: 3, c2: true, c3: new Date(0), c4: 'hello?' }, + { c1: 4, c2: false, c3: new Date(1), c4: 'hello!' }, + { c1: 5, c2: true, c3: new Date(2), c4: 'hello.' }, + ]); + + // ACT + const result = await dataStoreService.getManyRowsAndCount(dataStore1.id, {}); + + // ASSERT + expect(result).toEqual([ + 3, + [ + { c1: 3, c2: 1, c3: '1970-01-01T00:00:00.000Z', c4: 'hello?', id: 1 }, + { c1: 4, c2: 0, c3: '1970-01-01T00:00:00.001Z', c4: 'hello!', id: 2 }, + { c1: 5, c2: 1, c3: '1970-01-01T00:00:00.002Z', c4: 'hello.', id: 3 }, + ], + ]); + }); + }); }); diff --git a/packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts b/packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts index bdc1bedcfc1..4e6f0bad554 100644 --- a/packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts +++ b/packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts @@ -1,4 +1,9 @@ -import { createUserTableQuery, addColumnQuery, deleteColumnQuery } from '../utils/sql-utils'; +import { + createUserTableQuery, + addColumnQuery, + deleteColumnQuery, + insertIntoQuery, +} from '../utils/sql-utils'; import type { DataStoreColumn } from '../data-store.types'; describe('sql-utils', () => { @@ -10,22 +15,21 @@ describe('sql-utils', () => { { name: 'age', type: 'number' }, ] satisfies DataStoreColumn[]; - const [query, columnNames] = createUserTableQuery(tableName, columns); - + const [query, columnNames] = createUserTableQuery(tableName, columns, 'sqlite'); expect(query).toBe( - 'CREATE TABLE IF NOT EXISTS data_store_user_abc (id VARCHAR(36) PRIMARY KEY, `?` TEXT, `?` FLOAT)', + 'CREATE TABLE IF NOT EXISTS data_store_user_abc (id INTEGER PRIMARY KEY AUTOINCREMENT , `name` TEXT, `age` FLOAT)', ); - expect(columnNames).toEqual(['name', 'age']); + expect(columnNames).toEqual([]); }); it('should generate a valid SQL query for creating a user table without columns', () => { const tableName = 'data_store_user_abc'; const columns: [] = []; - const [query, columnNames] = createUserTableQuery(tableName, columns); + const [query, columnNames] = createUserTableQuery(tableName, columns, 'postgres'); expect(query).toBe( - 'CREATE TABLE IF NOT EXISTS data_store_user_abc (id VARCHAR(36) PRIMARY KEY)', + 'CREATE TABLE IF NOT EXISTS data_store_user_abc (id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY )', ); expect(columnNames).toEqual([]); }); @@ -38,8 +42,8 @@ describe('sql-utils', () => { const [query, columnNames] = addColumnQuery(tableName, column); - expect(query).toBe('ALTER TABLE data_store_user_abc ADD `?` FLOAT'); - expect(columnNames).toEqual(['email']); + expect(query).toBe('ALTER TABLE data_store_user_abc ADD `email` FLOAT'); + expect(columnNames).toEqual([]); }); }); @@ -50,8 +54,43 @@ describe('sql-utils', () => { const [query, columnNames] = deleteColumnQuery(tableName, column); - expect(query).toBe('ALTER TABLE data_store_user_abc DROP COLUMN ?'); - expect(columnNames).toEqual(['email']); + expect(query).toBe('ALTER TABLE data_store_user_abc DROP COLUMN `email`'); + expect(columnNames).toEqual([]); + }); + }); + + describe('insertIntoQuery', () => { + it('should generate a valid SQL query for inserting rows into a table', () => { + const tableName = 'data_store_user_abc'; + const rows = [ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 25 }, + ]; + + const [query, parameters] = insertIntoQuery(tableName, rows); + + expect(query).toBe('INSERT INTO data_store_user_abc (name,age) VALUES (?,?),(?,?)'); + expect(parameters).toEqual(['Alice', 30, 'Bob', 25]); + }); + + it('should return an empty query and parameters when rows are empty', () => { + const tableName = 'data_store_user_abc'; + const rows: [] = []; + + const [query, parameters] = insertIntoQuery(tableName, rows); + + expect(query).toBe(''); + expect(parameters).toEqual([]); + }); + + it('should return an empty query and parameters when rows have no keys', () => { + const tableName = 'data_store_user_abc'; + const rows = [{}]; + + const [query, parameters] = insertIntoQuery(tableName, rows); + + expect(query).toBe(''); + expect(parameters).toEqual([]); }); }); }); diff --git a/packages/cli/src/modules/data-store/data-store-column.repository.ts b/packages/cli/src/modules/data-store/data-store-column.repository.ts index 38698e7d6f7..2c2dbff0e9b 100644 --- a/packages/cli/src/modules/data-store/data-store-column.repository.ts +++ b/packages/cli/src/modules/data-store/data-store-column.repository.ts @@ -11,6 +11,12 @@ export class DataStoreColumnRepository extends Repository super(DataStoreColumnEntity, dataSource.manager); } + async getColumns(rawDataStoreId: string) { + return await this.createQueryBuilder('dataStoreColumns') + .where(`dataStoreColumns.dataStoreId = '${rawDataStoreId}'`) + .getMany(); + } + async addColumn(dataStoreId: DataStoreUserTableName, column: DataStoreColumn) { await this.manager.query(...addColumnQuery(dataStoreId, column)); } diff --git a/packages/cli/src/modules/data-store/data-store-rows.repository.ts b/packages/cli/src/modules/data-store/data-store-rows.repository.ts new file mode 100644 index 00000000000..81b475b2e76 --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store-rows.repository.ts @@ -0,0 +1,104 @@ +import { ListDataStoreContentQueryDto } from '@n8n/api-types'; +import { Service } from '@n8n/di'; +import { BaseEntity, DataSource, Entity, SelectQueryBuilder } from '@n8n/typeorm'; +import { DataStoreRows, DataStoreUserTableName } from './data-store.types'; +import { insertIntoQuery } from './utils/sql-utils'; + +// type QueryBuilder = SelectQueryBuilder>; +type QueryBuilder = SelectQueryBuilder; +// type QueryBuilder = ReturnType; + +@Entity() +class Emptity extends BaseEntity {} + +function valueToSQL(value: ListDataStoreContentQueryDto['filter']['filters'][number]['value']) { + if (value instanceof Date) { + return value.toISOString(); // @Review: this feels bad + } + switch (typeof value) { + case 'number': + case 'boolean': + return `${value}`; + case 'string': + return `'${value}'`; + } +} + +function getConditionSQL(filter: ListDataStoreContentQueryDto['filter']['filters'][number]) { + switch (filter.condition) { + case 'eq': + return `dataStore.${filter.columnName} == ${valueToSQL(filter.value)}`; + case 'neq': + return `dataStore.${filter.columnName} != ${valueToSQL(filter.value)}`; + } +} + +@Service() +export class DataStoreRowsRepository { + constructor(private dataSource: DataSource) {} + + async appendRows(tableName: DataStoreUserTableName, rows: DataStoreRows) { + await this.dataSource.query(...insertIntoQuery(tableName, rows)); + return true; + } + + // ALL THE MANY STUFF + async getManyAndCount( + dataStoreId: DataStoreUserTableName, + dto: Partial, + ) { + const [countQuery, query] = this.getManyQuery(dataStoreId, dto); + const result = await query.getRawMany(); + const totalCount = (await countQuery.getRawOne())['COUNT(*)'] as number | undefined; + return [totalCount ?? -1, result]; + } + + getManyQuery( + dataStoreId: DataStoreUserTableName, + dto: Partial, + ): [QueryBuilder, QueryBuilder] { + const query = this.dataSource.createQueryBuilder(); + + query.from(dataStoreId, 'dataStore'); + this.applyFilters(query, dto); + const countQuery = query.clone().select('COUNT(*)'); + query.select('*'); + this.applySorting(query, dto); + this.applyPagination(query, dto); + + return [countQuery, query]; + } + + private applyFilters(query: QueryBuilder, dto: Partial): void { + const conditions = dto.filter?.filters.map(getConditionSQL) ?? []; + if (dto.filter?.type === 'and') { + for (const condition of conditions) { + query.andWhere(condition); + } + } else if (dto.filter?.type === 'or') { + for (const condition of conditions) { + query.orWhere(condition); + } + } + } + + private applySorting(query: QueryBuilder, dto: Partial): void { + if (!dto.sortBy) { + // query.orderBy('dataStore.', 'DESC'); + return; + } + + const [field, order] = dto.sortBy; + this.applySortingByField(query, field, order); + } + + private applySortingByField(query: QueryBuilder, field: string, direction: 'DESC' | 'ASC'): void { + console.log(field); + query.orderBy(`${field}`, direction); + } + + private applyPagination(query: QueryBuilder, dto: Partial): void { + query.skip(dto.skip); + query.take(dto.take); + } +} diff --git a/packages/cli/src/modules/data-store/data-store.controller.ts b/packages/cli/src/modules/data-store/data-store.controller.ts index 68eef2c0602..d288359a30a 100644 --- a/packages/cli/src/modules/data-store/data-store.controller.ts +++ b/packages/cli/src/modules/data-store/data-store.controller.ts @@ -2,6 +2,7 @@ import { AddDataStoreColumnDto, CreateDataStoreDto, DeleteDataStoreColumnDto, + ListDataStoreContentQueryDto, ListDataStoreQueryDto, RenameDataStoreDto, } from '@n8n/api-types'; @@ -70,9 +71,12 @@ export class DataStoreController { } @Get('/:dataStoreId', { skipAuth: true }) - async getDataStoreContent( + async getDataStoreRows( _req: AuthenticatedRequest, _res: Response, @Param('dataStoreId') dataStoreId: string, - ) {} + @Body dto: ListDataStoreContentQueryDto, + ) { + return await this.dataStoreService.getManyRowsAndCount(dataStoreId, dto); + } } diff --git a/packages/cli/src/modules/data-store/data-store.repository.ts b/packages/cli/src/modules/data-store/data-store.repository.ts index 2fa9d593df7..0270fda4790 100644 --- a/packages/cli/src/modules/data-store/data-store.repository.ts +++ b/packages/cli/src/modules/data-store/data-store.repository.ts @@ -12,7 +12,8 @@ export class DataStoreRepository extends Repository { } async createUserTable(tableName: DataStoreUserTableName, columns: DataStoreColumn[]) { - await this.manager.query(...createUserTableQuery(tableName, columns)); + const dbType = this.manager.connection.options.type; + await this.manager.query(...createUserTableQuery(tableName, columns, dbType)); } async deleteUserTable(tableName: DataStoreUserTableName) { @@ -89,7 +90,6 @@ export class DataStoreRepository extends Repository { field: string, direction: 'DESC' | 'ASC', ): void { - console.log(field, direction); if (field === 'name') { query .addSelect('LOWER(dataStore.name)', 'dataStore_name_lower') diff --git a/packages/cli/src/modules/data-store/data-store.service.ts b/packages/cli/src/modules/data-store/data-store.service.ts index 4a3164f0ae6..4a5ad67e10f 100644 --- a/packages/cli/src/modules/data-store/data-store.service.ts +++ b/packages/cli/src/modules/data-store/data-store.service.ts @@ -2,6 +2,7 @@ import { AddDataStoreColumnDto, CreateDataStoreDto, DeleteDataStoreColumnDto, + ListDataStoreContentQueryDto, } from '@n8n/api-types'; import { RenameDataStoreDto } from '@n8n/api-types/src/dto/data-store/rename-data-store.dto'; import { Logger } from '@n8n/backend-common'; @@ -9,8 +10,13 @@ import { Service } from '@n8n/di'; import { DataStoreConfig } from './data-store'; import { DataStoreColumnRepository } from './data-store-column.repository'; +import { DataStoreRowsRepository } from './data-store-rows.repository'; import { DataStoreRepository } from './data-store.repository'; -import type { DataStoreListOptions, DataStoreUserTableName } from './data-store.types'; +import type { + DataStoreListOptions, + DataStoreRows, + DataStoreUserTableName, +} from './data-store.types'; function toTableName(dataStoreId: string): DataStoreUserTableName { return `data_store_user_${dataStoreId}`; @@ -27,6 +33,7 @@ export class DataStoreService { constructor( private readonly dataStoreRepository: DataStoreRepository, private readonly dataStoreColumnRepository: DataStoreColumnRepository, + private readonly dataStoreRowsRepository: DataStoreRowsRepository, private readonly logger: Logger, private readonly config: DataStoreConfig, ) { @@ -69,26 +76,6 @@ export class DataStoreService { return dataStore; } - // async getMetaData(dataStoreId: string) { - // const existingMatch = await this.dataStoreRepository.findBy({ - // id: dataStoreId, - // }); - - // if (!existingMatch) { - // return 'tried to rename non-existent table'; - // } - - // return existingMatch; - // } - - // async getMetaDataByProjectIds(projectIds: string[]) { - // return await this.dataStoreRepository.findBy(projectIds.map((projectId) => ({ projectId }))); - // } - - // async getMetaDataAll() { - // return await this.dataStoreRepository.find({}); - // } - async renameDataStore(dataStoreId: string, dto: RenameDataStoreDto) { const existingTable = await this.dataStoreRepository.findOneBy({ id: dataStoreId, @@ -114,7 +101,6 @@ export class DataStoreService { async deleteDataStoreAll() { const existingMatches = await this.dataStoreRepository.findBy({}); - console.log(existingMatches); let changed = false; for (const match of existingMatches) { const result = await this.deleteDataStore(match.id); @@ -196,4 +182,59 @@ export class DataStoreService { async getManyAndCount(options: DataStoreListOptions) { return await this.dataStoreRepository.getManyAndCount(options); } + + async getManyRowsAndCount(dataStoreId: string, dto: Partial) { + // unclear if we should validate here, only use case would be to reduce the chance of + // a renamed/removed column appearing here (or added column missing) if the store was + // modified between when the frontend sent the request and we received it + return await this.dataStoreRowsRepository.getManyAndCount(toTableName(dataStoreId), dto); + } + + private async validateRows(dataStoreId: string, rows: DataStoreRows) { + const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); + if (columns.length === 0) { + return 'no columns found for id'; + } + + const columnNames = new Set(columns.map((x) => x.name)); + const columnTypeMap = new Map(columns.map((x) => [x.name, x.type])); + for (const row of rows) { + const keys = Object.keys(row); + if (columns.length !== keys.length) { + return 'mismatched key count'; + } + for (const key of keys) { + if (!columnNames.has(key)) { + return 'unknown column name'; + } + const cell = row[key]; + if (cell === null) continue; + switch (columnTypeMap.get(key)) { + case 'boolean': + if (typeof cell !== 'boolean') return 'type mismatch'; + break; + case 'date': + if (!(cell instanceof Date)) return 'type mismatch'; + row[key] = cell.toISOString(); + break; + case 'string': + if (typeof cell !== 'string') return 'type mismatch'; + break; + case 'number': + if (typeof cell !== 'number') return 'type mismatch'; + break; + } + } + } + return true; + } + + async appendRows(dataStoreId: string, rows: DataStoreRows) { + const validationResult = await this.validateRows(dataStoreId, rows); + if (validationResult !== true) { + return validationResult; + } + + return await this.dataStoreRowsRepository.appendRows(toTableName(dataStoreId), rows); + } } diff --git a/packages/cli/src/modules/data-store/data-store.types.ts b/packages/cli/src/modules/data-store/data-store.types.ts index ffb779b21db..3bb31a3d04d 100644 --- a/packages/cli/src/modules/data-store/data-store.types.ts +++ b/packages/cli/src/modules/data-store/data-store.types.ts @@ -26,3 +26,5 @@ export type DataStoreListOptions = ListQuery.Options< never, ListDataStoreQuerySortOptions >; + +export type DataStoreRows = Array>; diff --git a/packages/cli/src/modules/data-store/utils/sql-utils.ts b/packages/cli/src/modules/data-store/utils/sql-utils.ts index 8dfdb6b07c6..b7497b05454 100644 --- a/packages/cli/src/modules/data-store/utils/sql-utils.ts +++ b/packages/cli/src/modules/data-store/utils/sql-utils.ts @@ -1,3 +1,4 @@ +import type { DataSourceOptions } from '@n8n/typeorm'; import { z } from 'zod'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; @@ -5,8 +6,10 @@ import { NotFoundError } from '@/errors/response-errors/not-found.error'; import type { DataStoreColumn, DataStoreColumnType, + DataStoreRows, DataStoreUserTableName, } from '../data-store.types'; +import { UnexpectedError } from 'n8n-workflow'; function dataStoreColumnTypeToSql(type: DataStoreColumnType) { switch (type) { @@ -23,36 +26,43 @@ function dataStoreColumnTypeToSql(type: DataStoreColumnType) { } } -function dataStoreColumnTypeToZod(fieldType: DataStoreColumnType) { - switch (fieldType) { - case 'string': - return z.string(); - case 'number': - return z.number(); - case 'boolean': - return z.boolean(); - case 'date': - return z.date(); - default: - throw new NotFoundError(`Unsupported field type: ${fieldType as string}`); - } -} - function columnToWildcardAndType(column: DataStoreColumn) { return `\`${column.name}\` ${dataStoreColumnTypeToSql(column.type)}`; } +function getPrimaryKeyAutoIncrement(dbType: DataSourceOptions['type']) { + switch (dbType) { + case 'sqlite': + case 'sqlite-pooled': + case 'better-sqlite3': + return 'INTEGER PRIMARY KEY AUTOINCREMENT'; + case 'postgres': + case 'aurora-postgres': + return 'INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY'; + case 'mysql': + case 'aurora-mysql': + case 'mariadb': + return 'INT AUTO_INCREMENT PRIMARY KEY'; + } + + throw new UnexpectedError('Unexpected database type'); +} + export function createUserTableQuery( tableName: DataStoreUserTableName, columns: DataStoreColumn[], + dbType: DataSourceOptions['type'], ): [string, string[]] { const columnSql = columns.map(columnToWildcardAndType); const columnsFieldQuery = columnSql.length > 0 ? `, ${columnSql.join(', ')}` : ''; + const primaryKeyType = getPrimaryKeyAutoIncrement(dbType); + // The tableName here is selected by us based on the automatically generated id, not user input + // @Review: Any way to insert columns using wildcards? return [ - `CREATE TABLE IF NOT EXISTS ${tableName} (id VARCHAR(36) PRIMARY KEY${columnsFieldQuery})`, - columns.map((x) => x.name), + `CREATE TABLE IF NOT EXISTS ${tableName} (id ${primaryKeyType} ${columnsFieldQuery})`, + [], ]; } @@ -76,3 +86,28 @@ export function deleteColumnQuery( ): [string, string[]] { return [`ALTER TABLE ${tableName} DROP COLUMN \`${column}\``, []]; } + +export function insertIntoQuery( + tableName: DataStoreUserTableName, + rows: DataStoreRows, +): [string, unknown[]] { + if (rows.length === 0) { + return ['', []]; + } + + const keys = Object.keys(rows[0]); + + if (keys.length === 0) { + return ['', []]; + } + + const wildcards = keys.map((_) => '?').join(','); + const rowsQuery = Array(rows.length).fill(`(${wildcards})`).join(','); + const parameters = Array(rows.length * keys.length); + for (let i = 0; i < keys.length; ++i) { + for (let j = 0; j < rows.length; ++j) { + parameters[j * keys.length + i] = rows[j][keys[i]]; + } + } + return [`INSERT INTO ${tableName} (${keys.join(',')}) VALUES ${rowsQuery}`, parameters]; +}