Add row-based operations

packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts

@@ -0,0 +1,94 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
+
+const FilterConditionSchema = z.union([z.literal('eq'), z.literal('neq')]);
+export type ListDataStoreContentFilterConditionType = z.infer<typeof FilterConditionSchema>;
+
+const filterRecord = z.object({
+	columnName: dataStoreColumnNameSchema,
+	condition: FilterConditionSchema.default('eq'),
+	value: z.union([z.string(), z.number(), z.boolean(), z.date()]),
+});
+
+const chainedFilterSchema = z.union([z.literal('and'), z.literal('or')]);
+
+export type ListDataStoreContentFilter = z.infer<typeof filterValidator>;
+
+// ---------------------
+// Parameter Validators
+// ---------------------
+
+// Filter parameter validation
+const filterValidator = z.object({
+	type: chainedFilterSchema.default('and'),
+	filters: z.array(filterRecord).default([]),
+});
+
+// Skip parameter validation
+const skipValidator = z
+	.string()
+	.optional()
+	.transform((val) => (val ? parseInt(val, 10) : 0))
+	.refine((val) => !isNaN(val), {
+		message: 'Skip must be a valid number',
+	});
+
+// Take parameter validation
+const takeValidator = z
+	.string()
+	.optional()
+	.transform((val) => (val ? parseInt(val, 10) : 10))
+	.refine((val) => !isNaN(val), {
+		message: 'Take must be a valid number',
+	});
+
+// SortBy parameter validation
+const sortByValidator = z
+	.string()
+	.optional()
+	.transform((val, ctx) => {
+		if (val === undefined) return val;
+
+		if (!val.includes(':')) {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid sort format, expected <columnName>:<asc/desc>',
+				path: ['sort'],
+			});
+			return z.NEVER;
+		}
+
+		let [column, direction] = val.split(':');
+
+		try {
+			column = dataStoreColumnNameSchema.parse(column);
+		} catch {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid sort columnName',
+				path: ['sort'],
+			});
+			return z.NEVER;
+		}
+
+		direction = direction?.toUpperCase();
+		if (direction !== 'ASC' && direction !== 'DESC') {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid sort direction',
+				path: ['sort'],
+			});
+
+			return z.NEVER;
+		}
+		return [column, direction] as const;
+	});
+
+export class ListDataStoreContentQueryDto extends Z.class({
+	filter: filterValidator,
+	skip: skipValidator,
+	take: takeValidator,
+	sortBy: sortByValidator,
+}) {}

packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts

@@ -76,41 +76,6 @@ const takeValidator = z
 		message: 'Take must be a valid number',
 	});
 
-// Select parameter validation
-const selectFieldsValidator = z.array(z.enum(VALID_SELECT_FIELDS));
-const selectValidator = z
-	.string()
-	.optional()
-	.transform((val, ctx) => {
-		if (!val) return undefined;
-		try {
-			const parsed: unknown = JSON.parse(val);
-			try {
-				const selectFields = selectFieldsValidator.parse(parsed);
-				if (selectFields.length === 0) return undefined;
-				type SelectField = (typeof VALID_SELECT_FIELDS)[number];
-				return selectFields.reduce<Record<SelectField, true>>(
-					(acc, field) => ({ ...acc, [field]: true }),
-					{} as Record<SelectField, true>,
-				);
-			} catch (e) {
-				ctx.addIssue({
-					code: z.ZodIssueCode.custom,
-					message: `Invalid select fields. Valid fields are: ${VALID_SELECT_FIELDS.join(', ')}`,
-					path: ['select'],
-				});
-				return z.NEVER;
-			}
-		} catch (e) {
-			ctx.addIssue({
-				code: z.ZodIssueCode.custom,
-				message: 'Invalid select format',
-				path: ['select'],
-			});
-			return z.NEVER;
-		}
-	});
-
 // SortBy parameter validation
 const sortByValidator = z
 	.enum(VALID_SORT_OPTIONS, { message: `sortBy must be one of: ${VALID_SORT_OPTIONS.join(', ')}` })
@@ -120,6 +85,5 @@ export class ListDataStoreQueryDto extends Z.class({
 	filter: filterValidator,
 	skip: skipValidator,
 	take: takeValidator,
-	select: selectValidator,
 	sortBy: sortByValidator,
 }) {}

packages/@n8n/api-types/src/dto/index.ts

@@ -83,6 +83,7 @@ export { OidcConfigDto } from './oidc/config.dto';
 export { CreateDataStoreDto } from './data-store/create-data-store.dto';
 export { RenameDataStoreDto } from './data-store/rename-data-store.dto';
 export { ListDataStoreQueryDto } from './data-store/list-data-store-query.dto';
+export { ListDataStoreContentQueryDto } from './data-store/list-data-store-content-query.dto';
 export { CreateDataStoreColumnDto } from './data-store/create-data-store-column.dto';
 export { AddDatastoreRecordsDto } from './data-store/add-data-store-records.dto';
 export { AddDataStoreColumnDto } from './data-store/add-data-store-column.dto';

packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts

@@ -1,4 +1,4 @@
-import type { AddDataStoreColumnDto, DataStore } from '@n8n/api-types';
+import type { AddDataStoreColumnDto } from '@n8n/api-types';
 import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils';
 import { Project } from '@n8n/db';
 import { Container } from '@n8n/di';
@@ -429,4 +429,126 @@ describe('dataStore', () => {
 			expect(sizeBytesDesc[0].map((x) => x.name)).toEqual(['ds0', 'ds1', 'ds2']);
 		});
 	});
+	describe('appendRows', () => {
+		it('appends a row to an existing table', async () => {
+			// ARRANGE
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } });
+
+			// ACT
+			const result = await dataStoreService.appendRows(dataStore1.id, [
+				{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
+				{ c1: 4, c2: false, c3: new Date(), c4: 'hello!' },
+				{ c1: 5, c2: true, c3: new Date(), c4: 'hello.' },
+			]);
+
+			// ASSERT
+			expect(result).toBe(true);
+		});
+
+		it('rejects a mismatched row with extra column', async () => {
+			// ARRANGE
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } });
+
+			// ACT
+			const result = await dataStoreService.appendRows(dataStore1.id, [
+				{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
+				{ cWrong: 3, c1: 4, c2: true, c3: new Date(), c4: 'hello?' },
+			]);
+
+			// ASSERT
+			expect(result).toBe('mismatched key count');
+		});
+		it('rejects a mismatched row with missing column', async () => {
+			// ARRANGE
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } });
+
+			// ACT
+			const result = await dataStoreService.appendRows(dataStore1.id, [
+				{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
+				{ c2: true, c3: new Date(), c4: 'hello?' },
+			]);
+
+			// ASSERT
+			expect(result).toBe('mismatched key count');
+		});
+		it('rejects a mismatched row with replaced column', async () => {
+			// ARRANGE
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } });
+
+			// ACT
+			const result = await dataStoreService.appendRows(dataStore1.id, [
+				{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
+				{ cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' },
+			]);
+
+			// ASSERT
+			expect(result).toBe('unknown column name');
+		});
+		it('rejects unknown data store id', async () => {
+			// ARRANGE
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } });
+
+			// ACT
+			const result = await dataStoreService.appendRows('this is not an id', [
+				{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
+				{ cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' },
+			]);
+
+			// ASSERT
+			expect(result).toBe('no columns found for id');
+		});
+		it('fails on type mismatch', async () => {
+			// ARRANGE
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } });
+
+			// ACT
+			const result = await dataStoreService.appendRows(dataStore1.id, [{ c1: 3 }, { c1: true }]);
+
+			// ASSERT
+			expect(result).toBe('type mismatch');
+		});
+	});
+	describe('getManyRowsAndCount', () => {
+		it('retrieves rows correctly', async () => {
+			// ARRANGE
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c1', type: 'number' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c2', type: 'boolean' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c3', type: 'date' } });
+			await dataStoreService.addColumn(dataStore1.id, { column: { name: 'c4', type: 'string' } });
+
+			await dataStoreService.appendRows(dataStore1.id, [
+				{ c1: 3, c2: true, c3: new Date(0), c4: 'hello?' },
+				{ c1: 4, c2: false, c3: new Date(1), c4: 'hello!' },
+				{ c1: 5, c2: true, c3: new Date(2), c4: 'hello.' },
+			]);
+
+			// ACT
+			const result = await dataStoreService.getManyRowsAndCount(dataStore1.id, {});
+
+			// ASSERT
+			expect(result).toEqual([
+				3,
+				[
+					{ c1: 3, c2: 1, c3: '1970-01-01T00:00:00.000Z', c4: 'hello?', id: 1 },
+					{ c1: 4, c2: 0, c3: '1970-01-01T00:00:00.001Z', c4: 'hello!', id: 2 },
+					{ c1: 5, c2: 1, c3: '1970-01-01T00:00:00.002Z', c4: 'hello.', id: 3 },
+				],
+			]);
+		});
+	});
 });

packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts

@@ -1,4 +1,9 @@
-import { createUserTableQuery, addColumnQuery, deleteColumnQuery } from '../utils/sql-utils';
+import {
+	createUserTableQuery,
+	addColumnQuery,
+	deleteColumnQuery,
+	insertIntoQuery,
+} from '../utils/sql-utils';
 import type { DataStoreColumn } from '../data-store.types';
 
 describe('sql-utils', () => {
@@ -10,22 +15,21 @@ describe('sql-utils', () => {
 				{ name: 'age', type: 'number' },
 			] satisfies DataStoreColumn[];
 
-			const [query, columnNames] = createUserTableQuery(tableName, columns);
-
+			const [query, columnNames] = createUserTableQuery(tableName, columns, 'sqlite');
 			expect(query).toBe(
-				'CREATE TABLE IF NOT EXISTS data_store_user_abc (id VARCHAR(36) PRIMARY KEY, `?` TEXT, `?` FLOAT)',
+				'CREATE TABLE IF NOT EXISTS data_store_user_abc (id INTEGER PRIMARY KEY AUTOINCREMENT , `name` TEXT, `age` FLOAT)',
 			);
-			expect(columnNames).toEqual(['name', 'age']);
+			expect(columnNames).toEqual([]);
 		});
 
 		it('should generate a valid SQL query for creating a user table without columns', () => {
 			const tableName = 'data_store_user_abc';
 			const columns: [] = [];
 
-			const [query, columnNames] = createUserTableQuery(tableName, columns);
+			const [query, columnNames] = createUserTableQuery(tableName, columns, 'postgres');
 
 			expect(query).toBe(
-				'CREATE TABLE IF NOT EXISTS data_store_user_abc (id VARCHAR(36) PRIMARY KEY)',
+				'CREATE TABLE IF NOT EXISTS data_store_user_abc (id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY )',
 			);
 			expect(columnNames).toEqual([]);
 		});
@@ -38,8 +42,8 @@ describe('sql-utils', () => {
 
 			const [query, columnNames] = addColumnQuery(tableName, column);
 
-			expect(query).toBe('ALTER TABLE data_store_user_abc ADD `?` FLOAT');
-			expect(columnNames).toEqual(['email']);
+			expect(query).toBe('ALTER TABLE data_store_user_abc ADD `email` FLOAT');
+			expect(columnNames).toEqual([]);
 		});
 	});
 
@@ -50,8 +54,43 @@ describe('sql-utils', () => {
 
 			const [query, columnNames] = deleteColumnQuery(tableName, column);
 
-			expect(query).toBe('ALTER TABLE data_store_user_abc DROP COLUMN ?');
-			expect(columnNames).toEqual(['email']);
+			expect(query).toBe('ALTER TABLE data_store_user_abc DROP COLUMN `email`');
+			expect(columnNames).toEqual([]);
+		});
+	});
+
+	describe('insertIntoQuery', () => {
+		it('should generate a valid SQL query for inserting rows into a table', () => {
+			const tableName = 'data_store_user_abc';
+			const rows = [
+				{ name: 'Alice', age: 30 },
+				{ name: 'Bob', age: 25 },
+			];
+
+			const [query, parameters] = insertIntoQuery(tableName, rows);
+
+			expect(query).toBe('INSERT INTO data_store_user_abc (name,age) VALUES (?,?),(?,?)');
+			expect(parameters).toEqual(['Alice', 30, 'Bob', 25]);
+		});
+
+		it('should return an empty query and parameters when rows are empty', () => {
+			const tableName = 'data_store_user_abc';
+			const rows: [] = [];
+
+			const [query, parameters] = insertIntoQuery(tableName, rows);
+
+			expect(query).toBe('');
+			expect(parameters).toEqual([]);
+		});
+
+		it('should return an empty query and parameters when rows have no keys', () => {
+			const tableName = 'data_store_user_abc';
+			const rows = [{}];
+
+			const [query, parameters] = insertIntoQuery(tableName, rows);
+
+			expect(query).toBe('');
+			expect(parameters).toEqual([]);
 		});
 	});
 });

packages/cli/src/modules/data-store/data-store-column.repository.ts

@@ -11,6 +11,12 @@ export class DataStoreColumnRepository extends Repository<DataStoreColumnEntity>
 		super(DataStoreColumnEntity, dataSource.manager);
 	}
 
+	async getColumns(rawDataStoreId: string) {
+		return await this.createQueryBuilder('dataStoreColumns')
+			.where(`dataStoreColumns.dataStoreId = '${rawDataStoreId}'`)
+			.getMany();
+	}
+
 	async addColumn(dataStoreId: DataStoreUserTableName, column: DataStoreColumn) {
 		await this.manager.query(...addColumnQuery(dataStoreId, column));
 	}

packages/cli/src/modules/data-store/data-store-rows.repository.ts

@@ -0,0 +1,104 @@
+import { ListDataStoreContentQueryDto } from '@n8n/api-types';
+import { Service } from '@n8n/di';
+import { BaseEntity, DataSource, Entity, SelectQueryBuilder } from '@n8n/typeorm';
+import { DataStoreRows, DataStoreUserTableName } from './data-store.types';
+import { insertIntoQuery } from './utils/sql-utils';
+
+// type QueryBuilder = SelectQueryBuilder<Record<PropertyKey, unknown>>;
+type QueryBuilder = SelectQueryBuilder<any>;
+// type QueryBuilder = ReturnType<EntityManager['createQueryBuilder']>;
+
+@Entity()
+class Emptity extends BaseEntity {}
+
+function valueToSQL(value: ListDataStoreContentQueryDto['filter']['filters'][number]['value']) {
+	if (value instanceof Date) {
+		return value.toISOString(); // @Review: this feels bad
+	}
+	switch (typeof value) {
+		case 'number':
+		case 'boolean':
+			return `${value}`;
+		case 'string':
+			return `'${value}'`;
+	}
+}
+
+function getConditionSQL(filter: ListDataStoreContentQueryDto['filter']['filters'][number]) {
+	switch (filter.condition) {
+		case 'eq':
+			return `dataStore.${filter.columnName} == ${valueToSQL(filter.value)}`;
+		case 'neq':
+			return `dataStore.${filter.columnName} != ${valueToSQL(filter.value)}`;
+	}
+}
+
+@Service()
+export class DataStoreRowsRepository {
+	constructor(private dataSource: DataSource) {}
+
+	async appendRows(tableName: DataStoreUserTableName, rows: DataStoreRows) {
+		await this.dataSource.query(...insertIntoQuery(tableName, rows));
+		return true;
+	}
+
+	// ALL THE MANY STUFF
+	async getManyAndCount(
+		dataStoreId: DataStoreUserTableName,
+		dto: Partial<ListDataStoreContentQueryDto>,
+	) {
+		const [countQuery, query] = this.getManyQuery(dataStoreId, dto);
+		const result = await query.getRawMany();
+		const totalCount = (await countQuery.getRawOne())['COUNT(*)'] as number | undefined;
+		return [totalCount ?? -1, result];
+	}
+
+	getManyQuery(
+		dataStoreId: DataStoreUserTableName,
+		dto: Partial<ListDataStoreContentQueryDto>,
+	): [QueryBuilder, QueryBuilder] {
+		const query = this.dataSource.createQueryBuilder();
+
+		query.from(dataStoreId, 'dataStore');
+		this.applyFilters(query, dto);
+		const countQuery = query.clone().select('COUNT(*)');
+		query.select('*');
+		this.applySorting(query, dto);
+		this.applyPagination(query, dto);
+
+		return [countQuery, query];
+	}
+
+	private applyFilters(query: QueryBuilder, dto: Partial<ListDataStoreContentQueryDto>): void {
+		const conditions = dto.filter?.filters.map(getConditionSQL) ?? [];
+		if (dto.filter?.type === 'and') {
+			for (const condition of conditions) {
+				query.andWhere(condition);
+			}
+		} else if (dto.filter?.type === 'or') {
+			for (const condition of conditions) {
+				query.orWhere(condition);
+			}
+		}
+	}
+
+	private applySorting(query: QueryBuilder, dto: Partial<ListDataStoreContentQueryDto>): void {
+		if (!dto.sortBy) {
+			// query.orderBy('dataStore.', 'DESC');
+			return;
+		}
+
+		const [field, order] = dto.sortBy;
+		this.applySortingByField(query, field, order);
+	}
+
+	private applySortingByField(query: QueryBuilder, field: string, direction: 'DESC' | 'ASC'): void {
+		console.log(field);
+		query.orderBy(`${field}`, direction);
+	}
+
+	private applyPagination(query: QueryBuilder, dto: Partial<ListDataStoreContentQueryDto>): void {
+		query.skip(dto.skip);
+		query.take(dto.take);
+	}
+}

packages/cli/src/modules/data-store/data-store.controller.ts

@@ -2,6 +2,7 @@ import {
 	AddDataStoreColumnDto,
 	CreateDataStoreDto,
 	DeleteDataStoreColumnDto,
+	ListDataStoreContentQueryDto,
 	ListDataStoreQueryDto,
 	RenameDataStoreDto,
 } from '@n8n/api-types';
@@ -70,9 +71,12 @@ export class DataStoreController {
 	}
 
 	@Get('/:dataStoreId', { skipAuth: true })
-	async getDataStoreContent(
+	async getDataStoreRows(
 		_req: AuthenticatedRequest,
 		_res: Response,
 		@Param('dataStoreId') dataStoreId: string,
-	) {}
+		@Body dto: ListDataStoreContentQueryDto,
+	) {
+		return await this.dataStoreService.getManyRowsAndCount(dataStoreId, dto);
+	}
 }

packages/cli/src/modules/data-store/data-store.repository.ts

@@ -12,7 +12,8 @@ export class DataStoreRepository extends Repository<DataStoreEntity> {
 	}
 
 	async createUserTable(tableName: DataStoreUserTableName, columns: DataStoreColumn[]) {
-		await this.manager.query(...createUserTableQuery(tableName, columns));
+		const dbType = this.manager.connection.options.type;
+		await this.manager.query(...createUserTableQuery(tableName, columns, dbType));
 	}
 
 	async deleteUserTable(tableName: DataStoreUserTableName) {
@@ -89,7 +90,6 @@ export class DataStoreRepository extends Repository<DataStoreEntity> {
 		field: string,
 		direction: 'DESC' | 'ASC',
 	): void {
-		console.log(field, direction);
 		if (field === 'name') {
 			query
 				.addSelect('LOWER(dataStore.name)', 'dataStore_name_lower')

packages/cli/src/modules/data-store/data-store.service.ts

@@ -2,6 +2,7 @@ import {
 	AddDataStoreColumnDto,
 	CreateDataStoreDto,
 	DeleteDataStoreColumnDto,
+	ListDataStoreContentQueryDto,
 } from '@n8n/api-types';
 import { RenameDataStoreDto } from '@n8n/api-types/src/dto/data-store/rename-data-store.dto';
 import { Logger } from '@n8n/backend-common';
@@ -9,8 +10,13 @@ import { Service } from '@n8n/di';
 
 import { DataStoreConfig } from './data-store';
 import { DataStoreColumnRepository } from './data-store-column.repository';
+import { DataStoreRowsRepository } from './data-store-rows.repository';
 import { DataStoreRepository } from './data-store.repository';
-import type { DataStoreListOptions, DataStoreUserTableName } from './data-store.types';
+import type {
+	DataStoreListOptions,
+	DataStoreRows,
+	DataStoreUserTableName,
+} from './data-store.types';
 
 function toTableName(dataStoreId: string): DataStoreUserTableName {
 	return `data_store_user_${dataStoreId}`;
@@ -27,6 +33,7 @@ export class DataStoreService {
 	constructor(
 		private readonly dataStoreRepository: DataStoreRepository,
 		private readonly dataStoreColumnRepository: DataStoreColumnRepository,
+		private readonly dataStoreRowsRepository: DataStoreRowsRepository,
 		private readonly logger: Logger,
 		private readonly config: DataStoreConfig,
 	) {
@@ -69,26 +76,6 @@ export class DataStoreService {
 		return dataStore;
 	}
 
-	// async getMetaData(dataStoreId: string) {
-	// 	const existingMatch = await this.dataStoreRepository.findBy({
-	// 		id: dataStoreId,
-	// 	});
-
-	// 	if (!existingMatch) {
-	// 		return 'tried to rename non-existent table';
-	// 	}
-
-	// 	return existingMatch;
-	// }
-
-	// async getMetaDataByProjectIds(projectIds: string[]) {
-	// 	return await this.dataStoreRepository.findBy(projectIds.map((projectId) => ({ projectId })));
-	// }
-
-	// async getMetaDataAll() {
-	// 	return await this.dataStoreRepository.find({});
-	// }
-
 	async renameDataStore(dataStoreId: string, dto: RenameDataStoreDto) {
 		const existingTable = await this.dataStoreRepository.findOneBy({
 			id: dataStoreId,
@@ -114,7 +101,6 @@ export class DataStoreService {
 
 	async deleteDataStoreAll() {
 		const existingMatches = await this.dataStoreRepository.findBy({});
-		console.log(existingMatches);
 		let changed = false;
 		for (const match of existingMatches) {
 			const result = await this.deleteDataStore(match.id);
@@ -196,4 +182,59 @@ export class DataStoreService {
 	async getManyAndCount(options: DataStoreListOptions) {
 		return await this.dataStoreRepository.getManyAndCount(options);
 	}
+
+	async getManyRowsAndCount(dataStoreId: string, dto: Partial<ListDataStoreContentQueryDto>) {
+		// unclear if we should validate here, only use case would be to reduce the chance of
+		// a renamed/removed column appearing here (or added column missing) if the store was
+		// modified between when the frontend sent the request and we received it
+		return await this.dataStoreRowsRepository.getManyAndCount(toTableName(dataStoreId), dto);
+	}
+
+	private async validateRows(dataStoreId: string, rows: DataStoreRows) {
+		const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
+		if (columns.length === 0) {
+			return 'no columns found for id';
+		}
+
+		const columnNames = new Set(columns.map((x) => x.name));
+		const columnTypeMap = new Map(columns.map((x) => [x.name, x.type]));
+		for (const row of rows) {
+			const keys = Object.keys(row);
+			if (columns.length !== keys.length) {
+				return 'mismatched key count';
+			}
+			for (const key of keys) {
+				if (!columnNames.has(key)) {
+					return 'unknown column name';
+				}
+				const cell = row[key];
+				if (cell === null) continue;
+				switch (columnTypeMap.get(key)) {
+					case 'boolean':
+						if (typeof cell !== 'boolean') return 'type mismatch';
+						break;
+					case 'date':
+						if (!(cell instanceof Date)) return 'type mismatch';
+						row[key] = cell.toISOString();
+						break;
+					case 'string':
+						if (typeof cell !== 'string') return 'type mismatch';
+						break;
+					case 'number':
+						if (typeof cell !== 'number') return 'type mismatch';
+						break;
+				}
+			}
+		}
+		return true;
+	}
+
+	async appendRows(dataStoreId: string, rows: DataStoreRows) {
+		const validationResult = await this.validateRows(dataStoreId, rows);
+		if (validationResult !== true) {
+			return validationResult;
+		}
+
+		return await this.dataStoreRowsRepository.appendRows(toTableName(dataStoreId), rows);
+	}
 }

packages/cli/src/modules/data-store/data-store.types.ts

@@ -26,3 +26,5 @@ export type DataStoreListOptions = ListQuery.Options<
 	never,
 	ListDataStoreQuerySortOptions
 >;
+
+export type DataStoreRows = Array<Record<PropertyKey, unknown>>;

packages/cli/src/modules/data-store/utils/sql-utils.ts

@@ -1,3 +1,4 @@
+import type { DataSourceOptions } from '@n8n/typeorm';
 import { z } from 'zod';
 
 import { NotFoundError } from '@/errors/response-errors/not-found.error';
@@ -5,8 +6,10 @@ import { NotFoundError } from '@/errors/response-errors/not-found.error';
 import type {
 	DataStoreColumn,
 	DataStoreColumnType,
+	DataStoreRows,
 	DataStoreUserTableName,
 } from '../data-store.types';
+import { UnexpectedError } from 'n8n-workflow';
 
 function dataStoreColumnTypeToSql(type: DataStoreColumnType) {
 	switch (type) {
@@ -23,36 +26,43 @@ function dataStoreColumnTypeToSql(type: DataStoreColumnType) {
 	}
 }
 
-function dataStoreColumnTypeToZod(fieldType: DataStoreColumnType) {
-	switch (fieldType) {
-		case 'string':
-			return z.string();
-		case 'number':
-			return z.number();
-		case 'boolean':
-			return z.boolean();
-		case 'date':
-			return z.date();
-		default:
-			throw new NotFoundError(`Unsupported field type: ${fieldType as string}`);
-	}
-}
-
 function columnToWildcardAndType(column: DataStoreColumn) {
 	return `\`${column.name}\` ${dataStoreColumnTypeToSql(column.type)}`;
 }
 
+function getPrimaryKeyAutoIncrement(dbType: DataSourceOptions['type']) {
+	switch (dbType) {
+		case 'sqlite':
+		case 'sqlite-pooled':
+		case 'better-sqlite3':
+			return 'INTEGER PRIMARY KEY AUTOINCREMENT';
+		case 'postgres':
+		case 'aurora-postgres':
+			return 'INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY';
+		case 'mysql':
+		case 'aurora-mysql':
+		case 'mariadb':
+			return 'INT AUTO_INCREMENT PRIMARY KEY';
+	}
+
+	throw new UnexpectedError('Unexpected database type');
+}
+
 export function createUserTableQuery(
 	tableName: DataStoreUserTableName,
 	columns: DataStoreColumn[],
+	dbType: DataSourceOptions['type'],
 ): [string, string[]] {
 	const columnSql = columns.map(columnToWildcardAndType);
 	const columnsFieldQuery = columnSql.length > 0 ? `, ${columnSql.join(', ')}` : '';
 
+	const primaryKeyType = getPrimaryKeyAutoIncrement(dbType);
+
 	// The tableName here is selected by us based on the automatically generated id, not user input
+	// @Review: Any way to insert columns using wildcards?
 	return [
-		`CREATE TABLE IF NOT EXISTS ${tableName} (id VARCHAR(36) PRIMARY KEY${columnsFieldQuery})`,
-		columns.map((x) => x.name),
+		`CREATE TABLE IF NOT EXISTS ${tableName} (id ${primaryKeyType} ${columnsFieldQuery})`,
+		[],
 	];
 }
 
@@ -76,3 +86,28 @@ export function deleteColumnQuery(
 ): [string, string[]] {
 	return [`ALTER TABLE ${tableName} DROP COLUMN \`${column}\``, []];
 }
+
+export function insertIntoQuery(
+	tableName: DataStoreUserTableName,
+	rows: DataStoreRows,
+): [string, unknown[]] {
+	if (rows.length === 0) {
+		return ['', []];
+	}
+
+	const keys = Object.keys(rows[0]);
+
+	if (keys.length === 0) {
+		return ['', []];
+	}
+
+	const wildcards = keys.map((_) => '?').join(',');
+	const rowsQuery = Array(rows.length).fill(`(${wildcards})`).join(',');
+	const parameters = Array(rows.length * keys.length);
+	for (let i = 0; i < keys.length; ++i) {
+		for (let j = 0; j < rows.length; ++j) {
+			parameters[j * keys.length + i] = rows[j][keys[i]];
+		}
+	}
+	return [`INSERT INTO ${tableName} (${keys.join(',')}) VALUES ${rowsQuery}`, parameters];
+}