n8n/packages/testing/playwright/playwright-projects.ts

import type { Project } from '@playwright/test';
import type { N8NConfig } from 'n8n-containers/stack';

import {
	CONTAINER_ONLY_CAPABILITIES,
	CONTAINER_ONLY_MODES,
	LICENSED_TAG,
} from './fixtures/capabilities';
import { getBackendUrl, getFrontendUrl } from './utils/url-helper';

// Tests that require container environment (won't run against local n8n).
// Matches:
// - @capability:X - add-on features (email, proxy, source-control, etc.)
// - @mode:X - infrastructure modes (postgres, queue, multi-main)
// - @licensed - enterprise license features (log streaming, SSO, etc.)
// - @db:reset - tests needing per-test database reset (requires isolated containers)
const CONTAINER_ONLY = new RegExp(
	[
		`@capability:(${CONTAINER_ONLY_CAPABILITIES.join('|')})`,
		`@mode:(${CONTAINER_ONLY_MODES.join('|')})`,
		`@${LICENSED_TAG}`,
		'@db:reset',
	].join('|'),
);

// Escape hatch: allow `@capability:*` tests to run against a pre-started local
// n8n. Fixtures that depend on container-provided services (proxy, mailpit,
// etc.) must detect the no-container case and skip or fall back to direct
// network calls. Used by `pnpm test:local:isolated` and similar workflows.
const ALLOW_CONTAINER_ONLY = process.env.PLAYWRIGHT_ALLOW_CONTAINER_ONLY === 'true';

const CONTAINER_CONFIGS: Array<{ name: string; config: N8NConfig }> = [
	{ name: 'sqlite', config: {} },
	{ name: 'postgres', config: { postgres: true } },
	{ name: 'queue', config: { workers: 1 } },
	{
		name: 'multi-main',
		config: { mains: 2, workers: 1, services: ['victoriaLogs', 'victoriaMetrics', 'vector'] },
	},
];

// --- Benchmark profiles ---
// Each profile represents a real-world n8n deployment configuration.
// ONE test file runs in ALL profiles — adding a profile auto-expands coverage.

const BENCHMARK_WORKER_COUNT = parseInt(process.env.KAFKA_LOAD_WORKERS ?? '3', 10);

// Resource profiles matching realistic AWS instance types:
// Main: m5.large (2 vCPU, 8GB RAM) — matches staging main
// Workers: t3.medium (2 vCPU, 4GB RAM) — matches staging worker limits
export const BENCHMARK_MAIN_RESOURCES = { memory: 8, cpu: 2 };
export const BENCHMARK_WORKER_RESOURCES = { memory: 4, cpu: 2 };

export const OBSERVABILITY_SERVICES = ['victoriaLogs', 'victoriaMetrics', 'vector'] as const;

const BENCHMARK_BASE_CONFIG: N8NConfig = {
	// Postgres exporter scrapes DB internals into VictoriaMetrics — only meaningful for benchmarks.
	services: [...OBSERVABILITY_SERVICES, 'postgresExporter'],
	postgres: true,
	resourceQuota: BENCHMARK_MAIN_RESOURCES,
	workerResourceQuota: BENCHMARK_WORKER_RESOURCES,
	env: {
		N8N_METRICS_INCLUDE_MESSAGE_EVENT_BUS_METRICS: 'true',
	},
};

type BenchmarkProfile = { name: string; config: N8NConfig };

// Benchmark profiles exercised in CI (see `test-e2e-infrastructure-reusable.yml`).
// They scan `tests/infrastructure/benchmarks/`.
const CI_BENCHMARK_PROFILES: BenchmarkProfile[] = [
	{
		name: 'direct',
		config: {
			...BENCHMARK_BASE_CONFIG,
			services: [...BENCHMARK_BASE_CONFIG.services!, 'kafka'],
			env: {
				...BENCHMARK_BASE_CONFIG.env,
				DB_POSTGRESDB_POOL_SIZE: '20',
			},
		},
	},
	{
		name: 'queue',
		config: {
			...BENCHMARK_BASE_CONFIG,
			services: [...BENCHMARK_BASE_CONFIG.services!, 'kafka'],
			workers: BENCHMARK_WORKER_COUNT,
			env: {
				...BENCHMARK_BASE_CONFIG.env,
				N8N_METRICS_INCLUDE_QUEUE_METRICS: 'true',
			},
		},
	},
	{
		name: 'queue-tuned',
		config: {
			...BENCHMARK_BASE_CONFIG,
			services: [...BENCHMARK_BASE_CONFIG.services!, 'kafka'],
			workers: BENCHMARK_WORKER_COUNT,
			env: {
				...BENCHMARK_BASE_CONFIG.env,
				N8N_METRICS_INCLUDE_QUEUE_METRICS: 'true',
				N8N_LOG_LEVEL: 'info',
				DB_POSTGRESDB_POOL_SIZE: '30',
				DB_POSTGRESDB_CONNECTION_TIMEOUT: '60000',
				N8N_CONCURRENCY_PRODUCTION_LIMIT: '20',
				EXECUTIONS_DATA_SAVE_ON_SUCCESS: 'none',
			},
		},
	},
];

// Benchmark profiles that host local-only tests (model API keys, long runtimes,
// reserved metric names). They scan `tests/infrastructure/benchmarks-local/` and
// are intentionally left out of the CI matrix.
const LOCAL_ONLY_BENCHMARK_PROFILES: BenchmarkProfile[] = [
	{
		name: 'memory-instanceai',
		config: {
			...BENCHMARK_BASE_CONFIG,
			services: [...OBSERVABILITY_SERVICES],
			env: {
				...BENCHMARK_BASE_CONFIG.env,
				// Instance-AI module & model config
				N8N_ENABLED_MODULES: 'instance-ai',
				N8N_INSTANCE_AI_MODEL: process.env.N8N_INSTANCE_AI_MODEL ?? 'openai/gpt-5.4-nano',
				// Forward API keys to the container
				...(process.env.N8N_AI_OPENAI_API_KEY && {
					N8N_AI_OPENAI_API_KEY: process.env.N8N_AI_OPENAI_API_KEY,
					OPENAI_API_KEY: process.env.N8N_AI_OPENAI_API_KEY,
				}),
				...(process.env.LANGSMITH_API_KEY && {
					LANGSMITH_API_KEY: process.env.LANGSMITH_API_KEY,
				}),
				...(process.env.ANTHROPIC_API_KEY && {
					ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
				}),
				...(process.env.OPENROUTER_API_KEY && {
					OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY,
				}),
				...(process.env.CEREBRAS_API_KEY && {
					CEREBRAS_API_KEY: process.env.CEREBRAS_API_KEY,
				}),
				// Sandbox config — forwarded from host env if present
				...(process.env.N8N_INSTANCE_AI_SANDBOX_ENABLED && {
					N8N_INSTANCE_AI_SANDBOX_ENABLED: process.env.N8N_INSTANCE_AI_SANDBOX_ENABLED,
					N8N_INSTANCE_AI_SANDBOX_PROVIDER:
						process.env.N8N_INSTANCE_AI_SANDBOX_PROVIDER ?? 'daytona',
					N8N_INSTANCE_AI_SANDBOX_IMAGE: process.env.N8N_INSTANCE_AI_SANDBOX_IMAGE ?? '',
				}),
				...(process.env.DAYTONA_API_URL && {
					DAYTONA_API_URL: process.env.DAYTONA_API_URL,
				}),
				...(process.env.DAYTONA_API_KEY && {
					DAYTONA_API_KEY: process.env.DAYTONA_API_KEY,
				}),
			},
		},
	},
];

export function getProjects(): Project[] {
	const isLocal = !!getBackendUrl();
	const projects: Project[] = [];

	if (isLocal) {
		projects.push({
			name: 'e2e',
			testDir: './tests/e2e',
			grepInvert: ALLOW_CONTAINER_ONLY ? undefined : CONTAINER_ONLY,
			fullyParallel: true,
			use: { baseURL: getFrontendUrl() },
		});
		projects.push({
			name: 'dev-server-smoke',
			testDir: './tests/dev-server-smoke',
			fullyParallel: false,
			// Vite dev cold-start can take 15-25s on the first navigation while it
			// pre-bundles deps. Subsequent navigations are sub-second.
			timeout: 90_000,
			use: { baseURL: getFrontendUrl(), navigationTimeout: 30_000 },
		});
	} else {
		for (const { name, config } of CONTAINER_CONFIGS) {
			projects.push(
				{
					name: `${name}:e2e`,
					testDir: './tests/e2e',
					timeout: name === 'sqlite' ? 60000 : 180000, // 60 seconds for sqlite container test, 180 for other modes to allow startup
					fullyParallel: true,
					use: { containerConfig: config },
				},
				{
					name: `${name}:infrastructure`,
					testDir: './tests/infrastructure',
					grep: new RegExp(`@mode:${name}|@capability:${name}`),
					workers: 1,
					timeout: 180000,
					use: { containerConfig: config },
				},
			);
		}

		projects.push({
			name: 'coverage',
			testDir: './tests/e2e',
			timeout: 60000,
			fullyParallel: true,
			use: { containerConfig: {} },
		});

		for (const { name, config } of CI_BENCHMARK_PROFILES) {
			projects.push({
				name: `benchmark-${name}:infrastructure`,
				testDir: './tests/infrastructure/benchmarks',
				workers: 1,
				timeout: 600_000,
				retries: 0,
				use: { containerConfig: config },
			});
		}

		for (const { name, config } of LOCAL_ONLY_BENCHMARK_PROFILES) {
			projects.push({
				name: `benchmark-${name}:infrastructure`,
				testDir: './tests/infrastructure/benchmarks-local',
				workers: 1,
				timeout: 600_000,
				retries: 0,
				use: { containerConfig: config },
			});
		}
	}

	projects.push({
		name: 'cli-workflows',
		testDir: './tests/cli-workflows',
		fullyParallel: true,
		timeout: 60000,
	});

	projects.push({
		name: 'performance',
		testDir: './tests/performance',
		workers: 1,
		timeout: 300000,
		retries: 0,
		use: {
			// Default container config for performance tests, equivalent to @cloud:starter
			containerConfig: { resourceQuota: { memory: 0.75, cpu: 0.5 }, env: { E2E_TESTS: 'true' } },
		},
	});

	return projects;
}