From 722d99e122eeeccf85ec0b1b77de34cf1e546153 Mon Sep 17 00:00:00 2001 From: RomanDavydchuk Date: Mon, 18 May 2026 17:07:27 +0300 Subject: [PATCH] feat(core): Persist and periodically fetch MCP servers from a remote API (#30298) --- .../modules/__tests__/module-registry.test.ts | 2 + .../src/modules/module-registry.ts | 1 + .../@n8n/config/src/configs/logging.config.ts | 1 + ...4000000005-CreateMcpRegistryServerTable.ts | 33 +++ .../db/src/migrations/postgresdb/index.ts | 2 + .../@n8n/db/src/migrations/sqlite/index.ts | 2 + .../decorators/src/pubsub/pubsub-metadata.ts | 1 + .../community-node-types-utils.test.ts | 4 +- .../community-node-types-utils.ts | 2 +- .../community-node-types.service.ts | 2 +- .../mcp-registry-node-loader.test.ts | 86 +++--- .../mcp-registry-test.controller.test.ts | 54 ++++ .../node-description-transform.test.ts | 6 +- .../mcp-registry/mcp-registry-node-loader.ts | 11 +- .../mcp-registry-test.controller.ts | 37 +++ .../mcp-registry/mcp-registry.module.ts | 21 +- .../__tests__/mcp-registry-api.client.test.ts | 215 +++++++++++++++ .../__tests__/mcp-registry.service.test.ts | 258 ++++++++++++++++++ .../registry/mcp-registry-api.client.ts | 71 +++++ .../registry/mcp-registry-server.entity.ts | 50 ++++ .../mcp-registry-server.repository.ts | 11 + .../registry/mcp-registry.service.test.ts | 52 ---- .../registry/mcp-registry.service.ts | 227 ++++++++++++++- .../registry/mcp-registry.types.ts | 43 ++- .../src/scaling/pubsub/pubsub.event-map.ts | 2 + .../cli/src/scaling/pubsub/pubsub.types.ts | 2 + .../__tests__/strapi-utils.test.ts | 0 .../strapi-utils.ts | 4 +- .../testing/playwright/services/api-helper.ts | 16 ++ .../e2e/mcp-registry/mcp-registry.spec.ts | 3 +- 30 files changed, 1091 insertions(+), 128 deletions(-) create mode 100644 packages/@n8n/db/src/migrations/common/1784000000005-CreateMcpRegistryServerTable.ts rename packages/cli/src/modules/mcp-registry/{ => __tests__}/mcp-registry-node-loader.test.ts (73%) create mode 100644 packages/cli/src/modules/mcp-registry/__tests__/mcp-registry-test.controller.test.ts rename packages/cli/src/modules/mcp-registry/{ => __tests__}/node-description-transform.test.ts (98%) create mode 100644 packages/cli/src/modules/mcp-registry/mcp-registry-test.controller.ts create mode 100644 packages/cli/src/modules/mcp-registry/registry/__tests__/mcp-registry-api.client.test.ts create mode 100644 packages/cli/src/modules/mcp-registry/registry/__tests__/mcp-registry.service.test.ts create mode 100644 packages/cli/src/modules/mcp-registry/registry/mcp-registry-api.client.ts create mode 100644 packages/cli/src/modules/mcp-registry/registry/mcp-registry-server.entity.ts create mode 100644 packages/cli/src/modules/mcp-registry/registry/mcp-registry-server.repository.ts delete mode 100644 packages/cli/src/modules/mcp-registry/registry/mcp-registry.service.test.ts rename packages/cli/src/{modules/community-packages => utils}/__tests__/strapi-utils.test.ts (100%) rename packages/cli/src/{modules/community-packages => utils}/strapi-utils.ts (93%) diff --git a/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts b/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts index da65a14eda8..a9a8963f34d 100644 --- a/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts +++ b/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts @@ -45,6 +45,7 @@ describe('eligibleModules', () => { 'encryption-key-manager', 'oauth-jwe', 'inbound-secrets', + 'mcp-registry', ]); }); @@ -76,6 +77,7 @@ describe('eligibleModules', () => { 'encryption-key-manager', 'oauth-jwe', 'inbound-secrets', + 'mcp-registry', 'instance-ai', ]); }); diff --git a/packages/@n8n/backend-common/src/modules/module-registry.ts b/packages/@n8n/backend-common/src/modules/module-registry.ts index 4536ba68e24..35e07c50dff 100644 --- a/packages/@n8n/backend-common/src/modules/module-registry.ts +++ b/packages/@n8n/backend-common/src/modules/module-registry.ts @@ -56,6 +56,7 @@ export class ModuleRegistry { 'encryption-key-manager', 'oauth-jwe', 'inbound-secrets', + 'mcp-registry', ]; private readonly activeModules: string[] = []; diff --git a/packages/@n8n/config/src/configs/logging.config.ts b/packages/@n8n/config/src/configs/logging.config.ts index 188a3f7ec66..9f9d07084b2 100644 --- a/packages/@n8n/config/src/configs/logging.config.ts +++ b/packages/@n8n/config/src/configs/logging.config.ts @@ -45,6 +45,7 @@ export const LOG_SCOPES = [ 'expression-engine', 'encryption-key-manager', 'oauth-jwe', + 'mcp-registry', ] as const; export type LogScope = (typeof LOG_SCOPES)[number]; diff --git a/packages/@n8n/db/src/migrations/common/1784000000005-CreateMcpRegistryServerTable.ts b/packages/@n8n/db/src/migrations/common/1784000000005-CreateMcpRegistryServerTable.ts new file mode 100644 index 00000000000..f9e209e8602 --- /dev/null +++ b/packages/@n8n/db/src/migrations/common/1784000000005-CreateMcpRegistryServerTable.ts @@ -0,0 +1,33 @@ +import type { MigrationContext, ReversibleMigration } from '../migration-types'; + +// NOTE: this migration that has a timestamp that's in the future at the time +// of writing. The reason is that previous migrations are also in the future, +// so using the actual timestamp would place this migration in between existing +// ones. This leads to different order of migrations depending on whether the +// database is empty or not. So to avoid unexpected behavior, this migration +// has a timestamp that's 1 ms after the previous migration's timestamp +export class CreateMcpRegistryServerTable1784000000005 implements ReversibleMigration { + async up({ schemaBuilder: { createTable, column } }: MigrationContext) { + await createTable('mcp_registry_server') + .withColumns( + column('id').int.primary, + column('slug').varchar(255).notNull, + column('status') + .varchar(50) + .notNull.withEnumCheck(['active', 'deprecated']) + .comment( + 'Server status in the MCP registry. Deprecated servers are not surfaced to users.', + ), + column('version').varchar(50).notNull, + column('registryUpdatedAt').timestampNoTimezone(3).notNull, + column('data') + .json.notNull.default("'{}'") + .comment('JSON object containing server metadata (icons, remotes, tools, etc.)'), + ) + .withUniqueConstraintOn(['slug']).withTimestamps; + } + + async down({ schemaBuilder: { dropTable } }: MigrationContext) { + await dropTable('mcp_registry_server'); + } +} diff --git a/packages/@n8n/db/src/migrations/postgresdb/index.ts b/packages/@n8n/db/src/migrations/postgresdb/index.ts index f9512718344..6ed3fc4c35b 100644 --- a/packages/@n8n/db/src/migrations/postgresdb/index.ts +++ b/packages/@n8n/db/src/migrations/postgresdb/index.ts @@ -180,6 +180,7 @@ import { CreateAgentObservationTables1784000000000 } from '../common/17840000000 import { ReplaceAgentObservationTables1784000000001 } from '../common/1784000000001-ReplaceAgentObservationTables'; import { DropAgentExecutionWorkingMemory1784000000002 } from '../common/1784000000002-DropAgentExecutionWorkingMemory'; import { AddInsightsRawTimestampIdIndex1784000000004 } from '../common/1784000000004-AddInsightsRawTimestampIdIndex'; +import { CreateMcpRegistryServerTable1784000000005 } from '../common/1784000000005-CreateMcpRegistryServerTable'; import type { Migration } from '../migration-types'; export const postgresMigrations: Migration[] = [ @@ -365,4 +366,5 @@ export const postgresMigrations: Migration[] = [ DropAgentExecutionWorkingMemory1784000000002, LimitWorkflowVersionTriggerToContent1784000000003, AddInsightsRawTimestampIdIndex1784000000004, + CreateMcpRegistryServerTable1784000000005, ]; diff --git a/packages/@n8n/db/src/migrations/sqlite/index.ts b/packages/@n8n/db/src/migrations/sqlite/index.ts index f01c8924dc1..9e7e8c582bc 100644 --- a/packages/@n8n/db/src/migrations/sqlite/index.ts +++ b/packages/@n8n/db/src/migrations/sqlite/index.ts @@ -173,6 +173,7 @@ import { CreateAgentObservationTables1784000000000 } from '../common/17840000000 import { ReplaceAgentObservationTables1784000000001 } from '../common/1784000000001-ReplaceAgentObservationTables'; import { DropAgentExecutionWorkingMemory1784000000002 } from '../common/1784000000002-DropAgentExecutionWorkingMemory'; import { AddInsightsRawTimestampIdIndex1784000000004 } from '../common/1784000000004-AddInsightsRawTimestampIdIndex'; +import { CreateMcpRegistryServerTable1784000000005 } from '../common/1784000000005-CreateMcpRegistryServerTable'; import type { Migration } from '../migration-types'; const sqliteMigrations: Migration[] = [ @@ -351,6 +352,7 @@ const sqliteMigrations: Migration[] = [ DropAgentExecutionWorkingMemory1784000000002, LimitWorkflowVersionTriggerToContent1784000000003, AddInsightsRawTimestampIdIndex1784000000004, + CreateMcpRegistryServerTable1784000000005, ]; export { sqliteMigrations }; diff --git a/packages/@n8n/decorators/src/pubsub/pubsub-metadata.ts b/packages/@n8n/decorators/src/pubsub/pubsub-metadata.ts index ca69f0b8d0d..819f4f5bcdb 100644 --- a/packages/@n8n/decorators/src/pubsub/pubsub-metadata.ts +++ b/packages/@n8n/decorators/src/pubsub/pubsub-metadata.ts @@ -27,6 +27,7 @@ export type PubSubEventName = | 'relay-chat-message-edit' | 'reload-sso-provisioning-configuration' | 'reload-source-control-config' + | 'reload-mcp-registry' | 'cancel-test-run' | 'cancel-collection' | 'agent-chat-integration-changed' diff --git a/packages/cli/src/modules/community-packages/__tests__/community-node-types-utils.test.ts b/packages/cli/src/modules/community-packages/__tests__/community-node-types-utils.test.ts index ef4b8dc8dff..5c1480ea0cb 100644 --- a/packages/cli/src/modules/community-packages/__tests__/community-node-types-utils.test.ts +++ b/packages/cli/src/modules/community-packages/__tests__/community-node-types-utils.test.ts @@ -1,7 +1,7 @@ import { getCommunityNodeTypes, getCommunityNodesMetadata } from '../community-node-types-utils'; -import { paginatedRequest } from '../strapi-utils'; +import { paginatedRequest } from '@/utils/strapi-utils'; -jest.mock('../strapi-utils', () => ({ +jest.mock('@/utils/strapi-utils', () => ({ paginatedRequest: jest.fn(), })); diff --git a/packages/cli/src/modules/community-packages/community-node-types-utils.ts b/packages/cli/src/modules/community-packages/community-node-types-utils.ts index 667679c5897..226dbbddff9 100644 --- a/packages/cli/src/modules/community-packages/community-node-types-utils.ts +++ b/packages/cli/src/modules/community-packages/community-node-types-utils.ts @@ -1,6 +1,6 @@ import type { INodeTypeDescription } from 'n8n-workflow'; -import { paginatedRequest, type StrapiFilters } from './strapi-utils'; +import { paginatedRequest, type StrapiFilters } from '@/utils/strapi-utils'; export type StrapiCommunityNodeType = { id: number; diff --git a/packages/cli/src/modules/community-packages/community-node-types.service.ts b/packages/cli/src/modules/community-packages/community-node-types.service.ts index 7fb44dcc9d8..d3646e62c04 100644 --- a/packages/cli/src/modules/community-packages/community-node-types.service.ts +++ b/packages/cli/src/modules/community-packages/community-node-types.service.ts @@ -12,7 +12,7 @@ import { } from './community-node-types-utils'; import { CommunityPackagesConfig } from './community-packages.config'; import { CommunityPackagesService } from './community-packages.service'; -import { buildStrapiUpdateQuery } from './strapi-utils'; +import { buildStrapiUpdateQuery } from '@/utils/strapi-utils'; const UPDATE_INTERVAL = 8 * 60 * 60 * 1000; const RETRY_INTERVAL = 5 * 60 * 1000; diff --git a/packages/cli/src/modules/mcp-registry/mcp-registry-node-loader.test.ts b/packages/cli/src/modules/mcp-registry/__tests__/mcp-registry-node-loader.test.ts similarity index 73% rename from packages/cli/src/modules/mcp-registry/mcp-registry-node-loader.test.ts rename to packages/cli/src/modules/mcp-registry/__tests__/mcp-registry-node-loader.test.ts index 83afdd1b19a..02158575dbb 100644 --- a/packages/cli/src/modules/mcp-registry/mcp-registry-node-loader.test.ts +++ b/packages/cli/src/modules/mcp-registry/__tests__/mcp-registry-node-loader.test.ts @@ -7,15 +7,14 @@ import type { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; const logger = mock(); -import { McpRegistryNodeLoader } from './mcp-registry-node-loader'; +import { McpRegistryNodeLoader } from '../mcp-registry-node-loader'; import { LANGCHAIN_PACKAGE_NAME, MCP_REGISTRY_BASE_NODE_NAME, MCP_REGISTRY_PACKAGE_NAME, -} from './node-description-transform'; -import type { McpRegistryService } from './registry/mcp-registry.service'; -import type { McpRegistryServer } from './registry/mcp-registry.types'; -import { notionMockServer } from './registry/mock-servers'; +} from '../node-description-transform'; +import type { McpRegistryServer } from '../registry/mcp-registry.types'; +import { notionMockServer } from '../registry/mock-servers'; const baseDescription: INodeTypeDescription = { displayName: 'MCP Registry Client (internal)', @@ -85,18 +84,11 @@ function createLoadNodesAndCredentials(options?: { return { loadNodesAndCredentials, baseNode, sourcePath }; } -function createServiceWithServers(servers: McpRegistryServer[]): McpRegistryService { - const service = mock(); - service.getAll.mockReturnValue(servers); - return service; -} - describe('McpRegistryNodeLoader', () => { describe('packageName', () => { it('matches MCP_REGISTRY_PACKAGE_NAME', () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); expect(loader.packageName).toBe(MCP_REGISTRY_PACKAGE_NAME); }); @@ -105,8 +97,8 @@ describe('McpRegistryNodeLoader', () => { describe('loadAll', () => { it('populates `types`, `known`, registers synthetic nodes and credentials for each supported server', async () => { const { loadNodesAndCredentials, sourcePath } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.loadAll(); @@ -142,8 +134,8 @@ describe('McpRegistryNodeLoader', () => { it('inherits prototype methods from the base node class on synthetic nodes', async () => { const { loadNodesAndCredentials, baseNode } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.loadAll(); @@ -161,8 +153,8 @@ describe('McpRegistryNodeLoader', () => { remotes: [], }; const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer, unsupportedServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer, unsupportedServer]); await loader.loadAll(); @@ -180,8 +172,8 @@ describe('McpRegistryNodeLoader', () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials({ withLangchainLoader: false, }); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.loadAll(); @@ -193,8 +185,8 @@ describe('McpRegistryNodeLoader', () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials({ withBaseNode: false, }); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.loadAll(); @@ -204,8 +196,8 @@ describe('McpRegistryNodeLoader', () => { it('resets prior state before loading', async () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.loadAll(); await loader.loadAll(); @@ -214,22 +206,28 @@ describe('McpRegistryNodeLoader', () => { expect(loader.types.credentials).toHaveLength(1); }); - it('requests deprecated servers from the registry so existing workflows keep loading', async () => { + it('loads deprecated servers when passed through setServers', async () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const deprecatedServer: McpRegistryServer = { + ...notionMockServer, + slug: 'deprecated-server', + status: 'deprecated', + }; + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([deprecatedServer]); await loader.loadAll(); - expect(service.getAll).toHaveBeenCalledWith({ includeDeprecated: true }); + expect(loader.types.nodes).toHaveLength(1); + expect(loader.types.nodes[0].name).toBe('deprecatedServer'); }); }); describe('getNode', () => { it('returns the synthetic LoadedClass for a known type', async () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.loadAll(); const result = loader.getNode('notion'); @@ -240,8 +238,7 @@ describe('McpRegistryNodeLoader', () => { it('throws UnrecognizedNodeTypeError for an unknown type', () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); expect(() => loader.getNode('unknown')).toThrow(UnrecognizedNodeTypeError); }); @@ -250,8 +247,8 @@ describe('McpRegistryNodeLoader', () => { describe('getCredential', () => { it('returns the credential for a known credential type', async () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.loadAll(); @@ -262,8 +259,7 @@ describe('McpRegistryNodeLoader', () => { it('throws UnrecognizedCredentialTypeError for an unknown credential type', () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); expect(() => loader.getCredential('unknown')).toThrow(UnrecognizedCredentialTypeError); }); @@ -272,8 +268,8 @@ describe('McpRegistryNodeLoader', () => { describe('state management', () => { it('reset clears known, types, and registered node types', async () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.loadAll(); loader.reset(); @@ -290,8 +286,8 @@ describe('McpRegistryNodeLoader', () => { it('releaseTypes only clears types', async () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.loadAll(); loader.releaseTypes(); @@ -304,14 +300,14 @@ describe('McpRegistryNodeLoader', () => { it('ensureTypesLoaded calls loadAll only when types are empty', async () => { const { loadNodesAndCredentials } = createLoadNodesAndCredentials(); - const service = createServiceWithServers([notionMockServer]); - const loader = new McpRegistryNodeLoader(service, loadNodesAndCredentials, logger); + const loader = new McpRegistryNodeLoader(loadNodesAndCredentials, logger); + loader.setServers([notionMockServer]); await loader.ensureTypesLoaded(); - expect(service.getAll).toHaveBeenCalledTimes(1); + expect(loader.types.nodes).toHaveLength(1); await loader.ensureTypesLoaded(); - expect(service.getAll).toHaveBeenCalledTimes(1); + expect(loader.types.nodes).toHaveLength(1); }); }); }); diff --git a/packages/cli/src/modules/mcp-registry/__tests__/mcp-registry-test.controller.test.ts b/packages/cli/src/modules/mcp-registry/__tests__/mcp-registry-test.controller.test.ts new file mode 100644 index 00000000000..3a8de4b6ec9 --- /dev/null +++ b/packages/cli/src/modules/mcp-registry/__tests__/mcp-registry-test.controller.test.ts @@ -0,0 +1,54 @@ +import { mock } from 'jest-mock-extended'; + +import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; + +import { McpRegistryTestController } from '../mcp-registry-test.controller'; +import type { McpRegistryServerRepository } from '../registry/mcp-registry-server.repository'; +import type { McpRegistryService } from '../registry/mcp-registry.service'; +import { toEntity } from '../registry/mcp-registry.types'; +import { notionMockServer, linearMockServer } from '../registry/mock-servers'; + +describe('McpRegistryTestController', () => { + const repository = mock(); + const service = mock(); + const controller = new McpRegistryTestController(repository, service); + + const originalEnv = process.env; + + beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...originalEnv, E2E_TESTS: 'true' }; + }); + + afterAll(() => { + process.env = originalEnv; + }); + + describe('seed', () => { + it('should upsert mock servers and trigger registry reload', async () => { + repository.upsert.mockResolvedValue({} as never); + service.handleReloadMcpRegistry.mockResolvedValue(); + + const result = await controller.seed(); + + expect(repository.upsert).toHaveBeenCalledWith( + [notionMockServer, linearMockServer].map(toEntity), + ['id'], + ); + expect(service.handleReloadMcpRegistry).toHaveBeenCalled(); + expect(result).toEqual({ ok: true, count: 2 }); + }); + + it('should throw ForbiddenError when E2E_TESTS is not set', async () => { + delete process.env.E2E_TESTS; + + await expect(controller.seed()).rejects.toThrow(ForbiddenError); + }); + + it('should throw ForbiddenError in production even when E2E_TESTS is set', async () => { + process.env.NODE_ENV = 'production'; + + await expect(controller.seed()).rejects.toThrow(ForbiddenError); + }); + }); +}); diff --git a/packages/cli/src/modules/mcp-registry/node-description-transform.test.ts b/packages/cli/src/modules/mcp-registry/__tests__/node-description-transform.test.ts similarity index 98% rename from packages/cli/src/modules/mcp-registry/node-description-transform.test.ts rename to packages/cli/src/modules/mcp-registry/__tests__/node-description-transform.test.ts index 6ded55a3c3f..658d9e353b2 100644 --- a/packages/cli/src/modules/mcp-registry/node-description-transform.test.ts +++ b/packages/cli/src/modules/mcp-registry/__tests__/node-description-transform.test.ts @@ -3,9 +3,9 @@ import { deepCopy, type INodeTypeDescription } from 'n8n-workflow'; import { serverToNodeDescription, serverToCredentialDescription, -} from './node-description-transform'; -import type { McpRegistryServer } from './registry/mcp-registry.types'; -import { notionMockServer } from './registry/mock-servers'; +} from '../node-description-transform'; +import type { McpRegistryServer } from '../registry/mcp-registry.types'; +import { notionMockServer } from '../registry/mock-servers'; const baseDescription: INodeTypeDescription = { displayName: 'MCP Registry Client (internal)', diff --git a/packages/cli/src/modules/mcp-registry/mcp-registry-node-loader.ts b/packages/cli/src/modules/mcp-registry/mcp-registry-node-loader.ts index fef4967d728..9a77929e699 100644 --- a/packages/cli/src/modules/mcp-registry/mcp-registry-node-loader.ts +++ b/packages/cli/src/modules/mcp-registry/mcp-registry-node-loader.ts @@ -24,7 +24,7 @@ import { serverToCredentialDescription, serverToNodeDescription, } from './node-description-transform'; -import type { McpRegistryService } from './registry/mcp-registry.service'; +import type { McpRegistryServer } from './registry/mcp-registry.types'; /** * Synthetic node loader: turns each registry server into a node type, all @@ -46,12 +46,17 @@ export class McpRegistryNodeLoader implements NodeLoader { private typesReleased = true; + private servers: McpRegistryServer[] = []; + constructor( - private readonly registry: McpRegistryService, private readonly loadNodesAndCredentials: LoadNodesAndCredentials, private readonly logger: Logger, ) {} + setServers(servers: McpRegistryServer[]): void { + this.servers = servers; + } + async loadAll(): Promise { this.reset(); @@ -62,7 +67,7 @@ export class McpRegistryNodeLoader implements NodeLoader { const { type: baseNode, sourcePath } = baseLoaded; const { description: baseDescription } = NodeHelpers.getVersionedNodeType(baseNode); - for (const server of this.registry.getAll({ includeDeprecated: true })) { + for (const server of this.servers) { const nodeDescription = serverToNodeDescription(server, baseDescription); const credentialDescription = serverToCredentialDescription(server); if (!nodeDescription || !credentialDescription) continue; diff --git a/packages/cli/src/modules/mcp-registry/mcp-registry-test.controller.ts b/packages/cli/src/modules/mcp-registry/mcp-registry-test.controller.ts new file mode 100644 index 00000000000..306055626cd --- /dev/null +++ b/packages/cli/src/modules/mcp-registry/mcp-registry-test.controller.ts @@ -0,0 +1,37 @@ +import { Post, RestController } from '@n8n/decorators'; + +import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; + +import { McpRegistryServerRepository } from './registry/mcp-registry-server.repository'; +import { McpRegistryService } from './registry/mcp-registry.service'; +import { toEntity } from './registry/mcp-registry.types'; +import { notionMockServer, linearMockServer } from './registry/mock-servers'; + +/** + * Test-only endpoints for seeding MCP registry data in E2E tests. + * Only registered when E2E_TESTS is set. + */ +@RestController('/mcp-registry') +export class McpRegistryTestController { + constructor( + private readonly repository: McpRegistryServerRepository, + private readonly service: McpRegistryService, + ) {} + + @Post('/test/seed', { skipAuth: true }) + async seed() { + this.assertE2ETestsEnabled(); + + const entities = [notionMockServer, linearMockServer].map(toEntity); + await this.repository.upsert(entities, ['id']); + await this.service.handleReloadMcpRegistry(); + + return { ok: true, count: entities.length }; + } + + private assertE2ETestsEnabled(): void { + if (process.env.E2E_TESTS !== 'true' || process.env.NODE_ENV === 'production') { + throw new ForbiddenError('MCP registry test endpoints are not enabled'); + } + } +} diff --git a/packages/cli/src/modules/mcp-registry/mcp-registry.module.ts b/packages/cli/src/modules/mcp-registry/mcp-registry.module.ts index c6b82e757e5..96ad7bf2bc0 100644 --- a/packages/cli/src/modules/mcp-registry/mcp-registry.module.ts +++ b/packages/cli/src/modules/mcp-registry/mcp-registry.module.ts @@ -7,16 +7,25 @@ import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; @BackendModule({ name: 'mcp-registry' }) export class McpRegistryModule implements ModuleInterface { - async nodeLoaders() { + async init() { const { McpRegistryService } = await import('./registry/mcp-registry.service'); + await Container.get(McpRegistryService).init(); + + if (process.env.E2E_TESTS === 'true' && process.env.NODE_ENV !== 'production') { + await import('./mcp-registry-test.controller'); + } + } + + async entities() { + const { McpRegistryServerEntity } = await import('./registry/mcp-registry-server.entity'); + return [McpRegistryServerEntity]; + } + + async nodeLoaders() { const { McpRegistryNodeLoader } = await import('./mcp-registry-node-loader'); return [ - new McpRegistryNodeLoader( - Container.get(McpRegistryService), - Container.get(LoadNodesAndCredentials), - Container.get(Logger), - ), + new McpRegistryNodeLoader(Container.get(LoadNodesAndCredentials), Container.get(Logger)), ]; } } diff --git a/packages/cli/src/modules/mcp-registry/registry/__tests__/mcp-registry-api.client.test.ts b/packages/cli/src/modules/mcp-registry/registry/__tests__/mcp-registry-api.client.test.ts new file mode 100644 index 00000000000..a2a9dee7cf1 --- /dev/null +++ b/packages/cli/src/modules/mcp-registry/registry/__tests__/mcp-registry-api.client.test.ts @@ -0,0 +1,215 @@ +import { paginatedRequest, buildStrapiUpdateQuery } from '@/utils/strapi-utils'; + +import { McpRegistryApiClient } from '../mcp-registry-api.client'; + +jest.mock('@/utils/strapi-utils', () => ({ + paginatedRequest: jest.fn(), + buildStrapiUpdateQuery: jest.requireActual('@/utils/strapi-utils').buildStrapiUpdateQuery, +})); + +const mockPaginatedRequest = paginatedRequest as jest.MockedFunction; + +const PRODUCTION_URL = 'https://api.n8n.io/api/mcp-servers'; +const STAGING_URL = 'https://api-staging.n8n.io/api/mcp-servers'; + +describe('McpRegistryApiClient', () => { + let client: McpRegistryApiClient; + const originalEnv = process.env.ENVIRONMENT; + + beforeEach(() => { + jest.clearAllMocks(); + delete process.env.ENVIRONMENT; + client = new McpRegistryApiClient(); + }); + + afterEach(() => { + if (originalEnv !== undefined) { + process.env.ENVIRONMENT = originalEnv; + } else { + delete process.env.ENVIRONMENT; + } + }); + + describe('getUrl', () => { + it('should use production URL by default', async () => { + mockPaginatedRequest.mockResolvedValue([]); + + await client.fetchAllServers(); + + expect(mockPaginatedRequest).toHaveBeenCalledWith( + PRODUCTION_URL, + expect.any(Object), + expect.any(Object), + ); + }); + + it('should use staging URL when ENVIRONMENT is staging', async () => { + process.env.ENVIRONMENT = 'staging'; + mockPaginatedRequest.mockResolvedValue([]); + + await client.fetchAllServers(); + + expect(mockPaginatedRequest).toHaveBeenCalledWith( + STAGING_URL, + expect.any(Object), + expect.any(Object), + ); + }); + + it('should use production URL when ENVIRONMENT is production', async () => { + process.env.ENVIRONMENT = 'production'; + mockPaginatedRequest.mockResolvedValue([]); + + await client.fetchAllServers(); + + expect(mockPaginatedRequest).toHaveBeenCalledWith( + PRODUCTION_URL, + expect.any(Object), + expect.any(Object), + ); + }); + }); + + describe('fetchAllServers', () => { + it('should call paginatedRequest with pageSize 25', async () => { + mockPaginatedRequest.mockResolvedValue([]); + + await client.fetchAllServers(); + + expect(mockPaginatedRequest).toHaveBeenCalledWith( + PRODUCTION_URL, + { + pagination: { page: 1, pageSize: 25 }, + }, + { throwOnError: true }, + ); + }); + + it('should return servers from paginatedRequest', async () => { + const mockServers = [ + { id: 1, name: 'server-a' }, + { id: 2, name: 'server-b' }, + ]; + mockPaginatedRequest.mockResolvedValue(mockServers); + + const result = await client.fetchAllServers(); + + expect(result).toEqual(mockServers); + }); + }); + + describe('fetchServersMetadata', () => { + it('should request only version and updatedAt fields with pageSize 500', async () => { + mockPaginatedRequest.mockResolvedValue([]); + + await client.fetchServersMetadata(); + + expect(mockPaginatedRequest).toHaveBeenCalledWith( + PRODUCTION_URL, + { + fields: ['version', 'updatedAt'], + pagination: { page: 1, pageSize: 500 }, + }, + { throwOnError: true }, + ); + }); + + it('should return metadata from paginatedRequest', async () => { + const mockMetadata = [ + { id: 1, version: '1.0.0', updatedAt: '2025-01-01' }, + { id: 2, version: '2.0.0', updatedAt: '2025-01-02' }, + ]; + mockPaginatedRequest.mockResolvedValue(mockMetadata); + + const result = await client.fetchServersMetadata(); + + expect(result).toEqual(mockMetadata); + }); + }); + + describe('fetchServersByIds', () => { + it('should fetch servers using filter query built from ids', async () => { + mockPaginatedRequest.mockResolvedValue([]); + + await client.fetchServersByIds([1, 2, 3]); + + expect(mockPaginatedRequest).toHaveBeenCalledWith( + PRODUCTION_URL, + { + ...buildStrapiUpdateQuery([1, 2, 3]), + pagination: { page: 1, pageSize: 25 }, + }, + { throwOnError: true }, + ); + }); + + it('should return fetched servers', async () => { + const mockServers = [{ id: 1, name: 'server-a' }]; + mockPaginatedRequest.mockResolvedValue(mockServers); + + const result = await client.fetchServersByIds([1]); + + expect(result).toEqual(mockServers); + }); + + it('should return empty array for empty ids', async () => { + const result = await client.fetchServersByIds([]); + + expect(result).toEqual([]); + expect(mockPaginatedRequest).not.toHaveBeenCalled(); + }); + + it('should batch ids in chunks of 100', async () => { + const ids = Array.from({ length: 250 }, (_, i) => i + 1); + mockPaginatedRequest.mockResolvedValue([]); + + await client.fetchServersByIds(ids); + + expect(mockPaginatedRequest).toHaveBeenCalledTimes(3); + + // First batch: ids 1-100 + expect(mockPaginatedRequest).toHaveBeenNthCalledWith( + 1, + PRODUCTION_URL, + { + ...buildStrapiUpdateQuery(ids.slice(0, 100)), + pagination: { page: 1, pageSize: 25 }, + }, + { throwOnError: true }, + ); + + // Second batch: ids 101-200 + expect(mockPaginatedRequest).toHaveBeenNthCalledWith( + 2, + PRODUCTION_URL, + { + ...buildStrapiUpdateQuery(ids.slice(100, 200)), + pagination: { page: 1, pageSize: 25 }, + }, + { throwOnError: true }, + ); + + // Third batch: ids 201-250 + expect(mockPaginatedRequest).toHaveBeenNthCalledWith( + 3, + PRODUCTION_URL, + { + ...buildStrapiUpdateQuery(ids.slice(200, 250)), + pagination: { page: 1, pageSize: 25 }, + }, + { throwOnError: true }, + ); + }); + + it('should concatenate results from all batches', async () => { + const ids = Array.from({ length: 150 }, (_, i) => i + 1); + const batch1 = [{ id: 1, name: 'server-1' }]; + const batch2 = [{ id: 101, name: 'server-101' }]; + mockPaginatedRequest.mockResolvedValueOnce(batch1).mockResolvedValueOnce(batch2); + + const result = await client.fetchServersByIds(ids); + + expect(result).toEqual([...batch1, ...batch2]); + }); + }); +}); diff --git a/packages/cli/src/modules/mcp-registry/registry/__tests__/mcp-registry.service.test.ts b/packages/cli/src/modules/mcp-registry/registry/__tests__/mcp-registry.service.test.ts new file mode 100644 index 00000000000..e3a9dfbda81 --- /dev/null +++ b/packages/cli/src/modules/mcp-registry/registry/__tests__/mcp-registry.service.test.ts @@ -0,0 +1,258 @@ +import type { Logger } from '@n8n/backend-common'; +import { mock } from 'jest-mock-extended'; +import type { InstanceSettings } from 'n8n-core'; + +import type { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import type { Push } from '@/push'; +import type { Publisher } from '@/scaling/pubsub/publisher.service'; + +import type { McpRegistryApiClient, McpRegistryServerMetadata } from '../mcp-registry-api.client'; +import type { McpRegistryServerEntity } from '../mcp-registry-server.entity'; +import type { McpRegistryServerRepository } from '../mcp-registry-server.repository'; +import { McpRegistryService } from '../mcp-registry.service'; +import type { McpRegistryServer } from '../mcp-registry.types'; +import { toEntity } from '../mcp-registry.types'; +import { linearMockServer, notionMockServer } from '../mock-servers'; + +function toMockEntity(server: McpRegistryServer): McpRegistryServerEntity { + const now = new Date(); + return { ...toEntity(server), createdAt: now, updatedAt: now } as McpRegistryServerEntity; +} + +type CreateServiceOptions = { + storedServers?: McpRegistryServer[] | null; + isLeader?: boolean; + instanceType?: 'main' | 'worker'; +}; + +function createService(options: CreateServiceOptions = {}) { + const logger = mock({ scoped: jest.fn().mockReturnThis() }); + const repository = mock(); + const apiClient = mock(); + const instanceSettings = mock({ + isLeader: options.isLeader ?? false, + instanceType: options.instanceType ?? 'main', + }); + const loadNodesAndCredentials = mock({ loaders: {} }); + const push = mock({ broadcast: jest.fn() }); + const publisher = mock({ publishCommand: jest.fn().mockResolvedValue(undefined) }); + + if (options.storedServers === null) { + repository.find.mockResolvedValue([]); + repository.findBy.mockResolvedValue([]); + } else { + const servers = options.storedServers ?? [notionMockServer, linearMockServer]; + const entities = servers.map(toMockEntity); + repository.find.mockResolvedValue(entities); + repository.findBy.mockImplementation(async (where) => { + if (where && 'status' in where) { + return entities.filter((e) => e.status === where.status); + } + return entities; + }); + repository.findOneBy.mockImplementation(async (where) => { + if (where && 'slug' in where) { + return entities.find((e) => e.slug === where.slug) ?? null; + } + return null; + }); + } + + apiClient.fetchServersMetadata.mockResolvedValue([]); + apiClient.fetchServersByIds.mockResolvedValue([]); + apiClient.fetchAllServers.mockResolvedValue([notionMockServer, linearMockServer]); + repository.upsert.mockResolvedValue({} as never); + + const service = new McpRegistryService( + logger, + repository, + apiClient, + instanceSettings, + loadNodesAndCredentials, + push, + publisher, + ); + + return { + service, + repository, + apiClient, + push, + publisher, + }; +} + +describe('McpRegistryService', () => { + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + describe('getAll / get', () => { + it('returns active servers by default', async () => { + const deprecated: McpRegistryServer = { + ...notionMockServer, + slug: 'old-notion', + status: 'deprecated', + }; + const { service } = createService({ + storedServers: [notionMockServer, linearMockServer, deprecated], + }); + + await service.init(); + const servers = await service.getAll(); + + expect(servers).toEqual([notionMockServer, linearMockServer]); + }); + + it('includes deprecated servers when includeDeprecated is true', async () => { + const deprecated: McpRegistryServer = { + ...notionMockServer, + slug: 'old-notion', + status: 'deprecated', + }; + const { service } = createService({ + storedServers: [notionMockServer, linearMockServer, deprecated], + }); + + await service.init(); + const servers = await service.getAll({ includeDeprecated: true }); + + expect(servers).toEqual([notionMockServer, linearMockServer, deprecated]); + }); + + it('returns server by slug and undefined for unknown slug', async () => { + const { service } = createService(); + + await service.init(); + const notion = await service.get('notion'); + const missing = await service.get('missing'); + + expect(notion).toEqual(notionMockServer); + expect(missing).toBeUndefined(); + }); + }); + + describe('refresh flow', () => { + it('init does not start periodic refresh on followers', async () => { + jest.useFakeTimers(); + const setIntervalSpy = jest.spyOn(global, 'setInterval'); + const { service, apiClient } = createService({ isLeader: false }); + + await service.init(); + + expect(setIntervalSpy).not.toHaveBeenCalled(); + expect(apiClient.fetchServersMetadata).not.toHaveBeenCalled(); + }); + + it('init starts periodic refresh and kicks off startup refresh on leaders', async () => { + jest.useFakeTimers(); + const setIntervalSpy = jest.spyOn(global, 'setInterval'); + const { service, apiClient } = createService({ isLeader: true }); + + await service.init(); + await Promise.resolve(); + + expect(setIntervalSpy).toHaveBeenCalledTimes(1); + expect(apiClient.fetchServersMetadata).toHaveBeenCalledTimes(1); + + service.shutdown(); + }); + + it('onLeaderTakeover skips write + notifications when metadata is unchanged', async () => { + jest.useFakeTimers(); + const setIntervalSpy = jest.spyOn(global, 'setInterval'); + const metadata: McpRegistryServerMetadata[] = [ + { + id: notionMockServer.id, + version: notionMockServer.version, + updatedAt: notionMockServer.updatedAt, + }, + { + id: linearMockServer.id, + version: linearMockServer.version, + updatedAt: linearMockServer.updatedAt, + }, + ]; + const { service, apiClient, repository, push, publisher } = createService(); + apiClient.fetchServersMetadata.mockResolvedValue(metadata); + + await service.onLeaderTakeover(); + + expect(apiClient.fetchServersByIds).not.toHaveBeenCalled(); + expect(repository.upsert).not.toHaveBeenCalled(); + expect(push.broadcast).not.toHaveBeenCalled(); + expect(publisher.publishCommand).not.toHaveBeenCalled(); + expect(setIntervalSpy).toHaveBeenCalledTimes(1); + + service.shutdown(); + }); + + it('onLeaderTakeover fetches only changed servers and publishes reload', async () => { + const staleNotion: McpRegistryServer = { + ...notionMockServer, + version: '1.1.0', + updatedAt: '2026-04-01T10:00:00.000Z', + }; + const metadata: McpRegistryServerMetadata[] = [ + { + id: notionMockServer.id, + version: notionMockServer.version, + updatedAt: notionMockServer.updatedAt, + }, + { + id: linearMockServer.id, + version: linearMockServer.version, + updatedAt: linearMockServer.updatedAt, + }, + ]; + const { service, apiClient, repository, push, publisher } = createService({ + storedServers: [staleNotion, linearMockServer], + }); + apiClient.fetchServersMetadata.mockResolvedValue(metadata); + apiClient.fetchServersByIds.mockResolvedValue([notionMockServer]); + + await service.onLeaderTakeover(); + + expect(apiClient.fetchAllServers).not.toHaveBeenCalled(); + expect(apiClient.fetchServersByIds).toHaveBeenCalledWith([notionMockServer.id]); + expect(repository.upsert).toHaveBeenCalledTimes(1); + const upsertEntities = repository.upsert.mock.calls[0][0]; + expect(upsertEntities).toEqual([notionMockServer].map(toEntity)); + expect(push.broadcast).toHaveBeenCalledWith({ type: 'nodeDescriptionUpdated', data: {} }); + expect(publisher.publishCommand).toHaveBeenCalledWith({ command: 'reload-mcp-registry' }); + + service.shutdown(); + }); + + it('onLeaderTakeover fetches all servers when no data is persisted', async () => { + const { service, apiClient, repository } = createService({ storedServers: null }); + + await service.onLeaderTakeover(); + + expect(apiClient.fetchAllServers).toHaveBeenCalledTimes(1); + expect(apiClient.fetchServersMetadata).not.toHaveBeenCalled(); + expect(repository.upsert).toHaveBeenCalledTimes(1); + + service.shutdown(); + }); + }); + + describe('handleReloadMcpRegistry', () => { + it('notifies the UI on the main instance', async () => { + const { service, push } = createService({ instanceType: 'main' }); + + await service.handleReloadMcpRegistry(); + + expect(push.broadcast).toHaveBeenCalledWith({ type: 'nodeDescriptionUpdated', data: {} }); + }); + + it('does not notify the UI on worker instances', async () => { + const { service, push } = createService({ instanceType: 'worker' }); + + await service.handleReloadMcpRegistry(); + + expect(push.broadcast).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/cli/src/modules/mcp-registry/registry/mcp-registry-api.client.ts b/packages/cli/src/modules/mcp-registry/registry/mcp-registry-api.client.ts new file mode 100644 index 00000000000..02e75540572 --- /dev/null +++ b/packages/cli/src/modules/mcp-registry/registry/mcp-registry-api.client.ts @@ -0,0 +1,71 @@ +import { Service } from '@n8n/di'; + +import { buildStrapiUpdateQuery, paginatedRequest } from '@/utils/strapi-utils'; + +import type { McpRegistryServer } from './mcp-registry.types'; + +export type McpRegistryServerMetadata = Pick; + +const MCP_SERVERS_STAGING_URL = 'https://api-staging.n8n.io/api/mcp-servers'; +const MCP_SERVERS_PRODUCTION_URL = 'https://api.n8n.io/api/mcp-servers'; + +/** Strapi's qs parser has an arrayLimit of 100 */ +const STRAPI_ARRAY_LIMIT = 100; + +@Service() +export class McpRegistryApiClient { + async fetchAllServers(): Promise { + return await paginatedRequest( + this.getUrl(), + { + pagination: { page: 1, pageSize: 25 }, + }, + { + throwOnError: true, + }, + ); + } + + async fetchServersMetadata(): Promise { + return await paginatedRequest( + this.getUrl(), + { + fields: ['version', 'updatedAt'], + pagination: { page: 1, pageSize: 500 }, + }, + { + throwOnError: true, + }, + ); + } + + async fetchServersByIds(ids: number[]): Promise { + const data: McpRegistryServer[] = []; + for (let i = 0; i < ids.length; i += STRAPI_ARRAY_LIMIT) { + const batch = ids.slice(i, i + STRAPI_ARRAY_LIMIT); + const qs = buildStrapiUpdateQuery(batch); + const batchData = await paginatedRequest( + this.getUrl(), + { + ...qs, + pagination: { page: 1, pageSize: 25 }, + }, + { + throwOnError: true, + }, + ); + data.push(...batchData); + } + + return data; + } + + private getUrl(): string { + const environment = process.env.ENVIRONMENT; + if (environment === 'staging') { + return MCP_SERVERS_STAGING_URL; + } + + return MCP_SERVERS_PRODUCTION_URL; + } +} diff --git a/packages/cli/src/modules/mcp-registry/registry/mcp-registry-server.entity.ts b/packages/cli/src/modules/mcp-registry/registry/mcp-registry-server.entity.ts new file mode 100644 index 00000000000..6261e4bf1e6 --- /dev/null +++ b/packages/cli/src/modules/mcp-registry/registry/mcp-registry-server.entity.ts @@ -0,0 +1,50 @@ +import { datetimeColumnType, JsonColumn, WithTimestamps } from '@n8n/db'; +import { Column, Entity, Index, PrimaryColumn } from '@n8n/typeorm'; + +export type McpRegistryServerData = { + name: string; + title: string; + tagline: string; + description: string; + authType: string; + origin: string; + isOfficial: boolean; + icons: Array<{ + src: string; + mimeType?: string; + theme?: string; + }>; + remotes: Array<{ + type: string; + url: string; + }>; + tools: Array<{ + name: string; + title?: string; + annotations?: { readOnlyHint?: boolean }; + }>; + websiteUrl?: string; + tags?: string[]; +}; + +@Entity('mcp_registry_server') +export class McpRegistryServerEntity extends WithTimestamps { + @PrimaryColumn('int') + id: number; + + @Index({ unique: true }) + @Column('varchar') + slug: string; + + @Column('varchar') + status: 'active' | 'deprecated'; + + @Column('varchar') + version: string; + + @Column(datetimeColumnType) + registryUpdatedAt: Date; + + @JsonColumn() + data: McpRegistryServerData; +} diff --git a/packages/cli/src/modules/mcp-registry/registry/mcp-registry-server.repository.ts b/packages/cli/src/modules/mcp-registry/registry/mcp-registry-server.repository.ts new file mode 100644 index 00000000000..4a172e7d1a3 --- /dev/null +++ b/packages/cli/src/modules/mcp-registry/registry/mcp-registry-server.repository.ts @@ -0,0 +1,11 @@ +import { Service } from '@n8n/di'; +import { DataSource, Repository } from '@n8n/typeorm'; + +import { McpRegistryServerEntity } from './mcp-registry-server.entity'; + +@Service() +export class McpRegistryServerRepository extends Repository { + constructor(dataSource: DataSource) { + super(McpRegistryServerEntity, dataSource.manager); + } +} diff --git a/packages/cli/src/modules/mcp-registry/registry/mcp-registry.service.test.ts b/packages/cli/src/modules/mcp-registry/registry/mcp-registry.service.test.ts deleted file mode 100644 index 292c26ac7b1..00000000000 --- a/packages/cli/src/modules/mcp-registry/registry/mcp-registry.service.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { McpRegistryService } from './mcp-registry.service'; -import type { McpRegistryServer } from './mcp-registry.types'; -import { linearMockServer, notionMockServer } from './mock-servers'; - -describe('McpRegistryService', () => { - let service: McpRegistryService; - - beforeEach(() => { - service = new McpRegistryService(); - }); - - function seedDeprecated(slug: string): McpRegistryServer { - const deprecated: McpRegistryServer = { ...notionMockServer, slug, status: 'deprecated' }; - (service as unknown as { servers: Map }).servers.set( - slug, - deprecated, - ); - return deprecated; - } - - describe('getAll', () => { - it('returns active servers by default', () => { - expect(service.getAll()).toEqual([notionMockServer, linearMockServer]); - }); - - it('omits deprecated servers by default', () => { - seedDeprecated('old'); - - expect(service.getAll()).toEqual([notionMockServer, linearMockServer]); - }); - - it('returns deprecated servers when includeDeprecated is true', () => { - const deprecated = seedDeprecated('old'); - - expect(service.getAll({ includeDeprecated: true })).toEqual([ - notionMockServer, - linearMockServer, - deprecated, - ]); - }); - }); - - describe('get', () => { - it('returns the matching server by slug', () => { - expect(service.get('notion')).toEqual(notionMockServer); - }); - - it('returns undefined when the slug is not registered', () => { - expect(service.get('does-not-exist')).toBeUndefined(); - }); - }); -}); diff --git a/packages/cli/src/modules/mcp-registry/registry/mcp-registry.service.ts b/packages/cli/src/modules/mcp-registry/registry/mcp-registry.service.ts index 8b21c087c8e..c69f8a2c28b 100644 --- a/packages/cli/src/modules/mcp-registry/registry/mcp-registry.service.ts +++ b/packages/cli/src/modules/mcp-registry/registry/mcp-registry.service.ts @@ -1,22 +1,227 @@ +import { Logger } from '@n8n/backend-common'; +import { Time } from '@n8n/constants'; +import { OnLeaderStepdown, OnLeaderTakeover, OnPubSubEvent, OnShutdown } from '@n8n/decorators'; import { Service } from '@n8n/di'; +import { InstanceSettings } from 'n8n-core'; +import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import { Push } from '@/push'; +import { Publisher } from '@/scaling/pubsub/publisher.service'; + +import { McpRegistryServerRepository } from './mcp-registry-server.repository'; +import { McpRegistryNodeLoader } from '../mcp-registry-node-loader'; +import type { McpRegistryServerMetadata } from './mcp-registry-api.client'; +import { McpRegistryApiClient } from './mcp-registry-api.client'; import type { McpRegistryServer } from './mcp-registry.types'; -import { notionMockServer, linearMockServer } from './mock-servers'; +import { toEntity, fromEntity } from './mcp-registry.types'; +import { MCP_REGISTRY_PACKAGE_NAME } from '../node-description-transform'; + +type RefreshReason = 'startup' | 'leader-takeover' | 'interval'; + +const REFRESH_INTERVAL_HOURS = 8; + +const REFRESH_INTERVAL_MS = REFRESH_INTERVAL_HOURS * Time.hours.toMilliseconds; @Service() export class McpRegistryService { - // TODO: Implement actual registry fetching and caching - private readonly servers = new Map([ - [notionMockServer.slug, notionMockServer], - [linearMockServer.slug, linearMockServer], - ]); + private refreshInterval: NodeJS.Timeout | undefined; - getAll({ includeDeprecated = false }: { includeDeprecated?: boolean } = {}): McpRegistryServer[] { - const all = Array.from(this.servers.values()); - return includeDeprecated ? all : all.filter((server) => server.status === 'active'); + private refreshPromise: Promise | undefined; + + private isShuttingDown = false; + + constructor( + private readonly logger: Logger, + private readonly repository: McpRegistryServerRepository, + private readonly apiClient: McpRegistryApiClient, + private readonly instanceSettings: InstanceSettings, + private readonly loadNodesAndCredentials: LoadNodesAndCredentials, + private readonly push: Push, + private readonly publisher: Publisher, + ) { + this.logger = logger.scoped('mcp-registry'); } - get(slug: string): McpRegistryServer | undefined { - return this.servers.get(slug); + async init(): Promise { + await this.refreshRegistryNodeTypes(false); + if (this.instanceSettings.isLeader) { + // don't want to wait for API calls to block on init + void this.refreshFromApi('startup'); + this.startPeriodicRefresh(); + } + } + + @OnLeaderTakeover() + async onLeaderTakeover(): Promise { + await this.refreshFromApi('leader-takeover'); + this.startPeriodicRefresh(); + } + + @OnLeaderStepdown() + onLeaderStepdown(): void { + this.stopPeriodicRefresh(); + } + + @OnShutdown() + shutdown(): void { + this.isShuttingDown = true; + this.stopPeriodicRefresh(); + } + + @OnPubSubEvent('reload-mcp-registry') + async handleReloadMcpRegistry(): Promise { + await this.refreshRegistryNodeTypes(true); + if (this.isMainInstance()) { + this.notifyNodeDescriptionsUpdated(); + } + } + + async getAll({ + includeDeprecated = false, + }: { includeDeprecated?: boolean } = {}): Promise { + const entities = includeDeprecated + ? await this.repository.find() + : await this.repository.findBy({ status: 'active' }); + return entities.map(fromEntity); + } + + async get(slug: string): Promise { + const entity = await this.repository.findOneBy({ slug }); + return entity ? fromEntity(entity) : undefined; + } + + private startPeriodicRefresh(): void { + if (this.isShuttingDown || this.refreshInterval) { + return; + } + + this.refreshInterval = setInterval(() => { + void this.refreshFromApi('interval'); + }, REFRESH_INTERVAL_MS); + + this.logger.debug('Scheduled MCP registry refresh', { + intervalHours: REFRESH_INTERVAL_HOURS, + }); + } + + private stopPeriodicRefresh(): void { + clearInterval(this.refreshInterval); + this.refreshInterval = undefined; + } + + private async refreshFromApi(reason: RefreshReason): Promise { + if (this.refreshPromise) { + await this.refreshPromise; + return; + } + + this.refreshPromise = this.refreshFromApiInternal(reason); + try { + await this.refreshPromise; + } finally { + this.refreshPromise = undefined; + } + } + + private async refreshFromApiInternal(reason: RefreshReason): Promise { + try { + const existingServers = await this.getAll({ includeDeprecated: true }); + let updatedServers: McpRegistryServer[]; + if (existingServers.length === 0) { + updatedServers = await this.apiClient.fetchAllServers(); + } else { + const result = await this.refreshUpdatedServers(existingServers); + if (result === null) { + this.logger.debug('MCP registry is up to date', { reason }); + return; + } + + updatedServers = result; + } + + await this.saveServers(updatedServers); + await this.refreshRegistryNodeTypes(true); + this.notifyNodeDescriptionsUpdated(); + await this.publishReloadCommand(); + + this.logger.debug('MCP registry refreshed', { + reason, + serverCount: updatedServers.length, + }); + } catch (error) { + this.logger.error('Failed to refresh MCP registry', { error, reason }); + } + } + + private async refreshUpdatedServers( + existingServers: McpRegistryServer[], + ): Promise { + const metadata = await this.apiClient.fetchServersMetadata(); + const existingById = new Map(existingServers.map((server) => [server.id, server])); + const idsToFetch = metadata + .filter((entry) => this.shouldFetchFullServer(entry, existingById.get(entry.id))) + .map(({ id }) => id); + if (idsToFetch.length === 0) { + return null; + } + + return await this.apiClient.fetchServersByIds(idsToFetch); + } + + private shouldFetchFullServer( + metadata: McpRegistryServerMetadata, + existing: McpRegistryServer | undefined, + ): boolean { + return ( + !existing || + existing.version !== metadata.version || + existing.updatedAt !== metadata.updatedAt + ); + } + + private async saveServers(servers: McpRegistryServer[]): Promise { + const entities = servers.map(toEntity); + // We don't delete any servers since they are used to + // generate node types. If some node types are removed, + // it will break workflows that use them. + // If we want to stop supporting a server, + // we will set its status to 'deprecated' instead. + await this.repository.upsert(entities, ['id']); + } + + private async refreshRegistryNodeTypes(releaseTypes: boolean): Promise { + const loader = this.loadNodesAndCredentials.loaders[MCP_REGISTRY_PACKAGE_NAME]; + if (!loader) { + return; + } + + if (!(loader instanceof McpRegistryNodeLoader)) { + this.logger.warn('Unexpected MCP registry loader instance type', { + loaderType: loader.constructor.name, + }); + return; + } + + const servers = await this.getAll({ includeDeprecated: true }); + loader.setServers(servers); + await loader.loadAll(); + await this.loadNodesAndCredentials.postProcessLoaders(); + if (releaseTypes) { + this.loadNodesAndCredentials.releaseTypes(); + } + + this.logger.debug('MCP registry loader done', { serverCount: servers.length }); + } + + private async publishReloadCommand(): Promise { + await this.publisher.publishCommand({ command: 'reload-mcp-registry' }); + } + + private notifyNodeDescriptionsUpdated() { + this.push.broadcast({ type: 'nodeDescriptionUpdated', data: {} }); + } + + private isMainInstance(): boolean { + return this.instanceSettings.instanceType === 'main'; } } diff --git a/packages/cli/src/modules/mcp-registry/registry/mcp-registry.types.ts b/packages/cli/src/modules/mcp-registry/registry/mcp-registry.types.ts index edc36281fd2..58809a07954 100644 --- a/packages/cli/src/modules/mcp-registry/registry/mcp-registry.types.ts +++ b/packages/cli/src/modules/mcp-registry/registry/mcp-registry.types.ts @@ -1,3 +1,14 @@ +import type { McpRegistryServerEntity } from './mcp-registry-server.entity'; + +type McpRegistryServerUpsertRow = Pick< + McpRegistryServerEntity, + 'id' | 'slug' | 'status' | 'version' | 'registryUpdatedAt' | 'data' +>; + +const serverStatuses = ['active', 'deprecated'] as const; + +type McpRegistryServerStatus = (typeof serverStatuses)[number]; + /** * The shape of an entry returned by the MCP server registry. */ @@ -17,7 +28,7 @@ export type McpRegistryServer = { tools: McpRegistryTool[]; isOfficial: boolean; origin: 'registry'; - status: 'active' | 'deprecated'; + status: McpRegistryServerStatus; tags?: string[]; }; @@ -43,3 +54,33 @@ export type McpRegistryTool = { title?: string; annotations?: McpRegistryToolAnnotations; }; + +export function toEntity(server: McpRegistryServer): McpRegistryServerUpsertRow { + const { id, slug, status, version, updatedAt, ...rest } = server; + let mappedStatus = status; + // make sure that unknown statuses get mapped to a valid value + if (!serverStatuses.includes(status)) { + mappedStatus = 'deprecated'; + } + + return { + id, + slug, + status: mappedStatus, + version, + registryUpdatedAt: new Date(updatedAt), + data: rest, + }; +} + +export function fromEntity(entity: McpRegistryServerEntity): McpRegistryServer { + const { id, slug, status, version, registryUpdatedAt, data } = entity; + return { + id, + slug, + status, + version, + updatedAt: registryUpdatedAt.toISOString(), + ...data, + } as McpRegistryServer; +} diff --git a/packages/cli/src/scaling/pubsub/pubsub.event-map.ts b/packages/cli/src/scaling/pubsub/pubsub.event-map.ts index 0e29a7ea4eb..302e1dafe54 100644 --- a/packages/cli/src/scaling/pubsub/pubsub.event-map.ts +++ b/packages/cli/src/scaling/pubsub/pubsub.event-map.ts @@ -33,6 +33,8 @@ export type PubSubCommandMap = { 'reload-source-control-config': never; + 'reload-mcp-registry': never; + // #region Community packages 'community-package-install': { diff --git a/packages/cli/src/scaling/pubsub/pubsub.types.ts b/packages/cli/src/scaling/pubsub/pubsub.types.ts index 96a8e1bfb4e..53299c9ab30 100644 --- a/packages/cli/src/scaling/pubsub/pubsub.types.ts +++ b/packages/cli/src/scaling/pubsub/pubsub.types.ts @@ -63,6 +63,7 @@ export namespace PubSub { export type ReloadSsoProvisioningConfiguration = ToCommand<'reload-sso-provisioning-configuration'>; export type ReloadSourceControlConfiguration = ToCommand<'reload-source-control-config'>; + export type ReloadMcpRegistry = ToCommand<'reload-mcp-registry'>; export type CancelTestRun = ToCommand<'cancel-test-run'>; export type CancelCollection = ToCommand<'cancel-collection'>; export type AgentChatIntegrationChanged = ToCommand<'agent-chat-integration-changed'>; @@ -94,6 +95,7 @@ export namespace PubSub { | Commands.ReloadCredentialsOverwrites | Commands.ReloadSsoProvisioningConfiguration | Commands.ReloadSourceControlConfiguration + | Commands.ReloadMcpRegistry | Commands.CancelTestRun | Commands.CancelCollection | Commands.AgentChatIntegrationChanged diff --git a/packages/cli/src/modules/community-packages/__tests__/strapi-utils.test.ts b/packages/cli/src/utils/__tests__/strapi-utils.test.ts similarity index 100% rename from packages/cli/src/modules/community-packages/__tests__/strapi-utils.test.ts rename to packages/cli/src/utils/__tests__/strapi-utils.test.ts diff --git a/packages/cli/src/modules/community-packages/strapi-utils.ts b/packages/cli/src/utils/strapi-utils.ts similarity index 93% rename from packages/cli/src/modules/community-packages/strapi-utils.ts rename to packages/cli/src/utils/strapi-utils.ts index add3c21159c..de2ef57b3f7 100644 --- a/packages/cli/src/modules/community-packages/strapi-utils.ts +++ b/packages/cli/src/utils/strapi-utils.ts @@ -61,10 +61,10 @@ export async function paginatedRequest( }); } catch (error) { Container.get(ErrorReporter).error(error, { - tags: { source: 'communityNodesPaginatedRequest' }, + tags: { source: 'strapiPaginatedRequest' }, }); Container.get(Logger).error( - `Error while fetching community nodes: ${(error as Error).message}`, + `Error fetching from Strapi API (${url}): ${(error as Error).message}`, ); if (options?.throwOnError) throw error; diff --git a/packages/testing/playwright/services/api-helper.ts b/packages/testing/playwright/services/api-helper.ts index 8633d44f2d8..4d5a598c663 100644 --- a/packages/testing/playwright/services/api-helper.ts +++ b/packages/testing/playwright/services/api-helper.ts @@ -456,6 +456,22 @@ export class ApiHelpers { } } + // ===== MCP REGISTRY METHODS ===== + + /** + * Seed the MCP registry with mock server data + * This inserts data into the mcp_registry_server table and triggers + * a node type refresh so the synthetic MCP nodes become available. + */ + async seedMcpRegistry(): Promise { + const response = await this.request.post('/rest/mcp-registry/test/seed'); + if (!response.ok()) { + throw new TestError( + `Failed to seed MCP registry: ${response.status()} ${await response.text()}`, + ); + } + } + // ===== MCP API KEY METHODS ===== /** diff --git a/packages/testing/playwright/tests/e2e/mcp-registry/mcp-registry.spec.ts b/packages/testing/playwright/tests/e2e/mcp-registry/mcp-registry.spec.ts index 1af99b2da2b..1d819b7719a 100644 --- a/packages/testing/playwright/tests/e2e/mcp-registry/mcp-registry.spec.ts +++ b/packages/testing/playwright/tests/e2e/mcp-registry/mcp-registry.spec.ts @@ -16,7 +16,8 @@ test.describe( annotation: [{ type: 'owner', description: 'AI' }], }, () => { - test('exposes Notion MCP as a tool with hidden connection fields', async ({ n8n }) => { + test('exposes Notion MCP as a tool with hidden connection fields', async ({ n8n, api }) => { + await api.seedMcpRegistry(); await n8n.start.fromBlankCanvas(); await n8n.canvas.addNode(AGENT_NODE_NAME, { closeNDV: true });