From 7784aa229888eb871453584cc1297264fee04e92 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Mon, 10 Aug 2026 13:31:07 +0100 Subject: [PATCH] fix(core): Show custom OpenAI-compatible models in Agents (no-changelog) (#35781) --- .../__tests__/model-discovery.test.ts | 14 +- .../ai-utilities/src/model-discovery/index.ts | 6 +- .../src/model-discovery/providers/openai.ts | 12 +- .../agent-model-catalog.service.test.ts | 149 +++++++++++---- .../agents/agent-model-catalog.service.ts | 73 ++++++-- .../builder-model-live-lookup.service.test.ts | 169 +++++++++++++++++- .../builder-model-live-lookup.service.ts | 138 ++++++++++---- 7 files changed, 471 insertions(+), 90 deletions(-) diff --git a/packages/@n8n/ai-utilities/src/model-discovery/__tests__/model-discovery.test.ts b/packages/@n8n/ai-utilities/src/model-discovery/__tests__/model-discovery.test.ts index 9c136bd59a4..04e8c9ba85a 100644 --- a/packages/@n8n/ai-utilities/src/model-discovery/__tests__/model-discovery.test.ts +++ b/packages/@n8n/ai-utilities/src/model-discovery/__tests__/model-discovery.test.ts @@ -1,4 +1,4 @@ -import { listModelsForProvider, MODEL_DISCOVERY_PROVIDERS } from '../index'; +import { isOpenAiCustomEndpoint, listModelsForProvider, MODEL_DISCOVERY_PROVIDERS } from '../index'; function mockFetch(body: unknown, ok = true, status = 200) { return vi.fn().mockResolvedValue({ @@ -53,6 +53,18 @@ describe('model-discovery', () => { }); describe('openai', () => { + describe('isOpenAiCustomEndpoint', () => { + it.each([ + [undefined, false], + ['https://api.openai.com/v1/', false], + ['https://ai-assistant.n8n.io/v1', false], + ['https://litellm.example.com/v1', true], + ['not-a-valid-url', true], + ] as const)('classifies %s as custom: %s', (baseURL, expected) => { + expect(isOpenAiCustomEndpoint(baseURL)).toBe(expected); + }); + }); + it('filters out non-chat models on the official API and sorts by id', async () => { const fetch = mockFetch({ data: [ diff --git a/packages/@n8n/ai-utilities/src/model-discovery/index.ts b/packages/@n8n/ai-utilities/src/model-discovery/index.ts index 44c81300c10..cd4b38c509c 100644 --- a/packages/@n8n/ai-utilities/src/model-discovery/index.ts +++ b/packages/@n8n/ai-utilities/src/model-discovery/index.ts @@ -58,7 +58,11 @@ export { listGoogleModels } from './providers/google'; export { listGroqModels } from './providers/groq'; export { listMistralModels } from './providers/mistral'; export { listNvidiaModels } from './providers/nvidia'; -export { listOpenAiModels, shouldIncludeOpenAiModel } from './providers/openai'; +export { + isOpenAiCustomEndpoint, + listOpenAiModels, + shouldIncludeOpenAiModel, +} from './providers/openai'; export { listOpenRouterModels } from './providers/openrouter'; export { listVercelModels } from './providers/vercel'; export { listXaiModels } from './providers/xai'; diff --git a/packages/@n8n/ai-utilities/src/model-discovery/providers/openai.ts b/packages/@n8n/ai-utilities/src/model-discovery/providers/openai.ts index 623df1fb910..667ae7fe748 100644 --- a/packages/@n8n/ai-utilities/src/model-discovery/providers/openai.ts +++ b/packages/@n8n/ai-utilities/src/model-discovery/providers/openai.ts @@ -3,6 +3,16 @@ import type { ListModelsFn } from '../types'; const OFFICIAL_OPENAI_HOSTNAMES = ['api.openai.com', 'ai-assistant.n8n.io']; +export function isOpenAiCustomEndpoint(baseURL?: string): boolean { + if (baseURL === undefined) return false; + + try { + return !OFFICIAL_OPENAI_HOSTNAMES.includes(new URL(baseURL).hostname); + } catch { + return true; + } +} + /** * Source: LMChatOpenAi `methods/loadModels.ts` — on the official API, exclude * non-chat model families; on custom (proxy/self-hosted) hosts include all. @@ -28,7 +38,7 @@ export function shouldIncludeOpenAiModel(modelId: string, isCustomApi: boolean): /** Source: LMChatOpenAi `methods/loadModels.ts` (GET /models, filtered, id asc). */ export const listOpenAiModels: ListModelsFn = async (options) => { const base = baseUrl(options, 'https://api.openai.com/v1'); - const isCustomApi = !OFFICIAL_OPENAI_HOSTNAMES.includes(new URL(base).hostname); + const isCustomApi = isOpenAiCustomEndpoint(base); const data = (await getJson(`${base}/models`, bearerHeaders(options), options, 'openai')) as { data?: IdItem[]; }; diff --git a/packages/cli/src/modules/agents/__tests__/agent-model-catalog.service.test.ts b/packages/cli/src/modules/agents/__tests__/agent-model-catalog.service.test.ts index 10ae6391719..8ea0e317fe9 100644 --- a/packages/cli/src/modules/agents/__tests__/agent-model-catalog.service.test.ts +++ b/packages/cli/src/modules/agents/__tests__/agent-model-catalog.service.test.ts @@ -50,8 +50,9 @@ const catalogFixture = { function makeService() { const lookupService = mock(); - const service = new AgentModelCatalogService(mockLogger(), lookupService); - return { service, lookupService }; + const logger = mockLogger(); + const service = new AgentModelCatalogService(logger, lookupService); + return { service, lookupService, logger }; } describe('AgentModelCatalogService', () => { @@ -63,9 +64,11 @@ describe('AgentModelCatalogService', () => { it('keeps catalog models the provider still reports live, with catalog metadata, and prunes the rest', async () => { const { service, lookupService } = makeService(); // Provider reports Sonnet but not Opus — Opus (retired) must be pruned. - lookupService.list.mockResolvedValue([ - { name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }, - ]); + lookupService.lookup.mockResolvedValue({ + status: 'success', + policy: 'curated', + models: [{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }], + }); const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); @@ -76,7 +79,7 @@ describe('AgentModelCatalogService', () => { name: 'Claude Sonnet 4.6', cost: { input: 3, output: 15 }, }); - expect(lookupService.list).toHaveBeenCalledWith( + expect(lookupService.lookup).toHaveBeenCalledWith( user, 'project-1', credentialId, @@ -88,9 +91,11 @@ describe('AgentModelCatalogService', () => { it('verifies against the gateway allowlist for the n8n Connect managed tag', async () => { const { service, lookupService } = makeService(); // Gateway serves only Sonnet — the retired/unsupported Opus must be pruned. - lookupService.list.mockResolvedValue([ - { name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }, - ]); + lookupService.lookup.mockResolvedValue({ + status: 'success', + policy: 'managed', + models: [{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }], + }); const result = await service.getProviderModels( user, @@ -101,7 +106,7 @@ describe('AgentModelCatalogService', () => { expect(result.verified).toBe(true); expect(result.models.map((m) => m.id)).toEqual(['claude-sonnet-4-6']); - expect(lookupService.list).toHaveBeenCalledWith( + expect(lookupService.lookup).toHaveBeenCalledWith( user, 'project-1', AI_GATEWAY_MANAGED_TAG, @@ -110,9 +115,9 @@ describe('AgentModelCatalogService', () => { ); }); - it('flags a failed managed lookup as unavailable instead of an empty allowlist', async () => { + it('does not use the static catalog when a managed lookup throws before returning a policy', async () => { const { service, lookupService } = makeService(); - lookupService.list.mockRejectedValue(new Error('gateway unreachable')); + lookupService.lookup.mockRejectedValue(new Error('gateway unreachable')); const result = await service.getProviderModels( user, @@ -123,8 +128,13 @@ describe('AgentModelCatalogService', () => { // No static-catalog fallback for a managed slot: it would offer models the // gateway won't serve. But an outage must not read as "allowlist is empty". - expect(result.models).toEqual([]); - expect(result.unavailable).toBe(true); + expect(result).toEqual({ + provider: 'anthropic', + verified: true, + unavailable: true, + models: [], + }); + expect(fetchProviderCatalog).not.toHaveBeenCalled(); }); it('uses the gateway exact (snapshot) id for the managed tag, not the catalog alias', async () => { @@ -144,9 +154,11 @@ describe('AgentModelCatalogService', () => { }, }); // The gateway returns the dated snapshot — its allowlist matches this, not the alias. - lookupService.list.mockResolvedValue([ - { name: 'Claude Haiku 4.5', value: 'claude-haiku-4-5-20251001' }, - ]); + lookupService.lookup.mockResolvedValue({ + status: 'success', + policy: 'managed', + models: [{ name: 'Claude Haiku 4.5', value: 'claude-haiku-4-5-20251001' }], + }); const result = await service.getProviderModels( user, @@ -164,7 +176,11 @@ describe('AgentModelCatalogService', () => { it('leaves reasoning support unknown for managed models missing from the catalog', async () => { const { service, lookupService } = makeService(); - lookupService.list.mockResolvedValue([{ name: 'Claude Brand New', value: 'claude-brand-new' }]); + lookupService.lookup.mockResolvedValue({ + status: 'success', + policy: 'managed', + models: [{ name: 'Claude Brand New', value: 'claude-brand-new' }], + }); const result = await service.getProviderModels( user, @@ -204,9 +220,11 @@ describe('AgentModelCatalogService', () => { }); // …while Anthropic's API lists the dated snapshot only. The snapshot must // verify its alias; the retired alias with no live counterpart stays pruned. - lookupService.list.mockResolvedValue([ - { name: 'Claude Haiku 4.5', value: 'claude-haiku-4-5-20251001' }, - ]); + lookupService.lookup.mockResolvedValue({ + status: 'success', + policy: 'curated', + models: [{ name: 'Claude Haiku 4.5', value: 'claude-haiku-4-5-20251001' }], + }); const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); @@ -217,10 +235,14 @@ describe('AgentModelCatalogService', () => { it('does not add live models that are missing from the catalog', async () => { const { service, lookupService } = makeService(); // Live list includes a model models.dev has no entry for, alongside a known one. - lookupService.list.mockResolvedValue([ - { name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }, - { name: 'Claude Brand New', value: 'claude-brand-new' }, - ]); + lookupService.lookup.mockResolvedValue({ + status: 'success', + policy: 'curated', + models: [ + { name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }, + { name: 'Claude Brand New', value: 'claude-brand-new' }, + ], + }); const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); @@ -231,9 +253,11 @@ describe('AgentModelCatalogService', () => { it('strips the "models/" prefix from google model ids before matching', async () => { const { service, lookupService } = makeService(); - lookupService.list.mockResolvedValue([ - { name: 'models/gemini-2.5-flash', value: 'models/gemini-2.5-flash' }, - ]); + lookupService.lookup.mockResolvedValue({ + status: 'success', + policy: 'curated', + models: [{ name: 'models/gemini-2.5-flash', value: 'models/gemini-2.5-flash' }], + }); const result = await service.getProviderModels(user, 'project-1', 'google', credentialId); @@ -244,7 +268,7 @@ describe('AgentModelCatalogService', () => { it('falls back to the catalog list (verified: false) when the live lookup fails', async () => { const { service, lookupService } = makeService(); - lookupService.list.mockRejectedValue(new Error('provider is down')); + lookupService.lookup.mockRejectedValue(new Error('provider is down')); const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); @@ -252,6 +276,55 @@ describe('AgentModelCatalogService', () => { expect(result.models.map((m) => m.id).sort()).toEqual(['claude-opus-4-0', 'claude-sonnet-4-6']); }); + it('uses only exact live models for a custom OpenAI-compatible endpoint', async () => { + const { service, lookupService } = makeService(); + lookupService.lookup.mockResolvedValue({ + status: 'success', + policy: 'endpoint-only', + models: [ + { name: 'Gemini 2.5 Flash', value: 'gemini-2.5-flash' }, + { name: 'GLM 4.5', value: 'glm-4.5' }, + { name: '', value: 'qwen3-coder' }, + ], + }); + + const result = await service.getProviderModels(user, 'project-1', 'openai', credentialId); + + expect(result).toEqual({ + provider: 'openai', + verified: true, + models: [ + { id: 'gemini-2.5-flash', name: 'Gemini 2.5 Flash', toolCall: true }, + { id: 'glm-4.5', name: 'GLM 4.5', toolCall: true }, + { id: 'qwen3-coder', name: 'qwen3-coder', toolCall: true }, + ], + }); + expect(fetchProviderCatalog).not.toHaveBeenCalled(); + }); + + it('reports a custom OpenAI-compatible endpoint as unavailable without catalog fallback', async () => { + const { service, lookupService, logger } = makeService(); + lookupService.lookup.mockResolvedValue({ + status: 'unavailable', + policy: 'endpoint-only', + error: new Error('endpoint unavailable'), + }); + + const result = await service.getProviderModels(user, 'project-1', 'openai', credentialId); + + expect(result).toEqual({ + provider: 'openai', + verified: true, + unavailable: true, + models: [], + }); + expect(fetchProviderCatalog).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('Live model list failed'), + expect.objectContaining({ provider: 'openai', error: 'endpoint unavailable' }), + ); + }); + it('falls back to the catalog list (verified: false) when no credential is provided', async () => { const { service, lookupService } = makeService(); @@ -259,7 +332,7 @@ describe('AgentModelCatalogService', () => { expect(result.verified).toBe(false); expect(result.models).toHaveLength(2); - expect(lookupService.list).not.toHaveBeenCalled(); + expect(lookupService.lookup).not.toHaveBeenCalled(); }); it('falls back to the catalog list for providers without a live lookup', async () => { @@ -283,15 +356,17 @@ describe('AgentModelCatalogService', () => { expect(result.verified).toBe(false); expect(result.models).toHaveLength(1); - expect(lookupService.list).not.toHaveBeenCalled(); + expect(lookupService.lookup).not.toHaveBeenCalled(); }); it('still returns live models (verified: true) when the catalog fetch fails', async () => { const { service, lookupService } = makeService(); fetchProviderCatalog.mockRejectedValue(new Error('models.dev unreachable')); - lookupService.list.mockResolvedValue([ - { name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }, - ]); + lookupService.lookup.mockResolvedValue({ + status: 'success', + policy: 'curated', + models: [{ name: 'Claude Sonnet 4.6', value: 'claude-sonnet-4-6' }], + }); const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); @@ -305,7 +380,11 @@ describe('AgentModelCatalogService', () => { it('returns an empty unverified list when both the lookup and the catalog fail', async () => { const { service, lookupService } = makeService(); fetchProviderCatalog.mockRejectedValue(new Error('models.dev unreachable')); - lookupService.list.mockRejectedValue(new Error('provider is down')); + lookupService.lookup.mockResolvedValue({ + status: 'unavailable', + policy: 'curated', + error: new Error('provider is down'), + }); const result = await service.getProviderModels(user, 'project-1', 'anthropic', credentialId); diff --git a/packages/cli/src/modules/agents/agent-model-catalog.service.ts b/packages/cli/src/modules/agents/agent-model-catalog.service.ts index 3c7ce510173..37f9fe98ab4 100644 --- a/packages/cli/src/modules/agents/agent-model-catalog.service.ts +++ b/packages/cli/src/modules/agents/agent-model-catalog.service.ts @@ -10,7 +10,10 @@ import { Service } from '@n8n/di'; import { isModelDiscoveryProvider } from '@n8n/ai-utilities/model-discovery'; -import { BuilderModelLiveLookupService } from './builder/builder-model-live-lookup.service'; +import { + BuilderModelLiveLookupService, + type LiveModelLookupResult, +} from './builder/builder-model-live-lookup.service'; /** Google's models API returns ids as `models/`; the AI SDK expects the bare id. */ const GOOGLE_MODEL_ID_PREFIX = 'models/'; @@ -45,15 +48,11 @@ function liveModelIdVariants(id: string): string[] { /** * Builds the model list offered in the agent model picker for one provider. * - * The curated models.dev catalog is the display list — it is the up-to-date set - * of chat models with names, cost, and limits. But it lags provider - * retirements, so a listed model can 404 at call time. When a credential is - * available we verify the catalog against the provider's own model API (via the - * shared `@n8n/ai-utilities/model-discovery` functions) and prune any catalog - * entry the provider no longer reports. We never add live-only - * models: provider `/models` endpoints return every variant/snapshot and would - * overload the picker. Without a credential, or when the provider has no list - * API wired up, the catalog list is returned unpruned with `verified: false`. + * For curated providers, models.dev is the display list and live discovery + * verifies/prunes it. Custom OpenAI-compatible endpoints are the exception: + * their live `/models` response is authoritative and models.dev is never read. + * The managed gateway also uses exact live ids, enriched from models.dev when + * available. Without live discovery, curated catalog models remain unverified. */ @Service() export class AgentModelCatalogService { @@ -62,22 +61,23 @@ export class AgentModelCatalogService { private readonly builderModelLiveLookupService: BuilderModelLiveLookupService, ) {} + /** Returns the provider's models according to the live lookup's catalog policy. */ async getProviderModels( user: User, projectId: string, provider: string, credentialId?: string, ): Promise { - const catalogModels = await this.getCatalogModels(provider); const credentialType = getProviderCredentialType(provider); if (!credentialId || !credentialType) { + const catalogModels = await this.getCatalogModels(provider); return { provider, verified: false, models: Object.values(catalogModels) }; } - let liveModels: Array<{ name: string; value: string }>; + let lookup: LiveModelLookupResult; try { - liveModels = await this.builderModelLiveLookupService.list( + lookup = await this.builderModelLiveLookupService.lookup( user, projectId, credentialId, @@ -85,10 +85,7 @@ export class AgentModelCatalogService { provider, ); } catch (error) { - this.logger.warn('Live model list failed — falling back to the static catalog', { - provider, - error: error instanceof Error ? error.message : String(error), - }); + this.logLiveLookupFailure(provider, error); // Managed slot: the gateway allowlist is the contract, so don't fall back // to the static catalog (it may list models the gateway won't serve). // Flagged unavailable so the picker distinguishes a failed lookup from a @@ -96,14 +93,47 @@ export class AgentModelCatalogService { if (credentialId === AI_GATEWAY_MANAGED_TAG) { return { provider, verified: true, unavailable: true, models: [] }; } + const catalogModels = await this.getCatalogModels(provider); return { provider, verified: false, models: Object.values(catalogModels) }; } + // A custom endpoint can expose any OpenAI-compatible model id. Its live + // response is authoritative, so keep exact ids and attach no inferred or + // models.dev metadata. + if (lookup.policy === 'endpoint-only') { + if (lookup.status === 'unavailable') { + this.logLiveLookupFailure(provider, lookup.error); + return { provider, verified: true, unavailable: true, models: [] }; + } + + return { + provider, + verified: true, + models: lookup.models.map((live) => ({ + id: live.value, + name: live.name || live.value, + toolCall: true, + })), + }; + } + + if (lookup.status === 'unavailable') { + this.logLiveLookupFailure(provider, lookup.error); + if (lookup.policy === 'managed' || credentialId === AI_GATEWAY_MANAGED_TAG) { + return { provider, verified: true, unavailable: true, models: [] }; + } + const catalogModels = await this.getCatalogModels(provider); + return { provider, verified: false, models: Object.values(catalogModels) }; + } + + const liveModels = lookup.models; + const catalogModels = await this.getCatalogModels(provider); + // n8n Connect managed slot: the gateway's `/models` is the authoritative, // allowlist-filtered set, and its exact ids are what the gateway's model // allowlist matches (e.g. a dated snapshot, not the catalog's versionless // alias). Use those ids verbatim, enriched with catalog display/metadata. - if (credentialId === AI_GATEWAY_MANAGED_TAG) { + if (lookup.policy === 'managed') { return { provider, verified: true, @@ -157,6 +187,13 @@ export class AgentModelCatalogService { }; } + private logLiveLookupFailure(provider: string, error: unknown): void { + this.logger.warn('Live model list failed', { + provider, + error: error instanceof Error ? error.message : String(error), + }); + } + private async getCatalogModels(provider: string): Promise> { try { const { fetchProviderCatalog } = await import('@n8n/agents'); diff --git a/packages/cli/src/modules/agents/builder/__tests__/builder-model-live-lookup.service.test.ts b/packages/cli/src/modules/agents/builder/__tests__/builder-model-live-lookup.service.test.ts index f4cc16bdb20..689cc1fbff2 100644 --- a/packages/cli/src/modules/agents/builder/__tests__/builder-model-live-lookup.service.test.ts +++ b/packages/cli/src/modules/agents/builder/__tests__/builder-model-live-lookup.service.test.ts @@ -10,7 +10,8 @@ import type { AiGatewayService } from '@/services/ai-gateway.service'; import { BuilderModelLiveLookupService } from '../builder-model-live-lookup.service'; const listModelsForProvider = vi.fn(); -vi.mock('@n8n/ai-utilities/model-discovery', () => ({ +vi.mock('@n8n/ai-utilities/model-discovery', async (importOriginal) => ({ + ...(await importOriginal()), listModelsForProvider: (...args: unknown[]) => listModelsForProvider(...args) as unknown, })); @@ -46,6 +47,172 @@ describe('BuilderModelLiveLookupService', () => { listModelsForProvider.mockReset(); }); + describe('lookup', () => { + it('returns endpoint-only models and forwards OpenAI credential request options', async () => { + const { service, credentialsService, credentialsFinderService } = makeService(); + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue( + usable('cred-1', 'openAiApi'), + ); + credentialsFinderService.findCredentialById.mockResolvedValue(mock()); + credentialsService.decrypt.mockResolvedValue({ + apiKey: 'sk-key', + url: 'https://openai-compatible.example/v1', + organizationId: 'org-123', + header: true, + headerName: 'authorization', + headerValue: 'Bearer custom-token', + }); + listModelsForProvider.mockResolvedValue([{ id: 'custom-model', name: 'Custom model' }]); + + const result = await service.lookup(user, projectId, 'cred-1', 'openAiApi', 'openai'); + + expect(result).toEqual({ + status: 'success', + policy: 'endpoint-only', + models: [{ name: 'Custom model', value: 'custom-model' }], + }); + expect(listModelsForProvider).toHaveBeenCalledWith( + 'openai', + expect.objectContaining({ + apiKey: 'sk-key', + baseURL: 'https://openai-compatible.example/v1', + headers: { + Authorization: 'Bearer custom-token', + 'OpenAI-Organization': 'org-123', + }, + }), + ); + }); + + it('lets a custom OpenAI organization header override the credential organization', async () => { + const { service, credentialsService, credentialsFinderService } = makeService(); + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue( + usable('cred-1', 'openAiApi'), + ); + credentialsFinderService.findCredentialById.mockResolvedValue(mock()); + credentialsService.decrypt.mockResolvedValue({ + apiKey: 'sk-key', + url: 'https://openai-compatible.example/v1', + organizationId: 'org-default', + header: true, + headerName: 'openai-organization', + headerValue: 'org-custom', + }); + listModelsForProvider.mockResolvedValue([{ id: 'custom-model', name: 'Custom model' }]); + + await service.lookup(user, projectId, 'cred-1', 'openAiApi', 'openai'); + + expect(listModelsForProvider).toHaveBeenCalledWith( + 'openai', + expect.objectContaining({ + headers: { 'OpenAI-Organization': 'org-custom' }, + }), + ); + }); + + it('returns curated models for the official OpenAI endpoint', async () => { + const { service, credentialsService, credentialsFinderService } = makeService(); + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue( + usable('cred-1', 'openAiApi'), + ); + credentialsFinderService.findCredentialById.mockResolvedValue(mock()); + credentialsService.decrypt.mockResolvedValue({ + apiKey: 'sk-key', + url: 'https://api.openai.com/v1', + }); + listModelsForProvider.mockResolvedValue([{ id: 'gpt-5', name: 'GPT-5' }]); + + const result = await service.lookup(user, projectId, 'cred-1', 'openAiApi', 'openai'); + + expect(result).toEqual({ + status: 'success', + policy: 'curated', + models: [{ name: 'GPT-5', value: 'gpt-5' }], + }); + }); + + it('preserves endpoint-only policy when custom OpenAI discovery fails', async () => { + const { service, credentialsService, credentialsFinderService } = makeService(); + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue( + usable('cred-1', 'openAiApi'), + ); + credentialsFinderService.findCredentialById.mockResolvedValue(mock()); + credentialsService.decrypt.mockResolvedValue({ + apiKey: 'sk-key', + url: 'https://openai-compatible.example/v1', + }); + const error = new Error('Model listing failed'); + listModelsForProvider.mockRejectedValue(error); + + const result = await service.lookup(user, projectId, 'cred-1', 'openAiApi', 'openai'); + + expect(result).toEqual({ status: 'unavailable', policy: 'endpoint-only', error }); + }); + + it('preserves endpoint-only policy when custom OpenAI discovery is empty', async () => { + const { service, credentialsService, credentialsFinderService } = makeService(); + credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue( + usable('cred-1', 'openAiApi'), + ); + credentialsFinderService.findCredentialById.mockResolvedValue(mock()); + credentialsService.decrypt.mockResolvedValue({ + apiKey: 'sk-key', + url: 'https://openai-compatible.example/v1', + }); + listModelsForProvider.mockResolvedValue([]); + + const result = await service.lookup(user, projectId, 'cred-1', 'openAiApi', 'openai'); + + expect(result).toEqual({ + status: 'unavailable', + policy: 'endpoint-only', + error: expect.objectContaining({ message: 'Provider openai returned no models' }), + }); + }); + + it('returns managed policy for the n8n Connect managed tag', async () => { + const { service, aiGatewayService } = makeService(); + aiGatewayService.getCredentialTypeForProvider.mockResolvedValue('openAiApi'); + aiGatewayService.getSyntheticCredential.mockResolvedValue({ + apiKey: 'gateway-jwt', + url: 'https://gw.example/v1/gateway/openai/v1', + }); + listModelsForProvider.mockResolvedValue([{ id: 'gpt-5-mini', name: 'GPT-5 mini' }]); + + const result = await service.lookup( + user, + projectId, + AI_GATEWAY_MANAGED_TAG, + 'openAiApi', + 'openai', + ); + + expect(result).toEqual({ + status: 'success', + policy: 'managed', + models: [{ name: 'GPT-5 mini', value: 'gpt-5-mini' }], + }); + }); + + it('keeps list compatible with successful and unavailable lookups', async () => { + const { service } = makeService(); + const models = [{ name: 'GPT-5', value: 'gpt-5' }]; + const error = new Error('Model listing failed'); + const lookup = vi + .spyOn(service, 'lookup') + .mockResolvedValueOnce({ status: 'success', policy: 'curated', models }) + .mockResolvedValueOnce({ status: 'unavailable', policy: 'curated', error }); + + await expect(service.list(user, projectId, 'cred-1', 'openAiApi', 'openai')).resolves.toBe( + models, + ); + await expect(service.list(user, projectId, 'cred-1', 'openAiApi', 'openai')).rejects.toBe( + error, + ); + expect(lookup).toHaveBeenCalledTimes(2); + }); + }); + it('lists models for a credential the user can use in the project', async () => { const { service, credentialsService, credentialsFinderService } = makeService(); credentialsService.getCredentialsAUserCanUseInAWorkflow.mockResolvedValue( diff --git a/packages/cli/src/modules/agents/builder/builder-model-live-lookup.service.ts b/packages/cli/src/modules/agents/builder/builder-model-live-lookup.service.ts index 3b311825cbc..6085a0716ea 100644 --- a/packages/cli/src/modules/agents/builder/builder-model-live-lookup.service.ts +++ b/packages/cli/src/modules/agents/builder/builder-model-live-lookup.service.ts @@ -10,6 +10,16 @@ import { createAiProxyFetch } from '@/utils/ai-proxy-fetch'; import { mapCredentialForProvider } from '../json-config/credential-field-mapping'; +export type ModelCatalogPolicy = 'curated' | 'endpoint-only' | 'managed'; + +export type LiveModelLookupResult = + | { + status: 'success'; + models: Array<{ name: string; value: string }>; + policy: ModelCatalogPolicy; + } + | { status: 'unavailable'; error: unknown; policy: ModelCatalogPolicy }; + /** * Fetches a provider's live chat-model list for a credential, via the shared * `@n8n/ai-utilities/model-discovery` functions (the same provider knowledge @@ -42,10 +52,20 @@ export class BuilderModelLiveLookupService { credentialType: string, provider: string, ): Promise> { - // n8n Connect managed slot: there is no stored credential to decrypt — - // resolve the synthetic gateway credential instead, then discover as usual. + const result = await this.lookup(user, projectId, credentialId, credentialType, provider); + if (result.status === 'unavailable') throw result.error; + return result.models; + } + + async lookup( + user: User, + projectId: string, + credentialId: string, + credentialType: string, + provider: string, + ): Promise { if (credentialId === AI_GATEWAY_MANAGED_TAG) { - return await this.listAiGatewayManagedModels(projectId, provider, user); + return await this.lookupAiGatewayManagedModels(projectId, provider, user); } const usableCredentials = await this.credentialsService.getCredentialsAUserCanUseInAWorkflow( @@ -67,55 +87,107 @@ export class BuilderModelLiveLookupService { } /** - * Lists the chat models n8n Connect (AI Gateway) allows for a provider, via - * the synthetic gateway credential — discovery hits the gateway's `/models`, - * already filtered to the allowlist. Throws if the gateway does not serve it. + * Looks up the chat models n8n Connect (AI Gateway) allows for a provider. + * Gateway resolution or discovery failures are returned as managed-policy + * unavailability so `list` can preserve its throwing behavior. */ - private async listAiGatewayManagedModels( + private async lookupAiGatewayManagedModels( projectId: string, provider: string, user?: User, - ): Promise> { - const credentialType = await this.aiGatewayService.getCredentialTypeForProvider(provider); - if (!credentialType) { - throw new Error(`n8n credits does not support the "${provider}" model provider`); + ): Promise { + try { + const credentialType = await this.aiGatewayService.getCredentialTypeForProvider(provider); + if (!credentialType) { + throw new Error(`n8n credits does not support the "${provider}" model provider`); + } + const raw = await this.aiGatewayService.getSyntheticCredential({ + credentialType, + userId: user?.id, + projectId, + }); + return await this.discoverModels(provider, raw, 'managed'); + } catch (error) { + return { status: 'unavailable', error, policy: 'managed' }; } - const raw = await this.aiGatewayService.getSyntheticCredential({ - credentialType, - userId: user?.id, - projectId, - }); - return await this.discoverModels(provider, raw); } /** * Runs provider model discovery against a raw credential record (real or the - * gateway synthetic credential), returning `{ name, value }` pairs. + * gateway synthetic credential), preserving the applicable catalog policy. */ private async discoverModels( provider: string, rawData: object, - ): Promise> { - const { listModelsForProvider } = await import('@n8n/ai-utilities/model-discovery'); - const mapped = mapCredentialForProvider(provider, { apiKey: '', ...rawData }); + policyOverride?: ModelCatalogPolicy, + ): Promise { + const { isOpenAiCustomEndpoint, listModelsForProvider } = await import( + '@n8n/ai-utilities/model-discovery' + ); + const credentialData: Record = { apiKey: '', ...rawData }; + const mapped = mapCredentialForProvider(provider, credentialData); const apiKey = typeof mapped.apiKey === 'string' ? mapped.apiKey : ''; const baseURL = typeof mapped.baseURL === 'string' && mapped.baseURL ? mapped.baseURL : undefined; + const policy = + policyOverride ?? + (provider === 'openai' && isOpenAiCustomEndpoint(baseURL) ? 'endpoint-only' : 'curated'); + const headers = this.getOpenAiHeaders(provider, credentialData); - const models = await listModelsForProvider(provider, { - apiKey, - baseURL, - fetch: createAiProxyFetch(this.outboundHttp) as typeof globalThis.fetch, - }); + try { + const models = await listModelsForProvider(provider, { + apiKey, + baseURL, + fetch: createAiProxyFetch(this.outboundHttp) as typeof globalThis.fetch, + ...(headers ? { headers } : {}), + }); - // Every supported chat provider offers models, so an empty list means a - // broken request or a drifted response shape, not a zero-model account. - // Throw so callers fall back (unverified catalog / lookup-failed) instead - // of treating "nothing" as a verified answer and pruning every model. - if (models.length === 0) { - throw new Error(`Provider ${provider} returned no models`); + // Every supported chat provider offers models, so an empty list means a + // broken request or a drifted response shape, not a zero-model account. + if (models.length === 0) { + throw new Error(`Provider ${provider} returned no models`); + } + + return { + status: 'success', + models: models.map((model) => ({ name: model.name, value: model.id })), + policy, + }; + } catch (error) { + return { status: 'unavailable', error, policy }; + } + } + + private getOpenAiHeaders( + provider: string, + credentialData: Record, + ): Record | undefined { + if (provider !== 'openai') return undefined; + + const headers: Record = {}; + if ( + typeof credentialData.organizationId === 'string' && + credentialData.organizationId.length > 0 + ) { + headers['OpenAI-Organization'] = credentialData.organizationId; } - return models.map((model) => ({ name: model.name, value: model.id })); + if ( + credentialData.header === true && + typeof credentialData.headerName === 'string' && + credentialData.headerName.length > 0 && + typeof credentialData.headerValue === 'string' + ) { + const normalizedName = credentialData.headerName.toLowerCase(); + const headerName = + normalizedName === 'authorization' + ? 'Authorization' + : normalizedName === 'openai-organization' + ? 'OpenAI-Organization' + : credentialData.headerName; + headers[headerName] = credentialData.headerValue; + } + + return Object.keys(headers).length > 0 ? headers : undefined; } }