From d9dc457d40e1b94f2bb29404e83a9497f2e89f53 Mon Sep 17 00:00:00 2001 From: Arvin A <51036481+DeveloperTheExplorer@users.noreply.github.com> Date: Mon, 21 Sep 2026 09:09:43 +0000 Subject: [PATCH] feat(core): Track credential description usage (no-changelog) (#38986) --- .../@n8n/instance-ai/evaluations/README.md | 12 ++ .../build-workflow-seed-cleanup.test.ts | 7 +- .../__tests__/data-workflows-schema.test.ts | 22 ++++ .../__tests__/langtracer-to-exported.test.ts | 20 +++ .../evaluations/__tests__/user-proxy.test.ts | 2 + .../clients/__tests__/n8n-client.test.ts | 22 ++++ .../evaluations/clients/n8n-client.ts | 3 +- .../credentials/__tests__/seeder.test.ts | 52 +++++++- .../evaluations/credentials/seeder.ts | 34 ++++-- .../instance-ai/evaluations/harness/schema.ts | 9 +- .../evaluations/langtracer/to-exported.ts | 3 + .../@n8n/instance-ai/evaluations/types.ts | 30 +---- .../@n8n/telemetry/credential-descriptions.md | 77 ++++++++++++ .../__tests__/credential-description.test.ts | 40 ++++++ .../@n8n/telemetry/src/events/credentials.ts | 54 ++++++++ .../__tests__/credentials.controller.test.ts | 39 +++++- .../src/credentials/credentials.controller.ts | 2 + .../__tests__/activity-event-relay.test.ts | 4 + .../log-streaming-event-relay.test.ts | 2 + .../__tests__/telemetry-event-relay.test.ts | 115 +++++++++++------- .../cli/src/events/maps/relay.event-map.ts | 2 + .../events/relays/telemetry.event-relay.ts | 14 ++- .../credentials.public.controller.ts | 1 + 23 files changed, 475 insertions(+), 91 deletions(-) create mode 100644 packages/@n8n/telemetry/credential-descriptions.md create mode 100644 packages/@n8n/telemetry/src/__tests__/credential-description.test.ts diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index 7f22456b15b..df101b51a32 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -800,6 +800,18 @@ Declared credentials are created for real (placeholder token; set the matching ` Each type needs a data template in `credentials/seeder.ts`; declaring an unknown type fails the build with a pointer there. +Each declared credential can also have a `description`. The loader applies the +shared credential description schema. It trims the text, enforces the length +limit, and converts blank text to `null`. The seeder stores the description as +credential metadata, outside the secret `data` object. Omit the field to test +credential choice without descriptions. + +The current LangTracer case-write schema accepts only `type`, `name`, and +`valid` on a credential. The push refuses a case with a description before it +writes anything. Keep these cases on disk until LangTracer stores descriptions. +The existing `blank` field also needs server support. The export check catches +a server that drops it after a write. + ### Seeded cases (conversation pre-seeding) A seeded case starts **mid-conversation**: prior history is restored into the build thread before the live turn, so the eval drives only the turn under test. Use it to replicate a real misbehaviour — restore the conversation up to the moment it went wrong, re-drive that turn, and assert what should happen instead. diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-seed-cleanup.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-seed-cleanup.test.ts index a9d8b457f75..e86171e8f09 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-seed-cleanup.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-seed-cleanup.test.ts @@ -248,7 +248,12 @@ describe('buildWorkflow declared credentials', () => { expect(build.success).toBe(true); // A blank credential models one the user saved without filling anything in, // so it is seeded with no data and must never resolve a test as passing. - expect(createCredential).toHaveBeenCalledWith(expect.any(String), 'httpHeaderAuth', {}); + expect(createCredential).toHaveBeenCalledWith( + expect.any(String), + 'httpHeaderAuth', + {}, + undefined, + ); expect(setThreadCredentialAllowlist).toHaveBeenCalledWith( expect.any(String), ['cred-blank'], diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts index e499c24bc68..d086d74d972 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts @@ -49,6 +49,28 @@ beforeEach(() => { }); describe('EvalTestCaseSchema', () => { + it.each([ + { description: undefined, expected: undefined }, + { description: null, expected: null }, + { description: ' \n ', expected: null }, + { description: ' Production reports ', expected: 'Production reports' }, + ])('normalizes credential description $description', ({ description, expected }) => { + const parsed = EvalTestCaseSchema.parse({ + ...validFixture(), + credentials: [{ type: 'slackApi', description }], + }); + expect(parsed.credentials?.[0].description).toBe(expected); + }); + + it('rejects a credential description above the storage limit', () => { + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + credentials: [{ type: 'slackApi', description: '🔑'.repeat(257) }], + }), + ).toThrow(); + }); + it('accepts a minimal valid fixture', () => { const parsed = EvalTestCaseSchema.parse(validFixture()); expect(parsed.executionScenarios).toHaveLength(1); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts index 638462d8d71..1aa8c0ff9b8 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts @@ -107,6 +107,26 @@ describe('diskCaseToLangTracerCreate', () => { }); describe('unsupportedPushReason', () => { + it.each([null, 'Production reports'])( + 'refuses description %s until the case-write API preserves it', + (description) => { + const input = diskCase({ credentials: [{ type: 'httpHeaderAuth', description }] }); + expect(unsupportedPushReason(input)).toContain('description'); + const body = diskCaseToLangTracerCreate(input, 'description-case', { + suiteId: 8, + setKind: 'regression', + synthetic: true, + }); + expect(body.credentials).toEqual(input.credentials); + }, + ); + + it('allows credentials without descriptions', () => { + expect( + unsupportedPushReason(diskCase({ credentials: [{ type: 'httpHeaderAuth' }] })), + ).toBeNull(); + }); + it('refuses a case whose prompt version would be lost', () => { expect(unsupportedPushReason(diskCase({ promptVersion: 'progressive@1' }))).toContain( 'promptVersion', diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts index f687fc7a7c4..275b71e3748 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts @@ -902,6 +902,7 @@ describe('UserProxyLlm.respondToConfirmation', () => { expect.any(String), 'slackApi', expect.any(Object), + undefined, ); // The allowlist call must include the pre-existing id, not just the new // one — setThreadCredentialAllowlist replaces the whole list. @@ -1472,6 +1473,7 @@ describe('UserProxyLlm.respondToConfirmation', () => { expect.any(String), 'slackApi', expect.any(Object), + undefined, ); expect(setThreadCredentialAllowlist).toHaveBeenCalledWith('thread-1', ['cred-fresh'], []); expect(response.kind).toBe('credentialSelection'); diff --git a/packages/@n8n/instance-ai/evaluations/clients/__tests__/n8n-client.test.ts b/packages/@n8n/instance-ai/evaluations/clients/__tests__/n8n-client.test.ts index 3e0b8153fee..12d2b7e05ef 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/__tests__/n8n-client.test.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/__tests__/n8n-client.test.ts @@ -15,6 +15,28 @@ import { const BASE_URL = 'http://localhost:5678'; +describe('N8nClient.createCredential', () => { + afterEach(() => vi.unstubAllGlobals()); + + it.each([undefined, null, 'Production reports'])( + 'sends description %s as optional credential metadata', + async (description) => { + const fetchMock = stubFetch({ data: { id: 'credential-1' } }); + const data = { accessToken: 'placeholder' }; + await new N8nClient(BASE_URL).createCredential('Account A', 'slackApi', data, description); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe(`${BASE_URL}/rest/credentials`); + if (typeof init?.body !== 'string') throw new Error('Expected a JSON request body.'); + expect(jsonParse(init.body)).toEqual({ + name: 'Account A', + type: 'slackApi', + data, + ...(description !== undefined ? { description } : {}), + }); + }, + ); +}); + describe('N8nClient.sendMessage', () => { afterEach(() => vi.unstubAllGlobals()); diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index 92ddb2286a4..69cfada6ce3 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -806,10 +806,11 @@ export class N8nClient { name: string, type: string, data: Record, + description?: string | null, ): Promise<{ id: string }> { const result = (await this.fetch('/rest/credentials', { method: 'POST', - body: { name, type, data }, + body: { name, type, data, ...(description !== undefined ? { description } : {}) }, })) as { data: { id: string } }; return { id: result.data.id }; } diff --git a/packages/@n8n/instance-ai/evaluations/credentials/__tests__/seeder.test.ts b/packages/@n8n/instance-ai/evaluations/credentials/__tests__/seeder.test.ts index 3e3726892ea..bf10e55f21e 100644 --- a/packages/@n8n/instance-ai/evaluations/credentials/__tests__/seeder.test.ts +++ b/packages/@n8n/instance-ai/evaluations/credentials/__tests__/seeder.test.ts @@ -4,7 +4,7 @@ import { vi } from 'vitest'; import type { N8nClient } from '../../clients/n8n-client'; import type { EvalLogger } from '../../harness/logger'; -import { createOneCredential } from '../seeder'; +import { createDeclaredCredentials, createOneCredential } from '../seeder'; const makeClient = (createCredential: N8nClient['createCredential']) => ({ createCredential }) as unknown as N8nClient; @@ -133,3 +133,53 @@ describe('createOneCredential — static template path (regression)', () => { expect(createCredential.mock.calls[0][1]).toBe('slackApi'); }); }); + +describe('credential descriptions', () => { + it('keeps descriptions with their declared credentials outside the secret data', async () => { + const createCredential = vi + .fn() + .mockResolvedValue({ id: 'cred-1' }); + const client = makeClient(createCredential); + + await createDeclaredCredentials(client, [ + { type: 'slackApi', name: 'Account A', description: 'Staging checks' }, + { type: 'slackApi', name: 'Account B', description: 'Production reports' }, + { type: 'slackApi', name: 'Account C' }, + { type: 'slackApi', name: 'Account D', description: null }, + ]); + + expect( + createCredential.mock.calls.map(([name, , , description]) => ({ name, description })), + ).toEqual([ + { name: 'Account A', description: 'Staging checks' }, + { name: 'Account B', description: 'Production reports' }, + { name: 'Account C', description: undefined }, + { name: 'Account D', description: null }, + ]); + for (const [, , data] of createCredential.mock.calls) { + expect(data).not.toHaveProperty('description'); + } + }); + + it('passes descriptions through custom authentication setup', async () => { + const createCredential = vi + .fn() + .mockResolvedValue({ id: 'cred-1' }); + await createOneCredential( + makeClient(createCredential), + 'httpTemplatedCustomAuth', + 'Account A', + new Map(), + { + setupHint: baseHint, + description: 'Production reports', + }, + ); + expect(createCredential).toHaveBeenCalledWith( + 'Account A', + 'httpTemplatedCustomAuth', + expect.any(Object), + 'Production reports', + ); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts b/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts index c3688836d6b..236f7e713bb 100644 --- a/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts +++ b/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts @@ -141,6 +141,7 @@ export async function createOneCredential( options?: { logger?: EvalLogger; setupHint?: InstanceAiCredentialSetupHint; + description?: string | null; /** Seed with no field values, modelling a credential the user saved empty. */ blank?: boolean; }, @@ -171,6 +172,7 @@ export async function createOneCredential( resolvedName, credentialType, options?.blank ? {} : template.buildData(token), + options?.description, ); return { id, name: resolvedName, type: credentialType }; } @@ -186,7 +188,11 @@ async function createTemplatedCustomAuthCredential( client: N8nClient, name: string | undefined, usedNames: Map, - options?: { logger?: EvalLogger; setupHint?: InstanceAiCredentialSetupHint }, + options?: { + logger?: EvalLogger; + setupHint?: InstanceAiCredentialSetupHint; + description?: string | null; + }, ): Promise { const hint = options?.setupHint; if (!hint) { @@ -218,16 +224,21 @@ async function createTemplatedCustomAuthCredential( ); options?.logger?.verbose(` Creating credential ${resolvedName} (httpTemplatedCustomAuth)`); - const { id } = await client.createCredential(resolvedName, 'httpTemplatedCustomAuth', { - template: JSON.stringify(hint.template), - placeholderDefs: JSON.stringify(hint.placeholders), - placeholderValues: JSON.stringify(placeholderValues), - serviceHost: hint.serviceHost ?? '', - serviceOrigin: hint.serviceOrigin ?? '', - docsUrl: hint.docsUrl ?? '', - testUrl: hint.testUrl ?? '', - acceptedStatusCodes: hint.acceptedStatusCodes ? JSON.stringify(hint.acceptedStatusCodes) : '', - }); + const { id } = await client.createCredential( + resolvedName, + 'httpTemplatedCustomAuth', + { + template: JSON.stringify(hint.template), + placeholderDefs: JSON.stringify(hint.placeholders), + placeholderValues: JSON.stringify(placeholderValues), + serviceHost: hint.serviceHost ?? '', + serviceOrigin: hint.serviceOrigin ?? '', + docsUrl: hint.docsUrl ?? '', + testUrl: hint.testUrl ?? '', + acceptedStatusCodes: hint.acceptedStatusCodes ? JSON.stringify(hint.acceptedStatusCodes) : '', + }, + options?.description, + ); return { id, name: resolvedName, type: 'httpTemplatedCustomAuth' }; } @@ -263,6 +274,7 @@ export async function createDeclaredCredentials( for (const decl of declared) { const cred = await createOneCredential(client, decl.type, decl.name, nameCounts, { logger, + ...(decl.description !== undefined ? { description: decl.description } : {}), ...(decl.blank ? { blank: true } : {}), }); options?.onCreated?.(cred.id); diff --git a/packages/@n8n/instance-ai/evaluations/harness/schema.ts b/packages/@n8n/instance-ai/evaluations/harness/schema.ts index 1a799f2d173..55b3d97d8f6 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/schema.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/schema.ts @@ -1,4 +1,8 @@ -import { findSeedFolderIssues, instanceAiEvalSeedDataTableSchema } from '@n8n/api-types'; +import { + credentialDescriptionSchema, + findSeedFolderIssues, + instanceAiEvalSeedDataTableSchema, +} from '@n8n/api-types'; import { z } from 'zod'; import { @@ -206,7 +210,10 @@ const evalTestCaseObjectSchema = z message: `unknown credential type — add a template to evaluations/credentials/seeder.ts (supported: ${[...SUPPORTED_CREDENTIAL_TYPES].join(', ')})`, }), name: z.string().min(1).optional(), + description: credentialDescriptionSchema.optional(), + // False lets the connection test fail for a credential that is already broken. valid: z.boolean().optional(), + // True seeds no field values and disables the connection-test bypass. blank: z.boolean().optional(), }), ) diff --git a/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts b/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts index 52bd33d528d..64f52919233 100644 --- a/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts +++ b/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts @@ -69,6 +69,9 @@ export interface ToLangTracerOptions { * already holds, and such a case is barred from suites anyway. Returns a * human-readable reason, else null. */ export function unsupportedPushReason(testCase: EvalTestCaseInput): string | null { + if (testCase.credentials?.some((credential) => credential.description !== undefined)) { + return 'seeds credential descriptions, which the current LangTracer case-write schema does not store. Keep the case on disk until that contract supports descriptions.'; + } if (testCase.promptVersion !== undefined) { return 'pins promptVersion, which the current case-write contract does not carry. Keep the case on disk.'; } diff --git a/packages/@n8n/instance-ai/evaluations/types.ts b/packages/@n8n/instance-ai/evaluations/types.ts index f5de59c2dd3..1786807157e 100644 --- a/packages/@n8n/instance-ai/evaluations/types.ts +++ b/packages/@n8n/instance-ai/evaluations/types.ts @@ -15,7 +15,7 @@ import type { z } from 'zod'; import type { CheckOutcome } from './binaryChecks/types'; import type { WorkflowResponse } from './clients/n8n-client'; import type { EvalAttribution } from './harness/attribution'; -import type { CaseSeed, ConversationTurnSchema } from './harness/schema'; +import type { CaseSeed, ConversationTurnSchema, EvalTestCaseInput } from './harness/schema'; // --------------------------------------------------------------------------- // Checklist items and verification @@ -208,32 +208,8 @@ export interface ExecutionScenario { export type ConversationTurn = z.infer; -export interface TestCaseCredential { - /** n8n credential type name, e.g. `slackApi`. Must have a template in credentials/seeder.ts. */ - type: string; - /** Display name; defaults to the template's name, auto-suffixed on duplicates. */ - name?: string; - /** Defaults to true. false models a credential that was already broken before - * the conversation started (expired/revoked/scope-changed) — left off the - * connection-test bypass list, so its real test runs and fails. Distinct from - * a credential set up on a card mid-conversation (UserProxyLlm), which always - * passes. */ - valid?: boolean; - /** Defaults to false. true models a credential the user saved without filling - * anything in — seeded with no field values, and kept off the connection-test - * bypass so nothing resolves it as working. The shape behind a re-offered - * empty generic-auth credential. - * - * DOES NOT SURVIVE A LANG-TRACER PUSH yet. Its case-write schema validates - * each credential against a non-strict `z.object({ type, name, valid })` - * (lang-tracer `packages/server/src/lib/case-writes.ts`), so this key is - * silently stripped and the suite copy seeds a FILLED credential instead — - * a case relying on it then fails in CI for a reason unrelated to the - * product. `eval:langtracer-push` catches it (`did not store credentials`, - * non-zero exit); until lang-tracer declares the field, a case using it - * lives on disk. */ - blank?: boolean; -} +/** Seeded credentials use the case schema. The push verifies that the suite retains each field. */ +export type TestCaseCredential = NonNullable[number]; export interface WorkflowTestCase { /** Optional human-readable note on what this case is testing (esp. for behaviour cases). */ diff --git a/packages/@n8n/telemetry/credential-descriptions.md b/packages/@n8n/telemetry/credential-descriptions.md new file mode 100644 index 00000000000..17ea86cb3b5 --- /dev/null +++ b/packages/@n8n/telemetry/credential-descriptions.md @@ -0,0 +1,77 @@ +# Credential description metrics + +## Adoption + +Measure the share of users who save at least one nonempty credential description +among users who create or update a credential through the internal credential API +in the same seven-day window. The editor uses this API. +Use identified users on instances with the instrumentation enabled. + +Use these backend events: + +- `User created credentials` with `source = backend` and `public_api = false`. +- `User updated credentials` with `source = backend`. + +Require both description properties on each event. +Existing frontend creation events have no `source` property. +Exclude frontend events and events that lack the required properties from both +counts. Join `user_id` with the instance identity from the event envelope. +Count each user once in the numerator and once in the denominator. +Do not use raw event counts as counts of users or credentials. + +Public API creation stays tracked with `public_api = true` but does not count in +this metric. Imports and writes that bypass these controllers are outside this +metric. Calls to the internal API can also come from scripts. These events do not +prove that a person used the editor. + +`has_description` reports whether the saved description is nonempty. +`description_length` reports the length of the trimmed, saved text in UTF-16 code +units. A missing or blank description has length zero. +The server emits these values after persistence. A later OAuth or connection +test result does not change them. A connection without a save does not count. +The events never include the description text. + +This measures use of descriptions among users who save credentials. +An update can preserve an existing description without changing its text. +It does not measure the share of all stored credentials that have a description. +The existing telemetry transport is best effort. This change does not guarantee +delivery after a server failure. + +## Correctness + +The correctness metric is the percentage of eval runs in which the assistant +selects the credential whose description matches the request when two +credentials share one type. + +Use neutral credential names. Give the credentials different purposes in their +descriptions. Ask for one purpose without naming the credential. +Keep the distinguishing context within the list previews from CONTEXT-150. +Check the selected credential, not just whether the assistant read a description. +Keep the credential order fixed within a comparison. Repeat the comparison with +the descriptions swapped to expose a preference for one position or name. + +Run the same cases before and after the CONTEXT-150 read changes. +Keep the model, prompt configuration, case revision, and run count the same. +Report successful selections and total completed runs with each percentage. +Report infrastructure failures separately. Do not count them as selections. + +## Do-not-harm + +Credential selection accuracy must not fall below the baseline on the same cases +when all descriptions are absent. +Use cases where the request and credential names identify the intended choice. +Do not assign an arbitrary correct answer to indistinguishable credentials. +Report the change in percentage points with the underlying counts. + +## Release evidence + +Record the following on CONTEXT-151 before shipment: + +- Case and suite links. +- Baseline and candidate commit IDs. +- Model and prompt configuration. +- Successful selections, completed runs, and infrastructure failures. +- Correctness percentages and the do-not-harm change. + +The telemetry change alone does not establish a correctness result. +The comparison requires the CONTEXT-150 read changes on the candidate instance. diff --git a/packages/@n8n/telemetry/src/__tests__/credential-description.test.ts b/packages/@n8n/telemetry/src/__tests__/credential-description.test.ts new file mode 100644 index 00000000000..b8935cfb74b --- /dev/null +++ b/packages/@n8n/telemetry/src/__tests__/credential-description.test.ts @@ -0,0 +1,40 @@ +import { TELEMETRY_EVENT } from '../index'; +import { getEventValidationError } from '../validate'; + +describe('credential description telemetry', () => { + it.each([ + TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS, + TELEMETRY_EVENT.CREDENTIALS.USER_UPDATED_CREDENTIALS, + ])('accepts backend metadata and rejects description text in $name', (event) => { + const properties = { + user_id: 'user-1', + credential_id: 'credential-1', + credential_type: 'postgres', + source: 'backend', + has_description: true, + description_length: 18, + is_private: false, + uses_external_secrets: false, + jwe_enabled: false, + credential_supports_managed_auth: false, + credential_uses_managed_auth: false, + ...(event === TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS + ? { public_api: false } + : {}), + }; + expect(getEventValidationError(event, properties)).toBeNull(); + expect( + getEventValidationError(event, { ...properties, description: 'Production reports' }), + ).toContain('description: unrecognized property'); + }); + + it('accepts existing frontend creation without source or description metadata', () => { + expect( + getEventValidationError(TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS, { + credential_id: 'credential-1', + credential_type: 'postgres', + workflow_id: 'workflow-1', + }), + ).toBeNull(); + }); +}); diff --git a/packages/@n8n/telemetry/src/events/credentials.ts b/packages/@n8n/telemetry/src/events/credentials.ts index 458fa0d606d..28529532d6f 100644 --- a/packages/@n8n/telemetry/src/events/credentials.ts +++ b/packages/@n8n/telemetry/src/events/credentials.ts @@ -2,7 +2,61 @@ import { z } from 'zod/v4'; import { defineTelemetryEvents } from '../define'; +const descriptionProperties = { + has_description: z.boolean().describe('Whether the saved credential has a nonempty description.'), + description_length: z + .number() + .int() + .nonnegative() + .describe('Length of the trimmed, saved description in UTF-16 code units. Zero when absent.'), +}; + export const CREDENTIALS_TELEMETRY = defineTelemetryEvents({ + USER_CREATED_CREDENTIALS: { + name: 'User created credentials', + description: + 'A credential was created. The server sets source to backend and reports description metadata. Existing editor events omit these properties. Description text is never included.', + properties: z.object({ + credential_id: z.string(), + credential_type: z.string(), + source: z + .literal('backend') + .optional() + .describe('Set by the server. Existing editor events omit it.'), + public_api: z.boolean().optional().describe('Whether the server used the public API.'), + has_description: descriptionProperties.has_description.optional(), + description_length: descriptionProperties.description_length.optional(), + user_id: z.string().optional(), + user_role: z.string().optional(), + project_id: z.string().optional(), + project_type: z.string().optional(), + workflow_id: z.string().nullable().optional(), + uiContext: z.string().optional().describe('Legacy property for the editor entry point.'), + is_private: z.boolean().optional(), + uses_external_secrets: z.boolean().optional(), + jwe_enabled: z.boolean().optional(), + credential_supports_managed_auth: z.boolean().optional(), + credential_uses_managed_auth: z.boolean().optional(), + }), + }, + USER_UPDATED_CREDENTIALS: { + name: 'User updated credentials', + description: + 'The server updated a credential through the internal credential API. Description metadata comes from the saved record. Description text is never included.', + properties: z.object({ + user_id: z.string(), + user_role: z.string().optional(), + credential_id: z.string(), + credential_type: z.string(), + source: z.literal('backend'), + ...descriptionProperties, + is_private: z.boolean(), + uses_external_secrets: z.boolean(), + jwe_enabled: z.boolean(), + credential_supports_managed_auth: z.boolean(), + credential_uses_managed_auth: z.boolean(), + }), + }, USER_STARTED_CREDENTIAL_CONNECTION: { name: 'User started credential connection', description: diff --git a/packages/cli/src/credentials/__tests__/credentials.controller.test.ts b/packages/cli/src/credentials/__tests__/credentials.controller.test.ts index 0898e8cd544..7871c80e7c5 100644 --- a/packages/cli/src/credentials/__tests__/credentials.controller.test.ts +++ b/packages/cli/src/credentials/__tests__/credentials.controller.test.ts @@ -178,6 +178,28 @@ describe('CredentialsController', () => { expect(newApiKey).toEqual(createdCredentials); }); + it.each([null, 'Production reports'])( + 'emits the saved description length for %s', + async (description) => { + const payload = { ...createNewCredentialsPayload(), description: 'Draft description' }; + const { data: _data, ...payloadWithoutData } = payload; + const created = { ...createdCredentialsWithScopes(payloadWithoutData), description }; + createUnmanagedCredentialSpy.mockResolvedValue(created); + findCredentialOwningProjectSpy.mockResolvedValue( + mock({ id: 'project-1', type: 'team' }), + ); + + await credentialsController.createCredentials(req, res, payload); + + const [, eventPayload] = emitSpy.mock.calls[0]; + expect(eventPayload).toMatchObject({ + credentialDescriptionLength: description?.length ?? 0, + }); + expect(eventPayload).not.toHaveProperty('description'); + expect(eventPayload).not.toHaveProperty('credentialDescription'); + }, + ); + it('should emit "credentials-created" with jweEnabled true when payload enables JWE', async () => { const newCredentialsPayload = createNewCredentialsPayload({ data: { clientId: 'cid', jweEnabled: true }, @@ -267,6 +289,7 @@ describe('CredentialsController', () => { const existingCredential = mock({ id: credentialId, name: 'Test Credential', + description: null, type: 'apiKey', isGlobal: false, isManaged: false, @@ -364,6 +387,7 @@ describe('CredentialsController', () => { credentialType: existingCredential.type, credentialId: existingCredential.id, credentialName: 'Updated Credential', + credentialDescriptionLength: 0, isDynamic: false, usesExternalSecrets: false, jweEnabled: false, @@ -524,13 +548,14 @@ describe('CredentialsController', () => { it.each([ [ 'writes a description the payload sends', - 'Read-only reporting key.', + ' Read-only reporting key. ', 'Read-only reporting key.', ], ['clears a description the payload blanks out', ' ', null], ])('%s', async (_label, sent, prepared) => { const req = updateRequest({ description: sent }); prepareUpdateDataSpy.mockResolvedValue({ ...req.body, description: prepared }); + updateSpy.mockResolvedValue({ ...existingCredential, description: prepared }); await credentialsController.updateCredentials(req); @@ -540,15 +565,27 @@ describe('CredentialsController', () => { expect.anything(), expect.any(Object), ); + expect(emitSpy).toHaveBeenCalledWith( + 'credentials-updated', + expect.objectContaining({ credentialDescriptionLength: prepared?.length ?? 0 }), + ); + const event = emitSpy.mock.calls.find(([name]) => name === 'credentials-updated')?.[1]; + expect(event).not.toHaveProperty('description'); + expect(event).not.toHaveProperty('credentialDescription'); }); it('leaves the stored description alone when the payload omits the field', async () => { const req = updateRequest({}); prepareUpdateDataSpy.mockResolvedValue({ ...req.body }); + updateSpy.mockResolvedValue({ ...existingCredential, description: 'Production reports' }); await credentialsController.updateCredentials(req); expect(updateSpy.mock.calls[0][1]).not.toHaveProperty('description'); + expect(emitSpy).toHaveBeenCalledWith( + 'credentials-updated', + expect.objectContaining({ credentialDescriptionLength: 18 }), + ); }); }); diff --git a/packages/cli/src/credentials/credentials.controller.ts b/packages/cli/src/credentials/credentials.controller.ts index f16cce4c18a..ba4f14b8b58 100644 --- a/packages/cli/src/credentials/credentials.controller.ts +++ b/packages/cli/src/credentials/credentials.controller.ts @@ -216,6 +216,7 @@ export class CredentialsController { credentialType: newCredential.type, credentialId: newCredential.id, credentialName: newCredential.name, + credentialDescriptionLength: newCredential.description?.length ?? 0, publicApi: false, projectId: project?.id, projectType: project?.type, @@ -383,6 +384,7 @@ export class CredentialsController { credentialId: credential.id, // The updated entity, so a rename records the new name rather than the one it replaced. credentialName: responseData.name, + credentialDescriptionLength: responseData.description?.length ?? 0, isDynamic: newCredentialData.isResolvable ?? false, usesExternalSecrets: getExternalSecretExpressionPaths(preparedCredentialData.data).length > 0, jweEnabled: updatedData.jweEnabled === true, diff --git a/packages/cli/src/events/__tests__/activity-event-relay.test.ts b/packages/cli/src/events/__tests__/activity-event-relay.test.ts index 157f27b7eda..efa5b26102b 100644 --- a/packages/cli/src/events/__tests__/activity-event-relay.test.ts +++ b/packages/cli/src/events/__tests__/activity-event-relay.test.ts @@ -204,6 +204,7 @@ describe('ActivityEventRelay', () => { credentialType: 'slackApi', credentialId: 'credential1', credentialName: 'Team Slack', + credentialDescriptionLength: 0, publicApi: false, projectId: 'project1', }), @@ -224,6 +225,7 @@ describe('ActivityEventRelay', () => { credentialType: 'slackApi', credentialId: 'credential1', credentialName: 'Team Slack', + credentialDescriptionLength: 0, }), { category: 'credential', @@ -313,6 +315,7 @@ describe('ActivityEventRelay', () => { credentialType: 'slackApi', credentialId: 'credential1', credentialName: 'Team Slack', + credentialDescriptionLength: 0, publicApi: true, projectId: undefined, }); @@ -543,6 +546,7 @@ describe('ActivityEventRelay', () => { credentialType: 'slackApi', credentialId: 'credential1', credentialName: 'Instance chat model', + credentialDescriptionLength: 0, }); await flushPromises(); diff --git a/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts b/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts index 388e4c28706..19a033e42ed 100644 --- a/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts +++ b/packages/cli/src/events/__tests__/log-streaming-event-relay.test.ts @@ -1282,6 +1282,7 @@ describe('LogStreamingEventRelay', () => { it('should log on `credentials-created` event', () => { const event: RelayEventMap['credentials-created'] = { credentialName: 'My GitHub account', + credentialDescriptionLength: 0, user: { id: 'user123', email: 'user@example.com', @@ -1349,6 +1350,7 @@ describe('LogStreamingEventRelay', () => { it('should log on `credentials-updated` event', () => { const event: RelayEventMap['credentials-updated'] = { credentialName: 'Rotated token', + credentialDescriptionLength: 0, user: { id: 'user808', email: 'updatecred@example.com', diff --git a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts index d0e16b277b9..bd6e2622c24 100644 --- a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts +++ b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts @@ -1504,42 +1504,58 @@ describe('TelemetryEventRelay', () => { }); describe('credentials events', () => { - it('should track on `credentials-created` event', () => { - const event: RelayEventMap['credentials-created'] = { - credentialName: 'My GitHub account', - user: { - id: 'user123', - email: 'user@example.com', - firstName: 'John', - lastName: 'Doe', - role: { slug: GLOBAL_OWNER_ROLE.slug }, - }, - credentialType: 'github', - credentialId: 'cred123', - publicApi: false, - projectId: 'project123', - projectType: 'personal', - isDynamic: false, - supportsManagedAuth: true, - usesManagedAuth: true, - }; + it.each([ + { descriptionLength: 0, publicApi: false }, + { descriptionLength: 18, publicApi: false }, + { descriptionLength: 0, publicApi: true }, + { descriptionLength: 18, publicApi: true }, + ])( + 'tracks creation with description length $descriptionLength and public API $publicApi', + ({ descriptionLength, publicApi }) => { + const event: RelayEventMap['credentials-created'] = { + credentialName: 'My GitHub account', + credentialDescriptionLength: descriptionLength, + user: { + id: 'user123', + email: 'user@example.com', + firstName: 'John', + lastName: 'Doe', + role: { slug: GLOBAL_OWNER_ROLE.slug }, + }, + credentialType: 'github', + credentialId: 'cred123', + publicApi, + projectId: 'project123', + projectType: 'personal', + isDynamic: false, + supportsManagedAuth: true, + usesManagedAuth: true, + }; - eventService.emit('credentials-created', event); + eventService.emit('credentials-created', event); - expect(telemetry.track).toHaveBeenCalledWith('User created credentials', { - user_id: 'user123', - user_role: GLOBAL_OWNER_ROLE.slug, - credential_type: 'github', - credential_id: 'cred123', - project_id: 'project123', - project_type: 'personal', - is_private: false, - uses_external_secrets: false, - jwe_enabled: false, - credential_supports_managed_auth: true, - credential_uses_managed_auth: true, - }); - }); + expect(telemetry.track).toHaveBeenCalledWith( + TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS, + { + source: 'backend', + public_api: publicApi, + user_id: 'user123', + user_role: GLOBAL_OWNER_ROLE.slug, + credential_type: 'github', + credential_id: 'cred123', + has_description: descriptionLength > 0, + description_length: descriptionLength, + project_id: 'project123', + project_type: 'personal', + is_private: false, + uses_external_secrets: false, + jwe_enabled: false, + credential_supports_managed_auth: true, + credential_uses_managed_auth: true, + }, + ); + }, + ); it('should track on `credentials-shared` event', () => { const event: RelayEventMap['credentials-shared'] = { @@ -1570,9 +1586,10 @@ describe('TelemetryEventRelay', () => { }); }); - it('should track on `credentials-updated` event', () => { + it.each([0, 18])('tracks description length %i on update', (descriptionLength) => { const event: RelayEventMap['credentials-updated'] = { credentialName: 'Rotated token', + credentialDescriptionLength: descriptionLength, user: { id: 'user123', email: 'user@example.com', @@ -1587,17 +1604,23 @@ describe('TelemetryEventRelay', () => { eventService.emit('credentials-updated', event); - expect(telemetry.track).toHaveBeenCalledWith('User updated credentials', { - user_id: 'user123', - user_role: GLOBAL_OWNER_ROLE.slug, - credential_type: 'github', - credential_id: 'cred123', - is_private: true, - uses_external_secrets: false, - jwe_enabled: false, - credential_supports_managed_auth: false, - credential_uses_managed_auth: false, - }); + expect(telemetry.track).toHaveBeenCalledWith( + TELEMETRY_EVENT.CREDENTIALS.USER_UPDATED_CREDENTIALS, + { + source: 'backend', + has_description: descriptionLength > 0, + description_length: descriptionLength, + user_id: 'user123', + user_role: GLOBAL_OWNER_ROLE.slug, + credential_type: 'github', + credential_id: 'cred123', + is_private: true, + uses_external_secrets: false, + jwe_enabled: false, + credential_supports_managed_auth: false, + credential_uses_managed_auth: false, + }, + ); }); it('should track on `credentials-deleted` event', () => { diff --git a/packages/cli/src/events/maps/relay.event-map.ts b/packages/cli/src/events/maps/relay.event-map.ts index fecef9a2416..6f350918b2b 100644 --- a/packages/cli/src/events/maps/relay.event-map.ts +++ b/packages/cli/src/events/maps/relay.event-map.ts @@ -480,6 +480,7 @@ export type RelayEventMap = { credentialType: string; credentialId: string; credentialName: string; + credentialDescriptionLength: number; publicApi: boolean; projectId?: string; projectType?: string; @@ -505,6 +506,7 @@ export type RelayEventMap = { credentialType: string; credentialId: string; credentialName: string; + credentialDescriptionLength: number; isDynamic?: boolean; usesExternalSecrets?: boolean; jweEnabled?: boolean; diff --git a/packages/cli/src/events/relays/telemetry.event-relay.ts b/packages/cli/src/events/relays/telemetry.event-relay.ts index 48236b961a6..e4a3f359bb7 100644 --- a/packages/cli/src/events/relays/telemetry.event-relay.ts +++ b/packages/cli/src/events/relays/telemetry.event-relay.ts @@ -860,6 +860,8 @@ export class TelemetryEventRelay extends EventRelay { user, credentialType, credentialId, + credentialDescriptionLength, + publicApi, projectId, projectType, uiContext, @@ -869,11 +871,15 @@ export class TelemetryEventRelay extends EventRelay { supportsManagedAuth, usesManagedAuth, }: RelayEventMap['credentials-created']) { - this.telemetry.track('User created credentials', { + this.telemetry.track(TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS, { + source: 'backend', + public_api: publicApi, user_id: user.id, user_role: user.role?.slug, credential_type: credentialType, credential_id: credentialId, + has_description: credentialDescriptionLength > 0, + description_length: credentialDescriptionLength, project_id: projectId, project_type: projectType, uiContext, @@ -908,17 +914,21 @@ export class TelemetryEventRelay extends EventRelay { user, credentialId, credentialType, + credentialDescriptionLength, isDynamic, usesExternalSecrets, jweEnabled, supportsManagedAuth, usesManagedAuth, }: RelayEventMap['credentials-updated']) { - this.telemetry.track('User updated credentials', { + this.telemetry.track(TELEMETRY_EVENT.CREDENTIALS.USER_UPDATED_CREDENTIALS, { + source: 'backend', user_id: user.id, user_role: user.role?.slug, credential_type: credentialType, credential_id: credentialId, + has_description: credentialDescriptionLength > 0, + description_length: credentialDescriptionLength, is_private: isDynamic ?? false, uses_external_secrets: usesExternalSecrets ?? false, jwe_enabled: jweEnabled ?? false, diff --git a/packages/cli/src/public-api/v1/controllers/credentials.public.controller.ts b/packages/cli/src/public-api/v1/controllers/credentials.public.controller.ts index b91902e3628..834ad601b0a 100644 --- a/packages/cli/src/public-api/v1/controllers/credentials.public.controller.ts +++ b/packages/cli/src/public-api/v1/controllers/credentials.public.controller.ts @@ -216,6 +216,7 @@ export class CredentialsPublicController { credentialType: credential.type, credentialId: credential.id, credentialName: credential.name, + credentialDescriptionLength: credential.description?.length ?? 0, publicApi: true, projectId: project?.id, projectType: project?.type,