feat(core): Track credential description usage (no-changelog) (#38986)

This commit is contained in:
Arvin A 2026-09-21 09:09:43 +00:00 committed by GitHub
parent 989b67ee95
commit d9dc457d40
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 475 additions and 91 deletions

View File

@ -800,6 +800,18 @@ Declared credentials are created for real (placeholder token; set the matching `
Each type needs a data template in `credentials/seeder.ts`; declaring an unknown type fails the build with a pointer there.
Each declared credential can also have a `description`. The loader applies the
shared credential description schema. It trims the text, enforces the length
limit, and converts blank text to `null`. The seeder stores the description as
credential metadata, outside the secret `data` object. Omit the field to test
credential choice without descriptions.
The current LangTracer case-write schema accepts only `type`, `name`, and
`valid` on a credential. The push refuses a case with a description before it
writes anything. Keep these cases on disk until LangTracer stores descriptions.
The existing `blank` field also needs server support. The export check catches
a server that drops it after a write.
### Seeded cases (conversation pre-seeding)
A seeded case starts **mid-conversation**: prior history is restored into the build thread before the live turn, so the eval drives only the turn under test. Use it to replicate a real misbehaviour — restore the conversation up to the moment it went wrong, re-drive that turn, and assert what should happen instead.

View File

@ -248,7 +248,12 @@ describe('buildWorkflow declared credentials', () => {
expect(build.success).toBe(true);
// A blank credential models one the user saved without filling anything in,
// so it is seeded with no data and must never resolve a test as passing.
expect(createCredential).toHaveBeenCalledWith(expect.any(String), 'httpHeaderAuth', {});
expect(createCredential).toHaveBeenCalledWith(
expect.any(String),
'httpHeaderAuth',
{},
undefined,
);
expect(setThreadCredentialAllowlist).toHaveBeenCalledWith(
expect.any(String),
['cred-blank'],

View File

@ -49,6 +49,28 @@ beforeEach(() => {
});
describe('EvalTestCaseSchema', () => {
it.each([
{ description: undefined, expected: undefined },
{ description: null, expected: null },
{ description: ' \n ', expected: null },
{ description: ' Production reports ', expected: 'Production reports' },
])('normalizes credential description $description', ({ description, expected }) => {
const parsed = EvalTestCaseSchema.parse({
...validFixture(),
credentials: [{ type: 'slackApi', description }],
});
expect(parsed.credentials?.[0].description).toBe(expected);
});
it('rejects a credential description above the storage limit', () => {
expect(() =>
EvalTestCaseSchema.parse({
...validFixture(),
credentials: [{ type: 'slackApi', description: '🔑'.repeat(257) }],
}),
).toThrow();
});
it('accepts a minimal valid fixture', () => {
const parsed = EvalTestCaseSchema.parse(validFixture());
expect(parsed.executionScenarios).toHaveLength(1);

View File

@ -107,6 +107,26 @@ describe('diskCaseToLangTracerCreate', () => {
});
describe('unsupportedPushReason', () => {
it.each([null, 'Production reports'])(
'refuses description %s until the case-write API preserves it',
(description) => {
const input = diskCase({ credentials: [{ type: 'httpHeaderAuth', description }] });
expect(unsupportedPushReason(input)).toContain('description');
const body = diskCaseToLangTracerCreate(input, 'description-case', {
suiteId: 8,
setKind: 'regression',
synthetic: true,
});
expect(body.credentials).toEqual(input.credentials);
},
);
it('allows credentials without descriptions', () => {
expect(
unsupportedPushReason(diskCase({ credentials: [{ type: 'httpHeaderAuth' }] })),
).toBeNull();
});
it('refuses a case whose prompt version would be lost', () => {
expect(unsupportedPushReason(diskCase({ promptVersion: 'progressive@1' }))).toContain(
'promptVersion',

View File

@ -902,6 +902,7 @@ describe('UserProxyLlm.respondToConfirmation', () => {
expect.any(String),
'slackApi',
expect.any(Object),
undefined,
);
// The allowlist call must include the pre-existing id, not just the new
// one — setThreadCredentialAllowlist replaces the whole list.
@ -1472,6 +1473,7 @@ describe('UserProxyLlm.respondToConfirmation', () => {
expect.any(String),
'slackApi',
expect.any(Object),
undefined,
);
expect(setThreadCredentialAllowlist).toHaveBeenCalledWith('thread-1', ['cred-fresh'], []);
expect(response.kind).toBe('credentialSelection');

View File

@ -15,6 +15,28 @@ import {
const BASE_URL = 'http://localhost:5678';
describe('N8nClient.createCredential', () => {
afterEach(() => vi.unstubAllGlobals());
it.each([undefined, null, 'Production reports'])(
'sends description %s as optional credential metadata',
async (description) => {
const fetchMock = stubFetch({ data: { id: 'credential-1' } });
const data = { accessToken: 'placeholder' };
await new N8nClient(BASE_URL).createCredential('Account A', 'slackApi', data, description);
const [url, init] = fetchMock.mock.calls[0];
expect(url).toBe(`${BASE_URL}/rest/credentials`);
if (typeof init?.body !== 'string') throw new Error('Expected a JSON request body.');
expect(jsonParse(init.body)).toEqual({
name: 'Account A',
type: 'slackApi',
data,
...(description !== undefined ? { description } : {}),
});
},
);
});
describe('N8nClient.sendMessage', () => {
afterEach(() => vi.unstubAllGlobals());

View File

@ -806,10 +806,11 @@ export class N8nClient {
name: string,
type: string,
data: Record<string, unknown>,
description?: string | null,
): Promise<{ id: string }> {
const result = (await this.fetch('/rest/credentials', {
method: 'POST',
body: { name, type, data },
body: { name, type, data, ...(description !== undefined ? { description } : {}) },
})) as { data: { id: string } };
return { id: result.data.id };
}

View File

@ -4,7 +4,7 @@ import { vi } from 'vitest';
import type { N8nClient } from '../../clients/n8n-client';
import type { EvalLogger } from '../../harness/logger';
import { createOneCredential } from '../seeder';
import { createDeclaredCredentials, createOneCredential } from '../seeder';
const makeClient = (createCredential: N8nClient['createCredential']) =>
({ createCredential }) as unknown as N8nClient;
@ -133,3 +133,53 @@ describe('createOneCredential — static template path (regression)', () => {
expect(createCredential.mock.calls[0][1]).toBe('slackApi');
});
});
describe('credential descriptions', () => {
it('keeps descriptions with their declared credentials outside the secret data', async () => {
const createCredential = vi
.fn<N8nClient['createCredential']>()
.mockResolvedValue({ id: 'cred-1' });
const client = makeClient(createCredential);
await createDeclaredCredentials(client, [
{ type: 'slackApi', name: 'Account A', description: 'Staging checks' },
{ type: 'slackApi', name: 'Account B', description: 'Production reports' },
{ type: 'slackApi', name: 'Account C' },
{ type: 'slackApi', name: 'Account D', description: null },
]);
expect(
createCredential.mock.calls.map(([name, , , description]) => ({ name, description })),
).toEqual([
{ name: 'Account A', description: 'Staging checks' },
{ name: 'Account B', description: 'Production reports' },
{ name: 'Account C', description: undefined },
{ name: 'Account D', description: null },
]);
for (const [, , data] of createCredential.mock.calls) {
expect(data).not.toHaveProperty('description');
}
});
it('passes descriptions through custom authentication setup', async () => {
const createCredential = vi
.fn<N8nClient['createCredential']>()
.mockResolvedValue({ id: 'cred-1' });
await createOneCredential(
makeClient(createCredential),
'httpTemplatedCustomAuth',
'Account A',
new Map(),
{
setupHint: baseHint,
description: 'Production reports',
},
);
expect(createCredential).toHaveBeenCalledWith(
'Account A',
'httpTemplatedCustomAuth',
expect.any(Object),
'Production reports',
);
});
});

View File

@ -141,6 +141,7 @@ export async function createOneCredential(
options?: {
logger?: EvalLogger;
setupHint?: InstanceAiCredentialSetupHint;
description?: string | null;
/** Seed with no field values, modelling a credential the user saved empty. */
blank?: boolean;
},
@ -171,6 +172,7 @@ export async function createOneCredential(
resolvedName,
credentialType,
options?.blank ? {} : template.buildData(token),
options?.description,
);
return { id, name: resolvedName, type: credentialType };
}
@ -186,7 +188,11 @@ async function createTemplatedCustomAuthCredential(
client: N8nClient,
name: string | undefined,
usedNames: Map<string, number>,
options?: { logger?: EvalLogger; setupHint?: InstanceAiCredentialSetupHint },
options?: {
logger?: EvalLogger;
setupHint?: InstanceAiCredentialSetupHint;
description?: string | null;
},
): Promise<CreatedCredential> {
const hint = options?.setupHint;
if (!hint) {
@ -218,16 +224,21 @@ async function createTemplatedCustomAuthCredential(
);
options?.logger?.verbose(` Creating credential ${resolvedName} (httpTemplatedCustomAuth)`);
const { id } = await client.createCredential(resolvedName, 'httpTemplatedCustomAuth', {
template: JSON.stringify(hint.template),
placeholderDefs: JSON.stringify(hint.placeholders),
placeholderValues: JSON.stringify(placeholderValues),
serviceHost: hint.serviceHost ?? '',
serviceOrigin: hint.serviceOrigin ?? '',
docsUrl: hint.docsUrl ?? '',
testUrl: hint.testUrl ?? '',
acceptedStatusCodes: hint.acceptedStatusCodes ? JSON.stringify(hint.acceptedStatusCodes) : '',
});
const { id } = await client.createCredential(
resolvedName,
'httpTemplatedCustomAuth',
{
template: JSON.stringify(hint.template),
placeholderDefs: JSON.stringify(hint.placeholders),
placeholderValues: JSON.stringify(placeholderValues),
serviceHost: hint.serviceHost ?? '',
serviceOrigin: hint.serviceOrigin ?? '',
docsUrl: hint.docsUrl ?? '',
testUrl: hint.testUrl ?? '',
acceptedStatusCodes: hint.acceptedStatusCodes ? JSON.stringify(hint.acceptedStatusCodes) : '',
},
options?.description,
);
return { id, name: resolvedName, type: 'httpTemplatedCustomAuth' };
}
@ -263,6 +274,7 @@ export async function createDeclaredCredentials(
for (const decl of declared) {
const cred = await createOneCredential(client, decl.type, decl.name, nameCounts, {
logger,
...(decl.description !== undefined ? { description: decl.description } : {}),
...(decl.blank ? { blank: true } : {}),
});
options?.onCreated?.(cred.id);

View File

@ -1,4 +1,8 @@
import { findSeedFolderIssues, instanceAiEvalSeedDataTableSchema } from '@n8n/api-types';
import {
credentialDescriptionSchema,
findSeedFolderIssues,
instanceAiEvalSeedDataTableSchema,
} from '@n8n/api-types';
import { z } from 'zod';
import {
@ -206,7 +210,10 @@ const evalTestCaseObjectSchema = z
message: `unknown credential type — add a template to evaluations/credentials/seeder.ts (supported: ${[...SUPPORTED_CREDENTIAL_TYPES].join(', ')})`,
}),
name: z.string().min(1).optional(),
description: credentialDescriptionSchema.optional(),
// False lets the connection test fail for a credential that is already broken.
valid: z.boolean().optional(),
// True seeds no field values and disables the connection-test bypass.
blank: z.boolean().optional(),
}),
)

View File

@ -69,6 +69,9 @@ export interface ToLangTracerOptions {
* already holds, and such a case is barred from suites anyway. Returns a
* human-readable reason, else null. */
export function unsupportedPushReason(testCase: EvalTestCaseInput): string | null {
if (testCase.credentials?.some((credential) => credential.description !== undefined)) {
return 'seeds credential descriptions, which the current LangTracer case-write schema does not store. Keep the case on disk until that contract supports descriptions.';
}
if (testCase.promptVersion !== undefined) {
return 'pins promptVersion, which the current case-write contract does not carry. Keep the case on disk.';
}

View File

@ -15,7 +15,7 @@ import type { z } from 'zod';
import type { CheckOutcome } from './binaryChecks/types';
import type { WorkflowResponse } from './clients/n8n-client';
import type { EvalAttribution } from './harness/attribution';
import type { CaseSeed, ConversationTurnSchema } from './harness/schema';
import type { CaseSeed, ConversationTurnSchema, EvalTestCaseInput } from './harness/schema';
// ---------------------------------------------------------------------------
// Checklist items and verification
@ -208,32 +208,8 @@ export interface ExecutionScenario {
export type ConversationTurn = z.infer<typeof ConversationTurnSchema>;
export interface TestCaseCredential {
/** n8n credential type name, e.g. `slackApi`. Must have a template in credentials/seeder.ts. */
type: string;
/** Display name; defaults to the template's name, auto-suffixed on duplicates. */
name?: string;
/** Defaults to true. false models a credential that was already broken before
* the conversation started (expired/revoked/scope-changed) — left off the
* connection-test bypass list, so its real test runs and fails. Distinct from
* a credential set up on a card mid-conversation (UserProxyLlm), which always
* passes. */
valid?: boolean;
/** Defaults to false. true models a credential the user saved without filling
* anything in — seeded with no field values, and kept off the connection-test
* bypass so nothing resolves it as working. The shape behind a re-offered
* empty generic-auth credential.
*
* DOES NOT SURVIVE A LANG-TRACER PUSH yet. Its case-write schema validates
* each credential against a non-strict `z.object({ type, name, valid })`
* (lang-tracer `packages/server/src/lib/case-writes.ts`), so this key is
* silently stripped and the suite copy seeds a FILLED credential instead —
* a case relying on it then fails in CI for a reason unrelated to the
* product. `eval:langtracer-push` catches it (`did not store credentials`,
* non-zero exit); until lang-tracer declares the field, a case using it
* lives on disk. */
blank?: boolean;
}
/** Seeded credentials use the case schema. The push verifies that the suite retains each field. */
export type TestCaseCredential = NonNullable<EvalTestCaseInput['credentials']>[number];
export interface WorkflowTestCase {
/** Optional human-readable note on what this case is testing (esp. for behaviour cases). */

View File

@ -0,0 +1,77 @@
# Credential description metrics
## Adoption
Measure the share of users who save at least one nonempty credential description
among users who create or update a credential through the internal credential API
in the same seven-day window. The editor uses this API.
Use identified users on instances with the instrumentation enabled.
Use these backend events:
- `User created credentials` with `source = backend` and `public_api = false`.
- `User updated credentials` with `source = backend`.
Require both description properties on each event.
Existing frontend creation events have no `source` property.
Exclude frontend events and events that lack the required properties from both
counts. Join `user_id` with the instance identity from the event envelope.
Count each user once in the numerator and once in the denominator.
Do not use raw event counts as counts of users or credentials.
Public API creation stays tracked with `public_api = true` but does not count in
this metric. Imports and writes that bypass these controllers are outside this
metric. Calls to the internal API can also come from scripts. These events do not
prove that a person used the editor.
`has_description` reports whether the saved description is nonempty.
`description_length` reports the length of the trimmed, saved text in UTF-16 code
units. A missing or blank description has length zero.
The server emits these values after persistence. A later OAuth or connection
test result does not change them. A connection without a save does not count.
The events never include the description text.
This measures use of descriptions among users who save credentials.
An update can preserve an existing description without changing its text.
It does not measure the share of all stored credentials that have a description.
The existing telemetry transport is best effort. This change does not guarantee
delivery after a server failure.
## Correctness
The correctness metric is the percentage of eval runs in which the assistant
selects the credential whose description matches the request when two
credentials share one type.
Use neutral credential names. Give the credentials different purposes in their
descriptions. Ask for one purpose without naming the credential.
Keep the distinguishing context within the list previews from CONTEXT-150.
Check the selected credential, not just whether the assistant read a description.
Keep the credential order fixed within a comparison. Repeat the comparison with
the descriptions swapped to expose a preference for one position or name.
Run the same cases before and after the CONTEXT-150 read changes.
Keep the model, prompt configuration, case revision, and run count the same.
Report successful selections and total completed runs with each percentage.
Report infrastructure failures separately. Do not count them as selections.
## Do-not-harm
Credential selection accuracy must not fall below the baseline on the same cases
when all descriptions are absent.
Use cases where the request and credential names identify the intended choice.
Do not assign an arbitrary correct answer to indistinguishable credentials.
Report the change in percentage points with the underlying counts.
## Release evidence
Record the following on CONTEXT-151 before shipment:
- Case and suite links.
- Baseline and candidate commit IDs.
- Model and prompt configuration.
- Successful selections, completed runs, and infrastructure failures.
- Correctness percentages and the do-not-harm change.
The telemetry change alone does not establish a correctness result.
The comparison requires the CONTEXT-150 read changes on the candidate instance.

View File

@ -0,0 +1,40 @@
import { TELEMETRY_EVENT } from '../index';
import { getEventValidationError } from '../validate';
describe('credential description telemetry', () => {
it.each([
TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS,
TELEMETRY_EVENT.CREDENTIALS.USER_UPDATED_CREDENTIALS,
])('accepts backend metadata and rejects description text in $name', (event) => {
const properties = {
user_id: 'user-1',
credential_id: 'credential-1',
credential_type: 'postgres',
source: 'backend',
has_description: true,
description_length: 18,
is_private: false,
uses_external_secrets: false,
jwe_enabled: false,
credential_supports_managed_auth: false,
credential_uses_managed_auth: false,
...(event === TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS
? { public_api: false }
: {}),
};
expect(getEventValidationError(event, properties)).toBeNull();
expect(
getEventValidationError(event, { ...properties, description: 'Production reports' }),
).toContain('description: unrecognized property');
});
it('accepts existing frontend creation without source or description metadata', () => {
expect(
getEventValidationError(TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS, {
credential_id: 'credential-1',
credential_type: 'postgres',
workflow_id: 'workflow-1',
}),
).toBeNull();
});
});

View File

@ -2,7 +2,61 @@ import { z } from 'zod/v4';
import { defineTelemetryEvents } from '../define';
const descriptionProperties = {
has_description: z.boolean().describe('Whether the saved credential has a nonempty description.'),
description_length: z
.number()
.int()
.nonnegative()
.describe('Length of the trimmed, saved description in UTF-16 code units. Zero when absent.'),
};
export const CREDENTIALS_TELEMETRY = defineTelemetryEvents({
USER_CREATED_CREDENTIALS: {
name: 'User created credentials',
description:
'A credential was created. The server sets source to backend and reports description metadata. Existing editor events omit these properties. Description text is never included.',
properties: z.object({
credential_id: z.string(),
credential_type: z.string(),
source: z
.literal('backend')
.optional()
.describe('Set by the server. Existing editor events omit it.'),
public_api: z.boolean().optional().describe('Whether the server used the public API.'),
has_description: descriptionProperties.has_description.optional(),
description_length: descriptionProperties.description_length.optional(),
user_id: z.string().optional(),
user_role: z.string().optional(),
project_id: z.string().optional(),
project_type: z.string().optional(),
workflow_id: z.string().nullable().optional(),
uiContext: z.string().optional().describe('Legacy property for the editor entry point.'),
is_private: z.boolean().optional(),
uses_external_secrets: z.boolean().optional(),
jwe_enabled: z.boolean().optional(),
credential_supports_managed_auth: z.boolean().optional(),
credential_uses_managed_auth: z.boolean().optional(),
}),
},
USER_UPDATED_CREDENTIALS: {
name: 'User updated credentials',
description:
'The server updated a credential through the internal credential API. Description metadata comes from the saved record. Description text is never included.',
properties: z.object({
user_id: z.string(),
user_role: z.string().optional(),
credential_id: z.string(),
credential_type: z.string(),
source: z.literal('backend'),
...descriptionProperties,
is_private: z.boolean(),
uses_external_secrets: z.boolean(),
jwe_enabled: z.boolean(),
credential_supports_managed_auth: z.boolean(),
credential_uses_managed_auth: z.boolean(),
}),
},
USER_STARTED_CREDENTIAL_CONNECTION: {
name: 'User started credential connection',
description:

View File

@ -178,6 +178,28 @@ describe('CredentialsController', () => {
expect(newApiKey).toEqual(createdCredentials);
});
it.each([null, 'Production reports'])(
'emits the saved description length for %s',
async (description) => {
const payload = { ...createNewCredentialsPayload(), description: 'Draft description' };
const { data: _data, ...payloadWithoutData } = payload;
const created = { ...createdCredentialsWithScopes(payloadWithoutData), description };
createUnmanagedCredentialSpy.mockResolvedValue(created);
findCredentialOwningProjectSpy.mockResolvedValue(
mock<Project>({ id: 'project-1', type: 'team' }),
);
await credentialsController.createCredentials(req, res, payload);
const [, eventPayload] = emitSpy.mock.calls[0];
expect(eventPayload).toMatchObject({
credentialDescriptionLength: description?.length ?? 0,
});
expect(eventPayload).not.toHaveProperty('description');
expect(eventPayload).not.toHaveProperty('credentialDescription');
},
);
it('should emit "credentials-created" with jweEnabled true when payload enables JWE', async () => {
const newCredentialsPayload = createNewCredentialsPayload({
data: { clientId: 'cid', jweEnabled: true },
@ -267,6 +289,7 @@ describe('CredentialsController', () => {
const existingCredential = mock<CredentialsEntity>({
id: credentialId,
name: 'Test Credential',
description: null,
type: 'apiKey',
isGlobal: false,
isManaged: false,
@ -364,6 +387,7 @@ describe('CredentialsController', () => {
credentialType: existingCredential.type,
credentialId: existingCredential.id,
credentialName: 'Updated Credential',
credentialDescriptionLength: 0,
isDynamic: false,
usesExternalSecrets: false,
jweEnabled: false,
@ -524,13 +548,14 @@ describe('CredentialsController', () => {
it.each([
[
'writes a description the payload sends',
'Read-only reporting key.',
' Read-only reporting key. ',
'Read-only reporting key.',
],
['clears a description the payload blanks out', ' ', null],
])('%s', async (_label, sent, prepared) => {
const req = updateRequest({ description: sent });
prepareUpdateDataSpy.mockResolvedValue({ ...req.body, description: prepared });
updateSpy.mockResolvedValue({ ...existingCredential, description: prepared });
await credentialsController.updateCredentials(req);
@ -540,15 +565,27 @@ describe('CredentialsController', () => {
expect.anything(),
expect.any(Object),
);
expect(emitSpy).toHaveBeenCalledWith(
'credentials-updated',
expect.objectContaining({ credentialDescriptionLength: prepared?.length ?? 0 }),
);
const event = emitSpy.mock.calls.find(([name]) => name === 'credentials-updated')?.[1];
expect(event).not.toHaveProperty('description');
expect(event).not.toHaveProperty('credentialDescription');
});
it('leaves the stored description alone when the payload omits the field', async () => {
const req = updateRequest({});
prepareUpdateDataSpy.mockResolvedValue({ ...req.body });
updateSpy.mockResolvedValue({ ...existingCredential, description: 'Production reports' });
await credentialsController.updateCredentials(req);
expect(updateSpy.mock.calls[0][1]).not.toHaveProperty('description');
expect(emitSpy).toHaveBeenCalledWith(
'credentials-updated',
expect.objectContaining({ credentialDescriptionLength: 18 }),
);
});
});

View File

@ -216,6 +216,7 @@ export class CredentialsController {
credentialType: newCredential.type,
credentialId: newCredential.id,
credentialName: newCredential.name,
credentialDescriptionLength: newCredential.description?.length ?? 0,
publicApi: false,
projectId: project?.id,
projectType: project?.type,
@ -383,6 +384,7 @@ export class CredentialsController {
credentialId: credential.id,
// The updated entity, so a rename records the new name rather than the one it replaced.
credentialName: responseData.name,
credentialDescriptionLength: responseData.description?.length ?? 0,
isDynamic: newCredentialData.isResolvable ?? false,
usesExternalSecrets: getExternalSecretExpressionPaths(preparedCredentialData.data).length > 0,
jweEnabled: updatedData.jweEnabled === true,

View File

@ -204,6 +204,7 @@ describe('ActivityEventRelay', () => {
credentialType: 'slackApi',
credentialId: 'credential1',
credentialName: 'Team Slack',
credentialDescriptionLength: 0,
publicApi: false,
projectId: 'project1',
}),
@ -224,6 +225,7 @@ describe('ActivityEventRelay', () => {
credentialType: 'slackApi',
credentialId: 'credential1',
credentialName: 'Team Slack',
credentialDescriptionLength: 0,
}),
{
category: 'credential',
@ -313,6 +315,7 @@ describe('ActivityEventRelay', () => {
credentialType: 'slackApi',
credentialId: 'credential1',
credentialName: 'Team Slack',
credentialDescriptionLength: 0,
publicApi: true,
projectId: undefined,
});
@ -543,6 +546,7 @@ describe('ActivityEventRelay', () => {
credentialType: 'slackApi',
credentialId: 'credential1',
credentialName: 'Instance chat model',
credentialDescriptionLength: 0,
});
await flushPromises();

View File

@ -1282,6 +1282,7 @@ describe('LogStreamingEventRelay', () => {
it('should log on `credentials-created` event', () => {
const event: RelayEventMap['credentials-created'] = {
credentialName: 'My GitHub account',
credentialDescriptionLength: 0,
user: {
id: 'user123',
email: 'user@example.com',
@ -1349,6 +1350,7 @@ describe('LogStreamingEventRelay', () => {
it('should log on `credentials-updated` event', () => {
const event: RelayEventMap['credentials-updated'] = {
credentialName: 'Rotated token',
credentialDescriptionLength: 0,
user: {
id: 'user808',
email: 'updatecred@example.com',

View File

@ -1504,42 +1504,58 @@ describe('TelemetryEventRelay', () => {
});
describe('credentials events', () => {
it('should track on `credentials-created` event', () => {
const event: RelayEventMap['credentials-created'] = {
credentialName: 'My GitHub account',
user: {
id: 'user123',
email: 'user@example.com',
firstName: 'John',
lastName: 'Doe',
role: { slug: GLOBAL_OWNER_ROLE.slug },
},
credentialType: 'github',
credentialId: 'cred123',
publicApi: false,
projectId: 'project123',
projectType: 'personal',
isDynamic: false,
supportsManagedAuth: true,
usesManagedAuth: true,
};
it.each([
{ descriptionLength: 0, publicApi: false },
{ descriptionLength: 18, publicApi: false },
{ descriptionLength: 0, publicApi: true },
{ descriptionLength: 18, publicApi: true },
])(
'tracks creation with description length $descriptionLength and public API $publicApi',
({ descriptionLength, publicApi }) => {
const event: RelayEventMap['credentials-created'] = {
credentialName: 'My GitHub account',
credentialDescriptionLength: descriptionLength,
user: {
id: 'user123',
email: 'user@example.com',
firstName: 'John',
lastName: 'Doe',
role: { slug: GLOBAL_OWNER_ROLE.slug },
},
credentialType: 'github',
credentialId: 'cred123',
publicApi,
projectId: 'project123',
projectType: 'personal',
isDynamic: false,
supportsManagedAuth: true,
usesManagedAuth: true,
};
eventService.emit('credentials-created', event);
eventService.emit('credentials-created', event);
expect(telemetry.track).toHaveBeenCalledWith('User created credentials', {
user_id: 'user123',
user_role: GLOBAL_OWNER_ROLE.slug,
credential_type: 'github',
credential_id: 'cred123',
project_id: 'project123',
project_type: 'personal',
is_private: false,
uses_external_secrets: false,
jwe_enabled: false,
credential_supports_managed_auth: true,
credential_uses_managed_auth: true,
});
});
expect(telemetry.track).toHaveBeenCalledWith(
TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS,
{
source: 'backend',
public_api: publicApi,
user_id: 'user123',
user_role: GLOBAL_OWNER_ROLE.slug,
credential_type: 'github',
credential_id: 'cred123',
has_description: descriptionLength > 0,
description_length: descriptionLength,
project_id: 'project123',
project_type: 'personal',
is_private: false,
uses_external_secrets: false,
jwe_enabled: false,
credential_supports_managed_auth: true,
credential_uses_managed_auth: true,
},
);
},
);
it('should track on `credentials-shared` event', () => {
const event: RelayEventMap['credentials-shared'] = {
@ -1570,9 +1586,10 @@ describe('TelemetryEventRelay', () => {
});
});
it('should track on `credentials-updated` event', () => {
it.each([0, 18])('tracks description length %i on update', (descriptionLength) => {
const event: RelayEventMap['credentials-updated'] = {
credentialName: 'Rotated token',
credentialDescriptionLength: descriptionLength,
user: {
id: 'user123',
email: 'user@example.com',
@ -1587,17 +1604,23 @@ describe('TelemetryEventRelay', () => {
eventService.emit('credentials-updated', event);
expect(telemetry.track).toHaveBeenCalledWith('User updated credentials', {
user_id: 'user123',
user_role: GLOBAL_OWNER_ROLE.slug,
credential_type: 'github',
credential_id: 'cred123',
is_private: true,
uses_external_secrets: false,
jwe_enabled: false,
credential_supports_managed_auth: false,
credential_uses_managed_auth: false,
});
expect(telemetry.track).toHaveBeenCalledWith(
TELEMETRY_EVENT.CREDENTIALS.USER_UPDATED_CREDENTIALS,
{
source: 'backend',
has_description: descriptionLength > 0,
description_length: descriptionLength,
user_id: 'user123',
user_role: GLOBAL_OWNER_ROLE.slug,
credential_type: 'github',
credential_id: 'cred123',
is_private: true,
uses_external_secrets: false,
jwe_enabled: false,
credential_supports_managed_auth: false,
credential_uses_managed_auth: false,
},
);
});
it('should track on `credentials-deleted` event', () => {

View File

@ -480,6 +480,7 @@ export type RelayEventMap = {
credentialType: string;
credentialId: string;
credentialName: string;
credentialDescriptionLength: number;
publicApi: boolean;
projectId?: string;
projectType?: string;
@ -505,6 +506,7 @@ export type RelayEventMap = {
credentialType: string;
credentialId: string;
credentialName: string;
credentialDescriptionLength: number;
isDynamic?: boolean;
usesExternalSecrets?: boolean;
jweEnabled?: boolean;

View File

@ -860,6 +860,8 @@ export class TelemetryEventRelay extends EventRelay {
user,
credentialType,
credentialId,
credentialDescriptionLength,
publicApi,
projectId,
projectType,
uiContext,
@ -869,11 +871,15 @@ export class TelemetryEventRelay extends EventRelay {
supportsManagedAuth,
usesManagedAuth,
}: RelayEventMap['credentials-created']) {
this.telemetry.track('User created credentials', {
this.telemetry.track(TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS, {
source: 'backend',
public_api: publicApi,
user_id: user.id,
user_role: user.role?.slug,
credential_type: credentialType,
credential_id: credentialId,
has_description: credentialDescriptionLength > 0,
description_length: credentialDescriptionLength,
project_id: projectId,
project_type: projectType,
uiContext,
@ -908,17 +914,21 @@ export class TelemetryEventRelay extends EventRelay {
user,
credentialId,
credentialType,
credentialDescriptionLength,
isDynamic,
usesExternalSecrets,
jweEnabled,
supportsManagedAuth,
usesManagedAuth,
}: RelayEventMap['credentials-updated']) {
this.telemetry.track('User updated credentials', {
this.telemetry.track(TELEMETRY_EVENT.CREDENTIALS.USER_UPDATED_CREDENTIALS, {
source: 'backend',
user_id: user.id,
user_role: user.role?.slug,
credential_type: credentialType,
credential_id: credentialId,
has_description: credentialDescriptionLength > 0,
description_length: credentialDescriptionLength,
is_private: isDynamic ?? false,
uses_external_secrets: usesExternalSecrets ?? false,
jwe_enabled: jweEnabled ?? false,

View File

@ -216,6 +216,7 @@ export class CredentialsPublicController {
credentialType: credential.type,
credentialId: credential.id,
credentialName: credential.name,
credentialDescriptionLength: credential.description?.length ?? 0,
publicApi: true,
projectId: project?.id,
projectType: project?.type,