mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 05:29:44 +02:00
feat(core): Track credential description usage (no-changelog) (#38986)
This commit is contained in:
parent
989b67ee95
commit
d9dc457d40
|
|
@ -800,6 +800,18 @@ Declared credentials are created for real (placeholder token; set the matching `
|
|||
|
||||
Each type needs a data template in `credentials/seeder.ts`; declaring an unknown type fails the build with a pointer there.
|
||||
|
||||
Each declared credential can also have a `description`. The loader applies the
|
||||
shared credential description schema. It trims the text, enforces the length
|
||||
limit, and converts blank text to `null`. The seeder stores the description as
|
||||
credential metadata, outside the secret `data` object. Omit the field to test
|
||||
credential choice without descriptions.
|
||||
|
||||
The current LangTracer case-write schema accepts only `type`, `name`, and
|
||||
`valid` on a credential. The push refuses a case with a description before it
|
||||
writes anything. Keep these cases on disk until LangTracer stores descriptions.
|
||||
The existing `blank` field also needs server support. The export check catches
|
||||
a server that drops it after a write.
|
||||
|
||||
### Seeded cases (conversation pre-seeding)
|
||||
|
||||
A seeded case starts **mid-conversation**: prior history is restored into the build thread before the live turn, so the eval drives only the turn under test. Use it to replicate a real misbehaviour — restore the conversation up to the moment it went wrong, re-drive that turn, and assert what should happen instead.
|
||||
|
|
|
|||
|
|
@ -248,7 +248,12 @@ describe('buildWorkflow declared credentials', () => {
|
|||
expect(build.success).toBe(true);
|
||||
// A blank credential models one the user saved without filling anything in,
|
||||
// so it is seeded with no data and must never resolve a test as passing.
|
||||
expect(createCredential).toHaveBeenCalledWith(expect.any(String), 'httpHeaderAuth', {});
|
||||
expect(createCredential).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
'httpHeaderAuth',
|
||||
{},
|
||||
undefined,
|
||||
);
|
||||
expect(setThreadCredentialAllowlist).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
['cred-blank'],
|
||||
|
|
|
|||
|
|
@ -49,6 +49,28 @@ beforeEach(() => {
|
|||
});
|
||||
|
||||
describe('EvalTestCaseSchema', () => {
|
||||
it.each([
|
||||
{ description: undefined, expected: undefined },
|
||||
{ description: null, expected: null },
|
||||
{ description: ' \n ', expected: null },
|
||||
{ description: ' Production reports ', expected: 'Production reports' },
|
||||
])('normalizes credential description $description', ({ description, expected }) => {
|
||||
const parsed = EvalTestCaseSchema.parse({
|
||||
...validFixture(),
|
||||
credentials: [{ type: 'slackApi', description }],
|
||||
});
|
||||
expect(parsed.credentials?.[0].description).toBe(expected);
|
||||
});
|
||||
|
||||
it('rejects a credential description above the storage limit', () => {
|
||||
expect(() =>
|
||||
EvalTestCaseSchema.parse({
|
||||
...validFixture(),
|
||||
credentials: [{ type: 'slackApi', description: '🔑'.repeat(257) }],
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it('accepts a minimal valid fixture', () => {
|
||||
const parsed = EvalTestCaseSchema.parse(validFixture());
|
||||
expect(parsed.executionScenarios).toHaveLength(1);
|
||||
|
|
|
|||
|
|
@ -107,6 +107,26 @@ describe('diskCaseToLangTracerCreate', () => {
|
|||
});
|
||||
|
||||
describe('unsupportedPushReason', () => {
|
||||
it.each([null, 'Production reports'])(
|
||||
'refuses description %s until the case-write API preserves it',
|
||||
(description) => {
|
||||
const input = diskCase({ credentials: [{ type: 'httpHeaderAuth', description }] });
|
||||
expect(unsupportedPushReason(input)).toContain('description');
|
||||
const body = diskCaseToLangTracerCreate(input, 'description-case', {
|
||||
suiteId: 8,
|
||||
setKind: 'regression',
|
||||
synthetic: true,
|
||||
});
|
||||
expect(body.credentials).toEqual(input.credentials);
|
||||
},
|
||||
);
|
||||
|
||||
it('allows credentials without descriptions', () => {
|
||||
expect(
|
||||
unsupportedPushReason(diskCase({ credentials: [{ type: 'httpHeaderAuth' }] })),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('refuses a case whose prompt version would be lost', () => {
|
||||
expect(unsupportedPushReason(diskCase({ promptVersion: 'progressive@1' }))).toContain(
|
||||
'promptVersion',
|
||||
|
|
|
|||
|
|
@ -902,6 +902,7 @@ describe('UserProxyLlm.respondToConfirmation', () => {
|
|||
expect.any(String),
|
||||
'slackApi',
|
||||
expect.any(Object),
|
||||
undefined,
|
||||
);
|
||||
// The allowlist call must include the pre-existing id, not just the new
|
||||
// one — setThreadCredentialAllowlist replaces the whole list.
|
||||
|
|
@ -1472,6 +1473,7 @@ describe('UserProxyLlm.respondToConfirmation', () => {
|
|||
expect.any(String),
|
||||
'slackApi',
|
||||
expect.any(Object),
|
||||
undefined,
|
||||
);
|
||||
expect(setThreadCredentialAllowlist).toHaveBeenCalledWith('thread-1', ['cred-fresh'], []);
|
||||
expect(response.kind).toBe('credentialSelection');
|
||||
|
|
|
|||
|
|
@ -15,6 +15,28 @@ import {
|
|||
|
||||
const BASE_URL = 'http://localhost:5678';
|
||||
|
||||
describe('N8nClient.createCredential', () => {
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
it.each([undefined, null, 'Production reports'])(
|
||||
'sends description %s as optional credential metadata',
|
||||
async (description) => {
|
||||
const fetchMock = stubFetch({ data: { id: 'credential-1' } });
|
||||
const data = { accessToken: 'placeholder' };
|
||||
await new N8nClient(BASE_URL).createCredential('Account A', 'slackApi', data, description);
|
||||
const [url, init] = fetchMock.mock.calls[0];
|
||||
expect(url).toBe(`${BASE_URL}/rest/credentials`);
|
||||
if (typeof init?.body !== 'string') throw new Error('Expected a JSON request body.');
|
||||
expect(jsonParse(init.body)).toEqual({
|
||||
name: 'Account A',
|
||||
type: 'slackApi',
|
||||
data,
|
||||
...(description !== undefined ? { description } : {}),
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('N8nClient.sendMessage', () => {
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
|
|
|
|||
|
|
@ -806,10 +806,11 @@ export class N8nClient {
|
|||
name: string,
|
||||
type: string,
|
||||
data: Record<string, unknown>,
|
||||
description?: string | null,
|
||||
): Promise<{ id: string }> {
|
||||
const result = (await this.fetch('/rest/credentials', {
|
||||
method: 'POST',
|
||||
body: { name, type, data },
|
||||
body: { name, type, data, ...(description !== undefined ? { description } : {}) },
|
||||
})) as { data: { id: string } };
|
||||
return { id: result.data.id };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { vi } from 'vitest';
|
|||
|
||||
import type { N8nClient } from '../../clients/n8n-client';
|
||||
import type { EvalLogger } from '../../harness/logger';
|
||||
import { createOneCredential } from '../seeder';
|
||||
import { createDeclaredCredentials, createOneCredential } from '../seeder';
|
||||
|
||||
const makeClient = (createCredential: N8nClient['createCredential']) =>
|
||||
({ createCredential }) as unknown as N8nClient;
|
||||
|
|
@ -133,3 +133,53 @@ describe('createOneCredential — static template path (regression)', () => {
|
|||
expect(createCredential.mock.calls[0][1]).toBe('slackApi');
|
||||
});
|
||||
});
|
||||
|
||||
describe('credential descriptions', () => {
|
||||
it('keeps descriptions with their declared credentials outside the secret data', async () => {
|
||||
const createCredential = vi
|
||||
.fn<N8nClient['createCredential']>()
|
||||
.mockResolvedValue({ id: 'cred-1' });
|
||||
const client = makeClient(createCredential);
|
||||
|
||||
await createDeclaredCredentials(client, [
|
||||
{ type: 'slackApi', name: 'Account A', description: 'Staging checks' },
|
||||
{ type: 'slackApi', name: 'Account B', description: 'Production reports' },
|
||||
{ type: 'slackApi', name: 'Account C' },
|
||||
{ type: 'slackApi', name: 'Account D', description: null },
|
||||
]);
|
||||
|
||||
expect(
|
||||
createCredential.mock.calls.map(([name, , , description]) => ({ name, description })),
|
||||
).toEqual([
|
||||
{ name: 'Account A', description: 'Staging checks' },
|
||||
{ name: 'Account B', description: 'Production reports' },
|
||||
{ name: 'Account C', description: undefined },
|
||||
{ name: 'Account D', description: null },
|
||||
]);
|
||||
for (const [, , data] of createCredential.mock.calls) {
|
||||
expect(data).not.toHaveProperty('description');
|
||||
}
|
||||
});
|
||||
|
||||
it('passes descriptions through custom authentication setup', async () => {
|
||||
const createCredential = vi
|
||||
.fn<N8nClient['createCredential']>()
|
||||
.mockResolvedValue({ id: 'cred-1' });
|
||||
await createOneCredential(
|
||||
makeClient(createCredential),
|
||||
'httpTemplatedCustomAuth',
|
||||
'Account A',
|
||||
new Map(),
|
||||
{
|
||||
setupHint: baseHint,
|
||||
description: 'Production reports',
|
||||
},
|
||||
);
|
||||
expect(createCredential).toHaveBeenCalledWith(
|
||||
'Account A',
|
||||
'httpTemplatedCustomAuth',
|
||||
expect.any(Object),
|
||||
'Production reports',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -141,6 +141,7 @@ export async function createOneCredential(
|
|||
options?: {
|
||||
logger?: EvalLogger;
|
||||
setupHint?: InstanceAiCredentialSetupHint;
|
||||
description?: string | null;
|
||||
/** Seed with no field values, modelling a credential the user saved empty. */
|
||||
blank?: boolean;
|
||||
},
|
||||
|
|
@ -171,6 +172,7 @@ export async function createOneCredential(
|
|||
resolvedName,
|
||||
credentialType,
|
||||
options?.blank ? {} : template.buildData(token),
|
||||
options?.description,
|
||||
);
|
||||
return { id, name: resolvedName, type: credentialType };
|
||||
}
|
||||
|
|
@ -186,7 +188,11 @@ async function createTemplatedCustomAuthCredential(
|
|||
client: N8nClient,
|
||||
name: string | undefined,
|
||||
usedNames: Map<string, number>,
|
||||
options?: { logger?: EvalLogger; setupHint?: InstanceAiCredentialSetupHint },
|
||||
options?: {
|
||||
logger?: EvalLogger;
|
||||
setupHint?: InstanceAiCredentialSetupHint;
|
||||
description?: string | null;
|
||||
},
|
||||
): Promise<CreatedCredential> {
|
||||
const hint = options?.setupHint;
|
||||
if (!hint) {
|
||||
|
|
@ -218,16 +224,21 @@ async function createTemplatedCustomAuthCredential(
|
|||
);
|
||||
|
||||
options?.logger?.verbose(` Creating credential ${resolvedName} (httpTemplatedCustomAuth)`);
|
||||
const { id } = await client.createCredential(resolvedName, 'httpTemplatedCustomAuth', {
|
||||
template: JSON.stringify(hint.template),
|
||||
placeholderDefs: JSON.stringify(hint.placeholders),
|
||||
placeholderValues: JSON.stringify(placeholderValues),
|
||||
serviceHost: hint.serviceHost ?? '',
|
||||
serviceOrigin: hint.serviceOrigin ?? '',
|
||||
docsUrl: hint.docsUrl ?? '',
|
||||
testUrl: hint.testUrl ?? '',
|
||||
acceptedStatusCodes: hint.acceptedStatusCodes ? JSON.stringify(hint.acceptedStatusCodes) : '',
|
||||
});
|
||||
const { id } = await client.createCredential(
|
||||
resolvedName,
|
||||
'httpTemplatedCustomAuth',
|
||||
{
|
||||
template: JSON.stringify(hint.template),
|
||||
placeholderDefs: JSON.stringify(hint.placeholders),
|
||||
placeholderValues: JSON.stringify(placeholderValues),
|
||||
serviceHost: hint.serviceHost ?? '',
|
||||
serviceOrigin: hint.serviceOrigin ?? '',
|
||||
docsUrl: hint.docsUrl ?? '',
|
||||
testUrl: hint.testUrl ?? '',
|
||||
acceptedStatusCodes: hint.acceptedStatusCodes ? JSON.stringify(hint.acceptedStatusCodes) : '',
|
||||
},
|
||||
options?.description,
|
||||
);
|
||||
return { id, name: resolvedName, type: 'httpTemplatedCustomAuth' };
|
||||
}
|
||||
|
||||
|
|
@ -263,6 +274,7 @@ export async function createDeclaredCredentials(
|
|||
for (const decl of declared) {
|
||||
const cred = await createOneCredential(client, decl.type, decl.name, nameCounts, {
|
||||
logger,
|
||||
...(decl.description !== undefined ? { description: decl.description } : {}),
|
||||
...(decl.blank ? { blank: true } : {}),
|
||||
});
|
||||
options?.onCreated?.(cred.id);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
import { findSeedFolderIssues, instanceAiEvalSeedDataTableSchema } from '@n8n/api-types';
|
||||
import {
|
||||
credentialDescriptionSchema,
|
||||
findSeedFolderIssues,
|
||||
instanceAiEvalSeedDataTableSchema,
|
||||
} from '@n8n/api-types';
|
||||
import { z } from 'zod';
|
||||
|
||||
import {
|
||||
|
|
@ -206,7 +210,10 @@ const evalTestCaseObjectSchema = z
|
|||
message: `unknown credential type — add a template to evaluations/credentials/seeder.ts (supported: ${[...SUPPORTED_CREDENTIAL_TYPES].join(', ')})`,
|
||||
}),
|
||||
name: z.string().min(1).optional(),
|
||||
description: credentialDescriptionSchema.optional(),
|
||||
// False lets the connection test fail for a credential that is already broken.
|
||||
valid: z.boolean().optional(),
|
||||
// True seeds no field values and disables the connection-test bypass.
|
||||
blank: z.boolean().optional(),
|
||||
}),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -69,6 +69,9 @@ export interface ToLangTracerOptions {
|
|||
* already holds, and such a case is barred from suites anyway. Returns a
|
||||
* human-readable reason, else null. */
|
||||
export function unsupportedPushReason(testCase: EvalTestCaseInput): string | null {
|
||||
if (testCase.credentials?.some((credential) => credential.description !== undefined)) {
|
||||
return 'seeds credential descriptions, which the current LangTracer case-write schema does not store. Keep the case on disk until that contract supports descriptions.';
|
||||
}
|
||||
if (testCase.promptVersion !== undefined) {
|
||||
return 'pins promptVersion, which the current case-write contract does not carry. Keep the case on disk.';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import type { z } from 'zod';
|
|||
import type { CheckOutcome } from './binaryChecks/types';
|
||||
import type { WorkflowResponse } from './clients/n8n-client';
|
||||
import type { EvalAttribution } from './harness/attribution';
|
||||
import type { CaseSeed, ConversationTurnSchema } from './harness/schema';
|
||||
import type { CaseSeed, ConversationTurnSchema, EvalTestCaseInput } from './harness/schema';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Checklist items and verification
|
||||
|
|
@ -208,32 +208,8 @@ export interface ExecutionScenario {
|
|||
|
||||
export type ConversationTurn = z.infer<typeof ConversationTurnSchema>;
|
||||
|
||||
export interface TestCaseCredential {
|
||||
/** n8n credential type name, e.g. `slackApi`. Must have a template in credentials/seeder.ts. */
|
||||
type: string;
|
||||
/** Display name; defaults to the template's name, auto-suffixed on duplicates. */
|
||||
name?: string;
|
||||
/** Defaults to true. false models a credential that was already broken before
|
||||
* the conversation started (expired/revoked/scope-changed) — left off the
|
||||
* connection-test bypass list, so its real test runs and fails. Distinct from
|
||||
* a credential set up on a card mid-conversation (UserProxyLlm), which always
|
||||
* passes. */
|
||||
valid?: boolean;
|
||||
/** Defaults to false. true models a credential the user saved without filling
|
||||
* anything in — seeded with no field values, and kept off the connection-test
|
||||
* bypass so nothing resolves it as working. The shape behind a re-offered
|
||||
* empty generic-auth credential.
|
||||
*
|
||||
* DOES NOT SURVIVE A LANG-TRACER PUSH yet. Its case-write schema validates
|
||||
* each credential against a non-strict `z.object({ type, name, valid })`
|
||||
* (lang-tracer `packages/server/src/lib/case-writes.ts`), so this key is
|
||||
* silently stripped and the suite copy seeds a FILLED credential instead —
|
||||
* a case relying on it then fails in CI for a reason unrelated to the
|
||||
* product. `eval:langtracer-push` catches it (`did not store credentials`,
|
||||
* non-zero exit); until lang-tracer declares the field, a case using it
|
||||
* lives on disk. */
|
||||
blank?: boolean;
|
||||
}
|
||||
/** Seeded credentials use the case schema. The push verifies that the suite retains each field. */
|
||||
export type TestCaseCredential = NonNullable<EvalTestCaseInput['credentials']>[number];
|
||||
|
||||
export interface WorkflowTestCase {
|
||||
/** Optional human-readable note on what this case is testing (esp. for behaviour cases). */
|
||||
|
|
|
|||
77
packages/@n8n/telemetry/credential-descriptions.md
Normal file
77
packages/@n8n/telemetry/credential-descriptions.md
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
# Credential description metrics
|
||||
|
||||
## Adoption
|
||||
|
||||
Measure the share of users who save at least one nonempty credential description
|
||||
among users who create or update a credential through the internal credential API
|
||||
in the same seven-day window. The editor uses this API.
|
||||
Use identified users on instances with the instrumentation enabled.
|
||||
|
||||
Use these backend events:
|
||||
|
||||
- `User created credentials` with `source = backend` and `public_api = false`.
|
||||
- `User updated credentials` with `source = backend`.
|
||||
|
||||
Require both description properties on each event.
|
||||
Existing frontend creation events have no `source` property.
|
||||
Exclude frontend events and events that lack the required properties from both
|
||||
counts. Join `user_id` with the instance identity from the event envelope.
|
||||
Count each user once in the numerator and once in the denominator.
|
||||
Do not use raw event counts as counts of users or credentials.
|
||||
|
||||
Public API creation stays tracked with `public_api = true` but does not count in
|
||||
this metric. Imports and writes that bypass these controllers are outside this
|
||||
metric. Calls to the internal API can also come from scripts. These events do not
|
||||
prove that a person used the editor.
|
||||
|
||||
`has_description` reports whether the saved description is nonempty.
|
||||
`description_length` reports the length of the trimmed, saved text in UTF-16 code
|
||||
units. A missing or blank description has length zero.
|
||||
The server emits these values after persistence. A later OAuth or connection
|
||||
test result does not change them. A connection without a save does not count.
|
||||
The events never include the description text.
|
||||
|
||||
This measures use of descriptions among users who save credentials.
|
||||
An update can preserve an existing description without changing its text.
|
||||
It does not measure the share of all stored credentials that have a description.
|
||||
The existing telemetry transport is best effort. This change does not guarantee
|
||||
delivery after a server failure.
|
||||
|
||||
## Correctness
|
||||
|
||||
The correctness metric is the percentage of eval runs in which the assistant
|
||||
selects the credential whose description matches the request when two
|
||||
credentials share one type.
|
||||
|
||||
Use neutral credential names. Give the credentials different purposes in their
|
||||
descriptions. Ask for one purpose without naming the credential.
|
||||
Keep the distinguishing context within the list previews from CONTEXT-150.
|
||||
Check the selected credential, not just whether the assistant read a description.
|
||||
Keep the credential order fixed within a comparison. Repeat the comparison with
|
||||
the descriptions swapped to expose a preference for one position or name.
|
||||
|
||||
Run the same cases before and after the CONTEXT-150 read changes.
|
||||
Keep the model, prompt configuration, case revision, and run count the same.
|
||||
Report successful selections and total completed runs with each percentage.
|
||||
Report infrastructure failures separately. Do not count them as selections.
|
||||
|
||||
## Do-not-harm
|
||||
|
||||
Credential selection accuracy must not fall below the baseline on the same cases
|
||||
when all descriptions are absent.
|
||||
Use cases where the request and credential names identify the intended choice.
|
||||
Do not assign an arbitrary correct answer to indistinguishable credentials.
|
||||
Report the change in percentage points with the underlying counts.
|
||||
|
||||
## Release evidence
|
||||
|
||||
Record the following on CONTEXT-151 before shipment:
|
||||
|
||||
- Case and suite links.
|
||||
- Baseline and candidate commit IDs.
|
||||
- Model and prompt configuration.
|
||||
- Successful selections, completed runs, and infrastructure failures.
|
||||
- Correctness percentages and the do-not-harm change.
|
||||
|
||||
The telemetry change alone does not establish a correctness result.
|
||||
The comparison requires the CONTEXT-150 read changes on the candidate instance.
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
import { TELEMETRY_EVENT } from '../index';
|
||||
import { getEventValidationError } from '../validate';
|
||||
|
||||
describe('credential description telemetry', () => {
|
||||
it.each([
|
||||
TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS,
|
||||
TELEMETRY_EVENT.CREDENTIALS.USER_UPDATED_CREDENTIALS,
|
||||
])('accepts backend metadata and rejects description text in $name', (event) => {
|
||||
const properties = {
|
||||
user_id: 'user-1',
|
||||
credential_id: 'credential-1',
|
||||
credential_type: 'postgres',
|
||||
source: 'backend',
|
||||
has_description: true,
|
||||
description_length: 18,
|
||||
is_private: false,
|
||||
uses_external_secrets: false,
|
||||
jwe_enabled: false,
|
||||
credential_supports_managed_auth: false,
|
||||
credential_uses_managed_auth: false,
|
||||
...(event === TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS
|
||||
? { public_api: false }
|
||||
: {}),
|
||||
};
|
||||
expect(getEventValidationError(event, properties)).toBeNull();
|
||||
expect(
|
||||
getEventValidationError(event, { ...properties, description: 'Production reports' }),
|
||||
).toContain('description: unrecognized property');
|
||||
});
|
||||
|
||||
it('accepts existing frontend creation without source or description metadata', () => {
|
||||
expect(
|
||||
getEventValidationError(TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS, {
|
||||
credential_id: 'credential-1',
|
||||
credential_type: 'postgres',
|
||||
workflow_id: 'workflow-1',
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -2,7 +2,61 @@ import { z } from 'zod/v4';
|
|||
|
||||
import { defineTelemetryEvents } from '../define';
|
||||
|
||||
const descriptionProperties = {
|
||||
has_description: z.boolean().describe('Whether the saved credential has a nonempty description.'),
|
||||
description_length: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.describe('Length of the trimmed, saved description in UTF-16 code units. Zero when absent.'),
|
||||
};
|
||||
|
||||
export const CREDENTIALS_TELEMETRY = defineTelemetryEvents({
|
||||
USER_CREATED_CREDENTIALS: {
|
||||
name: 'User created credentials',
|
||||
description:
|
||||
'A credential was created. The server sets source to backend and reports description metadata. Existing editor events omit these properties. Description text is never included.',
|
||||
properties: z.object({
|
||||
credential_id: z.string(),
|
||||
credential_type: z.string(),
|
||||
source: z
|
||||
.literal('backend')
|
||||
.optional()
|
||||
.describe('Set by the server. Existing editor events omit it.'),
|
||||
public_api: z.boolean().optional().describe('Whether the server used the public API.'),
|
||||
has_description: descriptionProperties.has_description.optional(),
|
||||
description_length: descriptionProperties.description_length.optional(),
|
||||
user_id: z.string().optional(),
|
||||
user_role: z.string().optional(),
|
||||
project_id: z.string().optional(),
|
||||
project_type: z.string().optional(),
|
||||
workflow_id: z.string().nullable().optional(),
|
||||
uiContext: z.string().optional().describe('Legacy property for the editor entry point.'),
|
||||
is_private: z.boolean().optional(),
|
||||
uses_external_secrets: z.boolean().optional(),
|
||||
jwe_enabled: z.boolean().optional(),
|
||||
credential_supports_managed_auth: z.boolean().optional(),
|
||||
credential_uses_managed_auth: z.boolean().optional(),
|
||||
}),
|
||||
},
|
||||
USER_UPDATED_CREDENTIALS: {
|
||||
name: 'User updated credentials',
|
||||
description:
|
||||
'The server updated a credential through the internal credential API. Description metadata comes from the saved record. Description text is never included.',
|
||||
properties: z.object({
|
||||
user_id: z.string(),
|
||||
user_role: z.string().optional(),
|
||||
credential_id: z.string(),
|
||||
credential_type: z.string(),
|
||||
source: z.literal('backend'),
|
||||
...descriptionProperties,
|
||||
is_private: z.boolean(),
|
||||
uses_external_secrets: z.boolean(),
|
||||
jwe_enabled: z.boolean(),
|
||||
credential_supports_managed_auth: z.boolean(),
|
||||
credential_uses_managed_auth: z.boolean(),
|
||||
}),
|
||||
},
|
||||
USER_STARTED_CREDENTIAL_CONNECTION: {
|
||||
name: 'User started credential connection',
|
||||
description:
|
||||
|
|
|
|||
|
|
@ -178,6 +178,28 @@ describe('CredentialsController', () => {
|
|||
expect(newApiKey).toEqual(createdCredentials);
|
||||
});
|
||||
|
||||
it.each([null, 'Production reports'])(
|
||||
'emits the saved description length for %s',
|
||||
async (description) => {
|
||||
const payload = { ...createNewCredentialsPayload(), description: 'Draft description' };
|
||||
const { data: _data, ...payloadWithoutData } = payload;
|
||||
const created = { ...createdCredentialsWithScopes(payloadWithoutData), description };
|
||||
createUnmanagedCredentialSpy.mockResolvedValue(created);
|
||||
findCredentialOwningProjectSpy.mockResolvedValue(
|
||||
mock<Project>({ id: 'project-1', type: 'team' }),
|
||||
);
|
||||
|
||||
await credentialsController.createCredentials(req, res, payload);
|
||||
|
||||
const [, eventPayload] = emitSpy.mock.calls[0];
|
||||
expect(eventPayload).toMatchObject({
|
||||
credentialDescriptionLength: description?.length ?? 0,
|
||||
});
|
||||
expect(eventPayload).not.toHaveProperty('description');
|
||||
expect(eventPayload).not.toHaveProperty('credentialDescription');
|
||||
},
|
||||
);
|
||||
|
||||
it('should emit "credentials-created" with jweEnabled true when payload enables JWE', async () => {
|
||||
const newCredentialsPayload = createNewCredentialsPayload({
|
||||
data: { clientId: 'cid', jweEnabled: true },
|
||||
|
|
@ -267,6 +289,7 @@ describe('CredentialsController', () => {
|
|||
const existingCredential = mock<CredentialsEntity>({
|
||||
id: credentialId,
|
||||
name: 'Test Credential',
|
||||
description: null,
|
||||
type: 'apiKey',
|
||||
isGlobal: false,
|
||||
isManaged: false,
|
||||
|
|
@ -364,6 +387,7 @@ describe('CredentialsController', () => {
|
|||
credentialType: existingCredential.type,
|
||||
credentialId: existingCredential.id,
|
||||
credentialName: 'Updated Credential',
|
||||
credentialDescriptionLength: 0,
|
||||
isDynamic: false,
|
||||
usesExternalSecrets: false,
|
||||
jweEnabled: false,
|
||||
|
|
@ -524,13 +548,14 @@ describe('CredentialsController', () => {
|
|||
it.each([
|
||||
[
|
||||
'writes a description the payload sends',
|
||||
'Read-only reporting key.',
|
||||
' Read-only reporting key. ',
|
||||
'Read-only reporting key.',
|
||||
],
|
||||
['clears a description the payload blanks out', ' ', null],
|
||||
])('%s', async (_label, sent, prepared) => {
|
||||
const req = updateRequest({ description: sent });
|
||||
prepareUpdateDataSpy.mockResolvedValue({ ...req.body, description: prepared });
|
||||
updateSpy.mockResolvedValue({ ...existingCredential, description: prepared });
|
||||
|
||||
await credentialsController.updateCredentials(req);
|
||||
|
||||
|
|
@ -540,15 +565,27 @@ describe('CredentialsController', () => {
|
|||
expect.anything(),
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(emitSpy).toHaveBeenCalledWith(
|
||||
'credentials-updated',
|
||||
expect.objectContaining({ credentialDescriptionLength: prepared?.length ?? 0 }),
|
||||
);
|
||||
const event = emitSpy.mock.calls.find(([name]) => name === 'credentials-updated')?.[1];
|
||||
expect(event).not.toHaveProperty('description');
|
||||
expect(event).not.toHaveProperty('credentialDescription');
|
||||
});
|
||||
|
||||
it('leaves the stored description alone when the payload omits the field', async () => {
|
||||
const req = updateRequest({});
|
||||
prepareUpdateDataSpy.mockResolvedValue({ ...req.body });
|
||||
updateSpy.mockResolvedValue({ ...existingCredential, description: 'Production reports' });
|
||||
|
||||
await credentialsController.updateCredentials(req);
|
||||
|
||||
expect(updateSpy.mock.calls[0][1]).not.toHaveProperty('description');
|
||||
expect(emitSpy).toHaveBeenCalledWith(
|
||||
'credentials-updated',
|
||||
expect.objectContaining({ credentialDescriptionLength: 18 }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -216,6 +216,7 @@ export class CredentialsController {
|
|||
credentialType: newCredential.type,
|
||||
credentialId: newCredential.id,
|
||||
credentialName: newCredential.name,
|
||||
credentialDescriptionLength: newCredential.description?.length ?? 0,
|
||||
publicApi: false,
|
||||
projectId: project?.id,
|
||||
projectType: project?.type,
|
||||
|
|
@ -383,6 +384,7 @@ export class CredentialsController {
|
|||
credentialId: credential.id,
|
||||
// The updated entity, so a rename records the new name rather than the one it replaced.
|
||||
credentialName: responseData.name,
|
||||
credentialDescriptionLength: responseData.description?.length ?? 0,
|
||||
isDynamic: newCredentialData.isResolvable ?? false,
|
||||
usesExternalSecrets: getExternalSecretExpressionPaths(preparedCredentialData.data).length > 0,
|
||||
jweEnabled: updatedData.jweEnabled === true,
|
||||
|
|
|
|||
|
|
@ -204,6 +204,7 @@ describe('ActivityEventRelay', () => {
|
|||
credentialType: 'slackApi',
|
||||
credentialId: 'credential1',
|
||||
credentialName: 'Team Slack',
|
||||
credentialDescriptionLength: 0,
|
||||
publicApi: false,
|
||||
projectId: 'project1',
|
||||
}),
|
||||
|
|
@ -224,6 +225,7 @@ describe('ActivityEventRelay', () => {
|
|||
credentialType: 'slackApi',
|
||||
credentialId: 'credential1',
|
||||
credentialName: 'Team Slack',
|
||||
credentialDescriptionLength: 0,
|
||||
}),
|
||||
{
|
||||
category: 'credential',
|
||||
|
|
@ -313,6 +315,7 @@ describe('ActivityEventRelay', () => {
|
|||
credentialType: 'slackApi',
|
||||
credentialId: 'credential1',
|
||||
credentialName: 'Team Slack',
|
||||
credentialDescriptionLength: 0,
|
||||
publicApi: true,
|
||||
projectId: undefined,
|
||||
});
|
||||
|
|
@ -543,6 +546,7 @@ describe('ActivityEventRelay', () => {
|
|||
credentialType: 'slackApi',
|
||||
credentialId: 'credential1',
|
||||
credentialName: 'Instance chat model',
|
||||
credentialDescriptionLength: 0,
|
||||
});
|
||||
await flushPromises();
|
||||
|
||||
|
|
|
|||
|
|
@ -1282,6 +1282,7 @@ describe('LogStreamingEventRelay', () => {
|
|||
it('should log on `credentials-created` event', () => {
|
||||
const event: RelayEventMap['credentials-created'] = {
|
||||
credentialName: 'My GitHub account',
|
||||
credentialDescriptionLength: 0,
|
||||
user: {
|
||||
id: 'user123',
|
||||
email: 'user@example.com',
|
||||
|
|
@ -1349,6 +1350,7 @@ describe('LogStreamingEventRelay', () => {
|
|||
it('should log on `credentials-updated` event', () => {
|
||||
const event: RelayEventMap['credentials-updated'] = {
|
||||
credentialName: 'Rotated token',
|
||||
credentialDescriptionLength: 0,
|
||||
user: {
|
||||
id: 'user808',
|
||||
email: 'updatecred@example.com',
|
||||
|
|
|
|||
|
|
@ -1504,42 +1504,58 @@ describe('TelemetryEventRelay', () => {
|
|||
});
|
||||
|
||||
describe('credentials events', () => {
|
||||
it('should track on `credentials-created` event', () => {
|
||||
const event: RelayEventMap['credentials-created'] = {
|
||||
credentialName: 'My GitHub account',
|
||||
user: {
|
||||
id: 'user123',
|
||||
email: 'user@example.com',
|
||||
firstName: 'John',
|
||||
lastName: 'Doe',
|
||||
role: { slug: GLOBAL_OWNER_ROLE.slug },
|
||||
},
|
||||
credentialType: 'github',
|
||||
credentialId: 'cred123',
|
||||
publicApi: false,
|
||||
projectId: 'project123',
|
||||
projectType: 'personal',
|
||||
isDynamic: false,
|
||||
supportsManagedAuth: true,
|
||||
usesManagedAuth: true,
|
||||
};
|
||||
it.each([
|
||||
{ descriptionLength: 0, publicApi: false },
|
||||
{ descriptionLength: 18, publicApi: false },
|
||||
{ descriptionLength: 0, publicApi: true },
|
||||
{ descriptionLength: 18, publicApi: true },
|
||||
])(
|
||||
'tracks creation with description length $descriptionLength and public API $publicApi',
|
||||
({ descriptionLength, publicApi }) => {
|
||||
const event: RelayEventMap['credentials-created'] = {
|
||||
credentialName: 'My GitHub account',
|
||||
credentialDescriptionLength: descriptionLength,
|
||||
user: {
|
||||
id: 'user123',
|
||||
email: 'user@example.com',
|
||||
firstName: 'John',
|
||||
lastName: 'Doe',
|
||||
role: { slug: GLOBAL_OWNER_ROLE.slug },
|
||||
},
|
||||
credentialType: 'github',
|
||||
credentialId: 'cred123',
|
||||
publicApi,
|
||||
projectId: 'project123',
|
||||
projectType: 'personal',
|
||||
isDynamic: false,
|
||||
supportsManagedAuth: true,
|
||||
usesManagedAuth: true,
|
||||
};
|
||||
|
||||
eventService.emit('credentials-created', event);
|
||||
eventService.emit('credentials-created', event);
|
||||
|
||||
expect(telemetry.track).toHaveBeenCalledWith('User created credentials', {
|
||||
user_id: 'user123',
|
||||
user_role: GLOBAL_OWNER_ROLE.slug,
|
||||
credential_type: 'github',
|
||||
credential_id: 'cred123',
|
||||
project_id: 'project123',
|
||||
project_type: 'personal',
|
||||
is_private: false,
|
||||
uses_external_secrets: false,
|
||||
jwe_enabled: false,
|
||||
credential_supports_managed_auth: true,
|
||||
credential_uses_managed_auth: true,
|
||||
});
|
||||
});
|
||||
expect(telemetry.track).toHaveBeenCalledWith(
|
||||
TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS,
|
||||
{
|
||||
source: 'backend',
|
||||
public_api: publicApi,
|
||||
user_id: 'user123',
|
||||
user_role: GLOBAL_OWNER_ROLE.slug,
|
||||
credential_type: 'github',
|
||||
credential_id: 'cred123',
|
||||
has_description: descriptionLength > 0,
|
||||
description_length: descriptionLength,
|
||||
project_id: 'project123',
|
||||
project_type: 'personal',
|
||||
is_private: false,
|
||||
uses_external_secrets: false,
|
||||
jwe_enabled: false,
|
||||
credential_supports_managed_auth: true,
|
||||
credential_uses_managed_auth: true,
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('should track on `credentials-shared` event', () => {
|
||||
const event: RelayEventMap['credentials-shared'] = {
|
||||
|
|
@ -1570,9 +1586,10 @@ describe('TelemetryEventRelay', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should track on `credentials-updated` event', () => {
|
||||
it.each([0, 18])('tracks description length %i on update', (descriptionLength) => {
|
||||
const event: RelayEventMap['credentials-updated'] = {
|
||||
credentialName: 'Rotated token',
|
||||
credentialDescriptionLength: descriptionLength,
|
||||
user: {
|
||||
id: 'user123',
|
||||
email: 'user@example.com',
|
||||
|
|
@ -1587,17 +1604,23 @@ describe('TelemetryEventRelay', () => {
|
|||
|
||||
eventService.emit('credentials-updated', event);
|
||||
|
||||
expect(telemetry.track).toHaveBeenCalledWith('User updated credentials', {
|
||||
user_id: 'user123',
|
||||
user_role: GLOBAL_OWNER_ROLE.slug,
|
||||
credential_type: 'github',
|
||||
credential_id: 'cred123',
|
||||
is_private: true,
|
||||
uses_external_secrets: false,
|
||||
jwe_enabled: false,
|
||||
credential_supports_managed_auth: false,
|
||||
credential_uses_managed_auth: false,
|
||||
});
|
||||
expect(telemetry.track).toHaveBeenCalledWith(
|
||||
TELEMETRY_EVENT.CREDENTIALS.USER_UPDATED_CREDENTIALS,
|
||||
{
|
||||
source: 'backend',
|
||||
has_description: descriptionLength > 0,
|
||||
description_length: descriptionLength,
|
||||
user_id: 'user123',
|
||||
user_role: GLOBAL_OWNER_ROLE.slug,
|
||||
credential_type: 'github',
|
||||
credential_id: 'cred123',
|
||||
is_private: true,
|
||||
uses_external_secrets: false,
|
||||
jwe_enabled: false,
|
||||
credential_supports_managed_auth: false,
|
||||
credential_uses_managed_auth: false,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('should track on `credentials-deleted` event', () => {
|
||||
|
|
|
|||
|
|
@ -480,6 +480,7 @@ export type RelayEventMap = {
|
|||
credentialType: string;
|
||||
credentialId: string;
|
||||
credentialName: string;
|
||||
credentialDescriptionLength: number;
|
||||
publicApi: boolean;
|
||||
projectId?: string;
|
||||
projectType?: string;
|
||||
|
|
@ -505,6 +506,7 @@ export type RelayEventMap = {
|
|||
credentialType: string;
|
||||
credentialId: string;
|
||||
credentialName: string;
|
||||
credentialDescriptionLength: number;
|
||||
isDynamic?: boolean;
|
||||
usesExternalSecrets?: boolean;
|
||||
jweEnabled?: boolean;
|
||||
|
|
|
|||
|
|
@ -860,6 +860,8 @@ export class TelemetryEventRelay extends EventRelay {
|
|||
user,
|
||||
credentialType,
|
||||
credentialId,
|
||||
credentialDescriptionLength,
|
||||
publicApi,
|
||||
projectId,
|
||||
projectType,
|
||||
uiContext,
|
||||
|
|
@ -869,11 +871,15 @@ export class TelemetryEventRelay extends EventRelay {
|
|||
supportsManagedAuth,
|
||||
usesManagedAuth,
|
||||
}: RelayEventMap['credentials-created']) {
|
||||
this.telemetry.track('User created credentials', {
|
||||
this.telemetry.track(TELEMETRY_EVENT.CREDENTIALS.USER_CREATED_CREDENTIALS, {
|
||||
source: 'backend',
|
||||
public_api: publicApi,
|
||||
user_id: user.id,
|
||||
user_role: user.role?.slug,
|
||||
credential_type: credentialType,
|
||||
credential_id: credentialId,
|
||||
has_description: credentialDescriptionLength > 0,
|
||||
description_length: credentialDescriptionLength,
|
||||
project_id: projectId,
|
||||
project_type: projectType,
|
||||
uiContext,
|
||||
|
|
@ -908,17 +914,21 @@ export class TelemetryEventRelay extends EventRelay {
|
|||
user,
|
||||
credentialId,
|
||||
credentialType,
|
||||
credentialDescriptionLength,
|
||||
isDynamic,
|
||||
usesExternalSecrets,
|
||||
jweEnabled,
|
||||
supportsManagedAuth,
|
||||
usesManagedAuth,
|
||||
}: RelayEventMap['credentials-updated']) {
|
||||
this.telemetry.track('User updated credentials', {
|
||||
this.telemetry.track(TELEMETRY_EVENT.CREDENTIALS.USER_UPDATED_CREDENTIALS, {
|
||||
source: 'backend',
|
||||
user_id: user.id,
|
||||
user_role: user.role?.slug,
|
||||
credential_type: credentialType,
|
||||
credential_id: credentialId,
|
||||
has_description: credentialDescriptionLength > 0,
|
||||
description_length: credentialDescriptionLength,
|
||||
is_private: isDynamic ?? false,
|
||||
uses_external_secrets: usesExternalSecrets ?? false,
|
||||
jwe_enabled: jweEnabled ?? false,
|
||||
|
|
|
|||
|
|
@ -216,6 +216,7 @@ export class CredentialsPublicController {
|
|||
credentialType: credential.type,
|
||||
credentialId: credential.id,
|
||||
credentialName: credential.name,
|
||||
credentialDescriptionLength: credential.description?.length ?? 0,
|
||||
publicApi: true,
|
||||
projectId: project?.id,
|
||||
projectType: project?.type,
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user