diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts index d95e06712ba..fdaee35aabf 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts @@ -293,6 +293,35 @@ describe('planPush', () => { expect(plan.toUpdate).toEqual([]); }); + it('treats a requiresMemoryCompaction addition as an update (the export omits the stored false)', () => { + const plan = planPush( + [item('c', { requiresMemoryCompaction: true })], + { 'c.json': body() }, + { c: 5 }, + ); + expect(plan.toUpdate.map((u) => u.id)).toEqual([5]); + expect(plan.unchanged).toEqual([]); + }); + + it('reports a case whose stored requiresMemoryCompaction matches as unchanged', () => { + const plan = planPush( + [item('c', { requiresMemoryCompaction: true })], + { 'c.json': body({ requiresMemoryCompaction: true }) }, + { c: 5 }, + ); + expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']); + }); + + it('reports a disk requiresMemoryCompaction: false as unchanged when the export omits the stored default', () => { + const plan = planPush( + [item('c', { requiresMemoryCompaction: false })], + { 'c.json': body() }, + { c: 5 }, + ); + expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']); + expect(plan.toUpdate).toEqual([]); + }); + it('treats a datasets difference as an update so tier edits re-sync', () => { const plan = planPush( [item('c', { datasets: ['mcp', 'pr', 'full'] })], diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts index fc4cd940a8c..5a200f1a676 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts @@ -55,6 +55,7 @@ describe('diskCaseToLangTracerCreate', () => { datasets: ['pr', 'full'], messageBudget: 4, credentials: [{ type: 'slackApi', name: 'Slack' }], + requiresMemoryCompaction: true, }), 'c', { suiteId: 1, setKind: 'regression', synthetic: true }, @@ -67,6 +68,7 @@ describe('diskCaseToLangTracerCreate', () => { expect(body.datasets).toEqual(['pr', 'full']); expect(body.messageBudget).toBe(4); expect(body.credentials).toEqual([{ type: 'slackApi', name: 'Slack' }]); + expect(body.requiresMemoryCompaction).toBe(true); }); it('omits optional keys that are absent on the disk case', () => { @@ -83,6 +85,7 @@ describe('diskCaseToLangTracerCreate', () => { expect('credentials' in body).toBe(false); expect('evalTriggerType' in body).toBe(false); expect('description' in body).toBe(false); + expect('requiresMemoryCompaction' in body).toBe(false); }); it('forwards typed scenario seed tables so lang-tracer stores their rows', () => { diff --git a/packages/@n8n/instance-ai/evaluations/langtracer/push.ts b/packages/@n8n/instance-ai/evaluations/langtracer/push.ts index 01a9e62c26e..bcc1ba7b17c 100644 --- a/packages/@n8n/instance-ai/evaluations/langtracer/push.ts +++ b/packages/@n8n/instance-ai/evaluations/langtracer/push.ts @@ -34,6 +34,7 @@ const COMPARED_KEYS = [ 'messageBudget', 'credentials', 'credentialFixture', + 'requiresMemoryCompaction', 'datasets', // Round-trips faithfully: PATCH /cases/:id reconciles scenario rows by name // (lang-tracer #48) and the export emits them back in disk shape. @@ -121,6 +122,9 @@ function projectComparable(src: unknown): Record { // The export only emits `messageBudget` for multi-turn cases (it's ignored for // single-turn auto-approve builds), so ignore it there to stay convergent. if (key === 'messageBudget' && !isMultiTurn) continue; + // The export omits a stored `false` for requiresMemoryCompaction (the column + // default), so a disk `false` folds to absent to keep re-pushes convergent. + if (key === 'requiresMemoryCompaction' && value === false) continue; // The loader defaults an absent disk `datasets` while the export omits (or // nulls) the stored default — fold the default to absent on both sides, and // compare order-insensitively since tiers are a set. diff --git a/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts b/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts index ee291cd0863..0aac17a1c85 100644 --- a/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts +++ b/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts @@ -43,6 +43,7 @@ export interface LangTracerCreateCaseBody { * promote/scrub over there, so pushing one would fabricate provenance. */ seed?: PushableSeed; credentialFixture?: string; + requiresMemoryCompaction?: boolean; } type InlineSeed = Extract; @@ -160,6 +161,9 @@ export function diskCaseToLangTracerCreate( // Replay never reaches here — `unsupportedPushReason` skips those cases upstream. if (testCase.seed?.mode === 'inline') body.seed = pushableSeed(testCase.seed); if (testCase.credentialFixture !== undefined) body.credentialFixture = testCase.credentialFixture; + if (testCase.requiresMemoryCompaction !== undefined) { + body.requiresMemoryCompaction = testCase.requiresMemoryCompaction; + } return body; }