import { test } from 'node:test' import assert from 'node:assert/strict' import { ThinkTagSplitter, normalizeNonStreamed, partialTagSuffix, splitThinkTags, } from '../../app/utils/think_stream.js' /** * Characterization tests for the tag splitter extracted out of * OllamaService.chatStream. The behaviour under test is the chunk-boundary * handling: a tag can arrive split across any number of stream chunks, and text * that might still turn out to be a tag must not be emitted early. */ /** Feed a whole string through the splitter one chunk at a time. */ function stream(chunks: string[]) { const splitter = new ThinkTagSplitter() let content = '' let thinking = '' for (const chunk of chunks) { const out = splitter.push(chunk) content += out.content thinking += out.thinking } const tail = splitter.flush() return { content: content + tail.content, thinking: thinking + tail.thinking } } test('partialTagSuffix finds how much trailing text could start a tag', () => { assert.equal(partialTagSuffix('', 'hello', 'hello<'), 1) assert.equal(partialTagSuffix('', 'hello'), 0) // A complete tag is not a *partial* tag — the caller handles that case via indexOf. assert.equal(partialTagSuffix('', ''), 0) }) test('passes through text with no tags', () => { assert.deepEqual(stream(['Hello ', 'world']), { content: 'Hello world', thinking: '' }) }) test('separates a complete think block arriving in one chunk', () => { assert.deepEqual(stream(['reasoninganswer']), { content: 'answer', thinking: 'reasoning', }) }) test('handles an opening tag split across chunks', () => { assert.deepEqual(stream(['reasoninganswer']), { content: 'answer', thinking: 'reasoning', }) }) test('handles a closing tag split across chunks', () => { assert.deepEqual(stream(['reasoninganswer']), { content: 'answer', thinking: 'reasoning', }) }) test('handles tags split one character at a time', () => { const chunks = 'abcxyz'.split('') assert.deepEqual(stream(chunks), { content: 'xyz', thinking: 'abc' }) }) test('never emits a partial tag as content before it resolves', () => { const splitter = new ThinkTagSplitter() // "", so it must be held back, not emitted. assert.deepEqual(splitter.push('answerhidden'), { content: '', thinking: 'hidden' }) }) test('flushes a dangling partial tag rather than swallowing it', () => { // " { assert.deepEqual(stream(['cut off mid-thought']), { content: '', thinking: 'cut off mid-thought', }) }) test('handles multiple think blocks in one stream', () => { assert.deepEqual(stream(['oneatwob']), { content: 'ab', thinking: 'onetwo', }) }) test('splitThinkTags handles the non-streaming case', () => { assert.deepEqual(splitThinkTags('whybecause'), { content: 'because', thinking: 'why', }) assert.deepEqual(splitThinkTags('no tags here'), { content: 'no tags here', thinking: '' }) }) // --- normalizeNonStreamed ---------------------------------------------------- // // The non-streaming path feeds every ancillary call: chat titles, suggestion // chips, and — the one that matters — the query that gets embedded and sent to // Qdrant. Reasoning reaching any of those is the bug this exists to close. test('normalizeNonStreamed: inline tags are split out of content', () => { assert.deepEqual(normalizeNonStreamed('weighing it upBoiling Water'), { content: 'Boiling Water', thinking: 'weighing it up', }) }) test('normalizeNonStreamed: a structured thinking field passes through untouched', () => { assert.deepEqual(normalizeNonStreamed('Boiling Water', 'weighing it up'), { content: 'Boiling Water', thinking: 'weighing it up', }) }) test('normalizeNonStreamed: structured and inline reasoning merge into one channel', () => { // Ollama reports reasoning natively AND the model emits literal tags in content. assert.deepEqual(normalizeNonStreamed('then thisAnswer', 'first this '), { content: 'Answer', thinking: 'first this then this', }) }) test('normalizeNonStreamed: reasoning truncated mid-thought leaves no content', () => { // What a reasoning model does under QUERY_REWRITE_MAX_TOKENS: it spends the whole // budget thinking and never closes the tag. Callers must treat this as "no answer" // rather than embedding the empty string. assert.deepEqual(normalizeNonStreamed('still working on it'), { content: '', thinking: 'still working on it', }) }) test('normalizeNonStreamed: text with no tags is passed through exactly', () => { assert.deepEqual(normalizeNonStreamed(' How do I purify water? '), { content: ' How do I purify water? ', thinking: '', }) }) test('normalizeNonStreamed: tolerates a missing content field', () => { assert.deepEqual(normalizeNonStreamed(undefined as unknown as string), { content: '', thinking: '', }) })