mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-10-17 05:14:19 +02:00
a013eb6394
planPrompt() returned the response reserve as num_predict, and the reserve was clamped by MAX_RESPONSE_RESERVE = 1024. Chat answers stopped at 1024 output tokens regardless of window size, often mid-sentence, and the normalized stream chunks dropped Ollama's done_reason, so neither the UI nor API clients could discern between a cut-off answer and a finished one. - Decouple the generation cap from the reserve. num_predict is now the space left in the window after the prompt, less a 5% (min 64 token) margin for estimator error, with the reserve as its floor. Raise MAX_RESPONSE_RESERVE to 8192 so it only bounds prompt hold-back. - Carry done_reason through all four OllamaService chat paths. Native forwards Ollama's done_reason; compat maps OpenAI's finish_reason. - Log length-limited stops in OllamaController with num_predict, num_ctx and tokens generated. - Show a truncation notice on cut-off answers, with a Continue button on the latest one that sends a normal follow-up turn. Generation evals before and after this change are not length comparable. Closes #1342
97 lines
3.2 KiB
TypeScript
97 lines
3.2 KiB
TypeScript
/**
|
|
* The stop reason has to survive chunk normalization (#1342).
|
|
*
|
|
* Both transports used to rebuild each chunk as {message, done, usage}, which
|
|
* dropped Ollama's done_reason and OpenAI's finish_reason. A reply cut off by
|
|
* num_predict then looked exactly like one that finished.
|
|
*
|
|
* A Japa spec: OllamaService imports the Adonis logger at module scope. Run with
|
|
* `node ace test unit`.
|
|
*/
|
|
import assert from 'node:assert/strict'
|
|
import { test } from '@japa/runner'
|
|
import { OllamaService } from '../../app/services/ollama_service.js'
|
|
|
|
async function collect<T>(iterable: AsyncIterable<T>): Promise<T[]> {
|
|
const out: T[] = []
|
|
for await (const item of iterable) out.push(item)
|
|
return out
|
|
}
|
|
|
|
function fakeNativeIterator(chunks: any[]) {
|
|
return {
|
|
abort() {},
|
|
async *[Symbol.asyncIterator]() {
|
|
yield* chunks
|
|
},
|
|
}
|
|
}
|
|
|
|
function serviceWithNative(chunks: any[], response?: any) {
|
|
const service = new OllamaService() as any
|
|
service.ollama = {
|
|
chat: async (req: { stream: boolean }) => (req.stream ? fakeNativeIterator(chunks) : response),
|
|
}
|
|
return service
|
|
}
|
|
|
|
function serviceWithCompat(chunks: any[], response?: any) {
|
|
const service = new OllamaService() as any
|
|
service.openai = {
|
|
chat: {
|
|
completions: {
|
|
create: async (params: { stream: boolean }) =>
|
|
params.stream
|
|
? (async function* () {
|
|
yield* chunks
|
|
})()
|
|
: response,
|
|
},
|
|
},
|
|
}
|
|
return service
|
|
}
|
|
|
|
const request = { model: 'm', messages: [{ role: 'user', content: 'hi' }], numPredict: 16 }
|
|
|
|
test('native stream forwards done_reason on the final chunk', async () => {
|
|
const service = serviceWithNative([
|
|
{ message: { content: 'Alternate Between' }, done: false },
|
|
{ message: { content: ' Annuals' }, done: true, done_reason: 'length', eval_count: 16 },
|
|
])
|
|
const chunks = await collect<any>(await service._chatStreamNative(request))
|
|
assert.equal(chunks[0].done_reason, undefined)
|
|
assert.equal(chunks.at(-1).done, true)
|
|
assert.equal(chunks.at(-1).done_reason, 'length')
|
|
assert.equal(chunks.at(-1).usage.completionTokens, 16)
|
|
})
|
|
|
|
test('compat stream maps finish_reason to done_reason', async () => {
|
|
const service = serviceWithCompat([
|
|
{ choices: [{ delta: { content: 'egg' }, finish_reason: null }] },
|
|
{ choices: [{ delta: { content: '' }, finish_reason: 'length' }] },
|
|
{ choices: [], usage: { prompt_tokens: 10, completion_tokens: 16 } },
|
|
])
|
|
const chunks = await collect<any>(await service._chatStreamCompat(request))
|
|
assert.equal(chunks[0].done_reason, undefined)
|
|
assert.equal(chunks[1].done, true)
|
|
assert.equal(chunks[1].done_reason, 'length')
|
|
})
|
|
|
|
test('non-streaming native and compat responses carry done_reason', async () => {
|
|
const native = serviceWithNative([], {
|
|
model: 'm',
|
|
message: { content: 'cut' },
|
|
done: true,
|
|
done_reason: 'length',
|
|
})
|
|
assert.equal((await native._chatNative(request)).done_reason, 'length')
|
|
|
|
const compat = serviceWithCompat([], {
|
|
model: 'm',
|
|
choices: [{ message: { content: 'done' }, finish_reason: 'stop' }],
|
|
usage: { prompt_tokens: 1, completion_tokens: 1 },
|
|
})
|
|
assert.equal((await compat._chatCompat(request)).done_reason, 'stop')
|
|
})
|