Spaces:
Build error
Build error
File size: 1,688 Bytes
5213b80 8c7e6f1 dd66861 8c7e6f1 f977d49 8c7e6f1 f977d49 38434c2 f977d49 2ac97e2 8c7e6f1 f977d49 2ac97e2 5213b80 35f4e45 5213b80 2ac97e2 f977d49 8c7e6f1 f977d49 38434c2 f977d49 35f4e45 38434c2 f977d49 35f4e45 f977d49 8c7e6f1 f977d49 8c7e6f1 dd66861 e9a4671 dd66861 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
import { type ChatCompletionInputMessage } from '@huggingface/tasks';
import { HfInference } from '@huggingface/inference';
import type { ModelEntryWithTokenizer } from '$lib/types';
export function createHfInference(token: string): HfInference {
return new HfInference(token);
}
export async function handleStreamingResponse(
hf: HfInference,
model: string,
messages: ChatCompletionInputMessage[],
temperature: number,
maxTokens: number,
onChunk: (content: string) => void,
abortController: AbortController
): Promise<void> {
let out = '';
try {
for await (const chunk of hf.chatCompletionStream(
{
model: model,
messages: messages,
temperature: temperature,
max_tokens: maxTokens
},
{ signal: abortController.signal }
)) {
if (chunk.choices && chunk.choices.length > 0 && chunk.choices[0]?.delta?.content) {
out += chunk.choices[0].delta.content;
onChunk(out);
}
}
} catch (error) {
if (error.name === 'AbortError') {
console.log('Stream aborted');
} else {
throw error;
}
}
}
export async function handleNonStreamingResponse(
hf: HfInference,
model: string,
messages: ChatCompletionInputMessage[],
temperature: number,
maxTokens: number
): Promise<ChatCompletionInputMessage> {
const response = await hf.chatCompletion({
model: model,
messages: messages,
temperature: temperature,
max_tokens: maxTokens
});
if (response.choices && response.choices.length > 0) {
return response.choices[0].message;
}
throw new Error('No response from the model');
}
export function isSystemPromptSupported(model: ModelEntryWithTokenizer) {
return model.tokenizerConfig?.chat_template?.includes('system');
}
|