Add reasoning streaming
This commit is contained in:
@@ -177,18 +177,21 @@ func (c *Client) completions(
|
|||||||
|
|
||||||
// StreamEvent wraps a message or delta produced during streaming.
|
// StreamEvent wraps a message or delta produced during streaming.
|
||||||
type StreamEvent struct {
|
type StreamEvent struct {
|
||||||
// Type identifies the event kind: delta, done, or message.
|
// Type identifies the event kind: delta, reasoning_delta, done, or message.
|
||||||
Type string
|
Type string
|
||||||
// Delta is a text fragment from the assistant response.
|
// Delta is a text fragment from the assistant response.
|
||||||
Delta string
|
Delta string
|
||||||
|
// ReasoningDelta is a reasoning text fragment from the assistant response.
|
||||||
|
ReasoningDelta string
|
||||||
// Message is the complete chat completion message.
|
// Message is the complete chat completion message.
|
||||||
Message Message
|
Message Message
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
streamEventDelta = "delta"
|
streamEventDelta = "delta"
|
||||||
streamEventDone = "done"
|
streamEventReasoningDelta = "reasoning_delta"
|
||||||
streamEventMessage = "message"
|
streamEventDone = "done"
|
||||||
|
streamEventMessage = "message"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CompletionsStream sends messages to the LLM using the specified model and
|
// CompletionsStream sends messages to the LLM using the specified model and
|
||||||
@@ -403,6 +406,10 @@ func (c *Client) completionsStream(ctx context.Context, messages []Message, mode
|
|||||||
acc.reasoning.WriteString(
|
acc.reasoning.WriteString(
|
||||||
delta.ReasoningContent,
|
delta.ReasoningContent,
|
||||||
)
|
)
|
||||||
|
events <- StreamEvent{
|
||||||
|
Type: streamEventReasoningDelta,
|
||||||
|
ReasoningDelta: delta.ReasoningContent,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Accumulate tool call deltas by index.
|
// Accumulate tool call deltas by index.
|
||||||
for _, tc := range delta.ToolCalls {
|
for _, tc := range delta.ToolCalls {
|
||||||
|
|||||||
@@ -585,6 +585,8 @@ type StreamMessage struct {
|
|||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
// Delta is a text fragment from the assistant response.
|
// Delta is a text fragment from the assistant response.
|
||||||
Delta string `json:"delta,omitempty"`
|
Delta string `json:"delta,omitempty"`
|
||||||
|
// ReasoningDelta is a reasoning text fragment from the assistant response.
|
||||||
|
ReasoningDelta string `json:"reasoning_delta,omitempty"`
|
||||||
// Message is the chat completion message.
|
// Message is the chat completion message.
|
||||||
Message *llm.Message `json:"message,omitempty"`
|
Message *llm.Message `json:"message,omitempty"`
|
||||||
// Provider is the LLM provider used for the response.
|
// Provider is the LLM provider used for the response.
|
||||||
@@ -709,9 +711,10 @@ func (svc *Service) chatStream(w http.ResponseWriter, r *http.Request) {
|
|||||||
// Consume events and send as SSE.
|
// Consume events and send as SSE.
|
||||||
for evt := range events {
|
for evt := range events {
|
||||||
msg := StreamMessage{
|
msg := StreamMessage{
|
||||||
Delta: evt.Delta,
|
Delta: evt.Delta,
|
||||||
Provider: provider,
|
ReasoningDelta: evt.ReasoningDelta,
|
||||||
Model: model,
|
Provider: provider,
|
||||||
|
Model: model,
|
||||||
}
|
}
|
||||||
if evt.Message.Role != "" {
|
if evt.Message.Role != "" {
|
||||||
msg.Message = &evt.Message
|
msg.Message = &evt.Message
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ const PROVIDER_KEY = 'odidere_provider';
|
|||||||
const STORAGE_KEY = 'odidere_history';
|
const STORAGE_KEY = 'odidere_history';
|
||||||
const STREAM_ENDPOINT = '/v1/chat/voice/stream';
|
const STREAM_ENDPOINT = '/v1/chat/voice/stream';
|
||||||
const StreamEventDelta = 'delta';
|
const StreamEventDelta = 'delta';
|
||||||
|
const StreamEventReasoningDelta = 'reasoning_delta';
|
||||||
const StreamEventDone = 'done';
|
const StreamEventDone = 'done';
|
||||||
const StreamEventMessage = 'message';
|
const StreamEventMessage = 'message';
|
||||||
const SYSTEM_MESSAGE_KEY = 'odidere_system_message';
|
const SYSTEM_MESSAGE_KEY = 'odidere_system_message';
|
||||||
@@ -1583,7 +1584,10 @@ class Odidere {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (eventType === StreamEventDelta) {
|
if (
|
||||||
|
eventType === StreamEventDelta ||
|
||||||
|
eventType === StreamEventReasoningDelta
|
||||||
|
) {
|
||||||
if (!streamingMessage) {
|
if (!streamingMessage) {
|
||||||
const message = {
|
const message = {
|
||||||
role: 'assistant',
|
role: 'assistant',
|
||||||
@@ -1598,8 +1602,21 @@ class Odidere {
|
|||||||
streamingMessage = { message: appended[0], $el };
|
streamingMessage = { message: appended[0], $el };
|
||||||
}
|
}
|
||||||
|
|
||||||
streamingMessage.message.content[0].text += event.delta || '';
|
if (eventType === StreamEventDelta) {
|
||||||
this.#appendStreamingDelta(streamingMessage.$el, event.delta || '');
|
streamingMessage.message.content[0].text += event.delta || '';
|
||||||
|
this.#appendStreamingDelta(
|
||||||
|
streamingMessage.$el,
|
||||||
|
event.delta || '',
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
streamingMessage.message.reasoning_content = `${
|
||||||
|
streamingMessage.message.reasoning_content || ''
|
||||||
|
}${event.reasoning_delta || ''}`;
|
||||||
|
this.#appendStreamingReasoningDelta(
|
||||||
|
streamingMessage.$el,
|
||||||
|
event.reasoning_delta || '',
|
||||||
|
);
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2263,6 +2280,24 @@ class Odidere {
|
|||||||
this.#scrollToBottom();
|
this.#scrollToBottom();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* #appendStreamingReasoningDelta appends reasoning text to a streaming
|
||||||
|
* assistant message.
|
||||||
|
* @param {HTMLElement} $msg
|
||||||
|
* @param {string} delta
|
||||||
|
*/
|
||||||
|
#appendStreamingReasoningDelta($msg, delta) {
|
||||||
|
let $reasoning = $msg.querySelector('.collapsible--reasoning');
|
||||||
|
if (!$reasoning) {
|
||||||
|
$reasoning = this.#createCollapsibleBlock('reasoning', 'Reasoning', '');
|
||||||
|
const $content = $msg.querySelector('.message__content');
|
||||||
|
$msg.querySelector('.message__body').insertBefore($reasoning, $content);
|
||||||
|
}
|
||||||
|
|
||||||
|
$reasoning.querySelector('pre').textContent += delta;
|
||||||
|
this.#scrollToBottom();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* #finalizeStreamingAssistantMessage replaces a streaming placeholder with
|
* #finalizeStreamingAssistantMessage replaces a streaming placeholder with
|
||||||
* a fully bound assistant message.
|
* a fully bound assistant message.
|
||||||
|
|||||||
Reference in New Issue
Block a user