branch: externals/ellama
commit a1e8ab196cd5ea793feac3fc64e6dc3ddc02a3b8
Author: Sergey Kostyaev <[email protected]>
Commit: Sergey Kostyaev <[email protected]>
Add support for skipping messages in streaming responses
Added a new option to the streaming method configuration that allows users
to
skip every N messages before processing one. This is useful for reducing
resource usage when dealing with high-frequency responses. The
implementation
includes a new handler that tracks message count and only processes messages
when the skip threshold is reached. The change also updates the streaming
logic
to handle integer values in the response processing pipeline, ensuring that
the
skip functionality works correctly with the existing streaming
infrastructure.
---
ellama.el | 47 +++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 45 insertions(+), 2 deletions(-)
diff --git a/ellama.el b/ellama.el
index 0a176edfcc..7d71fb4183 100644
--- a/ellama.el
+++ b/ellama.el
@@ -231,7 +231,8 @@ If the default streaming method is too resource-heavy, you
can try other
options."
:type `(choice
(const :tag "Streaming" streaming)
- (const :tag "Async" async)))
+ (const :tag "Async" async)
+ (integer :tag "Skip every N messages before process one")))
(defcustom ellama-define-word-prompt-template "Define %s"
"Prompt template for `ellama-define-word'."
@@ -1510,7 +1511,49 @@ failure (with BUFFER current).
(funcall errcb msg)
(setq ellama--current-request nil)
(ellama-request-mode -1)))
- t)))))
+ t))
+ ((pred integerp)
+ (let* ((cnt 0)
+ (skip-handler
+ (lambda (request)
+ (if (= cnt ellama-response-process-method)
+ (progn
+ (funcall handler request)
+ (setq cnt 0))
+ (cl-incf cnt)))))
+ (llm-chat-streaming
+ provider
+ llm-prompt
+ skip-handler
+ (lambda (response)
+ (let ((text (plist-get response :text))
+ (reasoning (plist-get response
:reasoning)))
+ (funcall handler response)
+ (when (or ellama--current-session
+ (not reasoning))
+ (kill-buffer reasoning-buffer))
+ (with-current-buffer buffer
+ (accept-change-group ellama--change-group)
+ (when ellama-spinner-enabled
+ (spinner-stop))
+ (if (and (listp donecb)
+ (functionp (car donecb)))
+ (mapc (lambda (fn) (funcall fn text))
+ donecb)
+ (funcall donecb text))
+ (when ellama-session-hide-org-quotes
+ (ellama-collapse-org-quotes))
+ (setq ellama--current-request nil)
+ (ellama-request-mode -1))))
+ (lambda (_ msg)
+ (with-current-buffer buffer
+ (cancel-change-group ellama--change-group)
+ (when ellama-spinner-enabled
+ (spinner-stop))
+ (funcall errcb msg)
+ (setq ellama--current-request nil)
+ (ellama-request-mode -1)))
+ t))))))
(with-current-buffer buffer
(setq ellama--current-request request)))))))
