branch: externals/ellama
commit 018c2151d1f231a3e3a15a5e99cf39157d758ee4
Merge: 8281a9847b 729df728b6
Author: Sergey Kostyaev <s-kosty...@users.noreply.github.com>
Commit: GitHub <nore...@github.com>
Merge pull request #327 from
s-kostyaev/fix-paragraph-filling-in-md-to-org-conversion
Refactor transformations and improve newline handling
---
NEWS.org | 17 +++++
ellama-transient.el | 3 +-
ellama.el | 171 ++++++++++++++++++++++++++++-----------------------
tests/test-ellama.el | 104 +++++++++++++++++++++++++++++++
4 files changed, 218 insertions(+), 77 deletions(-)
diff --git a/NEWS.org b/NEWS.org
index 2d10cfb1ba..f174b93143 100644
--- a/NEWS.org
+++ b/NEWS.org
@@ -1,3 +1,20 @@
+* Version 1.8.2
+- Refactored ~ellama--apply-transformations~ to use markers for stability
during
+ replacements. Added ~set-hard-newline~ properties to handle hard newlines
during
+ text filling. Updated test cases to include a new test for markdown list to
+ org list conversion.
+- Added check for function existence to avoid errors when ~llm-ollama-p~ is not
+ available.
+- Displayed ellama instant result buffer on done instead of visible reasoning
+ buffer.
+- Added two new tests for ellama functionality: ~test-ellama-lorem-ipsum~
+ verifies proper line wrapping and formatting of long text.
+ ~test-ellama-duplicate-strings~ checks handling of duplicate strings with
+ markdown formatting.
+- Added a check to skip processing when input text is empty, preventing
+ potential errors in subsequent operations.
+- Moved ~(require 'llm-ollama)~ from inside the function to top scope to ensure
+ package is loaded before use.
* Version 1.8.1
- Use direct interfacing with Ollama's API instead of local installation.
* Version 1.8.0
diff --git a/ellama-transient.el b/ellama-transient.el
index 458110be48..f000677c60 100644
--- a/ellama-transient.el
+++ b/ellama-transient.el
@@ -171,7 +171,8 @@ Otherwise, prompt the user to enter a system message."
(declare-function llm-ollama-chat-model "ext:llm-ollama")
(declare-function llm-ollama-default-chat-temperature "ext:llm-ollama")
(declare-function llm-ollama-default-chat-non-standard-params
"ext:llm-ollama")
- (when (llm-ollama-p provider)
+ (when (and (fboundp 'llm-ollama-p)
+ (llm-ollama-p provider))
(setq ellama-transient-ollama-model-name (llm-ollama-chat-model provider))
(setq ellama-transient-temperature (or
(llm-ollama-default-chat-temperature provider) 0.7))
(setq ellama-transient-host (llm-ollama-host provider))
diff --git a/ellama.el b/ellama.el
index ca420d9d7b..d08d4ed9d7 100644
--- a/ellama.el
+++ b/ellama.el
@@ -6,7 +6,7 @@
;; URL: http://github.com/s-kostyaev/ellama
;; Keywords: help local tools
;; Package-Requires: ((emacs "28.1") (llm "0.24.0") (plz "0.8") (transient
"0.7") (compat "29.1"))
-;; Version: 1.8.1
+;; Version: 1.8.2
;; SPDX-License-Identifier: GPL-3.0-or-later
;; Created: 8th Oct 2023
@@ -521,7 +521,7 @@ It should be a function with single argument generated text
string."
(defun ellama--replace-bad-code-blocks (text)
"Replace code src blocks in TEXT."
(with-temp-buffer
- (insert text)
+ (insert (propertize text 'hard t))
(goto-char (point-min))
;; skip good code blocks
(while (re-search-forward "#\\+BEGIN_SRC\\(.\\|\n\\)*?#\\+END_SRC" nil t))
@@ -539,40 +539,47 @@ It should be a function with single argument generated
text string."
(defun ellama--apply-transformations (beg end)
"Apply md to org transformations for region BEG END."
- ;; headings
- (ellama--replace "^# " "* " beg end)
- (ellama--replace "^## " "** " beg end)
- (ellama--replace "^### " "*** " beg end)
- (ellama--replace "^#### " "**** " beg end)
- (ellama--replace "^##### " "***** " beg end)
- (ellama--replace "^###### " "****** " beg end)
- ;; bold
- (ellama--replace "__\\(.+?\\)__" "*\\1*" beg end)
- (ellama--replace "\\*\\*\\(.+?\\)\\*\\*" "*\\1*" beg end)
- (ellama--replace "<b>\\(.+?\\)</b>" "*\\1*" beg end)
- (ellama--replace "<i>\\(.+?\\)</i>" "/\\1/" beg end)
- ;; underlined
- (ellama--replace "<u>\\(.+?\\)</u>" "_\\1_" beg end)
- ;; inline code
- (ellama--replace "`\\(.+?\\)`" "~\\1~" beg end)
- ;; italic
- (when ellama-translate-italic
- (ellama--replace "_\\(.+?\\)_" "/\\1/" beg end))
- ;; lists
- (ellama--replace "^\\* " "+ " beg end)
- ;; strikethrough
- (ellama--replace "~~\\(.+?\\)~~" "+\\1+" beg end)
- (ellama--replace "<s>\\(.+?\\)</s>" "+\\1+" beg end)
- ;; badges
- (ellama--replace "\\[\\!\\[.*?\\](\\(.*?\\))\\](\\(.*?\\))"
"[[\\2][file:\\1]]" beg end)
- ;;links
- (ellama--replace "\\[\\(.*?\\)\\](\\(.*?\\))" "[[\\2][\\1]]" beg end)
-
- ;; filling long lines
- (goto-char beg)
- (when ellama-fill-paragraphs
- (let ((use-hard-newlines t))
- (fill-region beg end nil t t))))
+ (let ((beg-pos (make-marker))
+ (end-pos (make-marker)))
+ (set-marker-insertion-type beg-pos t)
+ (set-marker-insertion-type end-pos t)
+ (set-marker beg-pos beg)
+ (set-marker end-pos end)
+ ;; bold
+ (ellama--replace "__\\(.+?\\)__" "*\\1*" beg-pos end-pos)
+ (ellama--replace "\\*\\*\\([^\*\n]+?\\)\\*\\*" "*\\1*" beg-pos end-pos)
+ (ellama--replace "<b>\\(.+?\\)</b>" "*\\1*" beg-pos end-pos)
+ (ellama--replace "<i>\\(.+?\\)</i>" "/\\1/" beg-pos end-pos)
+ ;; headings
+ (ellama--replace "^# " "* " beg-pos end-pos)
+ (ellama--replace "^## " "** " beg-pos end-pos)
+ (ellama--replace "^### " "*** " beg-pos end-pos)
+ (ellama--replace "^#### " "**** " beg-pos end-pos)
+ (ellama--replace "^##### " "***** " beg-pos end-pos)
+ (ellama--replace "^###### " "****** " beg-pos end-pos)
+ ;; underlined
+ (ellama--replace "<u>\\(.+?\\)</u>" "_\\1_" beg-pos end-pos)
+ ;; inline code
+ (ellama--replace "`\\(.+?\\)`" "~\\1~" beg-pos end-pos)
+ ;; italic
+ (when ellama-translate-italic
+ (ellama--replace "_\\(.+?\\)_" "/\\1/" beg-pos end-pos))
+ ;; lists
+ (ellama--replace "^\\* " "+ " beg-pos end-pos)
+ ;; strikethrough
+ (ellama--replace "~~\\(.+?\\)~~" "+\\1+" beg-pos end-pos)
+ (ellama--replace "<s>\\(.+?\\)</s>" "+\\1+" beg-pos end-pos)
+ ;; badges
+ (ellama--replace "\\[\\!\\[.*?\\](\\(.*?\\))\\](\\(.*?\\))"
"[[\\2][file:\\1]]" beg-pos end-pos)
+ ;;links
+ (ellama--replace "\\[\\(.*?\\)\\](\\(.*?\\))" "[[\\2][\\1]]" beg-pos
end-pos)
+
+ ;; filling long lines
+ (goto-char beg-pos)
+ (set-hard-newline-properties beg-pos end-pos)
+ (when ellama-fill-paragraphs
+ (let* ((use-hard-newlines t))
+ (fill-region beg end-pos nil t t)))))
(defun ellama--replace-outside-of-code-blocks (text)
"Replace markdown elements in TEXT with org equivalents.
@@ -1226,45 +1233,46 @@ FILTER is a function for text transformation."
(safe-common-prefix ""))
(lambda
(text)
- (with-current-buffer buffer
- (save-excursion
- (goto-char end-marker)
- (let* ((filtered-text
- (funcall filter text))
- (use-hard-newlines t)
- (common-prefix (concat
- safe-common-prefix
- (ellama-max-common-prefix
- (string-remove-prefix
- safe-common-prefix
- filtered-text)
- (string-remove-prefix
+ (when (not (string-empty-p text))
+ (with-current-buffer buffer
+ (save-excursion
+ (goto-char end-marker)
+ (let* ((filtered-text
+ (funcall filter text))
+ (use-hard-newlines t)
+ (common-prefix (concat
safe-common-prefix
- previous-filtered-text))))
- (wrong-chars-cnt (- (length previous-filtered-text)
- (length common-prefix)))
- (delta (string-remove-prefix common-prefix filtered-text)))
- (delete-char (- wrong-chars-cnt))
- (when delta (insert (propertize delta 'hard t))
- (when (and
- ellama-fill-paragraphs
- (pcase ellama-fill-paragraphs
- ((cl-type function) (funcall
ellama-fill-paragraphs))
- ((cl-type boolean) ellama-fill-paragraphs)
- ((cl-type list) (and (apply #'derived-mode-p
-
ellama-fill-paragraphs)))))
- (if (not (eq major-mode 'org-mode))
- (fill-paragraph)
- (when (not (save-excursion
- (re-search-backward
- "#\\+BEGIN_SRC"
- beg-marker t)))
- (org-fill-paragraph))))
- (set-marker end-marker (point))
- (when (and ellama-auto-scroll (not ellama--stop-scroll))
- (ellama--scroll buffer end-marker))
- (setq safe-common-prefix (ellama--string-without-last-line
common-prefix))
- (setq previous-filtered-text filtered-text)))))))))
+ (ellama-max-common-prefix
+ (string-remove-prefix
+ safe-common-prefix
+ filtered-text)
+ (string-remove-prefix
+ safe-common-prefix
+ previous-filtered-text))))
+ (wrong-chars-cnt (- (length previous-filtered-text)
+ (length common-prefix)))
+ (delta (string-remove-prefix common-prefix filtered-text)))
+ (delete-char (- wrong-chars-cnt))
+ (when delta (insert (propertize delta 'hard t))
+ (when (and
+ ellama-fill-paragraphs
+ (pcase ellama-fill-paragraphs
+ ((cl-type function) (funcall
ellama-fill-paragraphs))
+ ((cl-type boolean) ellama-fill-paragraphs)
+ ((cl-type list) (and (apply #'derived-mode-p
+
ellama-fill-paragraphs)))))
+ (if (not (eq major-mode 'org-mode))
+ (fill-paragraph)
+ (when (not (save-excursion
+ (re-search-backward
+ "#\\+BEGIN_SRC"
+ beg-marker t)))
+ (org-fill-paragraph))))
+ (set-marker end-marker (point))
+ (when (and ellama-auto-scroll (not ellama--stop-scroll))
+ (ellama--scroll buffer end-marker))
+ (setq safe-common-prefix
(ellama--string-without-last-line common-prefix))
+ (setq previous-filtered-text filtered-text))))))))))
(defun ellama--handle-partial (insert-text insert-reasoning reasoning-buffer)
"Handle partial llm callback.
@@ -1950,7 +1958,17 @@ ARGS contains keys for fine control.
(make-temp-name (concat buffer-name "
"))
buffer-name)))
(system (plist-get args :system))
- (donecb (plist-get args :on-done))
+ (donecb (lambda (text)
+ (let ((callback (plist-get args :on-done)))
+ (display-buffer buffer
+ (when
ellama-instant-display-action-function
+ `((ignore .
(,ellama-instant-display-action-function)))))
+ (when callback
+ (if (and (listp callback)
+ (functionp (car callback)))
+ (mapc (lambda (fn) (funcall fn text))
+ callback)
+ (funcall callback text))))))
filter)
(with-current-buffer buffer
(funcall ellama-major-mode)
@@ -2362,6 +2380,7 @@ Call CALLBACK on result list of strings. ARGS contains
keys for fine control.
(lambda (err)
(user-error err)))))
+(declare-function make-llm-ollama "ext:llm-ollama")
(defun ellama-get-ollama-model-names ()
"Get ollama model names."
(llm-models (or ellama-provider
@@ -2401,12 +2420,12 @@ Call CALLBACK on result list of strings. ARGS contains
keys for fine control.
(declare-function llm-ollama-p "ext:llm-ollama")
(declare-function llm-ollama-host "ext:llm-ollama")
(declare-function llm-ollama-port "ext:llm-ollama")
+ (require 'llm-ollama)
(let ((model-name (ellama-get-ollama-model-name))
(host (when (llm-ollama-p ellama-provider)
(llm-ollama-host ellama-provider)))
(port (when (llm-ollama-p ellama-provider)
(llm-ollama-port ellama-provider))))
- (require 'llm-ollama)
(if host
(make-llm-ollama
:chat-model model-name :embedding-model model-name :host host :port
port)
diff --git a/tests/test-ellama.el b/tests/test-ellama.el
index 5e7100e6c4..ce0a1ebf49 100644
--- a/tests/test-ellama.el
+++ b/tests/test-ellama.el
@@ -53,6 +53,62 @@
(ellama-code-improve)
(should (equal original (buffer-string)))))))
+(ert-deftest test-ellama-lorem-ipsum ()
+ (let ((fill-column 70)
+ (raw "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do
eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed do
eiusmod tempor incididunt [...]
+ (expected "Lorem ipsum dolor sit amet, consectetur adipiscing elit.
Sed do
+eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad
+minim veniam, quis nostrud exercitation ullamco laboris nisi ut
+aliquip ex ea commodo consequat. Duis aute irure dolor in
+reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
+pariatur. Excepteur sint occaecat cupidatat non proident, sunt in
+culpa qui officia deserunt mollit anim id est laborum. Sed do eiusmod
+tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
+veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex
+ea commodo consequat. Duis aute irure dolor in reprehenderit in
+voluptate velit esse cillum dolore eu fugiat nulla pariatur.")
+ (ellama-provider (make-llm-fake))
+ prev-lines)
+ (with-temp-buffer
+ (org-mode)
+ (cl-letf (((symbol-function 'llm-chat-streaming)
+ (lambda (_provider prompt partial-callback response-callback
_error-callback _multi-output)
+ (should (string-match "test" (llm-chat-prompt-to-text
prompt)))
+ (dolist (s (string-split raw " "))
+ (funcall partial-callback `(:text ,(concat prev-lines s)))
+ (setq prev-lines (concat prev-lines s)))
+ (funcall response-callback `(:text ,raw)))))
+ (ellama-write "test")
+ (should (equal expected (buffer-string)))))))
+
+(ert-deftest test-ellama-duplicate-strings ()
+ (let ((fill-column 80)
+ (raw "Great question! Whether you should start with **\"Natural
Language Processing with Transformers\"** (O’Reilly) or wait for **\"Build a
Large Language Model (From Scratch)\"** depends on your **goals, background,
and learning style**. Here’s a detailed comparison to help
+you decide:
+
+---
+
+")
+ (expected "Great question! Whether you should start with *\"Natural
Language Processing with
+Transformers\"* (O’Reilly) or wait for *\"Build a Large Language Model (From
+Scratch)\"* depends on your *goals, background, and learning style*. Here’s a
+detailed comparison to help you decide:
+
+---")
+ (ellama-provider (make-llm-fake))
+ prev-lines)
+ (with-temp-buffer
+ (org-mode)
+ (cl-letf (((symbol-function 'llm-chat-streaming)
+ (lambda (_provider prompt partial-callback response-callback
_error-callback _multi-output)
+ (should (string-match "test" (llm-chat-prompt-to-text
prompt)))
+ (dolist (s (string-split raw " "))
+ (funcall partial-callback `(:text ,(concat prev-lines s "
")))
+ (setq prev-lines (concat prev-lines s " ")))
+ (funcall response-callback `(:text ,raw)))))
+ (ellama-write "test")
+ (should (equal expected (buffer-string)))))))
+
(ert-deftest test-ellama-context-element-format-buffer-markdown ()
(let ((element (ellama-context-element-buffer :name "*scratch*")))
(should (equal "```emacs-lisp\n(display-buffer \"*scratch*\")\n```\n"
@@ -454,6 +510,54 @@ package main
1. *Initialization*: We create a boolean slice ~prime~ of size ~n+1~, where
each
index represents whether the number is prime (~true~) or not (~false~)."))))
+(ert-deftest test-ellama-md-to-org-lists ()
+ (let* ((fill-column 80)
+ (result (ellama--translate-markdown-to-org-filter "<think>Okay, the
user asked me to create a list of fruits. Let me think about how to approach
this.</think> Here’s a comprehensive list of fruits, categorized for clarity:
+
+---
+
+### **Common Fruits**
+1. **Apple**
+2. **Banana**
+3. **Orange**
+4. **Grape**
+5. **Strawberry**
+6. **Blueberry**
+
+---
+
+### **Additional Notes**
+- **Tomatoes** are technically fruits (part of the nightshade family).
+- **Coconut** is a tropical fruit, often used in cooking.
+- **Papaya** is a versatile fruit with nutritional value.
+
+Let me know if you'd like a simplified version or a specific category (e.g.,
by region, season, or type)! 🍎🍊")))
+ (should (string= result "#+BEGIN_QUOTE
+Okay, the user asked me to create a list of fruits. Let me think about how to
+approach this.
+#+END_QUOTE
+ Here’s a comprehensive list of fruits, categorized for clarity:
+
+---
+
+*** *Common Fruits*
+1. *Apple*
+2. *Banana*
+3. *Orange*
+4. *Grape*
+5. *Strawberry*
+6. *Blueberry*
+
+---
+
+*** *Additional Notes*
+- *Tomatoes* are technically fruits (part of the nightshade family).
+- *Coconut* is a tropical fruit, often used in cooking.
+- *Papaya* is a versatile fruit with nutritional value.
+
+Let me know if you'd like a simplified version or a specific category (e.g., by
+region, season, or type)! 🍎🍊"))))
+
(defun ellama-test-max-common-prefix ()
"Test the `ellama-max-common-prefix` function."
(should (equal (ellama-max-common-prefix "" "") ""))
