branch: externals/ellama
commit 4402eabcc5f728454c20118ee4f66b2d7509ae47
Author: Sergey Kostyaev <sskosty...@gmail.com>
Commit: Sergey Kostyaev <sskosty...@gmail.com>
Add semantic similarity reasoning with context
Added a new custom variable
`ellama-semantic-identity-reasoning-template` to define the template
for checking semantic similarity of two texts in a given context.
Implemented the function `ellama-make-semantic-similar-p-with-context`
to create a function that checks if two texts have the same meaning
using the provided context. Updated the test file to include a new
integration test for the semantic similarity reasoning feature.
---
ellama.el | 40 +++++++++++++++++++++++++++++++++++++++-
tests/integration-test-ellama.el | 18 ++++++++++++++++++
2 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/ellama.el b/ellama.el
index 241cb35963..5cd7f7b536 100644
--- a/ellama.el
+++ b/ellama.el
@@ -382,6 +382,26 @@ is not changed.
:group 'ellama
:type 'string)
+(defcustom ellama-semantic-identity-reasoning-template "Determine if two texts
have the same meaning. If they are similar but differ in key aspects, they are
not the same. Return the answer as a JSON object.
+<CONTEXT>
+%s
+</CONTEXT>
+<TEXT_1>
+%s
+</TEXT_1>
+<TEXT_2>
+%s
+</TEXT_2>
+<EXAMPLE>
+{
+ \"think\": \"Think if texts have same meaning in provided context\",
+ \"same\": true
+}
+</EXAMPLE>"
+ "Extract string list template with context and reasoning."
+ :group 'ellama
+ :type 'string)
+
(defcustom ellama-extraction-provider nil
"LLM provider for data extraction."
:group 'ellama
@@ -2244,6 +2264,24 @@ otherwise prompt user for URL to summarize."
(kill-region (point) (point-max))
(ellama-summarize))))
+(defun ellama-make-semantic-similar-p-with-context (context)
+ "Return function for checking semantic similarity of two texts in CONTEXT."
+ (lambda (text1 text2)
+ "Check if TEXT1 means the same as TEXT2."
+ (plist-get
+ (json-parse-string
+ (llm-chat
+ (or ellama-extraction-provider ellama-provider)
+ (llm-make-chat-prompt
+ (format ellama-semantic-identity-reasoning-template context text1 text2)
+ :response-format '(:type object :properties
+ (:think (:type string)
+ :same (:type boolean))
+ :required ["think" "same"])))
+ :object-type 'plist
+ :false-object nil)
+ :same)))
+
(defun ellama-semantic-similar-p (text1 text2)
"Check if TEXT1 means the same as TEXT2."
(plist-get
@@ -2254,7 +2292,7 @@ otherwise prompt user for URL to summarize."
(format ellama-semantic-identity-template text1 text2)
:response-format '(:type object :properties
(:same (:type boolean))
- :required (same))))
+ :required ["same"])))
:object-type 'plist
:false-object nil)
:same))
diff --git a/tests/integration-test-ellama.el b/tests/integration-test-ellama.el
index ab329dbc70..aaea94c719 100644
--- a/tests/integration-test-ellama.el
+++ b/tests/integration-test-ellama.el
@@ -48,6 +48,24 @@
'("How many r's in strawberry?"
"How many times letter e appears in word strawberry?"))))
+(ert-deftest ellama-semantic-similar-reasoning-test ()
+ "Check if `ellama-make-semantic-similar-p-with-context' works correctly."
+ (let ((testfn (ellama-make-semantic-similar-p-with-context
+ "Communication between user and assistant")))
+ (should (equal-including-properties
+ (let ((res))
+ (dolist (el '("How many r's in strawberry?"
+ "How many times letter r appears in word
strawberry?"
+ "How many r's in strawberry?"
+ "How many times letter e appears in word
strawberry?"
+ "Define RAPTOR"
+ "What does mean RAPTOR?"))
+ (cl-pushnew el res :test testfn))
+ (reverse res))
+ '("How many r's in strawberry?"
+ "How many times letter e appears in word strawberry?"
+ "Define RAPTOR")))))
+
(provide 'integration-test-ellama)
;;; integration-test-ellama.el ends here
