branch: externals/llm
commit d3b46890f3d081044a4f3422f5280b27fb356f5d
Author: Andrew Hyatt <ahy...@gmail.com>
Commit: GitHub <nore...@github.com>
Return nil for false values in tool calls. (#191)
This should fix https://github.com/ahyatt/llm/issues/173.
---
NEWS.org | 2 ++
README.org | 2 ++
llm-integration-test.el | 15 +++++++++++++++
llm-provider-utils-test.el | 12 ++++++++++++
llm-provider-utils.el | 22 +++++++++++++++++++++-
llm.el | 2 +-
6 files changed, 53 insertions(+), 2 deletions(-)
diff --git a/NEWS.org b/NEWS.org
index d7970e0278..3cdbd24dd9 100644
--- a/NEWS.org
+++ b/NEWS.org
@@ -1,3 +1,5 @@
+* Version 0.26.0
+- Call tools with =nil= when called with false JSON values
* Version 0.25.0
- Add =llm-ollama-authed= provider, which is like Ollama but takes a key.
- Set Gemini 2.5 Pro to be the default Gemini model
diff --git a/README.org b/README.org
index a510937688..4469a584bd 100644
--- a/README.org
+++ b/README.org
@@ -271,6 +271,8 @@ The various chat APIs will execute the functions defined in
=tools= slot with th
After the tool is called, the client could use the result, but if you want to
proceed with the conversation, or get a textual response that accompany the
function you should just send the prompt back with no modifications. This is
because the LLM gives the tool use to perform, and then expects to get back the
results of that tool use. The results were already executed at the end of the
call which returned the tools used, which also stores the result of that
execution in the prompt. Th [...]
+Tools will be called with vectors for array results, =nil= for false boolean
results, and plists for objects.
+
Be aware that there is no gaurantee that the tool will be called correctly.
While the LLMs mostly get this right, they are trained on Javascript functions,
so imitating Javascript names is recommended. So, "write_email" is a better
name for a function than "write-email".
Examples can be found in =llm-tester=. There is also a function call to
generate function calls from existing elisp functions in
=utilities/elisp-to-tool.el=.
diff --git a/llm-integration-test.el b/llm-integration-test.el
index bfc1631c95..8f7bfdd324 100644
--- a/llm-integration-test.el
+++ b/llm-integration-test.el
@@ -331,6 +331,21 @@ else. We really just want to see if it's in the right
ballpark."
;; Test that we can send the function back to the provider without error.
(llm-chat provider prompt))))
+(llm-def-integration-test llm-boolean-tool-use (provider)
+ (when (member 'tool-use (llm-capabilities provider))
+ (llm-chat provider (llm-make-chat-prompt
+ "Is Lyon the capital of France?"
+ :tools
+ (list (llm-make-tool
+ :function (lambda (result)
+ (should-not result))
+ :name "verifier"
+ :description "Test the LLM's decision on the
veracity of the statement."
+ :args '((:name "llm-decision"
+ :description "The decision on
the statement by the LLM."
+ :type boolean))
+ :async nil))))))
+
(llm-def-integration-test llm-tool-use-multi-output (provider)
(when (member 'tool-use (llm-capabilities provider))
(let* ((prompt (llm-integration-test-tool-use-prompt))
diff --git a/llm-provider-utils-test.el b/llm-provider-utils-test.el
index 06189ac8a6..faae8228cb 100644
--- a/llm-provider-utils-test.el
+++ b/llm-provider-utils-test.el
@@ -148,5 +148,17 @@
(should (equal '(:foo 3) (llm-provider-utils-streaming-accumulate '(:foo 1)
'(:foo 2))))
(should (equal '(:foo "foo bar baz")
(llm-provider-utils-streaming-accumulate '(:foo "foo bar") '(:foo " baz")))))
+(ert-deftest llm-provider-utils--normalize-args ()
+ (should-not (llm-provider-utils--normalize-args :false))
+ (should-not (llm-provider-utils--normalize-args :json-false))
+ (should (equal '(1 2 nil)
+ (llm-provider-utils--normalize-args '(1 2 :json-false))))
+ (should (equal [1 2 nil]
+ (llm-provider-utils--normalize-args [1 2 :json-false])))
+ (should (equal '(1 2 [t nil t])
+ (llm-provider-utils--normalize-args '(1 2 [t :false t]))))
+ (should (equal '(:a 1 :b nil)
+ (llm-provider-utils--normalize-args '(:a 1 :b :json-false)))))
+
(provide 'llm-provider-utils-test)
;;; llm-provider-utils-test.el ends here
diff --git a/llm-provider-utils.el b/llm-provider-utils.el
index 6fc1e113e5..1936744f2c 100644
--- a/llm-provider-utils.el
+++ b/llm-provider-utils.el
@@ -25,6 +25,7 @@
(require 'llm-request-plz)
(require 'llm-models)
(require 'seq)
+(require 'compat)
(cl-defstruct llm-standard-provider
"A struct indicating that this is a standard provider.
@@ -787,6 +788,24 @@ This transforms the plist so that:
value)
value))))
+(defun llm-provider-utils--normalize-args (args)
+ "Normalize ARGS to a form that can be passed to the user.
+
+This will convert all :json-false and :false values to nil."
+ (cond
+ ((vectorp args) (vconcat (mapcar #'llm-provider-utils--normalize-args
args)))
+ ((listp args) (mapcar #'llm-provider-utils--normalize-args args))
+ ((plistp args) (let (new-plist)
+ (map-do
+ (lambda (key value)
+ (setq new-plist
+ (plist-put new-plist
+ key
+ (llm-provider-utils--normalize-args
value))))
+ args)))
+ ((member args '(:json-false :false)) nil)
+ (t args)))
+
(defun llm-provider-utils-execute-tool-uses (provider prompt tool-uses
multi-output partial-result success-callback)
"Execute TOOL-USES, a list of `llm-provider-utils-tool-use'.
@@ -841,7 +860,8 @@ have returned results."
(if (llm-tool-async tool)
(apply (llm-tool-function tool)
(append (list end-func) call-args))
- (funcall end-func (apply (llm-tool-function tool) call-args)))))))
+ (funcall end-func (apply (llm-tool-function tool)
+ (llm-provider-utils--normalize-args
call-args))))))))
;; This is a useful method for getting out of the request buffer when it's time
diff --git a/llm.el b/llm.el
index 1008e12ee7..773be2ed18 100644
--- a/llm.el
+++ b/llm.el
@@ -4,7 +4,7 @@
;; Author: Andrew Hyatt <ahy...@gmail.com>
;; Homepage: https://github.com/ahyatt/llm
-;; Package-Requires: ((emacs "28.1") (plz "0.8") (plz-event-source "0.1.1")
(plz-media-type "0.2.1"))
+;; Package-Requires: ((emacs "28.1") (plz "0.8") (plz-event-source "0.1.1")
(plz-media-type "0.2.1") (compat "29.1"))
;; Package-Version: 0.25.0
;; SPDX-License-Identifier: GPL-3.0-or-later
;;
