branch: externals/elisa
commit 594dd9efdb57ce76a1d82a731db979c941e5b39f
Merge: 322d7eb839 f43953b4d7
Author: Sergey Kostyaev <s-kosty...@users.noreply.github.com>
Commit: GitHub <nore...@github.com>
Merge pull request #31 from s-kostyaev/add-embeddings-recalculation
Add embeddings recalculation
---
README.org | 23 ++++++++++++++++++-----
elisa.el | 44 ++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 60 insertions(+), 7 deletions(-)
diff --git a/README.org b/README.org
index 3fc9d4003a..29af50a377 100644
--- a/README.org
+++ b/README.org
@@ -88,7 +88,7 @@ I prefer this models:
#+begin_src shell
ollama pull gemma2:9b-instruct-q6_K
ollama pull qwen2.5:3b
- ollama pull chatfire/bge-m3:q8_0
+ ollama pull snowflake-arctic-embed2
#+end_src
**** Complex documents
@@ -203,6 +203,12 @@ Disable all collections.
Removes collection and all its data from index.
+*** elisa-async-recalculate-embeddings
+
+Recalculate embeddings asynchronously. Use it after changing
+~elisa-embeddings-provider~ variable. Can take some time. Works faster
+with ~elisa-batch-embeddings-enabled~.
+
** Configuration
Example configuration.
@@ -222,7 +228,7 @@ Example configuration.
;; (setopt elisa-chat-provider
;; (make-llm-ollama
;; :chat-model "gemma2:9b-instruct-q6_K"
- ;; :embedding-model "chatfire/bge-m3:q8_0"
+ ;; :embedding-model "snowflake-arctic-embed2"
;; ;; set context window to 8k
;; :default-chat-non-standard-params '(("num_ctx" . 8192))))
;;
@@ -230,11 +236,13 @@ Example configuration.
(setopt elisa-chat-provider
(make-llm-ollama
:chat-model "qwen2.5:3b"
- :embedding-model "chatfire/bge-m3:q8_0"
+ :embedding-model "snowflake-arctic-embed2"
:default-chat-temperature 0.1
:default-chat-non-standard-params '(("num_ctx" . 32768))))
;; this embedding model has stong multilingual capabilities
- (setopt elisa-embeddings-provider (make-llm-ollama :embedding-model
"chatfire/bge-m3:q8_0"))
+ (setopt elisa-embeddings-provider (make-llm-ollama :embedding-model
"snowflake-arctic-embed2"))
+ ;; enable batch embeddings for faster processing
+ (setopt elisa-batch-embeddings-enabled t)
:config
;; searxng works better than duckduckgo in my tests
(setopt elisa-web-search-function 'elisa-search-searxng))
@@ -293,7 +301,7 @@ Example configuration.
+ ~elisa-semantic-split-function~:
* Type: Function
- * Description: Function used to split text into semantically meaningful
chunks.
+ * Description: Function used to split text into semantically meaningful
chunks.
* Default: ~elisa-split-by-paragraph~
+ ~elisa-prompt-rewriting-enabled~:
@@ -313,6 +321,11 @@ Example configuration.
* Type: Boolean
* Description: Enable batch embeddings if supported.
++ ~elisa-batch-size~:
+ * Type: Integer
+ * Description: Batch size to send to provider during batch embeddings
calculation.
+ * Default: 300
+
**** Web Search and Integration
+ ~elisa-searxng-url~:
diff --git a/elisa.el b/elisa.el
index 1dd57a9a96..415f7455fa 100644
--- a/elisa.el
+++ b/elisa.el
@@ -1,6 +1,6 @@
;;; elisa.el --- Emacs Lisp Information System Assistant -*- lexical-binding:
t -*-
-;; Copyright (C) 2024 Free Software Foundation, Inc.
+;; Copyright (C) 2024, 2025 Free Software Foundation, Inc.
;; Author: Sergey Kostyaev <sskosty...@gmail.com>
;; URL: http://github.com/s-kostyaev/elisa
@@ -107,6 +107,7 @@
(require 'shr)
(require 'plz)
(require 'json)
+(require 'sqlite)
(defgroup elisa nil
"RAG implementation for `ellama'."
@@ -279,6 +280,10 @@ If set, all quotes with similarity less than threshold
will be filtered out."
"Enable batch embeddings if supported."
:type 'boolean)
+(defcustom elisa-batch-size 300
+ "Batch size to send to provider during batch embeddings calculation."
+ :type 'integer)
+
(defun elisa-supported-complex-document-p (path)
"Check if PATH contain supported complex document."
(cl-find (file-name-extension path)
@@ -350,6 +355,10 @@ database."
(format "CREATE VIRTUAL TABLE IF NOT EXISTS data_embeddings USING
vss0(embedding(%d));"
(elisa-get-embedding-size)))
+(defun elisa-data-embeddings-drop-table-sql ()
+ "Generate sql for drop data embeddings table."
+ "DROP TABLE IF EXISTS data_embeddings;")
+
(defun elisa-data-fts-create-table-sql ()
"Generate sql for create full text search table."
"CREATE VIRTUAL TABLE IF NOT EXISTS data_fts USING FTS5(data);")
@@ -473,7 +482,8 @@ Return list of vectors."
(let ((provider elisa-embeddings-provider))
(if (and elisa-batch-embeddings-enabled
(member 'embeddings-batch (llm-capabilities provider)))
- (llm-batch-embeddings provider chunks)
+ (let ((batches (seq-partition chunks elisa-batch-size)))
+ (flatten-list (mapcar (lambda (batch) (llm-batch-embeddings provider
batch)) batches)))
(mapcar (lambda (chunk) (llm-embedding provider chunk)) chunks))))
(defun elisa-parse-info-manual (name collection-name)
@@ -1265,6 +1275,7 @@ Call ON-DONE callback with result as an argument after
FUNC evaluation done."
,(async-inject-variables "elisa-tar-executable")
,(async-inject-variables "elisa-prompt-rewriting-enabled")
,(async-inject-variables "elisa-batch-embeddings-enabled")
+ ,(async-inject-variables "elisa-batch-size")
,(async-inject-variables "elisa-rewrite-prompt-template")
,(async-inject-variables "elisa-semantic-split-function")
,(async-inject-variables
"elisa-webpage-extraction-function")
@@ -1492,5 +1503,34 @@ Find similar quotes in COLLECTIONS and add it to
context."
(let ((cols (or collections elisa-enabled-collections)))
(elisa--rewrite-prompt prompt (elisa--gen-chat cols))))
+(defun elisa-recalculate-embeddings ()
+ "Recalculate and save new embeddings after embedding provider change."
+ (sqlite-execute elisa-db "DELETE FROM data WHERE data = '';") ;; remove rows
without data
+ (let* ((data-rows (sqlite-select elisa-db "SELECT rowid, data FROM data;"))
+ (texts (mapcar #'cadr data-rows))
+ (rowids (mapcar #'car data-rows))
+ (embeddings (elisa-embeddings texts))
+ (len (length rowids))
+ (i 0))
+ ;; Recreate embeddings table
+ (sqlite-execute elisa-db (elisa-data-embeddings-drop-table-sql))
+ (sqlite-execute elisa-db (elisa-data-embeddings-create-table-sql))
+ ;; Recalculate embeddings
+ (with-sqlite-transaction elisa-db
+ (while (< i len)
+ (let ((rowid (nth i rowids))
+ (embedding (nth i embeddings)))
+ (sqlite-execute
+ elisa-db
+ (format "INSERT INTO data_embeddings(rowid, embedding) VALUES (%s,
%s);"
+ rowid (elisa-vector-to-sqlite embedding)))
+ (setq i (1+ i)))))))
+
+;;;###autoload
+(defun elisa-async-recalculate-embeddings ()
+ "Recalculate embeddings asynchronously."
+ (interactive)
+ (elisa--async-do 'elisa-recalculate-embeddings))
+
(provide 'elisa)
;;; elisa.el ends here.
