branch: externals/plz-media-type
commit 6ad13f4320cbf57e5f81a9cbee4a012ba29ae20a
Author: Roman Scherer <[email protected]>
Commit: r0man <[email protected]>
Don't fail on blank lines when parsing application/x-ndjson
---
plz-media-type.el | 25 ++++++---
tests/plz-media-type-test.el | 2 +
.../application/x-ndjson/ollama-blank-lines.txt | 37 +++++++++++++
.../application/x-ndjson/ollama-broken.txt | 9 ++++
tests/test-plz-media-type.el | 62 +++++++++++++++++++++-
5 files changed, 128 insertions(+), 7 deletions(-)
diff --git a/plz-media-type.el b/plz-media-type.el
index 2bd2afc43d..bcf330e54c 100644
--- a/plz-media-type.el
+++ b/plz-media-type.el
@@ -277,6 +277,15 @@ STRING which is output just received from the process."
(when moving
(goto-char (process-mark process)))))))
+(defconst plz-media-type--blank-line-regexp
+ (rx (+ space) (or "\r\n" "\n" "\r"))
+ "Regular expression matching a blank line.")
+
+(defun plz-media-type--delete-blank-lines ()
+ "Delete the next blank lines following point."
+ (while (looking-at plz-media-type--blank-line-regexp)
+ (delete-region (match-beginning 0) (match-end 0))))
+
;; Content Type: application/octet-stream
(defclass plz-media-type:application/octet-stream (plz-media-type)
@@ -461,7 +470,8 @@ will always be set to nil.")
"Parse a single line of the newline delimited JSON MEDIA-TYPE."
(when (looking-at plz-media-type:application/x-ndjson--line-regexp)
(prog1 (plz-media-type--parse-json-object media-type)
- (delete-region (match-beginning 0) (match-end 0)))))
+ (when (< (match-beginning 0) (match-end 0))
+ (delete-region (match-beginning 0) (match-end 0))))))
(defun plz-media-type:application/x-ndjson--parse-stream (media-type)
"Parse all lines of the newline delimited JSON MEDIA-TYPE in the PROCESS
buffer."
@@ -470,11 +480,14 @@ will always be set to nil.")
(unless plz-media-type--position
(setq-local plz-media-type--position (point)))
(goto-char plz-media-type--position)
- (when-let (object (plz-media-type:application/x-ndjson--parse-line
media-type))
- (while object
- (setq-local plz-media-type--position (point))
- (push object objects)
- (setq object (plz-media-type:application/x-ndjson--parse-line
media-type))))
+ (plz-media-type--delete-blank-lines)
+ (condition-case nil
+ (when-let (object (plz-media-type:application/x-ndjson--parse-line
media-type))
+ (while object
+ (setq-local plz-media-type--position (point))
+ (push object objects)
+ (setq object (plz-media-type:application/x-ndjson--parse-line
media-type))))
+ (json-end-of-file))
objects)))
(cl-defmethod plz-media-type-process
diff --git a/tests/plz-media-type-test.el b/tests/plz-media-type-test.el
index 7c36aae39f..5ec0c1df74 100644
--- a/tests/plz-media-type-test.el
+++ b/tests/plz-media-type-test.el
@@ -63,8 +63,10 @@ If running httpbin locally, set to \"http://localhost\".";)
;; that something funny is going on...
(cl-loop for i upto times ;; 10 seconds
while (equal 'run (process-status process))
+ ;; TODO: sleep-for or sit-for?
do (sleep-for seconds))))
+
(cl-defmacro plz-deftest (name () &body docstring-keys-and-body)
"Like `ert-deftest', but defines tests for both HTTP/1.1 and HTTP/2.
Also defines local function `url' which returns its argument
diff --git a/tests/response/application/x-ndjson/ollama-blank-lines.txt
b/tests/response/application/x-ndjson/ollama-blank-lines.txt
new file mode 100644
index 0000000000..1bd5835041
--- /dev/null
+++ b/tests/response/application/x-ndjson/ollama-blank-lines.txt
@@ -0,0 +1,37 @@
+HTTP/1.1 200 OK
+Content-Type: application/x-ndjson
+Date: Tue, 12 Mar 2024 12:05:13 GMT
+Transfer-Encoding: chunked
+
+{"model":"llama2","created_at":"2024-03-12T12:05:13.747334659Z","response":"Hello","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:13.814191426Z","response":"
there","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:13.880926587Z","response":"!","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:13.947866055Z","response":"
It","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.015054376Z","response":"'","done":false}
+
+
+{"model":"llama2","created_at":"2024-03-12T12:05:14.082471215Z","response":"s","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.148577108Z","response":"
nice","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.214802148Z","response":"
to","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.281459481Z","response":"
meet","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.350610212Z","response":"
you","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.419490326Z","response":".","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.486487527Z","response":"
Is","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.553190097Z","response":"
there","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.623595043Z","response":"
something","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.694458171Z","response":"
I","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.76547139Z","response":"
can","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.833659175Z","response":"
help","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.903078162Z","response":"
you","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:14.97368534Z","response":"
with","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:15.046102396Z","response":"
or","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:15.117115422Z","response":"
would","done":false}
+
+
+
+{"model":"llama2","created_at":"2024-03-12T12:05:15.18784764Z","response":"
you","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:15.259555212Z","response":"
like","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:15.328392358Z","response":"
to","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:15.398189056Z","response":"
chat","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:15.467785437Z","response":"?","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:15.535938819Z","response":"","done":true,"context":[518,25580,29962,3532,14816,29903,29958,5299,829,14816,29903,6778,13,13,10994,518,29914,25580,29962,13,10994,727,29991,739,29915,29879,7575,304,5870,366,29889,1317,727,1554,306,508,1371,366,411,470,723,366,763,304,13563,29973],"total_duration":3569916695,"load_duration":782537698,"prompt_eval_count":21,"prompt_eval_duration":998427000,"eval_count":27,"eval_duration":1788606000}
diff --git a/tests/response/application/x-ndjson/ollama-broken.txt
b/tests/response/application/x-ndjson/ollama-broken.txt
new file mode 100644
index 0000000000..6f194344e3
--- /dev/null
+++ b/tests/response/application/x-ndjson/ollama-broken.txt
@@ -0,0 +1,9 @@
+HTTP/1.1 200 OK
+Content-Type: application/x-ndjson
+Date: Tue, 12 Mar 2024 12:05:13 GMT
+Transfer-Encoding: chunked
+
+{"model":"llama2","created_at":"2024-03-12T12:05:13.747334659Z","response":"Hello","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:13.814191426Z","response":"
there","done":false}
+{"model":"llama2","created_at":"2024-03-12T12:05:13.880926587Z","response":"!","done":false}
+{"model":"llama2","created_at":
diff --git a/tests/test-plz-media-type.el b/tests/test-plz-media-type.el
index 07b7cb2d75..374f77e695 100644
--- a/tests/test-plz-media-type.el
+++ b/tests/test-plz-media-type.el
@@ -226,13 +226,73 @@
(response . "Hello")
(done . :json-false))
(seq-elt objects 26)))
- ;; TODO: Fix parsing of last line :/
(should (equal '((model . "llama2")
(created_at . "2024-03-12T12:05:15.467785437Z")
(response . "?")
(done . :json-false))
(seq-elt objects 1))))))
+(ert-deftest
test-plz-media-type-request:application/x-ndjson:ollama-blank-lines ()
+ (plz-media-type-test-with-mock-response (plz-media-type-test-response
"application/x-ndjson/ollama-blank-lines.txt")
+ (let* ((else) (finally) (then) (objects)
+ (process (plz-media-type-request 'get "MOCK-URL"
+ :as `(media-types ((application/x-ndjson
+ .
,(plz-media-type:application/x-ndjson
+ :handler (lambda (object) (push
object objects))))))
+ :else (lambda (object) (push object else))
+ :finally (lambda () (push t finally))
+ :then (lambda (object) (push object then)))))
+ (plz-media-type-test-wait process)
+ (should (null else))
+ (should (equal '(t) finally))
+ (should (equal 1 (length then)))
+ (seq-doseq (response then)
+ (should (plz-response-p response))
+ (should (equal 200 (plz-response-status response)))
+ (should (null (plz-response-body response))))
+ (should (equal 27 (length objects)))
+ (should (equal '((model . "llama2")
+ (created_at . "2024-03-12T12:05:13.747334659Z")
+ (response . "Hello")
+ (done . :json-false))
+ (seq-elt objects 26)))
+ (should (equal '((model . "llama2")
+ (created_at . "2024-03-12T12:05:15.467785437Z")
+ (response . "?")
+ (done . :json-false))
+ (seq-elt objects 1))))))
+
+(ert-deftest test-plz-media-type-request:application/x-ndjson:ollama-broken ()
+ (plz-media-type-test-with-mock-response (plz-media-type-test-response
"application/x-ndjson/ollama-broken.txt")
+ (let* ((else) (finally) (then) (objects)
+ (process (plz-media-type-request 'get "MOCK-URL"
+ :as `(media-types ((application/x-ndjson
+ .
,(plz-media-type:application/x-ndjson
+ :handler (lambda (object)
+ (push object
objects))))))
+ :else (lambda (object) (push object else))
+ :finally (lambda () (push t finally))
+ :then (lambda (object) (push object then)))))
+ (plz-media-type-test-wait process)
+ (should (null else))
+ (should (equal '(t) finally))
+ (should (equal 1 (length then)))
+ (seq-doseq (response then)
+ (should (plz-response-p response))
+ (should (equal 200 (plz-response-status response)))
+ (should (null (plz-response-body response))))
+ (should (equal 3 (length objects)))
+ (should (equal '((model . "llama2")
+ (created_at . "2024-03-12T12:05:13.747334659Z")
+ (response . "Hello")
+ (done . :json-false))
+ (seq-elt objects 2)))
+ (should (equal '((model . "llama2")
+ (created_at . "2024-03-12T12:05:13.880926587Z")
+ (response . "!")
+ (done . :json-false))
+ (seq-elt objects 0))))))
+
(ert-deftest test-plz-media-type-request:application/x-ndjson:proxy-http ()
(plz-media-type-test-with-mock-response (plz-media-type-test-response
"application/x-ndjson/proxy-http.txt")
(let* ((else) (finally) (then) (objects)
