branch: elpa/hyperdrive
commit 063c2275d151a086ebf248f50288bd46a49367f9
Author: Joseph Turner <jos...@ushin.org>
Commit: Joseph Turner <jos...@ushin.org>
WIP: (-fill-version-ranges) Fill existent and nonexistent ranges
Two codepaths:
- fill existent entries in series
- fill nonexistent entries in parallel
Filling existent entries is logically recursive: fill entry, get its
range-start, then fill the entry at the range-end of the prior range.
Filling nonexistent entries is not recursive: In parallel, send
a request to every version of entry between
(hyperdrive-entry-version entry)
and
(- (hyperdrive-entry-version entry) hyperdrive-queue-size)
If the earliest requested version is existent, go back to the
"existent series" codepath. Otherwise, repeat the "nonexistent
parallel" codepath.
So good, so far.
The problem is that the finally callback passed to
`hyperdrive-fill-version-ranges' is not called in the correct place.
We check the earliest requested version at the end of the "nonexistent
parallel" codepath inside a separate queue finalizer for the codepath.
Since the "nonexistent parallel" async requests are not made in the
main queue used by "existent series" codepath, the main queue runs
whenever the "existent series" codepath hands control over to the
"nonexistent parallel" codepath.
What about multiple queues per request, each with its own finalizer?
---
hyperdrive-history.el | 1 +
hyperdrive-lib.el | 89 +++++++++++++++++++++++++++++++++++----------------
2 files changed, 63 insertions(+), 27 deletions(-)
diff --git a/hyperdrive-history.el b/hyperdrive-history.el
index 8ed090f8fd..b0fc207ca6 100644
--- a/hyperdrive-history.el
+++ b/hyperdrive-history.el
@@ -227,6 +227,7 @@ Universal prefix argument \\[universal-argument] forces
(setf (hyperdrive-entry-version range-end-entry) range-end)
(hyperdrive-fill-version-ranges range-end-entry
:finally (lambda ()
+ (message "FINALLY")
;; TODO: Should we open the history buffer for entry
;; or range-end-entry or...?
(hyperdrive-history entry)))))
diff --git a/hyperdrive-lib.el b/hyperdrive-lib.el
index faceaa7987..ecfc92201d 100644
--- a/hyperdrive-lib.el
+++ b/hyperdrive-lib.el
@@ -813,33 +813,68 @@ The QUEUE argument is used in recursive calls."
(unless queue
(setf queue (make-plz-queue :limit hyperdrive-queue-size
:finally (when finally finally))))
- (let ((copy-entry (hyperdrive-copy-tree entry t)))
- (hyperdrive-fill copy-entry
- :then (lambda (filled-entry)
- (when (cl-plusp limit)
- ;; Don't use `hyperdrive-entry-previous' here, since it makes
a sync request
- (pcase-let ((`(,range-start . ,_plist)
(hyperdrive-entry-version-range filled-entry)))
- (setf (hyperdrive-entry-version filled-entry) (1-
range-start))
- (when (eq 'unknown (hyperdrive-entry-exists-p filled-entry))
- ;; Recurse backward through history, filling unknown
entries.
- ;; Stop recursing at known nonexistent entry or at the
limit.
- (hyperdrive-fill-version-ranges filled-entry
- :limit (1- limit) :queue queue)))))
- :else (lambda (err)
- (pcase (plz-response-status (plz-error-response err))
- ;; FIXME: If plz-error is a curl-error, this block will fail.
- (404
- ;; ENTRY is known nonexistent: send LIMIT number of
- ;; parallel requests for entries before ENTRY's version.
- (cl-dotimes (i limit)
- (cl-decf (hyperdrive-entry-version copy-entry))
- (unless (and (cl-plusp (hyperdrive-entry-version
copy-entry))
- (eq 'unknown (hyperdrive-entry-exists-p
copy-entry)))
- (cl-return))
- (hyperdrive-fill-version-ranges copy-entry
- :limit 0 :queue queue)))
- (_ (signal (car err) (cdr err)))))
- :queue queue)))
+ (cl-labels ((fill-existent (entry limit)
+ ;; For existent entries, send requests in series.
+ (when (cl-plusp limit)
+ ;; Don't use `hyperdrive-entry-previous' here, since it
makes a sync request
+ (pcase-let ((`(,range-start . ,_plist)
(hyperdrive-entry-version-range entry)))
+ (setf (hyperdrive-entry-version entry) (1- range-start))
+ (when (eq 'unknown (hyperdrive-entry-exists-p entry))
+ ;; Recurse backward through history.
+ (hyperdrive-fill-version-ranges entry
+ :limit (1- limit) :queue queue)))))
+ (fill-nonexistent (copy-entry limit)
+ (let ((nonexistent-queue (make-plz-queue
+ :limit hyperdrive-queue-size
+ :finally (lambda ()
+ (let ((new-limit (- limit
hyperdrive-queue-size))
+
(last-requested-entry (hyperdrive-copy-tree entry t)))
+ (cl-incf
(hyperdrive-entry-version last-requested-entry))
+ ;; (message "ENTRY2: %s
%s" (hyperdrive-entry-version entry) (hyperdrive-entry-exists-p
last-requested-entry))
+ (if
(hyperdrive-entry-exists-p last-requested-entry)
+ (fill-existent
entry new-limit)
+ (fill-nonexistent
entry new-limit)))))))
+ ;; For nonexistent entries, send requests in parallel.
+ (cl-dotimes (i hyperdrive-queue-size)
+ ;; Send the maximum number of simultaneous requests.
+ (cl-decf (hyperdrive-entry-version entry))
+ ;; (message "ENTRY0: %s %s %s %s"
(hyperdrive-entry-version entry) (hyperdrive-entry-exists-p entry) limit i)
+ (unless (and (cl-plusp (hyperdrive-entry-version entry))
+ (eq 'unknown (hyperdrive-entry-exists-p
entry))
+ (> limit i))
+ ;; Stop at the beginning of the history, at a known
+ ;; existent/nonexistent entry, or at the limit.
+ (cl-return))
+ ;; (message "ENTRY1: %s %s" (hyperdrive-entry-version
entry) (hyperdrive-entry-exists-p entry))
+
+ (hyperdrive-fill (hyperdrive-copy-tree entry t)
+ ;; `hyperdrive-fill' is only used to fill the version
ranges;
+ ;; the filled-entry is thrown away.
+ :then (lambda (_filled-entry)
+ ;; (message "KNOWN-EXISTENT: %s"
(hyperdrive-entry-version filled-entry))
+ (message "THEN")
+ )
+ :else (lambda (err)
+ (message "KNOWN-NONEXISTENT: %s"
(hyperdrive-entry-version entry))
+ ;; TODO: Better error handling.
+ (pcase (plz-response-status (plz-error-response
err))
+ ;; FIXME: If plz-error is a curl-error, this
block will fail.
+ (404 nil)
+ (_ (signal (car err) (cdr err)))))
+ :queue nonexistent-queue)))))
+ (let ((copy-entry (hyperdrive-copy-tree entry t)))
+ (hyperdrive-fill copy-entry
+ ;; `hyperdrive-fill' is only used to fill the version ranges;
+ ;; the filled-entry is thrown away.
+ :then (lambda (_filled-entry)
+ (fill-existent copy-entry limit))
+ :else (lambda (err)
+ (pcase (plz-response-status (plz-error-response err))
+ ;; FIXME: If plz-error is a curl-error, this block will fail.
+ (404
+ (fill-nonexistent copy-entry limit))
+ (_ (signal (car err) (cdr err)))))
+ :queue queue))))
(defun hyperdrive-fill-metadata (hyperdrive)
"Fill HYPERDRIVE's public metadata and return it.
