branch: externals/parser-generator
commit 48bc7edaa35a4c55e216856c9fbb35f1e8006954
Author: Christian Johansson <christ...@cvj.se>
Commit: Christian Johansson <christ...@cvj.se>
More work on buffered lexer
---
TODO.md | 1 +
parser-generator-lex-analyzer.el | 80 ++++++++++++++++++------------
test/parser-generator-lex-analyzer-test.el | 3 +-
test/parser-generator-lr-test.el | 11 +++-
4 files changed, 59 insertions(+), 36 deletions(-)
diff --git a/TODO.md b/TODO.md
index 3bb48e06b6..9834a9a305 100644
--- a/TODO.md
+++ b/TODO.md
@@ -14,6 +14,7 @@ Functions (with validations) to set global variables:
* Verify that parser-generator-lex-analyzer--index is used in exported
lex-analyzers
* Verify that parser-generator-lex-analyzer--state-init is used in exported
lex-analyzers
* Use buffer when lexing in case more tokens are return than what are needed
+* Use buffer when lexing in exported parsers as well
## LR-Parser
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index f4ef62fa68..6f4878b2e9 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -133,22 +133,15 @@
(nth 0 result-list))
(new-index
(nth 2 result-list)))
- (if token
- (progn
- (push
- token
- look-ahead)
- (setq
- look-ahead-length
- (1+ look-ahead-length))
- (setq
- index
- new-index))
-
- ;; Fill up look-ahead with EOF-identifier if we found nothing
- (push (list parser-generator--eof-identifier) look-ahead)
- (setq look-ahead-length (1+ look-ahead-length))
- (setq index (1+ index)))))
+ (push
+ token
+ look-ahead)
+ (setq
+ look-ahead-length
+ (1+ look-ahead-length))
+ (setq
+ index
+ new-index)))
(nreverse look-ahead)))
@@ -174,9 +167,9 @@
(setq-local
parser-generator-lex-analyzer--state
new-state)
- (if token
- (list token)
- nil)))
+ (if (equal token (list parser-generator--eof-identifier))
+ nil
+ (list token))))
(defun parser-generator-lex-analyzer--reset ()
"Reset lex-analyzer."
@@ -228,31 +221,52 @@
(if tokens
- (unless (listp (car tokens))
- (setq tokens (list tokens)))
+ (progn
+ (unless (listp (car tokens))
+ (setq tokens (list tokens)))
+
+ (let* ((first-token (car tokens))
+ (first-token-start (car (cdr first-token)))
+ (first-token-end (cdr (cdr first-token))))
+ (when (< index first-token-start)
+ (let ((token-start index))
+ (while (< token-start first-token-start)
+ (puthash
+ token-start
+ (list
+ first-token
+ nil
+ first-token-end
+ nil)
+
parser-generator-lex-analyzer--buffered-response)
+ (setq
+ token-start
+ (1+ token-start))))))
+
+ (dolist (token tokens)
+ (let ((token-start (car (cdr token)))
+ (token-end (cdr (cdr token))))
+ (puthash
+ token-start
+ (list token nil token-end new-state)
+
parser-generator-lex-analyzer--buffered-response))))
;; Fill up look-ahead with EOF-identifier if we found
nothing
- (push
- (list parser-generator--eof-identifier)
- tokens))
-
- (dolist (token tokens)
- (let ((token-start (car (cdr token)))
- (token-end (cdr (cdr token))))
- (puthash
- token-start
- (list token nil token-end new-state)
- parser-generator-lex-analyzer--buffered-response)))
+ (puthash
+ index
+ (list (list parser-generator--eof-identifier) nil (1+
index) nil)
+ parser-generator-lex-analyzer--buffered-response))
(setq
continue
nil))))
+
(error
(error
"Lex-analyze failed to get next token at: %s in state: %s, error:
%s"
index
state
- (car (cdr error))))))))
+ error))))))
(gethash
index
diff --git a/test/parser-generator-lex-analyzer-test.el
b/test/parser-generator-lex-analyzer-test.el
index 43bbd7a89a..48bfb640fb 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -119,12 +119,11 @@
(list (nreverse tokens) nil new-index nil))))
(should-error
(parser-generator-lex-analyzer--pop-token))
- (parser-generator-lex-analyzer--reset)
(message "Passed failing lex analysis 2")
- (parser-generator-lex-analyzer--reset)
(setq parser-generator--look-ahead-number 1)
+ (parser-generator-lex-analyzer--reset)
(should
(equal
diff --git a/test/parser-generator-lr-test.el b/test/parser-generator-lr-test.el
index 4784455fe5..8941bf9d32 100644
--- a/test/parser-generator-lr-test.el
+++ b/test/parser-generator-lr-test.el
@@ -353,7 +353,7 @@
token
`(,symbol ,(match-beginning 0) . ,(match-end 0)))))
(t (error "Unexpected input at %d!" index)))
- (list token nil new-index nil) )))))
+ (list token nil nil nil) )))))
(setq
parser-generator-lex-analyzer--get-function
@@ -562,6 +562,15 @@
translate)))
(message "Passed correct precedence of 10/5+2 => (10/5)+2 = 4")
+ (kill-region (point-min) (point-max))
+ (insert "-55\n")
+ (let ((translate (parser-generator-lr-translate)))
+ (should
+ (equal
+ -55.0
+ translate)))
+ (message "Passed correct precedence of -55")
+
(kill-region (point-min) (point-max))
(insert "- 55\n")
(let ((translate (parser-generator-lr-translate)))
