branch: externals/parser-generator
commit 096436cccd98c098bfb92272900518007b980117
Author: Christian Johansson <christ...@cvj.se>
Commit: Christian Johansson <christ...@cvj.se>
Added unit test for state-based lexer
---
TODO.md | 1 -
parser-generator-lex-analyzer.el | 9 ++-
test/parser-generator-lex-analyzer-test.el | 124 ++++++++++++++++++++++++++++-
3 files changed, 130 insertions(+), 4 deletions(-)
diff --git a/TODO.md b/TODO.md
index bc6e33d65e..b72f9c8b94 100644
--- a/TODO.md
+++ b/TODO.md
@@ -15,7 +15,6 @@ Functions (with validations) to set global variables:
* Change lex-function return value count
* Verify that parser-generator-lex-analyzer--index is used in exported
lex-analyzers
* Verify that parser-generator-lex-analyzer--state-init is used in exported
lex-analyzers
-* Use buffer when lexing in case more tokens are return than what are needed
* Use buffer when lexing in exported parsers as well
## LR-Parser
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index f3d4c323bc..52a2e841a9 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -132,7 +132,9 @@
(token
(nth 0 result-list))
(new-index
- (nth 2 result-list)))
+ (nth 2 result-list))
+ (new-state
+ (nth 3 result-list)))
(push
token
look-ahead)
@@ -141,7 +143,10 @@
(1+ look-ahead-length))
(setq
index
- new-index)))
+ new-index)
+ (setq
+ state
+ new-state)))
(nreverse look-ahead)))
diff --git a/test/parser-generator-lex-analyzer-test.el
b/test/parser-generator-lex-analyzer-test.el
index b14d3734b7..eac8a1eb69 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -88,7 +88,58 @@
(parser-generator-lex-analyzer--reset)
(should-error
- (parser-generator-lex-analyzer--peek-next-look-ahead))
+ (parser-generator-lex-analyzer--peek-next-look-ahead))
+
+ (setq parser-generator--look-ahead-number 6)
+ (setq
+ parser-generator-lex-analyzer--function
+ (lambda (index state)
+ (let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5) ("e" 5 .
6)))
+ (string-length (length string))
+ (max-index index)
+ (tokens)
+ (new-index)
+ (new-state)
+ (next-token))
+ (while (and
+ (< (1- index) string-length)
+ (< (1- index) max-index))
+ (setq next-token (nth (1- index) string))
+ (cond
+ ((equal state nil)
+ (if (string= (car next-token) "c")
+ (progn
+ (push next-token tokens)
+ (setq index (1+ index))
+ (setq next-token (nth (1- index) string))
+ (push next-token tokens)
+ (setq index (1+ index))
+ (setq new-state (list 'epislon)))
+ (if (string= (car next-token) "e")
+ (error "Invalid token: %s" next-token)
+ (setq next-token (nth (1- index) string))
+ (when (string= (car next-token) "d")
+ (error "Invalid token: %s" next-token))
+ (setq new-index (cdr (cdr (nth (1- index) string))))
+ (push next-token tokens)
+ (setq index (1+ index)))))
+ ((equal state (list 'epislon))
+ (setq next-token (nth (1- index) string))
+ (when (string= (car next-token) "d")
+ (error "Invalid token: %s" next-token))
+ (setq new-index (cdr (cdr (nth (1- index) string))))
+ (push next-token tokens)
+ (setq index (1+ index)))
+ (t
+ (error "Invalid state: %s" state))))
+ (list (nreverse tokens) nil nil new-state))))
+ (parser-generator-lex-analyzer--reset)
+ (should
+ (equal
+ '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5) ("e" 5 . 6) ($))
+ (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+ (message "Passed peek for state-based lexer")
(message "Ended tests for
(parser-generator-lex-analyzer--peek-next-look-ahead)"))
@@ -138,6 +189,77 @@
'(($))
(parser-generator-lex-analyzer--pop-token)))
+ (setq parser-generator--look-ahead-number 1)
+ (setq
+ parser-generator-lex-analyzer--function
+ (lambda (index state)
+ (let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5) ("e" 5 .
6)))
+ (string-length (length string))
+ (max-index index)
+ (tokens)
+ (new-index)
+ (new-state)
+ (next-token))
+ (while (and
+ (< (1- index) string-length)
+ (< (1- index) max-index))
+ (setq next-token (nth (1- index) string))
+ (cond
+ ((equal state nil)
+ (if (string= (car next-token) "c")
+ (progn
+ (push next-token tokens)
+ (setq index (1+ index))
+ (setq next-token (nth (1- index) string))
+ (push next-token tokens)
+ (setq index (1+ index))
+ (setq new-state (list 'epislon)))
+ (if (string= (car next-token) "e")
+ (error "Invalid token: %s" next-token)
+ (setq next-token (nth (1- index) string))
+ (when (string= (car next-token) "d")
+ (error "Invalid token: %s" next-token))
+ (setq new-index (cdr (cdr (nth (1- index) string))))
+ (push next-token tokens)
+ (setq index (1+ index)))))
+ ((equal state (list 'epislon))
+ (setq next-token (nth (1- index) string))
+ (when (string= (car next-token) "d")
+ (error "Invalid token: %s" next-token))
+ (setq new-index (cdr (cdr (nth (1- index) string))))
+ (push next-token tokens)
+ (setq index (1+ index)))
+ (t
+ (error "Invalid state: %s" state))))
+ (list (nreverse tokens) nil nil new-state))))
+ (parser-generator-lex-analyzer--reset)
+ (should
+ (equal
+ '(("a" 1 . 2))
+ (parser-generator-lex-analyzer--pop-token)))
+ (should
+ (equal
+ '(("b" 2 . 3))
+ (parser-generator-lex-analyzer--pop-token)))
+ (should
+ (equal
+ '(("c" 3 . 4))
+ (parser-generator-lex-analyzer--pop-token)))
+ (should
+ (equal
+ '(("d" 4 . 5))
+ (parser-generator-lex-analyzer--pop-token)))
+ (should
+ (equal
+ '(("e" 5 . 6))
+ (parser-generator-lex-analyzer--pop-token)))
+ (should
+ (equal
+ '(($))
+ (parser-generator-lex-analyzer--pop-token)))
+
+ (message "Passed pop for state-based lexer")
+
(message "Ended tests for (parser-generator-lex-analyzer--pop-token)"))
(defun parser-generator-lex-analyzer-test ()
