branch: externals/parser-generator
commit 9073a387ba81a02a0254e35d0e09c20e63a701d2
Author: Christian Johansson <christ...@cvj.se>
Commit: Christian Johansson <christ...@cvj.se>
More fixes for state-based lexer
---
TODO.md | 8 +++-----
docs/Lexical-Analysis.md | 8 ++++----
parser-generator-lex-analyzer.el | 2 +-
test/parser-generator-lex-analyzer-test.el | 10 +++++-----
test/parser-generator-lr-test.el | 2 +-
5 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/TODO.md b/TODO.md
index b72f9c8b94..65054c83de 100644
--- a/TODO.md
+++ b/TODO.md
@@ -11,11 +11,9 @@ Functions (with validations) to set global variables:
## Lex-analyzer
* State-based lex-analyzer
-* Added unit test for state-based lex-analyzer
-* Change lex-function return value count
-* Verify that parser-generator-lex-analyzer--index is used in exported
lex-analyzers
-* Verify that parser-generator-lex-analyzer--state-init is used in exported
lex-analyzers
-* Use buffer when lexing in exported parsers as well
+** Verify that parser-generator-lex-analyzer--index is used in exported
lex-analyzers
+** Verify that parser-generator-lex-analyzer--state-init is used in exported
lex-analyzers
+** Use buffered lexer when lexing in exported parsers as well
## LR-Parser
diff --git a/docs/Lexical-Analysis.md b/docs/Lexical-Analysis.md
index 842c19a02f..aae2dacbd0 100644
--- a/docs/Lexical-Analysis.md
+++ b/docs/Lexical-Analysis.md
@@ -4,7 +4,7 @@ Set lexical analysis function by setting variable
`parser-generator-lex-analyzer
The lexical analysis is internally indexed on a local variable
`parser-generator-lex-analyzer--index` and has it optional state in the local
variable `parser-generation-lex-analyzer--state`. The initial values for the
index and state can be set in variables
`parser-generation-lex-analyzer--index-init` and
`parser-generator-lex-analyzer--state-init`.
-All parsers expect a list as response from lexical-analysis, the first item in
the list should be a list of one or more tokens. The second is "move
index"-flag, if it is non-nil it is expected to be a integer representing the
index to temporarily move the index to and perform a new lex. Third item is not
used. The fourth item is the new state after the lex.
+All parsers expect a list as response from the lexical-analysis, the first
item in the list should be a list of one or more tokens. The second is "move
index"-flag, if it is non-nil it is expected to be a integer representing the
index to temporarily move the index to and perform a new lex at. The third item
is the new state after the lex. Return values 2 and 3 are optional.
To enable exporting, the functions need to be specified in a way that the
entire body is within the same block, do that using `(let)` or `(progn)` for
example.
@@ -21,7 +21,7 @@ To enable exporting, the functions need to be specified in a
way that the entire
(< (1- index) max-index))
(push (nth (1- index) string) tokens)
(setq index (1+ index)))
- (list tokens nil nil nil))))
+ (list tokens))))
```
## Token
@@ -62,7 +62,7 @@ Returns the look-ahead number of next terminals in stream, if
end of stream is r
(setq new-index (cdr (cdr (nth (1- index) string))))
(push next-token tokens)
(setq index (1+ index)))
- (list (nreverse tokens) nil nil nil))))
+ (list (nreverse tokens)))))
(parser-generator-lex-analyzer--reset)
(setq parser-generator--look-ahead-number 1)
@@ -104,7 +104,7 @@ Returns the next token in stream and moves the lexical
analyzer index one point
(setq new-index (cdr (cdr (nth (1- index) string))))
(push (nth (1- index) string) tokens)
(setq index (1+ index)))
- (list (nreverse tokens) nil nil nil))))
+ (list (nreverse tokens)))))
(parser-generator-lex-analyzer--reset)
(setq parser-generator--look-ahead-number 1)
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index 52a2e841a9..474139ccbe 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -212,7 +212,7 @@
(move-to-index-flag
(nth 1 result-list))
(new-state
- (nth 3 result-list)))
+ (nth 2 result-list)))
(if move-to-index-flag
(progn
(setq
diff --git a/test/parser-generator-lex-analyzer-test.el
b/test/parser-generator-lex-analyzer-test.el
index eac8a1eb69..df3f674671 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -38,7 +38,7 @@
(setq new-index (cdr (cdr (nth (1- index) string))))
(push next-token tokens)
(setq index (1+ index)))
- (list (nreverse tokens) nil new-index nil))))
+ (list (nreverse tokens)))))
(should-error
(parser-generator-lex-analyzer--peek-next-look-ahead))
(parser-generator-lex-analyzer--reset)
@@ -84,7 +84,7 @@
(setq new-index (cdr (cdr (nth (1- index) string))))
(push next-token tokens)
(setq index (1+ index)))
- (list (nreverse tokens) nil new-index nil))))
+ (list (nreverse tokens)))))
(parser-generator-lex-analyzer--reset)
(should-error
@@ -132,7 +132,7 @@
(setq index (1+ index)))
(t
(error "Invalid state: %s" state))))
- (list (nreverse tokens) nil nil new-state))))
+ (list (nreverse tokens) nil new-state))))
(parser-generator-lex-analyzer--reset)
(should
(equal
@@ -167,7 +167,7 @@
(push (nth (1- index) string) tokens)
(setq new-index (cdr (cdr (nth (1- index) string))))
(setq index (1+ index)))
- (list (nreverse tokens) nil new-index nil))))
+ (list (nreverse tokens)))))
(should-error
(parser-generator-lex-analyzer--pop-token))
@@ -231,7 +231,7 @@
(setq index (1+ index)))
(t
(error "Invalid state: %s" state))))
- (list (nreverse tokens) nil nil new-state))))
+ (list (nreverse tokens) nil new-state))))
(parser-generator-lex-analyzer--reset)
(should
(equal
diff --git a/test/parser-generator-lr-test.el b/test/parser-generator-lr-test.el
index 8941bf9d32..687a493a19 100644
--- a/test/parser-generator-lr-test.el
+++ b/test/parser-generator-lr-test.el
@@ -353,7 +353,7 @@
token
`(,symbol ,(match-beginning 0) . ,(match-end 0)))))
(t (error "Unexpected input at %d!" index)))
- (list token nil nil nil) )))))
+ (list token))))))
(setq
parser-generator-lex-analyzer--get-function
