branch: externals/parser-generator
commit 0416ca9ed4549cfd2c1b281763878e940fc21698
Author: Christian Johansson <christ...@cvj.se>
Commit: Christian Johansson <christ...@cvj.se>
Added information about lex-analyzer in README
---
README.md | 70 +++++++++++++++++++++++++++++-
test/parser-generator-lex-analyzer-test.el | 48 +++++++++++++++-----
2 files changed, 107 insertions(+), 11 deletions(-)
diff --git a/README.md b/README.md
index cbff6e6..7a87dbe 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,75 @@ This is just started, so most stuff are *WIP*.
## Lexical Analysis
-We use a regular-language based lexical analyzer that can be implemented by a
finite-state-machine (FSM).
+We use a regular-language based lexical analyzer that can be implemented by a
finite-state-machine (FSM). Set lexical analysis function by setting variable
`parser-generator-lex-analyzer--function`. Optionally set reset function by
setting variable `parser-generator-lex-analyzer--reset-function`. The lexical
analysis is indexed on variable `parser-generator-lex-analyzer--index`. All
parsers expect a list of tokens as response from lexical-analysis.
+
+### Peek next look-ahead
+
+Returns the look-ahead number of next terminals in stream.
+
+``` emacs-lisp
+(require 'ert)
+(setq
+ parser-generator-lex-analyzer--function
+ (lambda (index length)
+ (let* ((string '(a a b b b))
+ (string-length (length string))
+ (max-index (+ index length))
+ (tokens))
+ (while (and
+ (< index string-length)
+ (< index max-index))
+ (push (nth index string) tokens)
+ (setq index (1+ index)))
+ (nreverse tokens))))
+(parser-generator-lex-analyzer--reset)
+(setq parser-generator--look-ahead-number 1)
+ (should
+ (equal
+ '(a)
+ (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+ (setq parser-generator--look-ahead-number 2)
+ (should
+ (equal
+ '(a b)
+ (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+```
+
+### Pop token
+
+Returns the next token in stream and moves index one forward.
+
+``` emacs-lisp
+(require 'ert)
+(setq
+ parser-generator-lex-analyzer--function
+ (lambda (index length)
+ (let* ((string '(a b))
+ (string-length (length string))
+ (max-index (+ index length))
+ (tokens))
+ (while (and
+ (< index string-length)
+ (< index max-index))
+ (push (nth index string) tokens)
+ (setq index (1+ index)))
+ (nreverse tokens))))
+(parser-generator-lex-analyzer--reset)
+(should
+ (equal
+ '(a)
+ (parser-generator-lex-analyzer--pop-token)))
+ (should
+ (equal
+ '(b)
+ (parser-generator-lex-analyzer--pop-token)))
+ (should
+ (equal
+ nil
+ (parser-generator-lex-analyzer--pop-token)))
+```
## Syntax Analysis / Parsing
diff --git a/test/parser-generator-lex-analyzer-test.el
b/test/parser-generator-lex-analyzer-test.el
index 0638e47..f68722c 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -20,7 +20,19 @@
(should-error
(parser-generator-lex-analyzer--peek-next-look-ahead))
- (setq parser-generator-lex-analyzer--function (lambda (index length)
(substring "abcdefghijklmnopqrstuvxz" index (+ index length))))
+ (setq
+ parser-generator-lex-analyzer--function
+ (lambda (index length)
+ (let* ((string '(a b c d))
+ (string-length (length string))
+ (max-index (+ index length))
+ (tokens))
+ (while (and
+ (< index string-length)
+ (< index max-index))
+ (push (nth index string) tokens)
+ (setq index (1+ index)))
+ (nreverse tokens))))
(should-error
(parser-generator-lex-analyzer--peek-next-look-ahead))
(parser-generator-lex-analyzer--reset)
@@ -30,7 +42,19 @@
(setq parser-generator--look-ahead-number 1)
(should
(equal
- "a"
+ '(a)
+ (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+ (setq parser-generator--look-ahead-number 2)
+ (should
+ (equal
+ '(a b)
+ (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+ (setq parser-generator--look-ahead-number 10)
+ (should
+ (equal
+ '(a b c d)
(parser-generator-lex-analyzer--peek-next-look-ahead)))
(message "Ended tests for
(parser-generator-lex-analyzer--peek-next-look-ahead)"))
@@ -48,23 +72,27 @@
(setq
parser-generator-lex-analyzer--function
(lambda (index length)
- (let* ((string "ab")
- (string-length (length string)))
- (when (> string-length index)
- (if (>= string-length (+ index length))
- (substring string index (+ index length))
- (substring string index (1- string-length)))))))
+ (let* ((string '(a b))
+ (string-length (length string))
+ (max-index (+ index length))
+ (tokens))
+ (while (and
+ (< index string-length)
+ (< index max-index))
+ (push (nth index string) tokens)
+ (setq index (1+ index)))
+ (nreverse tokens))))
(should-error
(parser-generator-lex-analyzer--pop-token))
(parser-generator-lex-analyzer--reset)
(should
(equal
- "a"
+ '(a)
(parser-generator-lex-analyzer--pop-token)))
(should
(equal
- "b"
+ '(b)
(parser-generator-lex-analyzer--pop-token)))
(should
(equal
