branch: externals/parser-generator
commit edfb7b451a08d87cf4b6e103dfb32b2e6e0f3d35
Author: Christian Johansson <christ...@cvj.se>
Commit: Christian Johansson <christ...@cvj.se>
Moved lex-analyzer to separate file
---
Makefile | 8 ++-
parser-generator-lex-analyzer.el | 68 ++++++++++++++++++++++++
parser-generator.el | 20 +------
test/parser-generator-lex-analyzer-test.el | 84 ++++++++++++++++++++++++++++++
test/parser-generator-lr-test.el | 2 +
test/parser-generator-test.el | 2 +
6 files changed, 164 insertions(+), 20 deletions(-)
diff --git a/Makefile b/Makefile
index 0cd694e..999e63b 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ ifdef emacs
endif
EMACS_CMD := $(EMACS) -Q -batch -L . -L test/
-EL := parser-generator.el parser-generator-lr.el
test/parser-generator-test.el test/parser-generator-lr-test.el
+EL := parser-generator.el parser-generator-lex-analyzer.el
parser-generator-lr.el test/parser-generator-test.el
test/parser-generator-lex-analyzer-test.el test/parser-generator-lr-test.el
ELC := $(EL:.el=.elc)
.PHONY: clean
@@ -19,9 +19,13 @@ compile:
test:
$(EMACS_CMD) -l test/parser-generator-test.el -f "parser-generator-test"
+.PHONY: test-lex-analyzer
+test-lex-analyzer:
+ $(EMACS_CMD) -l test/parser-generator-lex-analyzer-test.el -f
"parser-generator-lex-analyzer-test"
+
.PHONY: test-lr
test-lr:
$(EMACS_CMD) -l test/parser-generator-lr-test.el -f
"parser-generator-lr-test"
.PHONY: tests
-tests: test test-lr
+tests: test test-lex-analyzer test-lr
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
new file mode 100644
index 0000000..dcd0db2
--- /dev/null
+++ b/parser-generator-lex-analyzer.el
@@ -0,0 +1,68 @@
+;;; parser-generator-lex-analyzer.el --- Lex-analyzer library -*-
lexical-binding: t -*-
+
+
+;;; Commentary:
+
+
+;;; Code:
+
+
+(require 'parser-generator)
+
+
+;;; Variables:
+
+
+(defvar parser-generator-lex-analyzer--function
+ nil
+ "Function used as lex-analyzer.")
+
+(defvar parser-generator-lex-analyzer--index
+ nil
+ "Index in lex-analyzer.")
+
+(defvar parser-generator-lex-analyzer--reset-function
+ nil
+ "Function used when resetting lex-analyzer.")
+
+
+;; Functions
+
+
+(defun parser-generator-lex-analyzer--peek-next-look-ahead ()
+ "Peek next look-ahead number of tokens via lex-analyzer."
+ (unless parser-generator-lex-analyzer--index
+ (error "Missing lex-analyzer index!"))
+ (unless parser-generator-lex-analyzer--function
+ (error "Missing lex-analyzer function!"))
+ (unless parser-generator--look-ahead-number
+ (error "Missing look-ahead number!"))
+ (funcall
+ parser-generator-lex-analyzer--function
+ parser-generator-lex-analyzer--index
+ parser-generator--look-ahead-number))
+
+(defun parser-generator-lex-analyzer--pop-token ()
+ "Pop next token via lex-analyzer."
+ (unless parser-generator-lex-analyzer--index
+ (error "Missing lex-analyzer index!"))
+ (unless parser-generator-lex-analyzer--function
+ (error "Missing lex-analyzer function!"))
+ (let ((token (funcall
+ parser-generator-lex-analyzer--function
+ parser-generator-lex-analyzer--index
+ 1)))
+ (setq parser-generator-lex-analyzer--index
+ (1+ parser-generator-lex-analyzer--index))
+ token))
+
+(defun parser-generator-lex-analyzer--reset ()
+ "Reset lex-analyzer."
+ (setq parser-generator-lex-analyzer--index 0)
+ (when parser-generator-lex-analyzer--reset-function
+ (funcall parser-generator-lex-analyzer--reset-function)))
+
+
+(provide 'parser-generator-lex-analyzer)
+
+;;; parser-generator-lex-analyzer.el ends here
diff --git a/parser-generator.el b/parser-generator.el
index 3467008..fd29fc1 100644
--- a/parser-generator.el
+++ b/parser-generator.el
@@ -9,6 +9,7 @@
;;; Variables:
+
(defvar parser-generator--allow-e-productions
nil
"Flag whether e-productions is allowed or not.")
@@ -33,10 +34,6 @@
nil
"Generated e-free F-sets for grammar.")
-(defvar parser-generator--lex-analyzer-function
- nil
- "Function used as lex-analyzer.")
-
(defvar parser-generator--look-ahead-number
nil
"Current look-ahead number used.")
@@ -293,20 +290,6 @@
(parser-generator--clear-cache)
(parser-generator--load-symbols))
-(defun parser-generator--load-next-look-ahead ()
- "Load next look-ahead number of tokens via lex-analyzer."
- (unless parser-generator--lex-analyzer-function
- (error "Missing lex-analyzer function!"))
- (let ((left parser-generator--look-ahead-number)
- (look-ahead))
- (while (> left 0)
- (let ((token (funcall parser-generator--lex-analyzer-function)))
- (if token
- (push token look-ahead)
- (push parser-generator--e-identifier look-ahead)))
- (setq left (1- left)))
- look-ahead))
-
(defun parser-generator--sort-list (a b)
"Return non-nil if a element in A is greater than a element in B in
lexicographic order."
(let ((length (min (length a) (length b)))
@@ -860,6 +843,7 @@
(setq follow-set (parser-generator--distinct follow-set)))
follow-set))
+
(provide 'parser-generator)
;;; parser-generator.el ends here
diff --git a/test/parser-generator-lex-analyzer-test.el
b/test/parser-generator-lex-analyzer-test.el
new file mode 100644
index 0000000..0638e47
--- /dev/null
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -0,0 +1,84 @@
+;;; parser-generator-lex-analyzer-test.el --- Tests for lex-analyzer -*-
lexical-binding: t -*-
+
+
+;;; Commentary:
+
+
+;;; Code:
+
+
+(require 'parser-generator-lex-analyzer)
+(require 'ert)
+
+(defun parser-generator-lex-analyzer-test--peek-next-look-ahead ()
+ "Test `parser-generator-lex-analyzer--peek-next-look-ahead'."
+ (message "Starting tests for
(parser-generator-lex-analyzer--peek-next-look-ahead)")
+
+ (setq parser-generator-lex-analyzer--index nil)
+ (setq parser-generator-lex-analyzer--function nil)
+ (setq parser-generator--look-ahead-number nil)
+
+ (should-error
+ (parser-generator-lex-analyzer--peek-next-look-ahead))
+ (setq parser-generator-lex-analyzer--function (lambda (index length)
(substring "abcdefghijklmnopqrstuvxz" index (+ index length))))
+ (should-error
+ (parser-generator-lex-analyzer--peek-next-look-ahead))
+ (parser-generator-lex-analyzer--reset)
+ (should-error
+ (parser-generator-lex-analyzer--peek-next-look-ahead))
+
+ (setq parser-generator--look-ahead-number 1)
+ (should
+ (equal
+ "a"
+ (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+ (message "Ended tests for
(parser-generator-lex-analyzer--peek-next-look-ahead)"))
+
+(defun parser-generator-lex-analyzer-test--pop-token ()
+ "Test `parser-generator-lex-analyzer--pop-token'."
+ (message "Starting tests for (parser-generator-lex-analyzer--pop-token)")
+
+ (setq parser-generator-lex-analyzer--index nil)
+ (setq parser-generator-lex-analyzer--function nil)
+ (setq parser-generator--look-ahead-number nil)
+
+ (should-error
+ (parser-generator-lex-analyzer--pop-token))
+ (setq
+ parser-generator-lex-analyzer--function
+ (lambda (index length)
+ (let* ((string "ab")
+ (string-length (length string)))
+ (when (> string-length index)
+ (if (>= string-length (+ index length))
+ (substring string index (+ index length))
+ (substring string index (1- string-length)))))))
+ (should-error
+ (parser-generator-lex-analyzer--pop-token))
+ (parser-generator-lex-analyzer--reset)
+
+ (should
+ (equal
+ "a"
+ (parser-generator-lex-analyzer--pop-token)))
+ (should
+ (equal
+ "b"
+ (parser-generator-lex-analyzer--pop-token)))
+ (should
+ (equal
+ nil
+ (parser-generator-lex-analyzer--pop-token)))
+
+ (message "Ended tests for (parser-generator-lex-analyzer--pop-token)"))
+
+(defun parser-generator-lex-analyzer-test ()
+ "Run test."
+ (parser-generator-lex-analyzer-test--peek-next-look-ahead)
+ (parser-generator-lex-analyzer-test--pop-token))
+
+
+(provide 'parser-generator-lex-analyzer-test)
+
+;;; parser-generator-lex-analyzer-test.el ends here
diff --git a/test/parser-generator-lr-test.el b/test/parser-generator-lr-test.el
index ac3d441..4252bbf 100644
--- a/test/parser-generator-lr-test.el
+++ b/test/parser-generator-lr-test.el
@@ -6,6 +6,7 @@
;;; Code:
+
(require 'parser-generator-lr)
(require 'ert)
@@ -244,6 +245,7 @@
(parser-generator-lr-test--generate-action-tables)
(parser-generator-lr-test--parse))
+
(provide 'parser-generator-lr-test)
;;; parser-generator-lr-test.el ends here
diff --git a/test/parser-generator-test.el b/test/parser-generator-test.el
index 824bced..554ed82 100644
--- a/test/parser-generator-test.el
+++ b/test/parser-generator-test.el
@@ -6,6 +6,7 @@
;;; Code:
+
(require 'parser-generator)
(require 'ert)
@@ -538,6 +539,7 @@
(parser-generator-test--e-free-first)
(parser-generator-test--follow))
+
(provide 'parser-generator-test)
;;; parser-generator-test.el ends here
