branch: externals/phps-mode
commit d6caab9576d02b0513219e312440aa8a7781e1d6
Author: Christian Johansson <christ...@cvj.se>
Commit: Christian Johansson <christ...@cvj.se>
Added support for coalesce_equal token and made lexer structure more
similar to re2c
---
phps-mode-lexer.el | 272 ++++++++++++++++++++++++++++++++----------------
phps-mode-test-lexer.el | 8 +-
2 files changed, 191 insertions(+), 89 deletions(-)
diff --git a/phps-mode-lexer.el b/phps-mode-lexer.el
index 0c9b4d9..f7f1223 100644
--- a/phps-mode-lexer.el
+++ b/phps-mode-lexer.el
@@ -348,6 +348,7 @@
(string= token 'T_AND_EQUAL)
(string= token 'T_OR_EQUAL)
(string= token 'T_XOR_EQUAL)
+ (string= token 'T_COALESCE_EQUAL)
(string= token 'T_BOOLEAN_OR)
(string= token 'T_BOOLEAN_AND)
(string= token 'T_BOOLEAN_XOR)
@@ -424,85 +425,123 @@
(ST_VAR_OFFSET (= phps-mode-lexer-STATE
phps-mode-lexer-ST_VAR_OFFSET)))
(cond
- ;; ST_IN_SCRIPTING
((and ST_IN_SCRIPTING (looking-at (concat "exit"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_EXIT (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "die"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_DIE (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "function"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_FUNCTION (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "const"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_CONST (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "return"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_RETURN (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "yield"
phps-mode-lexer-WHITESPACE "from" "[^a-zA-Z0-9_\x80-\xff]")))
(phps-mode-lexer-RETURN_TOKEN 'T_YIELD_FROM (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at (concat "yield"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_YIELD (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "try"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_TRY (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "catch"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_CATCH (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "finally"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_FINALLY (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "throw"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_THROW (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "if"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_IF (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "elseif"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ELSEIF (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "endif"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ENDIF (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "else"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ELSE (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "while"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_WHILE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "endwhile"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ENDWHILE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "do"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_DO (match-beginning 0) (1- (match-end
0))))
- ((and ST_IN_SCRIPTING (looking-at (concat "foreach"
phps-mode-lexer-NOT-LABEL)))
- (phps-mode-lexer-RETURN_TOKEN 'T_FOREACH (match-beginning 0) (1-
(match-end 0))))
- ((and ST_IN_SCRIPTING (looking-at (concat "endforeach"
phps-mode-lexer-NOT-LABEL)))
- (phps-mode-lexer-RETURN_TOKEN 'T_ENDFOREACH (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "for"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_FOR (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "endfor"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ENDFOR (match-beginning 0) (1-
(match-end 0))))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat "foreach"
phps-mode-lexer-NOT-LABEL)))
+ (phps-mode-lexer-RETURN_TOKEN 'T_FOREACH (match-beginning 0) (1-
(match-end 0))))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat "endforeach"
phps-mode-lexer-NOT-LABEL)))
+ (phps-mode-lexer-RETURN_TOKEN 'T_ENDFOREACH (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "declare"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_DECLARE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "enddeclare"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ENDDECLARE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "instanceof"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_INSTANCEOF (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "as"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_AS (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "switch"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_SWITCH (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "endswitch"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ENDSWITCH (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "case"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_CASE (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "default"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_DEFAULT (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "break"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_BREAK (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "continue"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_CONTINUE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "goto"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_GOTO (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "echo"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ECHO (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "print"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_PRINT (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "class"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_CLASS (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "interface"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_INTERFACE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "trait"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_TRAIT (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "extends"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_EXTENDS (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "implements"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_IMPLEMENTS (match-beginning 0) (1-
(match-end 0))))
@@ -510,176 +549,271 @@
(phps-mode-lexer-yy_push_state phps-mode-lexer-ST_LOOKING_FOR_PROPERTY)
(phps-mode-lexer-RETURN_TOKEN 'T_OBJECT_OPERATOR (match-beginning 0)
(match-end 0)))
- ((and ST_IN_SCRIPTING (looking-at "\\?>\n?"))
- (let ((start (match-beginning 0))
- (end (match-end 0)))
- (when (= (- end start) 3)
- (setq end (1- end)))
- (phps-mode-lexer-BEGIN phps-mode-lexer-ST_INITIAL)
- (when phps-mode-lexer-PARSER_MODE
- (phps-mode-lexer-RETURN_TOKEN ";" start end))
- (phps-mode-lexer-RETURN_TOKEN 'T_CLOSE_TAG start end)))
-
- ;; HEREDOC and NOWDOC
- ((and ST_IN_SCRIPTING (looking-at (concat "<<<"
phps-mode-lexer-TABS_AND_SPACES "\\(" phps-mode-lexer-LABEL "\\|'"
phps-mode-lexer-LABEL "'\\|\"" phps-mode-lexer-LABEL "\"\\)"
phps-mode-lexer-NEWLINE)))
+ ((and (or ST_IN_SCRIPTING ST_LOOKING_FOR_PROPERTY)
+ (looking-at phps-mode-lexer-WHITESPACE))
(let* ((start (match-beginning 0))
(end (match-end 0))
- (data (buffer-substring-no-properties (match-beginning 1)
(match-end 1)))
- (heredoc_label))
-
- ;; Determine if it's HEREDOC or NOWDOC and extract label here
- (if (string= (substring data 0 1) "'")
- (progn
- (setq heredoc_label (substring data 1 (- (length data) 1)))
- (phps-mode-lexer-BEGIN phps-mode-lexer-ST_NOWDOC))
- (progn
- (if (string= (substring data 0 1) "\"")
- (setq heredoc_label (substring data 1 (- (length data) 1)))
- (setq heredoc_label data))
- (phps-mode-lexer-BEGIN phps-mode-lexer-ST_HEREDOC)))
+ (data (buffer-substring-no-properties start end)))
+ (if phps-mode-lexer-PARSER_MODE
+ (phps-mode-lexer-MOVE_FORWARD end)
+ (phps-mode-lexer-RETURN_TOKEN data start end))))
- ;; Check for ending label on the next line
- (when (string= (buffer-substring-no-properties end (+ end (length
heredoc_label))) heredoc_label)
- (phps-mode-lexer-BEGIN phps-mode-lexer-ST_END_HEREDOC))
+ ((and ST_LOOKING_FOR_PROPERTY (looking-at "->"))
+ (phps-mode-lexer-RETURN_TOKEN 'T_OBJECT_OPERATOR (match-beginning 0)
(match-end 0)))
- (push heredoc_label phps-mode-lexer-heredoc_label_stack)
- ;; (message "Found heredoc or nowdoc at %s with label %s" data
heredoc_label)
+ ((and ST_LOOKING_FOR_PROPERTY (looking-at phps-mode-lexer-LABEL))
+ (let ((start (match-beginning 0))
+ (end (match-end 0)))
+ (phps-mode-lexer-yy_pop_state)
+ (phps-mode-lexer-RETURN_TOKEN 'T_STRING start end)))
- (phps-mode-lexer-RETURN_TOKEN 'T_START_HEREDOC start end)))
+ ((and ST_LOOKING_FOR_PROPERTY (looking-at phps-mode-lexer-ANY_CHAR))
+ (let ((_start (match-beginning 0))
+ (end (match-end 0)))
+ (phps-mode-lexer-yy_pop_state)
+ ;; TODO goto restart here?
+ ;; (message "Restart here")
+ (phps-mode-lexer-MOVE_FORWARD end)))
((and ST_IN_SCRIPTING (looking-at "::"))
(phps-mode-lexer-RETURN_TOKEN 'T_PAAMAYIM_NEKUDOTAYIM (match-beginning
0) (match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "\\\\"))
(phps-mode-lexer-RETURN_TOKEN 'T_NS_SEPARATOR (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "\\.\\.\\."))
(phps-mode-lexer-RETURN_TOKEN 'T_ELLIPSIS (match-beginning 0) (match-end
0)))
- ((and ST_IN_SCRIPTING (looking-at "\\?\\?"))
- (phps-mode-lexer-RETURN_TOKEN 'T_COALESCE (match-beginning 0) (match-end
0)))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat "\\?\\?" "[^\\=]")))
+ (phps-mode-lexer-RETURN_TOKEN 'T_COALESCE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "new"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_NEW (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "clone"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_CLONE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "var"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_VAR (match-beginning 0) (1- (match-end
0))))
- ((and ST_IN_SCRIPTING (looking-at (concat "("
phps-mode-lexer-TABS_AND_SPACES "\\(integer\\|int\\)"
phps-mode-lexer-TABS_AND_SPACES ")")))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat "("
phps-mode-lexer-TABS_AND_SPACES "\\(int\\|integer\\)"
phps-mode-lexer-TABS_AND_SPACES ")")))
(phps-mode-lexer-RETURN_TOKEN 'T_INT_CAST (match-beginning 0) (match-end
0)))
+
((and ST_IN_SCRIPTING (looking-at (concat "("
phps-mode-lexer-TABS_AND_SPACES "\\(real\\|double\\|float\\)"
phps-mode-lexer-TABS_AND_SPACES ")")))
(phps-mode-lexer-RETURN_TOKEN 'T_DOUBLE_CAST (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at (concat "("
phps-mode-lexer-TABS_AND_SPACES "\\(string\\|binary\\)"
phps-mode-lexer-TABS_AND_SPACES ")")))
(phps-mode-lexer-RETURN_TOKEN 'T_STRING_CAST (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at (concat "("
phps-mode-lexer-TABS_AND_SPACES "array" phps-mode-lexer-TABS_AND_SPACES ")")))
(phps-mode-lexer-RETURN_TOKEN 'T_ARRAY_CAST (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at (concat "("
phps-mode-lexer-TABS_AND_SPACES "object" phps-mode-lexer-TABS_AND_SPACES ")")))
(phps-mode-lexer-RETURN_TOKEN 'T_OBJECT_CAST (match-beginning 0)
(match-end 0)))
- ((and ST_IN_SCRIPTING (looking-at (concat "("
phps-mode-lexer-TABS_AND_SPACES "\\(boolean\\|bool\\)"
phps-mode-lexer-TABS_AND_SPACES ")")))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat "("
phps-mode-lexer-TABS_AND_SPACES "\\(bool\\|boolean\\)"
phps-mode-lexer-TABS_AND_SPACES ")")))
(phps-mode-lexer-RETURN_TOKEN 'T_BOOL_CAST (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at (concat "("
phps-mode-lexer-TABS_AND_SPACES "unset" phps-mode-lexer-TABS_AND_SPACES ")")))
(phps-mode-lexer-RETURN_TOKEN 'T_UNSET_CAST (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at (concat "eval"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_EVAL (match-beginning 0) (1- (match-end
0))))
- ((and ST_IN_SCRIPTING (looking-at (concat "include_once"
phps-mode-lexer-NOT-LABEL)))
- (phps-mode-lexer-RETURN_TOKEN 'T_INCLUDE_ONCE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "include"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_INCLUDE (match-beginning 0) (1-
(match-end 0))))
- ((and ST_IN_SCRIPTING (looking-at (concat "require_once"
phps-mode-lexer-NOT-LABEL)))
- (phps-mode-lexer-RETURN_TOKEN 'T_REQUIRE_ONCE (match-beginning 0) (1-
(match-end 0))))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat "include_once"
phps-mode-lexer-NOT-LABEL)))
+ (phps-mode-lexer-RETURN_TOKEN 'T_INCLUDE_ONCE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "require"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_REQUIRE (match-beginning 0) (1-
(match-end 0))))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat "require_once"
phps-mode-lexer-NOT-LABEL)))
+ (phps-mode-lexer-RETURN_TOKEN 'T_REQUIRE_ONCE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "namespace"
phps-mode-lexer-NOT-LABEL)))
(setq phps-mode-lexer-declaring_namespace t)
(phps-mode-lexer-RETURN_TOKEN 'T_NAMESPACE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "use"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_USE (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "insteadof"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_INSTEADOF (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "global"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_GLOBAL (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "isset"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ISSET (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "empty"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_EMPTY (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "__halt_compiler"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_HALT_COMPILER (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "static"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_STATIC (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "abstract"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ABSTRACT (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "final"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_FINAL (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "private"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_PRIVATE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "protected"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_PROTECTED (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "public"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_PUBLIC (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "unset"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_UNSET (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at "=>"))
(phps-mode-lexer-RETURN_TOKEN 'T_DOUBLE_ARROW (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at (concat "list"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_LIST (match-beginning 0) (1- (match-end
0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "array"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_ARRAY (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "callable"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_CALLABLE (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at "\\+\\+"))
(phps-mode-lexer-RETURN_TOKEN 'T_INC (match-beginning 0) (match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "--"))
(phps-mode-lexer-RETURN_TOKEN 'T_DEC (match-beginning 0) (match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "==="))
(phps-mode-lexer-RETURN_TOKEN 'T_IS_IDENTICAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "!=="))
(phps-mode-lexer-RETURN_TOKEN 'T_IS_NOT_IDENTICAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "=="))
(phps-mode-lexer-RETURN_TOKEN 'T_IS_EQUAL (match-beginning 0) (match-end
0)))
+
((and ST_IN_SCRIPTING (looking-at "\\(!=\\|<>\\)"))
(phps-mode-lexer-RETURN_TOKEN 'T_IS_NOT_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "<=>"))
(phps-mode-lexer-RETURN_TOKEN 'T_SPACESHIP (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "<="))
(phps-mode-lexer-RETURN_TOKEN 'T_IS_SMALLER_OR_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at ">="))
(phps-mode-lexer-RETURN_TOKEN 'T_IS_GREATER_OR_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "\\+="))
(phps-mode-lexer-RETURN_TOKEN 'T_PLUS_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "-="))
(phps-mode-lexer-RETURN_TOKEN 'T_MINUS_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "\\*="))
(phps-mode-lexer-RETURN_TOKEN 'T_MUL_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "\\*\\\\\\*="))
(phps-mode-lexer-RETURN_TOKEN 'T_POW_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "\\*\\\\\\*"))
(phps-mode-lexer-RETURN_TOKEN 'T_POW (match-beginning 0) (match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "/="))
(phps-mode-lexer-RETURN_TOKEN 'T_DIV_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "\\.="))
(phps-mode-lexer-RETURN_TOKEN 'T_CONCAT_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "%="))
(phps-mode-lexer-RETURN_TOKEN 'T_MOD_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "<<="))
(phps-mode-lexer-RETURN_TOKEN 'T_SL_EQUAL (match-beginning 0) (match-end
0)))
+
((and ST_IN_SCRIPTING (looking-at ">>="))
(phps-mode-lexer-RETURN_TOKEN 'T_SR_EQUAL (match-beginning 0) (match-end
0)))
+
((and ST_IN_SCRIPTING (looking-at "&="))
(phps-mode-lexer-RETURN_TOKEN 'T_AND_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "|="))
(phps-mode-lexer-RETURN_TOKEN 'T_OR_EQUAL (match-beginning 0) (match-end
0)))
+
((and ST_IN_SCRIPTING (looking-at "\\^="))
(phps-mode-lexer-RETURN_TOKEN 'T_XOR_EQUAL (match-beginning 0)
(match-end 0)))
+
+ ((and ST_IN_SCRIPTING (looking-at "\\?\\?="))
+ (phps-mode-lexer-RETURN_TOKEN 'T_COALESCE_EQUAL (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "||"))
(phps-mode-lexer-RETURN_TOKEN 'T_BOOLEAN_OR (match-beginning 0)
(match-end 0)))
+
((and ST_IN_SCRIPTING (looking-at "&&"))
(phps-mode-lexer-RETURN_TOKEN 'T_BOOLEAN_AND (match-beginning 0)
(match-end 0)))
- ((and ST_IN_SCRIPTING (looking-at (concat "XOR"
phps-mode-lexer-NOT-LABEL)))
- (phps-mode-lexer-RETURN_TOKEN 'T_LOGICAL_XOR (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "OR"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_LOGICAL_OR (match-beginning 0) (1-
(match-end 0))))
+
((and ST_IN_SCRIPTING (looking-at (concat "AND"
phps-mode-lexer-NOT-LABEL)))
(phps-mode-lexer-RETURN_TOKEN 'T_LOGICAL_AND (match-beginning 0) (1-
(match-end 0))))
- ((and ST_IN_SCRIPTING (looking-at "<<"))
- (phps-mode-lexer-RETURN_TOKEN 'T_SL (match-beginning 0) (match-end 0)))
- ((and ST_IN_SCRIPTING (looking-at ">>"))
- (phps-mode-lexer-RETURN_TOKEN 'T_SR (match-beginning 0) (match-end 0)))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat "XOR"
phps-mode-lexer-NOT-LABEL)))
+ (phps-mode-lexer-RETURN_TOKEN 'T_LOGICAL_XOR (match-beginning 0) (1-
(match-end 0))))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat "<<" "[^<]")))
+ (phps-mode-lexer-RETURN_TOKEN 'T_SL (match-beginning 0) (1- (match-end
0))))
+
+ ((and ST_IN_SCRIPTING (looking-at (concat ">>" "[^>]")))
+ (phps-mode-lexer-RETURN_TOKEN 'T_SR (match-beginning 0) (1- (match-end
0))))
+
+ ((and ST_IN_SCRIPTING (looking-at "\\?>\n?"))
+ (let ((start (match-beginning 0))
+ (end (match-end 0)))
+ (when (= (- end start) 3)
+ (setq end (1- end)))
+ (phps-mode-lexer-BEGIN phps-mode-lexer-ST_INITIAL)
+ (when phps-mode-lexer-PARSER_MODE
+ (phps-mode-lexer-RETURN_TOKEN ";" start end))
+ (phps-mode-lexer-RETURN_TOKEN 'T_CLOSE_TAG start end)))
+
+ ;; HEREDOC and NOWDOC
+ ((and ST_IN_SCRIPTING (looking-at (concat "<<<"
phps-mode-lexer-TABS_AND_SPACES "\\(" phps-mode-lexer-LABEL "\\|'"
phps-mode-lexer-LABEL "'\\|\"" phps-mode-lexer-LABEL "\"\\)"
phps-mode-lexer-NEWLINE)))
+ (let* ((start (match-beginning 0))
+ (end (match-end 0))
+ (data (buffer-substring-no-properties (match-beginning 1)
(match-end 1)))
+ (heredoc_label))
+
+ ;; Determine if it's HEREDOC or NOWDOC and extract label here
+ (if (string= (substring data 0 1) "'")
+ (progn
+ (setq heredoc_label (substring data 1 (- (length data) 1)))
+ (phps-mode-lexer-BEGIN phps-mode-lexer-ST_NOWDOC))
+ (progn
+ (if (string= (substring data 0 1) "\"")
+ (setq heredoc_label (substring data 1 (- (length data) 1)))
+ (setq heredoc_label data))
+ (phps-mode-lexer-BEGIN phps-mode-lexer-ST_HEREDOC)))
+
+ ;; Check for ending label on the next line
+ (when (string= (buffer-substring-no-properties end (+ end (length
heredoc_label))) heredoc_label)
+ (phps-mode-lexer-BEGIN phps-mode-lexer-ST_END_HEREDOC))
+
+ (push heredoc_label phps-mode-lexer-heredoc_label_stack)
+ ;; (message "Found heredoc or nowdoc at %s with label %s" data
heredoc_label)
+
+ (phps-mode-lexer-RETURN_TOKEN 'T_START_HEREDOC start end)))
+
((and ST_IN_SCRIPTING (looking-at "{"))
(phps-mode-lexer-yy_push_state phps-mode-lexer-ST_IN_SCRIPTING)
@@ -875,13 +1009,6 @@
(phps-mode-lexer-BEGIN phps-mode-lexer-ST_BACKQUOTE)
(phps-mode-lexer-RETURN_TOKEN "`" (match-beginning 0) (match-end 0)))
- ((and ST_IN_SCRIPTING (looking-at phps-mode-lexer-WHITESPACE))
- (let* ((start (match-beginning 0))
- (end (match-end 0))
- (data (buffer-substring-no-properties start end)))
- (if phps-mode-lexer-PARSER_MODE
- (phps-mode-lexer-MOVE_FORWARD end)
- (phps-mode-lexer-RETURN_TOKEN data start end))))
((and ST_IN_SCRIPTING (looking-at phps-mode-lexer-LABEL))
;; (message "Adding T_STRING from %s to %s" (match-beginning 0)
(match-end 0))
@@ -936,36 +1063,7 @@
(phps-mode-lexer-RETURN_TOKEN 'T_INLINE_HTML start (-
string-start 2))
(phps-mode-lexer-RETURN_TOKEN 'T_INLINE_HTML start (point-max))))))
-
- ;; ST_LOOKING_FOR_PROPERTY
-
-
- ((and ST_LOOKING_FOR_PROPERTY (looking-at phps-mode-lexer-WHITESPACE))
- (let* ((start (match-beginning 0))
- (end (match-end 0))
- (_data (buffer-substring-no-properties start end)))
- (if phps-mode-lexer-PARSER_MODE
- (phps-mode-lexer-MOVE_FORWARD end)
- (phps-mode-lexer-RETURN_TOKEN 'T_WHITESPACE start end))))
-
- ((and ST_LOOKING_FOR_PROPERTY (looking-at "->"))
- (phps-mode-lexer-RETURN_TOKEN 'T_OBJECT_OPERATOR (match-beginning 0)
(match-end 0)))
-
- ((and ST_LOOKING_FOR_PROPERTY (looking-at phps-mode-lexer-LABEL))
- (let ((start (match-beginning 0))
- (end (match-end 0)))
- (phps-mode-lexer-yy_pop_state)
- (phps-mode-lexer-RETURN_TOKEN 'T_STRING start end)))
-
- ((and ST_LOOKING_FOR_PROPERTY (looking-at phps-mode-lexer-ANY_CHAR))
- (let ((_start (match-beginning 0))
- (end (match-end 0)))
- (phps-mode-lexer-yy_pop_state)
- ;; TODO goto restart here?
- ;; (message "Restart here")
- (phps-mode-lexer-MOVE_FORWARD end)))
-
-
+
;; ST_DOUBLE_QUOTES
diff --git a/phps-mode-test-lexer.el b/phps-mode-test-lexer.el
index 603ee80..d247375 100644
--- a/phps-mode-test-lexer.el
+++ b/phps-mode-test-lexer.el
@@ -155,11 +155,14 @@
(should (equal (phps-mode-lexer-get-tokens)
'((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_ISSET 12 . 17) ("("
17 . 18) (T_VARIABLE 18 . 28) ("[" 28 . 29) (T_LNUMBER 29 . 30) ("]" 30 . 31)
("[" 31 . 32) (T_CONSTANT_ENCAPSED_STRING 32 . 38) ("]" 38 . 39) (")" 39 . 40)
("?" 41 . 42) (T_CONSTANT_ENCAPSED_STRING 43 . 48) (":" 49 . 50)
(T_CONSTANT_ENCAPSED_STRING 51 . 55) (";" 55 . 56) (T_COMMENT 57 . 75)
(T_COMMENT 76 . 101) (T_DOC_COMMENT 102 . 134) (T_ECHO 135 . 139) (T_VARIABLE
140 . 150) (";" 150 . 151) (";" 152 . 154) [...]
+ ;; TODO Add test for ??= token
+
(phps-mode-test-with-buffer
- "<?php $var EXIT die function return yield from yield try catch finally
throw if elseif endif else while endwhile do for endfor foreach endforeach
declare enddeclare instanceof as switch endswitch case default break continue
goto echo print class interface trait extends implements :: \\ ... ?? new clone
var (int) (integer) (real) (double) (float) (string) (binary) (array) (object)
(boolean) (bool) (unset) eval include include_once require require_once
namespace use insteadof global is [...]
+ "<?php ??= $var EXIT die function return yield from yield try catch finally
throw if elseif endif else while endwhile do for endfor foreach endforeach
declare enddeclare instanceof as switch endswitch case default break continue
goto echo print class interface trait extends implements :: \\ ... ?? new clone
var (int) (integer) (real) (double) (float) (string) (binary) (array) (object)
(boolean) (bool) (unset) eval include include_once require require_once
namespace use insteadof globa [...]
"All PHP tokens after each other"
+ ;; (message "Tokens: %s" (phps-mode-lexer-get-tokens))
(should (equal (phps-mode-lexer-get-tokens)
- '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 11) (T_EXIT 12 . 16)
(T_DIE 17 . 20) (T_FUNCTION 21 . 29) (T_RETURN 30 . 36) (T_YIELD_FROM 37 . 48)
(T_YIELD 48 . 53) (T_TRY 54 . 57) (T_CATCH 58 . 63) (T_FINALLY 64 . 71)
(T_THROW 72 . 77) (T_IF 78 . 80) (T_ELSEIF 81 . 87) (T_ENDIF 88 . 93) (T_ELSE
94 . 98) (T_WHILE 99 . 104) (T_ENDWHILE 105 . 113) (T_DO 114 . 116) (T_FOR 117
. 120) (T_ENDFOR 121 . 127) (T_FOREACH 128 . 135) (T_ENDFOREACH 136 . 146)
(T_DECLARE 147 . 154) (T_ENDDECLA [...]
+ '((T_OPEN_TAG 1 . 7) (T_COALESCE_EQUAL 7 . 10) (T_VARIABLE
11 . 15) (T_EXIT 16 . 20) (T_DIE 21 . 24) (T_FUNCTION 25 . 33) (T_RETURN 34 .
40) (T_YIELD_FROM 41 . 52) (T_YIELD 52 . 57) (T_TRY 58 . 61) (T_CATCH 62 . 67)
(T_FINALLY 68 . 75) (T_THROW 76 . 81) (T_IF 82 . 84) (T_ELSEIF 85 . 91)
(T_ENDIF 92 . 97) (T_ELSE 98 . 102) (T_WHILE 103 . 108) (T_ENDWHILE 109 . 117)
(T_DO 118 . 120) (T_FOR 121 . 124) (T_ENDFOR 125 . 131) (T_FOREACH 132 . 139)
(T_ENDFOREACH 140 . 150) (T_D [...]
(phps-mode-test-with-buffer
"<?php forgerarray($arg1, $arg2)"
@@ -404,6 +407,7 @@
"Run test for lexer."
;; (message "-- Running all tests for lexer... --\n")
;; (setq debug-on-error t)
+ ;; (setq phps-mode-functions-verbose t)
(phps-mode-test-lexer-script-boundaries)
(phps-mode-test-lexer-simple-tokens)
(phps-mode-test-lexer-complex-tokens)
