[PATCH] D107291: [clang][lexer] adds a way for tokens to track their physical spelling

[Christopher Di Bella via Phabricator via cfe-commits]

cjdb created this revision.
cjdb added a reviewer: aaron.ballman.
cjdb requested review of this revision.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
Tokens don't currently have a way to track their source-based spelling,
which makes implementing some warnings difficult.

This patch is the basis for two independent warnings, and should be
a non-functional change.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D107291

Files:
  clang/include/clang/Lex/Token.h
  clang/lib/Lex/Lexer.cpp


Index: clang/lib/Lex/Lexer.cpp
===================================================================
--- clang/lib/Lex/Lexer.cpp
+++ clang/lib/Lex/Lexer.cpp
@@ -3264,6 +3264,7 @@
 
   // Read a character, advancing over it.
   char Char = getAndAdvanceChar(CurPtr, Result);
+  Result.setSpelling(CurPtr - 1);
   tok::TokenKind Kind;
 
   if (!isVerticalWhitespace(Char))
@@ -3590,12 +3591,14 @@
     Char = getCharAndSize(CurPtr, SizeTmp);
     if (Char == '&') {
       Kind = tok::ampamp;
+      // Result.setSpelling(CurPtr - 1);
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else if (Char == '=') {
       Kind = tok::ampequal;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else {
       Kind = tok::amp;
+      // Result.setSpelling(CurPtr - 1);
     }
     break;
   case '*':
@@ -3639,6 +3642,7 @@
     }
     break;
   case '~':
+    // Result.setSpelling(CurPtr - 1);
     Kind = tok::tilde;
     break;
   case '!':
@@ -3646,6 +3650,7 @@
       Kind = tok::exclaimequal;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else {
+      // Result.setSpelling(CurPtr - 1);
       Kind = tok::exclaim;
     }
     break;
@@ -3846,6 +3851,7 @@
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
       Kind = tok::caretcaret;
     } else {
+      // Result.setSpelling(CurPtr - 1);
       Kind = tok::caret;
     }
     break;
@@ -3858,9 +3864,11 @@
       // If this is '|||||||' and we're in a conflict marker, ignore it.
       if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1))
         goto LexNextToken;
+      // Result.setSpelling(CurPtr - 1);
       Kind = tok::pipepipe;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else {
+      // Result.setSpelling(CurPtr - 1);
       Kind = tok::pipe;
     }
     break;
Index: clang/include/clang/Lex/Token.h
===================================================================
--- clang/include/clang/Lex/Token.h
+++ clang/include/clang/Lex/Token.h
@@ -67,6 +67,9 @@
   /// Flags - Bits we track about this token, members of the TokenFlags enum.
   unsigned short Flags;
 
+  /// Spelling - How a token is spelt in the source.
+  const char *Spelling;
+
 public:
   // Various flags set per token:
   enum TokenFlags {
@@ -237,6 +240,10 @@
     Flags |= Flag;
   }
 
+  StringRef getSpelling() const { return {Spelling, getLength()}; }
+
+  void setSpelling(const char *S) { Spelling = S; }
+
   /// Get the specified flag.
   bool getFlag(TokenFlags Flag) const {
     return (Flags & Flag) != 0;



Index: clang/lib/Lex/Lexer.cpp
===================================================================
--- clang/lib/Lex/Lexer.cpp
+++ clang/lib/Lex/Lexer.cpp
@@ -3264,6 +3264,7 @@
 
   // Read a character, advancing over it.
   char Char = getAndAdvanceChar(CurPtr, Result);
+  Result.setSpelling(CurPtr - 1);
   tok::TokenKind Kind;
 
   if (!isVerticalWhitespace(Char))
@@ -3590,12 +3591,14 @@
     Char = getCharAndSize(CurPtr, SizeTmp);
     if (Char == '&') {
       Kind = tok::ampamp;
+      // Result.setSpelling(CurPtr - 1);
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else if (Char == '=') {
       Kind = tok::ampequal;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else {
       Kind = tok::amp;
+      // Result.setSpelling(CurPtr - 1);
     }
     break;
   case '*':
@@ -3639,6 +3642,7 @@
     }
     break;
   case '~':
+    // Result.setSpelling(CurPtr - 1);
     Kind = tok::tilde;
     break;
   case '!':
@@ -3646,6 +3650,7 @@
       Kind = tok::exclaimequal;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else {
+      // Result.setSpelling(CurPtr - 1);
       Kind = tok::exclaim;
     }
     break;
@@ -3846,6 +3851,7 @@
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
       Kind = tok::caretcaret;
     } else {
+      // Result.setSpelling(CurPtr - 1);
       Kind = tok::caret;
     }
     break;
@@ -3858,9 +3864,11 @@
       // If this is '|||||||' and we're in a conflict marker, ignore it.
       if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1))
         goto LexNextToken;
+      // Result.setSpelling(CurPtr - 1);
       Kind = tok::pipepipe;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else {
+      // Result.setSpelling(CurPtr - 1);
       Kind = tok::pipe;
     }
     break;
Index: clang/include/clang/Lex/Token.h
===================================================================
--- clang/include/clang/Lex/Token.h
+++ clang/include/clang/Lex/Token.h
@@ -67,6 +67,9 @@
   /// Flags - Bits we track about this token, members of the TokenFlags enum.
   unsigned short Flags;
 
+  /// Spelling - How a token is spelt in the source.
+  const char *Spelling;
+
 public:
   // Various flags set per token:
   enum TokenFlags {
@@ -237,6 +240,10 @@
     Flags |= Flag;
   }
 
+  StringRef getSpelling() const { return {Spelling, getLength()}; }
+
+  void setSpelling(const char *S) { Spelling = S; }
+
   /// Get the specified flag.
   bool getFlag(TokenFlags Flag) const {
     return (Flags & Flag) != 0;

_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits