/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/include/clang/Lex/TokenLexer.h
Line | Count | Source |
1 | | //===- TokenLexer.h - Lex from a token buffer -------------------*- C++ -*-===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This file defines the TokenLexer interface. |
10 | | // |
11 | | //===----------------------------------------------------------------------===// |
12 | | |
13 | | #ifndef LLVM_CLANG_LEX_TOKENLEXER_H |
14 | | #define LLVM_CLANG_LEX_TOKENLEXER_H |
15 | | |
16 | | #include "clang/Basic/SourceLocation.h" |
17 | | #include "llvm/ADT/ArrayRef.h" |
18 | | |
19 | | namespace clang { |
20 | | |
21 | | class MacroArgs; |
22 | | class MacroInfo; |
23 | | class Preprocessor; |
24 | | class Token; |
25 | | class VAOptExpansionContext; |
26 | | |
27 | | /// TokenLexer - This implements a lexer that returns tokens from a macro body |
28 | | /// or token stream instead of lexing from a character buffer. This is used for |
29 | | /// macro expansion and _Pragma handling, for example. |
30 | | class TokenLexer { |
31 | | friend class Preprocessor; |
32 | | |
33 | | /// The macro we are expanding from. This is null if expanding a token stream. |
34 | | MacroInfo *Macro = nullptr; |
35 | | |
36 | | /// The actual arguments specified for a function-like macro, or null. The |
37 | | /// TokenLexer owns the pointed-to object. |
38 | | MacroArgs *ActualArgs = nullptr; |
39 | | |
40 | | /// The current preprocessor object we are expanding for. |
41 | | Preprocessor &PP; |
42 | | |
43 | | /// This is the pointer to an array of tokens that the macro is |
44 | | /// defined to, with arguments expanded for function-like macros. If this is |
45 | | /// a token stream, these are the tokens we are returning. This points into |
46 | | /// the macro definition we are lexing from, a cache buffer that is owned by |
47 | | /// the preprocessor, or some other buffer that we may or may not own |
48 | | /// (depending on OwnsTokens). |
49 | | /// Note that if it points into Preprocessor's cache buffer, the Preprocessor |
50 | | /// may update the pointer as needed. |
51 | | const Token *Tokens; |
52 | | |
53 | | /// This is the length of the Tokens array. |
54 | | unsigned NumTokens; |
55 | | |
56 | | /// This is the index of the next token that Lex will return. |
57 | | unsigned CurTokenIdx; |
58 | | |
59 | | /// The source location range where this macro was expanded. |
60 | | SourceLocation ExpandLocStart, ExpandLocEnd; |
61 | | |
62 | | /// Source location pointing at the source location entry chunk that |
63 | | /// was reserved for the current macro expansion. |
64 | | SourceLocation MacroExpansionStart; |
65 | | |
66 | | /// The offset of the macro expansion in the |
67 | | /// "source location address space". |
68 | | unsigned MacroStartSLocOffset; |
69 | | |
70 | | /// Location of the macro definition. |
71 | | SourceLocation MacroDefStart; |
72 | | |
73 | | /// Length of the macro definition. |
74 | | unsigned MacroDefLength; |
75 | | |
76 | | /// Lexical information about the expansion point of the macro: the identifier |
77 | | /// that the macro expanded from had these properties. |
78 | | bool AtStartOfLine : 1; |
79 | | bool HasLeadingSpace : 1; |
80 | | |
81 | | // When this is true, the next token appended to the |
82 | | // output list during function argument expansion will get a leading space, |
83 | | // regardless of whether it had one to begin with or not. This is used for |
84 | | // placemarker support. If still true after function argument expansion, the |
85 | | // leading space will be applied to the first token following the macro |
86 | | // expansion. |
87 | | bool NextTokGetsSpace : 1; |
88 | | |
89 | | /// This is true if this TokenLexer allocated the Tokens |
90 | | /// array, and thus needs to free it when destroyed. For simple object-like |
91 | | /// macros (for example) we just point into the token buffer of the macro |
92 | | /// definition, we don't make a copy of it. |
93 | | bool OwnsTokens : 1; |
94 | | |
95 | | /// This is true when tokens lexed from the TokenLexer |
96 | | /// should not be subject to further macro expansion. |
97 | | bool DisableMacroExpansion : 1; |
98 | | |
99 | | /// When true, the produced tokens have Token::IsReinjected flag set. |
100 | | /// See the flag documentation for details. |
101 | | bool IsReinject : 1; |
102 | | |
103 | | public: |
104 | | /// Create a TokenLexer for the specified macro with the specified actual |
105 | | /// arguments. Note that this ctor takes ownership of the ActualArgs pointer. |
106 | | /// ILEnd specifies the location of the ')' for a function-like macro or the |
107 | | /// identifier for an object-like macro. |
108 | | TokenLexer(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, |
109 | | MacroArgs *ActualArgs, Preprocessor &pp) |
110 | 44.5k | : PP(pp), OwnsTokens(false) { |
111 | 44.5k | Init(Tok, ILEnd, MI, ActualArgs); |
112 | 44.5k | } |
113 | | |
114 | | /// Create a TokenLexer for the specified token stream. If 'OwnsTokens' is |
115 | | /// specified, this takes ownership of the tokens and delete[]'s them when |
116 | | /// the token lexer is empty. |
117 | | TokenLexer(const Token *TokArray, unsigned NumToks, bool DisableExpansion, |
118 | | bool ownsTokens, bool isReinject, Preprocessor &pp) |
119 | 476k | : PP(pp), OwnsTokens(false) { |
120 | 476k | Init(TokArray, NumToks, DisableExpansion, ownsTokens, isReinject); |
121 | 476k | } |
122 | | |
123 | | TokenLexer(const TokenLexer &) = delete; |
124 | | TokenLexer &operator=(const TokenLexer &) = delete; |
125 | 509k | ~TokenLexer() { destroy(); } |
126 | | |
127 | | /// Initialize this TokenLexer to expand from the specified macro |
128 | | /// with the specified argument information. Note that this ctor takes |
129 | | /// ownership of the ActualArgs pointer. ILEnd specifies the location of the |
130 | | /// ')' for a function-like macro or the identifier for an object-like macro. |
131 | | void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, |
132 | | MacroArgs *Actuals); |
133 | | |
134 | | /// Initialize this TokenLexer with the specified token stream. |
135 | | /// This does not take ownership of the specified token vector. |
136 | | /// |
137 | | /// DisableExpansion is true when macro expansion of tokens lexed from this |
138 | | /// stream should be disabled. |
139 | | void Init(const Token *TokArray, unsigned NumToks, bool DisableMacroExpansion, |
140 | | bool OwnsTokens, bool IsReinject); |
141 | | |
142 | | /// If the next token lexed will pop this macro off the |
143 | | /// expansion stack, return 2. If the next unexpanded token is a '(', return |
144 | | /// 1, otherwise return 0. |
145 | | unsigned isNextTokenLParen() const; |
146 | | |
147 | | /// Lex and return a token from this macro stream. |
148 | | bool Lex(Token &Tok); |
149 | | |
150 | | /// isParsingPreprocessorDirective - Return true if we are in the middle of a |
151 | | /// preprocessor directive. |
152 | | bool isParsingPreprocessorDirective() const; |
153 | | |
154 | | private: |
155 | | void destroy(); |
156 | | |
157 | | /// Return true if the next lex call will pop this macro off the include |
158 | | /// stack. |
159 | 1.24G | bool isAtEnd() const { |
160 | 1.24G | return CurTokenIdx == NumTokens; |
161 | 1.24G | } |
162 | | |
163 | | /// Concatenates the next (sub-)sequence of \p Tokens separated by '##' |
164 | | /// starting with LHSTok - stopping when we encounter a token that is neither |
165 | | /// '##' nor preceded by '##'. Places the result back into \p LHSTok and sets |
166 | | /// \p CurIdx to point to the token following the last one that was pasted. |
167 | | /// |
168 | | /// Also performs the MSVC extension wide-literal token pasting involved with: |
169 | | /// \code L #macro-arg. \endcode |
170 | | /// |
171 | | /// \param[in,out] LHSTok - Contains the token to the left of '##' in \p |
172 | | /// Tokens upon entry and will contain the resulting concatenated Token upon |
173 | | /// exit. |
174 | | /// |
175 | | /// \param[in] TokenStream - The stream of Tokens we are lexing from. |
176 | | /// |
177 | | /// \param[in,out] CurIdx - Upon entry, \pTokens[\pCurIdx] must equal '##' |
178 | | /// (with the exception of the MSVC extension mentioned above). Upon exit, it |
179 | | /// is set to the index of the token following the last token that was |
180 | | /// concatenated together. |
181 | | /// |
182 | | /// \returns If this returns true, the caller should immediately return the |
183 | | /// token. |
184 | | bool pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream, |
185 | | unsigned int &CurIdx); |
186 | | |
187 | | /// Calls pasteTokens above, passing in the '*this' object's Tokens and |
188 | | /// CurTokenIdx data members. |
189 | | bool pasteTokens(Token &Tok); |
190 | | |
191 | | |
192 | | /// Takes the tail sequence of tokens within ReplacementToks that represent |
193 | | /// the just expanded __VA_OPT__ tokens (possibly zero tokens) and transforms |
194 | | /// them into a string. \p VCtx is used to determine which token represents |
195 | | /// the first __VA_OPT__ replacement token. |
196 | | /// |
197 | | /// \param[in,out] ResultToks - Contains the current Replacement Tokens |
198 | | /// (prior to rescanning and token pasting), the tail end of which represents |
199 | | /// the tokens just expanded through __VA_OPT__ processing. These (sub) |
200 | | /// sequence of tokens are folded into one stringified token. |
201 | | /// |
202 | | /// \param[in] VCtx - contains relevant contextual information about the |
203 | | /// state of the tokens around and including the __VA_OPT__ token, necessary |
204 | | /// for stringification. |
205 | | void stringifyVAOPTContents(SmallVectorImpl<Token> &ResultToks, |
206 | | const VAOptExpansionContext &VCtx, |
207 | | SourceLocation VAOPTClosingParenLoc); |
208 | | |
209 | | /// Expand the arguments of a function-like macro so that we can quickly |
210 | | /// return preexpanded tokens from Tokens. |
211 | | void ExpandFunctionArguments(); |
212 | | |
213 | | /// In microsoft compatibility mode, /##/ pastes |
214 | | /// together to form a comment that comments out everything in the current |
215 | | /// macro, other active macros, and anything left on the current physical |
216 | | /// source line of the expanded buffer. Handle this by returning the |
217 | | /// first token on the next line. |
218 | | void HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc); |
219 | | |
220 | | /// If \p loc is a FileID and points inside the current macro |
221 | | /// definition, returns the appropriate source location pointing at the |
222 | | /// macro expansion source location entry. |
223 | | SourceLocation getExpansionLocForMacroDefLoc(SourceLocation loc) const; |
224 | | |
225 | | /// Creates SLocEntries and updates the locations of macro argument |
226 | | /// tokens to their new expanded locations. |
227 | | /// |
228 | | /// \param ArgIdSpellLoc the location of the macro argument id inside the |
229 | | /// macro definition. |
230 | | void updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc, |
231 | | Token *begin_tokens, Token *end_tokens); |
232 | | |
233 | | /// Remove comma ahead of __VA_ARGS__, if present, according to compiler |
234 | | /// dialect settings. Returns true if the comma is removed. |
235 | | bool MaybeRemoveCommaBeforeVaArgs(SmallVectorImpl<Token> &ResultToks, |
236 | | bool HasPasteOperator, |
237 | | MacroInfo *Macro, unsigned MacroArgNo, |
238 | | Preprocessor &PP); |
239 | | |
240 | | void PropagateLineStartLeadingSpaceInfo(Token &Result); |
241 | | }; |
242 | | |
243 | | } // namespace clang |
244 | | |
245 | | #endif // LLVM_CLANG_LEX_TOKENLEXER_H |