/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Lex/TokenLexer.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- TokenLexer.cpp - Lex from a token stream ---------------------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This file implements the TokenLexer interface. |
10 | | // |
11 | | //===----------------------------------------------------------------------===// |
12 | | |
13 | | #include "clang/Lex/TokenLexer.h" |
14 | | #include "clang/Basic/Diagnostic.h" |
15 | | #include "clang/Basic/IdentifierTable.h" |
16 | | #include "clang/Basic/LangOptions.h" |
17 | | #include "clang/Basic/SourceLocation.h" |
18 | | #include "clang/Basic/SourceManager.h" |
19 | | #include "clang/Basic/TokenKinds.h" |
20 | | #include "clang/Lex/LexDiagnostic.h" |
21 | | #include "clang/Lex/Lexer.h" |
22 | | #include "clang/Lex/MacroArgs.h" |
23 | | #include "clang/Lex/MacroInfo.h" |
24 | | #include "clang/Lex/Preprocessor.h" |
25 | | #include "clang/Lex/Token.h" |
26 | | #include "clang/Lex/VariadicMacroSupport.h" |
27 | | #include "llvm/ADT/ArrayRef.h" |
28 | | #include "llvm/ADT/SmallString.h" |
29 | | #include "llvm/ADT/SmallVector.h" |
30 | | #include "llvm/ADT/iterator_range.h" |
31 | | #include <cassert> |
32 | | #include <cstring> |
33 | | |
34 | | using namespace clang; |
35 | | |
36 | | /// Create a TokenLexer for the specified macro with the specified actual |
37 | | /// arguments. Note that this ctor takes ownership of the ActualArgs pointer. |
38 | | void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, |
39 | 44.4M | MacroArgs *Actuals) { |
40 | | // If the client is reusing a TokenLexer, make sure to free any memory |
41 | | // associated with it. |
42 | 44.4M | destroy(); |
43 | | |
44 | 44.4M | Macro = MI; |
45 | 44.4M | ActualArgs = Actuals; |
46 | 44.4M | CurTokenIdx = 0; |
47 | | |
48 | 44.4M | ExpandLocStart = Tok.getLocation(); |
49 | 44.4M | ExpandLocEnd = ELEnd; |
50 | 44.4M | AtStartOfLine = Tok.isAtStartOfLine(); |
51 | 44.4M | HasLeadingSpace = Tok.hasLeadingSpace(); |
52 | 44.4M | NextTokGetsSpace = false; |
53 | 44.4M | Tokens = &*Macro->tokens_begin(); |
54 | 44.4M | OwnsTokens = false; |
55 | 44.4M | DisableMacroExpansion = false; |
56 | 44.4M | IsReinject = false; |
57 | 44.4M | NumTokens = Macro->tokens_end()-Macro->tokens_begin(); |
58 | 44.4M | MacroExpansionStart = SourceLocation(); |
59 | | |
60 | 44.4M | SourceManager &SM = PP.getSourceManager(); |
61 | 44.4M | MacroStartSLocOffset = SM.getNextLocalOffset(); |
62 | | |
63 | 44.4M | if (NumTokens > 0) { |
64 | 44.4M | assert(Tokens[0].getLocation().isValid()); |
65 | 44.4M | assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) && |
66 | 44.4M | "Macro defined in macro?"); |
67 | 44.4M | assert(ExpandLocStart.isValid()); |
68 | | |
69 | | // Reserve a source location entry chunk for the length of the macro |
70 | | // definition. Tokens that get lexed directly from the definition will |
71 | | // have their locations pointing inside this chunk. This is to avoid |
72 | | // creating separate source location entries for each token. |
73 | 44.4M | MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation()); |
74 | 44.4M | MacroDefLength = Macro->getDefinitionLength(SM); |
75 | 44.4M | MacroExpansionStart = SM.createExpansionLoc(MacroDefStart, |
76 | 44.4M | ExpandLocStart, |
77 | 44.4M | ExpandLocEnd, |
78 | 44.4M | MacroDefLength); |
79 | 44.4M | } |
80 | | |
81 | | // If this is a function-like macro, expand the arguments and change |
82 | | // Tokens to point to the expanded tokens. |
83 | 44.4M | if (Macro->isFunctionLike() && Macro->getNumParams()24.8M ) |
84 | 24.8M | ExpandFunctionArguments(); |
85 | | |
86 | | // Mark the macro as currently disabled, so that it is not recursively |
87 | | // expanded. The macro must be disabled only after argument pre-expansion of |
88 | | // function-like macro arguments occurs. |
89 | 44.4M | Macro->DisableMacro(); |
90 | 44.4M | } |
91 | | |
92 | | /// Create a TokenLexer for the specified token stream. This does not |
93 | | /// take ownership of the specified token vector. |
94 | | void TokenLexer::Init(const Token *TokArray, unsigned NumToks, |
95 | | bool disableMacroExpansion, bool ownsTokens, |
96 | 4.94M | bool isReinject) { |
97 | 4.94M | assert(!isReinject || disableMacroExpansion); |
98 | | // If the client is reusing a TokenLexer, make sure to free any memory |
99 | | // associated with it. |
100 | 4.94M | destroy(); |
101 | | |
102 | 4.94M | Macro = nullptr; |
103 | 4.94M | ActualArgs = nullptr; |
104 | 4.94M | Tokens = TokArray; |
105 | 4.94M | OwnsTokens = ownsTokens; |
106 | 4.94M | DisableMacroExpansion = disableMacroExpansion; |
107 | 4.94M | IsReinject = isReinject; |
108 | 4.94M | NumTokens = NumToks; |
109 | 4.94M | CurTokenIdx = 0; |
110 | 4.94M | ExpandLocStart = ExpandLocEnd = SourceLocation(); |
111 | 4.94M | AtStartOfLine = false; |
112 | 4.94M | HasLeadingSpace = false; |
113 | 4.94M | NextTokGetsSpace = false; |
114 | 4.94M | MacroExpansionStart = SourceLocation(); |
115 | | |
116 | | // Set HasLeadingSpace/AtStartOfLine so that the first token will be |
117 | | // returned unmodified. |
118 | 4.94M | if (NumToks != 0) { |
119 | 4.94M | AtStartOfLine = TokArray[0].isAtStartOfLine(); |
120 | 4.94M | HasLeadingSpace = TokArray[0].hasLeadingSpace(); |
121 | 4.94M | } |
122 | 4.94M | } |
123 | | |
124 | 49.7M | void TokenLexer::destroy() { |
125 | | // If this was a function-like macro that actually uses its arguments, delete |
126 | | // the expanded tokens. |
127 | 49.7M | if (OwnsTokens) { |
128 | 317k | delete [] Tokens; |
129 | 317k | Tokens = nullptr; |
130 | 317k | OwnsTokens = false; |
131 | 317k | } |
132 | | |
133 | | // TokenLexer owns its formal arguments. |
134 | 49.7M | if (ActualArgs) ActualArgs->destroy(PP)24.8M ; |
135 | 49.7M | } |
136 | | |
137 | | bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( |
138 | | SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro, |
139 | 204k | unsigned MacroArgNo, Preprocessor &PP) { |
140 | | // Is the macro argument __VA_ARGS__? |
141 | 204k | if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1203k ) |
142 | 180k | return false; |
143 | | |
144 | | // In Microsoft-compatibility mode, a comma is removed in the expansion |
145 | | // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is |
146 | | // not supported by gcc. |
147 | 23.1k | if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat17.2k ) |
148 | 17.2k | return false; |
149 | | |
150 | | // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if |
151 | | // __VA_ARGS__ is empty, but not in strict C99 mode where there are no |
152 | | // named arguments, where it remains. In all other modes, including C99 |
153 | | // with GNU extensions, it is removed regardless of named arguments. |
154 | | // Microsoft also appears to support this extension, unofficially. |
155 | 5.90k | if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode40 |
156 | 9 | && Macro->getNumParams() < 2) |
157 | 3 | return false; |
158 | | |
159 | | // Is a comma available to be removed? |
160 | 5.89k | if (ResultToks.empty() || !ResultToks.back().is(tok::comma)) |
161 | 3 | return false; |
162 | | |
163 | | // Issue an extension diagnostic for the paste operator. |
164 | 5.89k | if (HasPasteOperator) |
165 | 5.89k | PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); |
166 | | |
167 | | // Remove the comma. |
168 | 5.89k | ResultToks.pop_back(); |
169 | | |
170 | 5.89k | if (!ResultToks.empty()) { |
171 | | // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"), |
172 | | // then removal of the comma should produce a placemarker token (in C99 |
173 | | // terms) which we model by popping off the previous ##, giving us a plain |
174 | | // "X" when __VA_ARGS__ is empty. |
175 | 5.89k | if (ResultToks.back().is(tok::hashhash)) |
176 | 1 | ResultToks.pop_back(); |
177 | | |
178 | | // Remember that this comma was elided. |
179 | 5.89k | ResultToks.back().setFlag(Token::CommaAfterElided); |
180 | 5.89k | } |
181 | | |
182 | | // Never add a space, even if the comma, ##, or arg had a space. |
183 | 5.89k | NextTokGetsSpace = false; |
184 | 5.89k | return true; |
185 | 5.89k | } |
186 | | |
187 | | void TokenLexer::stringifyVAOPTContents( |
188 | | SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx, |
189 | 15 | const SourceLocation VAOPTClosingParenLoc) { |
190 | 15 | const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt(); |
191 | 15 | const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt; |
192 | 15 | Token *const VAOPTTokens = |
193 | 10 | NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt]5 : nullptr; |
194 | | |
195 | 15 | SmallVector<Token, 64> ConcatenatedVAOPTResultToks; |
196 | | // FIXME: Should we keep track within VCtx that we did or didnot |
197 | | // encounter pasting - and only then perform this loop. |
198 | | |
199 | | // Perform token pasting (concatenation) prior to stringization. |
200 | 56 | for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens; |
201 | 41 | ++CurTokenIdx) { |
202 | 41 | if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) { |
203 | 6 | assert(CurTokenIdx != 0 && |
204 | 6 | "Can not have __VAOPT__ contents begin with a ##"); |
205 | 6 | Token &LHS = VAOPTTokens[CurTokenIdx - 1]; |
206 | 6 | pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens), |
207 | 6 | CurTokenIdx); |
208 | | // Replace the token prior to the first ## in this iteration. |
209 | 6 | ConcatenatedVAOPTResultToks.back() = LHS; |
210 | 6 | if (CurTokenIdx == NumVAOptTokens) |
211 | 0 | break; |
212 | 41 | } |
213 | 41 | ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]); |
214 | 41 | } |
215 | | |
216 | 15 | ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok()); |
217 | | // Get the SourceLocation that represents the start location within |
218 | | // the macro definition that marks where this string is substituted |
219 | | // into: i.e. the __VA_OPT__ and the ')' within the spelling of the |
220 | | // macro definition, and use it to indicate that the stringified token |
221 | | // was generated from that location. |
222 | 15 | const SourceLocation ExpansionLocStartWithinMacro = |
223 | 15 | getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc()); |
224 | 15 | const SourceLocation ExpansionLocEndWithinMacro = |
225 | 15 | getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc); |
226 | | |
227 | 15 | Token StringifiedVAOPT = MacroArgs::StringifyArgument( |
228 | 15 | &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/, |
229 | 15 | ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro); |
230 | | |
231 | 15 | if (VCtx.getLeadingSpaceForStringifiedToken()) |
232 | 10 | StringifiedVAOPT.setFlag(Token::LeadingSpace); |
233 | | |
234 | 15 | StringifiedVAOPT.setFlag(Token::StringifiedInMacro); |
235 | | // Resize (shrink) the token stream to just capture this stringified token. |
236 | 15 | ResultToks.resize(NumToksPriorToVAOpt + 1); |
237 | 15 | ResultToks.back() = StringifiedVAOPT; |
238 | 15 | } |
239 | | |
240 | | /// Expand the arguments of a function-like macro so that we can quickly |
241 | | /// return preexpanded tokens from Tokens. |
242 | 24.8M | void TokenLexer::ExpandFunctionArguments() { |
243 | 24.8M | SmallVector<Token, 128> ResultToks; |
244 | | |
245 | | // Loop through 'Tokens', expanding them into ResultToks. Keep |
246 | | // track of whether we change anything. If not, no need to keep them. If so, |
247 | | // we install the newly expanded sequence as the new 'Tokens' list. |
248 | 24.8M | bool MadeChange = false; |
249 | | |
250 | 24.8M | Optional<bool> CalledWithVariadicArguments; |
251 | | |
252 | 24.8M | VAOptExpansionContext VCtx(PP); |
253 | | |
254 | 257M | for (unsigned I = 0, E = NumTokens; I != E; ++I232M ) { |
255 | 232M | const Token &CurTok = Tokens[I]; |
256 | | // We don't want a space for the next token after a paste |
257 | | // operator. In valid code, the token will get smooshed onto the |
258 | | // preceding one anyway. In assembler-with-cpp mode, invalid |
259 | | // pastes are allowed through: in this case, we do not want the |
260 | | // extra whitespace to be added. For example, we want ". ## foo" |
261 | | // -> ".foo" not ". foo". |
262 | 232M | if (I != 0 && !Tokens[I-1].is(tok::hashhash)207M && CurTok.hasLeadingSpace()198M ) |
263 | 23.1M | NextTokGetsSpace = true; |
264 | | |
265 | 232M | if (VCtx.isVAOptToken(CurTok)) { |
266 | 47 | MadeChange = true; |
267 | 47 | assert(Tokens[I + 1].is(tok::l_paren) && |
268 | 47 | "__VA_OPT__ must be followed by '('"); |
269 | | |
270 | 47 | ++I; // Skip the l_paren |
271 | 47 | VCtx.sawVAOptFollowedByOpeningParens(CurTok.getLocation(), |
272 | 47 | ResultToks.size()); |
273 | | |
274 | 47 | continue; |
275 | 47 | } |
276 | | |
277 | | // We have entered into the __VA_OPT__ context, so handle tokens |
278 | | // appropriately. |
279 | 232M | if (VCtx.isInVAOpt()) { |
280 | | // If we are about to process a token that is either an argument to |
281 | | // __VA_OPT__ or its closing rparen, then: |
282 | | // 1) If the token is the closing rparen that exits us out of __VA_OPT__, |
283 | | // perform any necessary stringification or placemarker processing, |
284 | | // and/or skip to the next token. |
285 | | // 2) else if macro was invoked without variadic arguments skip this |
286 | | // token. |
287 | | // 3) else (macro was invoked with variadic arguments) process the token |
288 | | // normally. |
289 | | |
290 | 226 | if (Tokens[I].is(tok::l_paren)) |
291 | 2 | VCtx.sawOpeningParen(Tokens[I].getLocation()); |
292 | | // Continue skipping tokens within __VA_OPT__ if the macro was not |
293 | | // called with variadic arguments, else let the rest of the loop handle |
294 | | // this token. Note sawClosingParen() returns true only if the r_paren matches |
295 | | // the closing r_paren of the __VA_OPT__. |
296 | 226 | if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()49 ) { |
297 | | // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__. |
298 | 179 | if (!CalledWithVariadicArguments.hasValue()) { |
299 | 35 | CalledWithVariadicArguments = |
300 | 35 | ActualArgs->invokedWithVariadicArgument(Macro, PP); |
301 | 35 | } |
302 | 179 | if (!*CalledWithVariadicArguments) { |
303 | | // Skip this token. |
304 | 83 | continue; |
305 | 83 | } |
306 | | // ... else the macro was called with variadic arguments, and we do not |
307 | | // have a closing rparen - so process this token normally. |
308 | 47 | } else { |
309 | | // Current token is the closing r_paren which marks the end of the |
310 | | // __VA_OPT__ invocation, so handle any place-marker pasting (if |
311 | | // empty) by removing hashhash either before (if exists) or after. And |
312 | | // also stringify the entire contents if VAOPT was preceded by a hash, |
313 | | // but do so only after any token concatenation that needs to occur |
314 | | // within the contents of VAOPT. |
315 | | |
316 | 47 | if (VCtx.hasStringifyOrCharifyBefore()) { |
317 | | // Replace all the tokens just added from within VAOPT into a single |
318 | | // stringified token. This requires token-pasting to eagerly occur |
319 | | // within these tokens. If either the contents of VAOPT were empty |
320 | | // or the macro wasn't called with any variadic arguments, the result |
321 | | // is a token that represents an empty string. |
322 | 15 | stringifyVAOPTContents(ResultToks, VCtx, |
323 | 15 | /*ClosingParenLoc*/ Tokens[I].getLocation()); |
324 | | |
325 | 32 | } else if (/*No tokens within VAOPT*/ |
326 | 32 | ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) { |
327 | | // Treat VAOPT as a placemarker token. Eat either the '##' before the |
328 | | // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that |
329 | | // hashhash was not a placemarker) or the '##' |
330 | | // after VAOPT, but not both. |
331 | | |
332 | 15 | if (ResultToks.size() && ResultToks.back().is(tok::hashhash)11 ) { |
333 | 4 | ResultToks.pop_back(); |
334 | 11 | } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)9 ) { |
335 | 2 | ++I; // Skip the following hashhash. |
336 | 2 | } |
337 | 17 | } else { |
338 | | // If there's a ## before the __VA_OPT__, we might have discovered |
339 | | // that the __VA_OPT__ begins with a placeholder. We delay action on |
340 | | // that to now to avoid messing up our stashed count of tokens before |
341 | | // __VA_OPT__. |
342 | 17 | if (VCtx.beginsWithPlaceholder()) { |
343 | 3 | assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 && |
344 | 3 | ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() && |
345 | 3 | ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is( |
346 | 3 | tok::hashhash) && |
347 | 3 | "no token paste before __VA_OPT__"); |
348 | 3 | ResultToks.erase(ResultToks.begin() + |
349 | 3 | VCtx.getNumberOfTokensPriorToVAOpt() - 1); |
350 | 3 | } |
351 | | // If the expansion of __VA_OPT__ ends with a placeholder, eat any |
352 | | // following '##' token. |
353 | 17 | if (VCtx.endsWithPlaceholder() && I + 1 != E3 && |
354 | 3 | Tokens[I + 1].is(tok::hashhash)) { |
355 | 3 | ++I; |
356 | 3 | } |
357 | 17 | } |
358 | 47 | VCtx.reset(); |
359 | | // We processed __VA_OPT__'s closing paren (and the exit out of |
360 | | // __VA_OPT__), so skip to the next token. |
361 | 47 | continue; |
362 | 47 | } |
363 | 232M | } |
364 | | |
365 | | // If we found the stringify operator, get the argument stringified. The |
366 | | // preprocessor already verified that the following token is a macro |
367 | | // parameter or __VA_OPT__ when the #define was lexed. |
368 | | |
369 | 232M | if (CurTok.isOneOf(tok::hash, tok::hashat)) { |
370 | 129k | int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo()); |
371 | 129k | assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) && |
372 | 129k | "Token following # is not an argument or __VA_OPT__!"); |
373 | | |
374 | 129k | if (ArgNo == -1) { |
375 | | // Handle the __VA_OPT__ case. |
376 | 15 | VCtx.sawHashOrHashAtBefore(NextTokGetsSpace, |
377 | 15 | CurTok.is(tok::hashat)); |
378 | 15 | continue; |
379 | 15 | } |
380 | | // Else handle the simple argument case. |
381 | 129k | SourceLocation ExpansionLocStart = |
382 | 129k | getExpansionLocForMacroDefLoc(CurTok.getLocation()); |
383 | 129k | SourceLocation ExpansionLocEnd = |
384 | 129k | getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation()); |
385 | | |
386 | 129k | bool Charify = CurTok.is(tok::hashat); |
387 | 129k | const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo); |
388 | 129k | Token Res = MacroArgs::StringifyArgument( |
389 | 129k | UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd); |
390 | 129k | Res.setFlag(Token::StringifiedInMacro); |
391 | | |
392 | | // The stringified/charified string leading space flag gets set to match |
393 | | // the #/#@ operator. |
394 | 129k | if (NextTokGetsSpace) |
395 | 58.0k | Res.setFlag(Token::LeadingSpace); |
396 | | |
397 | 129k | ResultToks.push_back(Res); |
398 | 129k | MadeChange = true; |
399 | 129k | ++I; // Skip arg name. |
400 | 129k | NextTokGetsSpace = false; |
401 | 129k | continue; |
402 | 129k | } |
403 | | |
404 | | // Find out if there is a paste (##) operator before or after the token. |
405 | 232M | bool NonEmptyPasteBefore = |
406 | 232M | !ResultToks.empty() && ResultToks.back().is(tok::hashhash)207M ; |
407 | 232M | bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash)207M ; |
408 | 232M | bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash)207M ; |
409 | 232M | bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren)207M ; |
410 | | |
411 | 232M | assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) && |
412 | 232M | "unexpected ## in ResultToks"); |
413 | | |
414 | | // Otherwise, if this is not an argument token, just add the token to the |
415 | | // output buffer. |
416 | 232M | IdentifierInfo *II = CurTok.getIdentifierInfo(); |
417 | 126M | int ArgNo = II ? Macro->getParameterNum(II)105M : -1; |
418 | 232M | if (ArgNo == -1) { |
419 | | // This isn't an argument, just add it. |
420 | 194M | ResultToks.push_back(CurTok); |
421 | | |
422 | 194M | if (NextTokGetsSpace) { |
423 | 22.0M | ResultToks.back().setFlag(Token::LeadingSpace); |
424 | 22.0M | NextTokGetsSpace = false; |
425 | 172M | } else if (PasteBefore && !NonEmptyPasteBefore673k ) |
426 | 6 | ResultToks.back().clearFlag(Token::LeadingSpace); |
427 | | |
428 | 194M | continue; |
429 | 194M | } |
430 | | |
431 | | // An argument is expanded somehow, the result is different than the |
432 | | // input. |
433 | 38.3M | MadeChange = true; |
434 | | |
435 | | // Otherwise, this is a use of the argument. |
436 | | |
437 | | // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there |
438 | | // are no trailing commas if __VA_ARGS__ is empty. |
439 | 38.3M | if (!PasteBefore && ActualArgs->isVarargsElidedUse()29.8M && |
440 | 198k | MaybeRemoveCommaBeforeVaArgs(ResultToks, |
441 | 198k | /*HasPasteOperator=*/false, |
442 | 198k | Macro, ArgNo, PP)) |
443 | 3 | continue; |
444 | | |
445 | | // If it is not the LHS/RHS of a ## operator, we must pre-expand the |
446 | | // argument and substitute the expanded tokens into the result. This is |
447 | | // C99 6.10.3.1p1. |
448 | 38.3M | if (!PasteBefore && !PasteAfter29.8M ) { |
449 | 29.3M | const Token *ResultArgToks; |
450 | | |
451 | | // Only preexpand the argument if it could possibly need it. This |
452 | | // avoids some work in common cases. |
453 | 29.3M | const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); |
454 | 29.3M | if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) |
455 | 3.63M | ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0]; |
456 | 25.7M | else |
457 | 25.7M | ResultArgToks = ArgTok; // Use non-preexpanded tokens. |
458 | | |
459 | | // If the arg token expanded into anything, append it. |
460 | 29.3M | if (ResultArgToks->isNot(tok::eof)) { |
461 | 28.7M | size_t FirstResult = ResultToks.size(); |
462 | 28.7M | unsigned NumToks = MacroArgs::getArgLength(ResultArgToks); |
463 | 28.7M | ResultToks.append(ResultArgToks, ResultArgToks+NumToks); |
464 | | |
465 | | // In Microsoft-compatibility mode, we follow MSVC's preprocessing |
466 | | // behavior by not considering single commas from nested macro |
467 | | // expansions as argument separators. Set a flag on the token so we can |
468 | | // test for this later when the macro expansion is processed. |
469 | 28.7M | if (PP.getLangOpts().MSVCCompat && NumToks == 112.3k && |
470 | 7.99k | ResultToks.back().is(tok::comma)) |
471 | 6 | ResultToks.back().setFlag(Token::IgnoredComma); |
472 | | |
473 | | // If the '##' came from expanding an argument, turn it into 'unknown' |
474 | | // to avoid pasting. |
475 | 28.7M | for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult, |
476 | 103M | ResultToks.end())) { |
477 | 103M | if (Tok.is(tok::hashhash)) |
478 | 2 | Tok.setKind(tok::unknown); |
479 | 103M | } |
480 | | |
481 | 28.7M | if(ExpandLocStart.isValid()) { |
482 | 28.7M | updateLocForMacroArgTokens(CurTok.getLocation(), |
483 | 28.7M | ResultToks.begin()+FirstResult, |
484 | 28.7M | ResultToks.end()); |
485 | 28.7M | } |
486 | | |
487 | | // If any tokens were substituted from the argument, the whitespace |
488 | | // before the first token should match the whitespace of the arg |
489 | | // identifier. |
490 | 28.7M | ResultToks[FirstResult].setFlagValue(Token::LeadingSpace, |
491 | 28.7M | NextTokGetsSpace); |
492 | 28.7M | ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false); |
493 | 28.7M | NextTokGetsSpace = false; |
494 | 645k | } else { |
495 | | // We're creating a placeholder token. Usually this doesn't matter, |
496 | | // but it can affect paste behavior when at the start or end of a |
497 | | // __VA_OPT__. |
498 | 645k | if (NonEmptyPasteBefore) { |
499 | | // We're imagining a placeholder token is inserted here. If this is |
500 | | // the first token in a __VA_OPT__ after a ##, delete the ##. |
501 | 1 | assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__"); |
502 | 1 | VCtx.hasPlaceholderAfterHashhashAtStart(); |
503 | 1 | } |
504 | 645k | if (RParenAfter) |
505 | 381k | VCtx.hasPlaceholderBeforeRParen(); |
506 | 645k | } |
507 | 29.3M | continue; |
508 | 29.3M | } |
509 | | |
510 | | // Okay, we have a token that is either the LHS or RHS of a paste (##) |
511 | | // argument. It gets substituted as its non-pre-expanded tokens. |
512 | 8.93M | const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo); |
513 | 8.93M | unsigned NumToks = MacroArgs::getArgLength(ArgToks); |
514 | 8.93M | if (NumToks) { // Not an empty argument? |
515 | 8.90M | bool VaArgsPseudoPaste = false; |
516 | | // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned |
517 | | // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when |
518 | | // the expander tries to paste ',' with the first token of the __VA_ARGS__ |
519 | | // expansion. |
520 | 8.90M | if (NonEmptyPasteBefore && ResultToks.size() >= 28.42M && |
521 | 8.42M | ResultToks[ResultToks.size()-2].is(tok::comma) && |
522 | 31.0k | (unsigned)ArgNo == Macro->getNumParams()-1 && |
523 | 31.0k | Macro->isVariadic()) { |
524 | 31.0k | VaArgsPseudoPaste = true; |
525 | | // Remove the paste operator, report use of the extension. |
526 | 31.0k | PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma); |
527 | 31.0k | } |
528 | | |
529 | 8.90M | ResultToks.append(ArgToks, ArgToks+NumToks); |
530 | | |
531 | | // If the '##' came from expanding an argument, turn it into 'unknown' |
532 | | // to avoid pasting. |
533 | 8.90M | for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks, |
534 | 24.7M | ResultToks.end())) { |
535 | 24.7M | if (Tok.is(tok::hashhash)) |
536 | 0 | Tok.setKind(tok::unknown); |
537 | 24.7M | } |
538 | | |
539 | 8.90M | if (ExpandLocStart.isValid()) { |
540 | 8.90M | updateLocForMacroArgTokens(CurTok.getLocation(), |
541 | 8.90M | ResultToks.end()-NumToks, ResultToks.end()); |
542 | 8.90M | } |
543 | | |
544 | | // Transfer the leading whitespace information from the token |
545 | | // (the macro argument) onto the first token of the |
546 | | // expansion. Note that we don't do this for the GNU |
547 | | // pseudo-paste extension ", ## __VA_ARGS__". |
548 | 8.90M | if (!VaArgsPseudoPaste) { |
549 | 8.87M | ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine, |
550 | 8.87M | false); |
551 | 8.87M | ResultToks[ResultToks.size() - NumToks].setFlagValue( |
552 | 8.87M | Token::LeadingSpace, NextTokGetsSpace); |
553 | 8.87M | } |
554 | | |
555 | 8.90M | NextTokGetsSpace = false; |
556 | 8.90M | continue; |
557 | 8.90M | } |
558 | | |
559 | | // If an empty argument is on the LHS or RHS of a paste, the standard (C99 |
560 | | // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We |
561 | | // implement this by eating ## operators when a LHS or RHS expands to |
562 | | // empty. |
563 | 31.2k | if (PasteAfter) { |
564 | | // Discard the argument token and skip (don't copy to the expansion |
565 | | // buffer) the paste operator after it. |
566 | 5.95k | ++I; |
567 | 5.95k | continue; |
568 | 5.95k | } |
569 | | |
570 | 25.2k | if (RParenAfter) |
571 | 5.87k | VCtx.hasPlaceholderBeforeRParen(); |
572 | | |
573 | | // If this is on the RHS of a paste operator, we've already copied the |
574 | | // paste operator to the ResultToks list, unless the LHS was empty too. |
575 | | // Remove it. |
576 | 25.2k | assert(PasteBefore); |
577 | 25.2k | if (NonEmptyPasteBefore) { |
578 | 25.2k | assert(ResultToks.back().is(tok::hashhash)); |
579 | | // Do not remove the paste operator if it is the one before __VA_OPT__ |
580 | | // (and we are still processing tokens within VA_OPT). We handle the case |
581 | | // of removing the paste operator if __VA_OPT__ reduces to the notional |
582 | | // placemarker above when we encounter the closing paren of VA_OPT. |
583 | 25.2k | if (!VCtx.isInVAOpt() || |
584 | 2 | ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt()) |
585 | 25.2k | ResultToks.pop_back(); |
586 | 2 | else |
587 | 2 | VCtx.hasPlaceholderAfterHashhashAtStart(); |
588 | 25.2k | } |
589 | | |
590 | | // If this is the __VA_ARGS__ token, and if the argument wasn't provided, |
591 | | // and if the macro had at least one real argument, and if the token before |
592 | | // the ## was a comma, remove the comma. This is a GCC extension which is |
593 | | // disabled when using -std=c99. |
594 | 25.2k | if (ActualArgs->isVarargsElidedUse()) |
595 | 5.90k | MaybeRemoveCommaBeforeVaArgs(ResultToks, |
596 | 5.90k | /*HasPasteOperator=*/true, |
597 | 5.90k | Macro, ArgNo, PP); |
598 | 25.2k | } |
599 | | |
600 | | // If anything changed, install this as the new Tokens list. |
601 | 24.8M | if (MadeChange) { |
602 | 24.8M | assert(!OwnsTokens && "This would leak if we already own the token list"); |
603 | | // This is deleted in the dtor. |
604 | 24.8M | NumTokens = ResultToks.size(); |
605 | | // The tokens will be added to Preprocessor's cache and will be removed |
606 | | // when this TokenLexer finishes lexing them. |
607 | 24.8M | Tokens = PP.cacheMacroExpandedTokens(this, ResultToks); |
608 | | |
609 | | // The preprocessor cache of macro expanded tokens owns these tokens,not us. |
610 | 24.8M | OwnsTokens = false; |
611 | 24.8M | } |
612 | 24.8M | } |
613 | | |
614 | | /// Checks if two tokens form wide string literal. |
615 | | static bool isWideStringLiteralFromMacro(const Token &FirstTok, |
616 | 1.01M | const Token &SecondTok) { |
617 | 1.01M | return FirstTok.is(tok::identifier) && |
618 | 242k | FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral()3 && |
619 | 3 | SecondTok.stringifiedInMacro(); |
620 | 1.01M | } |
621 | | |
622 | | /// Lex - Lex and return a token from this macro stream. |
623 | 542M | bool TokenLexer::Lex(Token &Tok) { |
624 | | // Lexing off the end of the macro, pop this macro off the expansion stack. |
625 | 542M | if (isAtEnd()) { |
626 | | // If this is a macro (not a token stream), mark the macro enabled now |
627 | | // that it is no longer being expanded. |
628 | 45.8M | if (Macro) Macro->EnableMacro()44.4M ; |
629 | | |
630 | 45.8M | Tok.startToken(); |
631 | 45.8M | Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); |
632 | 45.8M | Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace45.4M ); |
633 | 45.8M | if (CurTokenIdx == 0) |
634 | 262k | Tok.setFlag(Token::LeadingEmptyMacro); |
635 | 45.8M | return PP.HandleEndOfTokenLexer(Tok); |
636 | 45.8M | } |
637 | | |
638 | 496M | SourceManager &SM = PP.getSourceManager(); |
639 | | |
640 | | // If this is the first token of the expanded result, we inherit spacing |
641 | | // properties later. |
642 | 496M | bool isFirstToken = CurTokenIdx == 0; |
643 | | |
644 | | // Get the next token to return. |
645 | 496M | Tok = Tokens[CurTokenIdx++]; |
646 | 496M | if (IsReinject) |
647 | 12.3M | Tok.setFlag(Token::IsReinjected); |
648 | | |
649 | 496M | bool TokenIsFromPaste = false; |
650 | | |
651 | | // If this token is followed by a token paste (##) operator, paste the tokens! |
652 | | // Note that ## is a normal token when not expanding a macro. |
653 | 496M | if (!isAtEnd() && Macro449M && |
654 | 430M | (Tokens[CurTokenIdx].is(tok::hashhash) || |
655 | | // Special processing of L#x macros in -fms-compatibility mode. |
656 | | // Microsoft compiler is able to form a wide string literal from |
657 | | // 'L#macro_arg' construct in a function-like macro. |
658 | 422M | (PP.getLangOpts().MSVCCompat && |
659 | 7.78M | isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])1.01M ))) { |
660 | | // When handling the microsoft /##/ extension, the final token is |
661 | | // returned by pasteTokens, not the pasted token. |
662 | 7.78M | if (pasteTokens(Tok)) |
663 | 4 | return true; |
664 | | |
665 | 7.78M | TokenIsFromPaste = true; |
666 | 7.78M | } |
667 | | |
668 | | // The token's current location indicate where the token was lexed from. We |
669 | | // need this information to compute the spelling of the token, but any |
670 | | // diagnostics for the expanded token should appear as if they came from |
671 | | // ExpansionLoc. Pull this information together into a new SourceLocation |
672 | | // that captures all of this. |
673 | 496M | if (ExpandLocStart.isValid() && // Don't do this for token streams. |
674 | | // Check that the token's location was not already set properly. |
675 | 472M | SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) { |
676 | 345M | SourceLocation instLoc; |
677 | 345M | if (Tok.is(tok::comment)) { |
678 | 3 | instLoc = SM.createExpansionLoc(Tok.getLocation(), |
679 | 3 | ExpandLocStart, |
680 | 3 | ExpandLocEnd, |
681 | 3 | Tok.getLength()); |
682 | 345M | } else { |
683 | 345M | instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation()); |
684 | 345M | } |
685 | | |
686 | 345M | Tok.setLocation(instLoc); |
687 | 345M | } |
688 | | |
689 | | // If this is the first token, set the lexical properties of the token to |
690 | | // match the lexical properties of the macro identifier. |
691 | 496M | if (isFirstToken) { |
692 | 49.1M | Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); |
693 | 49.1M | Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace); |
694 | 447M | } else { |
695 | | // If this is not the first token, we may still need to pass through |
696 | | // leading whitespace if we've expanded a macro. |
697 | 447M | if (AtStartOfLine) Tok.setFlag(Token::StartOfLine)287k ; |
698 | 447M | if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace)340k ; |
699 | 447M | } |
700 | 496M | AtStartOfLine = false; |
701 | 496M | HasLeadingSpace = false; |
702 | | |
703 | | // Handle recursive expansion! |
704 | 496M | if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr495M ) { |
705 | | // Change the kind of this identifier to the appropriate token kind, e.g. |
706 | | // turning "for" into a keyword. |
707 | 177M | IdentifierInfo *II = Tok.getIdentifierInfo(); |
708 | 177M | Tok.setKind(II->getTokenID()); |
709 | | |
710 | | // If this identifier was poisoned and from a paste, emit an error. This |
711 | | // won't be handled by Preprocessor::HandleIdentifier because this is coming |
712 | | // from a macro expansion. |
713 | 177M | if (II->isPoisoned() && TokenIsFromPaste4 ) { |
714 | 2 | PP.HandlePoisonedIdentifier(Tok); |
715 | 2 | } |
716 | | |
717 | 177M | if (!DisableMacroExpansion && II->isHandleIdentifierCase()173M ) |
718 | 52.4M | return PP.HandleIdentifier(Tok); |
719 | 444M | } |
720 | | |
721 | | // Otherwise, return a normal token. |
722 | 444M | return true; |
723 | 444M | } |
724 | | |
725 | 7.78M | bool TokenLexer::pasteTokens(Token &Tok) { |
726 | 7.78M | return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx); |
727 | 7.78M | } |
728 | | |
729 | | /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ## |
730 | | /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there |
731 | | /// are more ## after it, chomp them iteratively. Return the result as LHSTok. |
732 | | /// If this returns true, the caller should immediately return the token. |
733 | | bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream, |
734 | 7.78M | unsigned int &CurIdx) { |
735 | 7.78M | assert(CurIdx > 0 && "## can not be the first token within tokens"); |
736 | 7.78M | assert((TokenStream[CurIdx].is(tok::hashhash) || |
737 | 7.78M | (PP.getLangOpts().MSVCCompat && |
738 | 7.78M | isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) && |
739 | 7.78M | "Token at this Index must be ## or part of the MSVC 'L " |
740 | 7.78M | "#macro-arg' pasting pair"); |
741 | | |
742 | | // MSVC: If previous token was pasted, this must be a recovery from an invalid |
743 | | // paste operation. Ignore spaces before this token to mimic MSVC output. |
744 | | // Required for generating valid UUID strings in some MS headers. |
745 | 7.78M | if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2)1.99k && |
746 | 475 | TokenStream[CurIdx - 2].is(tok::hashhash)) |
747 | 8 | LHSTok.clearFlag(Token::LeadingSpace); |
748 | | |
749 | 7.78M | SmallString<128> Buffer; |
750 | 7.78M | const char *ResultTokStrPtr = nullptr; |
751 | 7.78M | SourceLocation StartLoc = LHSTok.getLocation(); |
752 | 7.78M | SourceLocation PasteOpLoc; |
753 | | |
754 | 18.1M | auto IsAtEnd = [&TokenStream, &CurIdx] { |
755 | 18.1M | return TokenStream.size() == CurIdx; |
756 | 18.1M | }; |
757 | | |
758 | 9.07M | do { |
759 | | // Consume the ## operator if any. |
760 | 9.07M | PasteOpLoc = TokenStream[CurIdx].getLocation(); |
761 | 9.07M | if (TokenStream[CurIdx].is(tok::hashhash)) |
762 | 9.07M | ++CurIdx; |
763 | 9.07M | assert(!IsAtEnd() && "No token on the RHS of a paste operator!"); |
764 | | |
765 | | // Get the RHS token. |
766 | 9.07M | const Token &RHS = TokenStream[CurIdx]; |
767 | | |
768 | | // Allocate space for the result token. This is guaranteed to be enough for |
769 | | // the two tokens. |
770 | 9.07M | Buffer.resize(LHSTok.getLength() + RHS.getLength()); |
771 | | |
772 | | // Get the spelling of the LHS token in Buffer. |
773 | 9.07M | const char *BufPtr = &Buffer[0]; |
774 | 9.07M | bool Invalid = false; |
775 | 9.07M | unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid); |
776 | 9.07M | if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer! |
777 | 9.07M | memcpy(&Buffer[0], BufPtr, LHSLen); |
778 | 9.07M | if (Invalid) |
779 | 0 | return true; |
780 | | |
781 | 9.07M | BufPtr = Buffer.data() + LHSLen; |
782 | 9.07M | unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid); |
783 | 9.07M | if (Invalid) |
784 | 0 | return true; |
785 | 9.07M | if (RHSLen && BufPtr != &Buffer[LHSLen]9.07M ) |
786 | | // Really, we want the chars in Buffer! |
787 | 9.07M | memcpy(&Buffer[LHSLen], BufPtr, RHSLen); |
788 | | |
789 | | // Trim excess space. |
790 | 9.07M | Buffer.resize(LHSLen+RHSLen); |
791 | | |
792 | | // Plop the pasted result (including the trailing newline and null) into a |
793 | | // scratch buffer where we can lex it. |
794 | 9.07M | Token ResultTokTmp; |
795 | 9.07M | ResultTokTmp.startToken(); |
796 | | |
797 | | // Claim that the tmp token is a string_literal so that we can get the |
798 | | // character pointer back from CreateString in getLiteralData(). |
799 | 9.07M | ResultTokTmp.setKind(tok::string_literal); |
800 | 9.07M | PP.CreateString(Buffer, ResultTokTmp); |
801 | 9.07M | SourceLocation ResultTokLoc = ResultTokTmp.getLocation(); |
802 | 9.07M | ResultTokStrPtr = ResultTokTmp.getLiteralData(); |
803 | | |
804 | | // Lex the resultant pasted token into Result. |
805 | 9.07M | Token Result; |
806 | | |
807 | 9.07M | if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()9.07M ) { |
808 | | // Common paste case: identifier+identifier = identifier. Avoid creating |
809 | | // a lexer and other overhead. |
810 | 8.60M | PP.IncrementPasteCounter(true); |
811 | 8.60M | Result.startToken(); |
812 | 8.60M | Result.setKind(tok::raw_identifier); |
813 | 8.60M | Result.setRawIdentifierData(ResultTokStrPtr); |
814 | 8.60M | Result.setLocation(ResultTokLoc); |
815 | 8.60M | Result.setLength(LHSLen+RHSLen); |
816 | 462k | } else { |
817 | 462k | PP.IncrementPasteCounter(false); |
818 | | |
819 | 462k | assert(ResultTokLoc.isFileID() && |
820 | 462k | "Should be a raw location into scratch buffer"); |
821 | 462k | SourceManager &SourceMgr = PP.getSourceManager(); |
822 | 462k | FileID LocFileID = SourceMgr.getFileID(ResultTokLoc); |
823 | | |
824 | 462k | bool Invalid = false; |
825 | 462k | const char *ScratchBufStart |
826 | 462k | = SourceMgr.getBufferData(LocFileID, &Invalid).data(); |
827 | 462k | if (Invalid) |
828 | 0 | return false; |
829 | | |
830 | | // Make a lexer to lex this string from. Lex just this one token. |
831 | | // Make a lexer object so that we lex and expand the paste result. |
832 | 462k | Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID), |
833 | 462k | PP.getLangOpts(), ScratchBufStart, |
834 | 462k | ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen); |
835 | | |
836 | | // Lex a token in raw mode. This way it won't look up identifiers |
837 | | // automatically, lexing off the end will return an eof token, and |
838 | | // warnings are disabled. This returns true if the result token is the |
839 | | // entire buffer. |
840 | 462k | bool isInvalid = !TL.LexFromRawLexer(Result); |
841 | | |
842 | | // If we got an EOF token, we didn't form even ONE token. For example, we |
843 | | // did "/ ## /" to get "//". |
844 | 462k | isInvalid |= Result.is(tok::eof); |
845 | | |
846 | | // If pasting the two tokens didn't form a full new token, this is an |
847 | | // error. This occurs with "x ## +" and other stuff. Return with LHSTok |
848 | | // unmodified and with RHS as the next token to lex. |
849 | 462k | if (isInvalid) { |
850 | | // Explicitly convert the token location to have proper expansion |
851 | | // information so that the user knows where it came from. |
852 | 30 | SourceManager &SM = PP.getSourceManager(); |
853 | 30 | SourceLocation Loc = |
854 | 30 | SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2); |
855 | | |
856 | | // Test for the Microsoft extension of /##/ turning into // here on the |
857 | | // error path. |
858 | 30 | if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash)17 && |
859 | 4 | RHS.is(tok::slash)) { |
860 | 4 | HandleMicrosoftCommentPaste(LHSTok, Loc); |
861 | 4 | return true; |
862 | 4 | } |
863 | | |
864 | | // Do not emit the error when preprocessing assembler code. |
865 | 26 | if (!PP.getLangOpts().AsmPreprocessor) { |
866 | | // If we're in microsoft extensions mode, downgrade this from a hard |
867 | | // error to an extension that defaults to an error. This allows |
868 | | // disabling it. |
869 | 12 | PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms |
870 | 5 | : diag::err_pp_bad_paste) |
871 | 17 | << Buffer; |
872 | 17 | } |
873 | | |
874 | | // An error has occurred so exit loop. |
875 | 26 | break; |
876 | 26 | } |
877 | | |
878 | | // Turn ## into 'unknown' to avoid # ## # from looking like a paste |
879 | | // operator. |
880 | 462k | if (Result.is(tok::hashhash)) |
881 | 2 | Result.setKind(tok::unknown); |
882 | 462k | } |
883 | | |
884 | | // Transfer properties of the LHS over the Result. |
885 | 9.07M | Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine()); |
886 | 9.07M | Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace()); |
887 | | |
888 | | // Finally, replace LHS with the result, consume the RHS, and iterate. |
889 | 9.07M | ++CurIdx; |
890 | 9.07M | LHSTok = Result; |
891 | 9.07M | } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash)7.64M ); |
892 | | |
893 | 7.78M | SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation(); |
894 | | |
895 | | // The token's current location indicate where the token was lexed from. We |
896 | | // need this information to compute the spelling of the token, but any |
897 | | // diagnostics for the expanded token should appear as if the token was |
898 | | // expanded from the full ## expression. Pull this information together into |
899 | | // a new SourceLocation that captures all of this. |
900 | 7.78M | SourceManager &SM = PP.getSourceManager(); |
901 | 7.78M | if (StartLoc.isFileID()) |
902 | 7.31M | StartLoc = getExpansionLocForMacroDefLoc(StartLoc); |
903 | 7.78M | if (EndLoc.isFileID()) |
904 | 41.5k | EndLoc = getExpansionLocForMacroDefLoc(EndLoc); |
905 | 7.78M | FileID MacroFID = SM.getFileID(MacroExpansionStart); |
906 | 8.24M | while (SM.getFileID(StartLoc) != MacroFID) |
907 | 464k | StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin(); |
908 | 15.5M | while (SM.getFileID(EndLoc) != MacroFID) |
909 | 7.74M | EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd(); |
910 | | |
911 | 7.78M | LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc, |
912 | 7.78M | LHSTok.getLength())); |
913 | | |
914 | | // Now that we got the result token, it will be subject to expansion. Since |
915 | | // token pasting re-lexes the result token in raw mode, identifier information |
916 | | // isn't looked up. As such, if the result is an identifier, look up id info. |
917 | 7.78M | if (LHSTok.is(tok::raw_identifier)) { |
918 | | // Look up the identifier info for the token. We disabled identifier lookup |
919 | | // by saying we're skipping contents, so we need to do this manually. |
920 | 7.78M | PP.LookUpIdentifierInfo(LHSTok); |
921 | 7.78M | } |
922 | 7.78M | return false; |
923 | 7.78M | } |
924 | | |
925 | | /// isNextTokenLParen - If the next token lexed will pop this macro off the |
926 | | /// expansion stack, return 2. If the next unexpanded token is a '(', return |
927 | | /// 1, otherwise return 0. |
928 | 27.1M | unsigned TokenLexer::isNextTokenLParen() const { |
929 | | // Out of tokens? |
930 | 27.1M | if (isAtEnd()) |
931 | 2.99M | return 2; |
932 | 24.1M | return Tokens[CurTokenIdx].is(tok::l_paren); |
933 | 24.1M | } |
934 | | |
935 | | /// isParsingPreprocessorDirective - Return true if we are in the middle of a |
936 | | /// preprocessor directive. |
937 | 574k | bool TokenLexer::isParsingPreprocessorDirective() const { |
938 | 574k | return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd()25 ; |
939 | 574k | } |
940 | | |
941 | | /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes |
942 | | /// together to form a comment that comments out everything in the current |
943 | | /// macro, other active macros, and anything left on the current physical |
944 | | /// source line of the expanded buffer. Handle this by returning the |
945 | | /// first token on the next line. |
946 | 4 | void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) { |
947 | 4 | PP.Diag(OpLoc, diag::ext_comment_paste_microsoft); |
948 | | |
949 | | // We 'comment out' the rest of this macro by just ignoring the rest of the |
950 | | // tokens that have not been lexed yet, if any. |
951 | | |
952 | | // Since this must be a macro, mark the macro enabled now that it is no longer |
953 | | // being expanded. |
954 | 4 | assert(Macro && "Token streams can't paste comments"); |
955 | 4 | Macro->EnableMacro(); |
956 | | |
957 | 4 | PP.HandleMicrosoftCommentPaste(Tok); |
958 | 4 | } |
959 | | |
960 | | /// If \arg loc is a file ID and points inside the current macro |
961 | | /// definition, returns the appropriate source location pointing at the |
962 | | /// macro expansion source location entry, otherwise it returns an invalid |
963 | | /// SourceLocation. |
964 | | SourceLocation |
965 | 391M | TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const { |
966 | 391M | assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() && |
967 | 391M | "Not appropriate for token streams"); |
968 | 391M | assert(loc.isValid() && loc.isFileID()); |
969 | | |
970 | 391M | SourceManager &SM = PP.getSourceManager(); |
971 | 391M | assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) && |
972 | 391M | "Expected loc to come from the macro definition"); |
973 | | |
974 | 391M | unsigned relativeOffset = 0; |
975 | 391M | SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset); |
976 | 391M | return MacroExpansionStart.getLocWithOffset(relativeOffset); |
977 | 391M | } |
978 | | |
979 | | /// Finds the tokens that are consecutive (from the same FileID) |
980 | | /// creates a single SLocEntry, and assigns SourceLocations to each token that |
981 | | /// point to that SLocEntry. e.g for |
982 | | /// assert(foo == bar); |
983 | | /// There will be a single SLocEntry for the "foo == bar" chunk and locations |
984 | | /// for the 'foo', '==', 'bar' tokens will point inside that chunk. |
985 | | /// |
986 | | /// \arg begin_tokens will be updated to a position past all the found |
987 | | /// consecutive tokens. |
988 | | static void updateConsecutiveMacroArgTokens(SourceManager &SM, |
989 | | SourceLocation InstLoc, |
990 | | Token *&begin_tokens, |
991 | 21.8M | Token * end_tokens) { |
992 | 21.8M | assert(begin_tokens < end_tokens); |
993 | | |
994 | 21.8M | SourceLocation FirstLoc = begin_tokens->getLocation(); |
995 | 21.8M | SourceLocation CurLoc = FirstLoc; |
996 | | |
997 | | // Compare the source location offset of tokens and group together tokens that |
998 | | // are close, even if their locations point to different FileIDs. e.g. |
999 | | // |
1000 | | // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs) |
1001 | | // ^ ^ |
1002 | | // |bar foo cake| (one SLocEntry chunk for all tokens) |
1003 | | // |
1004 | | // we can perform this "merge" since the token's spelling location depends |
1005 | | // on the relative offset. |
1006 | | |
1007 | 21.8M | Token *NextTok = begin_tokens + 1; |
1008 | 103M | for (; NextTok < end_tokens; ++NextTok81.8M ) { |
1009 | 90.3M | SourceLocation NextLoc = NextTok->getLocation(); |
1010 | 90.3M | if (CurLoc.isFileID() != NextLoc.isFileID()) |
1011 | 288k | break; // Token from different kind of FileID. |
1012 | | |
1013 | 90.0M | int RelOffs; |
1014 | 90.0M | if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs)) |
1015 | 0 | break; // Token from different local/loaded location. |
1016 | | // Check that token is not before the previous token or more than 50 |
1017 | | // "characters" away. |
1018 | 90.0M | if (RelOffs < 0 || RelOffs > 5088.0M ) |
1019 | 2.21M | break; |
1020 | | |
1021 | 87.8M | if (CurLoc.isMacroID() && !SM.isWrittenInSameFile(CurLoc, NextLoc)46.9M ) |
1022 | 5.93M | break; // Token from a different macro. |
1023 | | |
1024 | 81.8M | CurLoc = NextLoc; |
1025 | 81.8M | } |
1026 | | |
1027 | | // For the consecutive tokens, find the length of the SLocEntry to contain |
1028 | | // all of them. |
1029 | 21.8M | Token &LastConsecutiveTok = *(NextTok-1); |
1030 | 21.8M | int LastRelOffs = 0; |
1031 | 21.8M | SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(), |
1032 | 21.8M | &LastRelOffs); |
1033 | 21.8M | unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength(); |
1034 | | |
1035 | | // Create a macro expansion SLocEntry that will "contain" all of the tokens. |
1036 | 21.8M | SourceLocation Expansion = |
1037 | 21.8M | SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength); |
1038 | | |
1039 | | // Change the location of the tokens from the spelling location to the new |
1040 | | // expanded location. |
1041 | 125M | for (; begin_tokens < NextTok; ++begin_tokens103M ) { |
1042 | 103M | Token &Tok = *begin_tokens; |
1043 | 103M | int RelOffs = 0; |
1044 | 103M | SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs); |
1045 | 103M | Tok.setLocation(Expansion.getLocWithOffset(RelOffs)); |
1046 | 103M | } |
1047 | 21.8M | } |
1048 | | |
1049 | | /// Creates SLocEntries and updates the locations of macro argument |
1050 | | /// tokens to their new expanded locations. |
1051 | | /// |
1052 | | /// \param ArgIdSpellLoc the location of the macro argument id inside the macro |
1053 | | /// definition. |
1054 | | void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc, |
1055 | | Token *begin_tokens, |
1056 | 37.6M | Token *end_tokens) { |
1057 | 37.6M | SourceManager &SM = PP.getSourceManager(); |
1058 | | |
1059 | 37.6M | SourceLocation InstLoc = |
1060 | 37.6M | getExpansionLocForMacroDefLoc(ArgIdSpellLoc); |
1061 | | |
1062 | 59.4M | while (begin_tokens < end_tokens) { |
1063 | | // If there's only one token just create a SLocEntry for it. |
1064 | 46.0M | if (end_tokens - begin_tokens == 1) { |
1065 | 24.2M | Token &Tok = *begin_tokens; |
1066 | 24.2M | Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(), |
1067 | 24.2M | InstLoc, |
1068 | 24.2M | Tok.getLength())); |
1069 | 24.2M | return; |
1070 | 24.2M | } |
1071 | | |
1072 | 21.8M | updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens); |
1073 | 21.8M | } |
1074 | 37.6M | } |
1075 | | |
1076 | 28.3M | void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { |
1077 | 28.3M | AtStartOfLine = Result.isAtStartOfLine(); |
1078 | 28.3M | HasLeadingSpace = Result.hasLeadingSpace(); |
1079 | 28.3M | } |