Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/lib/Lex/TokenLexer.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements the TokenLexer interface.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "clang/Lex/TokenLexer.h"
14
#include "clang/Basic/Diagnostic.h"
15
#include "clang/Basic/IdentifierTable.h"
16
#include "clang/Basic/LangOptions.h"
17
#include "clang/Basic/SourceLocation.h"
18
#include "clang/Basic/SourceManager.h"
19
#include "clang/Basic/TokenKinds.h"
20
#include "clang/Lex/LexDiagnostic.h"
21
#include "clang/Lex/Lexer.h"
22
#include "clang/Lex/MacroArgs.h"
23
#include "clang/Lex/MacroInfo.h"
24
#include "clang/Lex/Preprocessor.h"
25
#include "clang/Lex/Token.h"
26
#include "clang/Lex/VariadicMacroSupport.h"
27
#include "llvm/ADT/ArrayRef.h"
28
#include "llvm/ADT/SmallString.h"
29
#include "llvm/ADT/SmallVector.h"
30
#include "llvm/ADT/iterator_range.h"
31
#include <cassert>
32
#include <cstring>
33
34
using namespace clang;
35
36
/// Create a TokenLexer for the specified macro with the specified actual
37
/// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
38
void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
39
12.9M
                      MacroArgs *Actuals) {
40
12.9M
  // If the client is reusing a TokenLexer, make sure to free any memory
41
12.9M
  // associated with it.
42
12.9M
  destroy();
43
12.9M
44
12.9M
  Macro = MI;
45
12.9M
  ActualArgs = Actuals;
46
12.9M
  CurTokenIdx = 0;
47
12.9M
48
12.9M
  ExpandLocStart = Tok.getLocation();
49
12.9M
  ExpandLocEnd = ELEnd;
50
12.9M
  AtStartOfLine = Tok.isAtStartOfLine();
51
12.9M
  HasLeadingSpace = Tok.hasLeadingSpace();
52
12.9M
  NextTokGetsSpace = false;
53
12.9M
  Tokens = &*Macro->tokens_begin();
54
12.9M
  OwnsTokens = false;
55
12.9M
  DisableMacroExpansion = false;
56
12.9M
  IsReinject = false;
57
12.9M
  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
58
12.9M
  MacroExpansionStart = SourceLocation();
59
12.9M
60
12.9M
  SourceManager &SM = PP.getSourceManager();
61
12.9M
  MacroStartSLocOffset = SM.getNextLocalOffset();
62
12.9M
63
12.9M
  if (NumTokens > 0) {
64
12.9M
    assert(Tokens[0].getLocation().isValid());
65
12.9M
    assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66
12.9M
           "Macro defined in macro?");
67
12.9M
    assert(ExpandLocStart.isValid());
68
12.9M
69
12.9M
    // Reserve a source location entry chunk for the length of the macro
70
12.9M
    // definition. Tokens that get lexed directly from the definition will
71
12.9M
    // have their locations pointing inside this chunk. This is to avoid
72
12.9M
    // creating separate source location entries for each token.
73
12.9M
    MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
74
12.9M
    MacroDefLength = Macro->getDefinitionLength(SM);
75
12.9M
    MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
76
12.9M
                                                ExpandLocStart,
77
12.9M
                                                ExpandLocEnd,
78
12.9M
                                                MacroDefLength);
79
12.9M
  }
80
12.9M
81
12.9M
  // If this is a function-like macro, expand the arguments and change
82
12.9M
  // Tokens to point to the expanded tokens.
83
12.9M
  if (Macro->isFunctionLike() && 
Macro->getNumParams()4.38M
)
84
4.34M
    ExpandFunctionArguments();
85
12.9M
86
12.9M
  // Mark the macro as currently disabled, so that it is not recursively
87
12.9M
  // expanded.  The macro must be disabled only after argument pre-expansion of
88
12.9M
  // function-like macro arguments occurs.
89
12.9M
  Macro->DisableMacro();
90
12.9M
}
91
92
/// Create a TokenLexer for the specified token stream.  This does not
93
/// take ownership of the specified token vector.
94
void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
95
                      bool disableMacroExpansion, bool ownsTokens,
96
2.10M
                      bool isReinject) {
97
2.10M
  assert(!isReinject || disableMacroExpansion);
98
2.10M
  // If the client is reusing a TokenLexer, make sure to free any memory
99
2.10M
  // associated with it.
100
2.10M
  destroy();
101
2.10M
102
2.10M
  Macro = nullptr;
103
2.10M
  ActualArgs = nullptr;
104
2.10M
  Tokens = TokArray;
105
2.10M
  OwnsTokens = ownsTokens;
106
2.10M
  DisableMacroExpansion = disableMacroExpansion;
107
2.10M
  IsReinject = isReinject;
108
2.10M
  NumTokens = NumToks;
109
2.10M
  CurTokenIdx = 0;
110
2.10M
  ExpandLocStart = ExpandLocEnd = SourceLocation();
111
2.10M
  AtStartOfLine = false;
112
2.10M
  HasLeadingSpace = false;
113
2.10M
  NextTokGetsSpace = false;
114
2.10M
  MacroExpansionStart = SourceLocation();
115
2.10M
116
2.10M
  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
117
2.10M
  // returned unmodified.
118
2.10M
  if (NumToks != 0) {
119
2.10M
    AtStartOfLine   = TokArray[0].isAtStartOfLine();
120
2.10M
    HasLeadingSpace = TokArray[0].hasLeadingSpace();
121
2.10M
  }
122
2.10M
}
123
124
15.5M
void TokenLexer::destroy() {
125
15.5M
  // If this was a function-like macro that actually uses its arguments, delete
126
15.5M
  // the expanded tokens.
127
15.5M
  if (OwnsTokens) {
128
124k
    delete [] Tokens;
129
124k
    Tokens = nullptr;
130
124k
    OwnsTokens = false;
131
124k
  }
132
15.5M
133
15.5M
  // TokenLexer owns its formal arguments.
134
15.5M
  if (ActualArgs) 
ActualArgs->destroy(PP)4.36M
;
135
15.5M
}
136
137
bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
138
    SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
139
3.93k
    unsigned MacroArgNo, Preprocessor &PP) {
140
3.93k
  // Is the macro argument __VA_ARGS__?
141
3.93k
  if (!Macro->isVariadic() || 
MacroArgNo != Macro->getNumParams()-13.93k
)
142
2.50k
    return false;
143
1.43k
144
1.43k
  // In Microsoft-compatibility mode, a comma is removed in the expansion
145
1.43k
  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty.  This extension is
146
1.43k
  // not supported by gcc.
147
1.43k
  if (!HasPasteOperator && 
!PP.getLangOpts().MSVCCompat771
)
148
765
    return false;
149
666
150
666
  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
151
666
  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
152
666
  // named arguments, where it remains.  In all other modes, including C99
153
666
  // with GNU extensions, it is removed regardless of named arguments.
154
666
  // Microsoft also appears to support this extension, unofficially.
155
666
  if (PP.getLangOpts().C99 && 
!PP.getLangOpts().GNUMode104
156
666
        && 
Macro->getNumParams() < 29
)
157
3
    return false;
158
663
159
663
  // Is a comma available to be removed?
160
663
  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
161
3
    return false;
162
660
163
660
  // Issue an extension diagnostic for the paste operator.
164
660
  if (HasPasteOperator)
165
657
    PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
166
660
167
660
  // Remove the comma.
168
660
  ResultToks.pop_back();
169
660
170
660
  if (!ResultToks.empty()) {
171
660
    // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
172
660
    // then removal of the comma should produce a placemarker token (in C99
173
660
    // terms) which we model by popping off the previous ##, giving us a plain
174
660
    // "X" when __VA_ARGS__ is empty.
175
660
    if (ResultToks.back().is(tok::hashhash))
176
1
      ResultToks.pop_back();
177
660
178
660
    // Remember that this comma was elided.
179
660
    ResultToks.back().setFlag(Token::CommaAfterElided);
180
660
  }
181
660
182
660
  // Never add a space, even if the comma, ##, or arg had a space.
183
660
  NextTokGetsSpace = false;
184
660
  return true;
185
660
}
186
187
void TokenLexer::stringifyVAOPTContents(
188
    SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
189
15
    const SourceLocation VAOPTClosingParenLoc) {
190
15
  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
191
15
  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
192
15
  Token *const VAOPTTokens =
193
15
      NumVAOptTokens ? 
&ResultToks[NumToksPriorToVAOpt]5
:
nullptr10
;
194
15
195
15
  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
196
15
  // FIXME: Should we keep track within VCtx that we did or didnot
197
15
  // encounter pasting - and only then perform this loop.
198
15
199
15
  // Perform token pasting (concatenation) prior to stringization.
200
56
  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
201
41
       ++CurTokenIdx) {
202
41
    if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
203
6
      assert(CurTokenIdx != 0 &&
204
6
             "Can not have __VAOPT__ contents begin with a ##");
205
6
      Token &LHS = VAOPTTokens[CurTokenIdx - 1];
206
6
      pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
207
6
                  CurTokenIdx);
208
6
      // Replace the token prior to the first ## in this iteration.
209
6
      ConcatenatedVAOPTResultToks.back() = LHS;
210
6
      if (CurTokenIdx == NumVAOptTokens)
211
0
        break;
212
41
    }
213
41
    ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
214
41
  }
215
15
216
15
  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
217
15
  // Get the SourceLocation that represents the start location within
218
15
  // the macro definition that marks where this string is substituted
219
15
  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
220
15
  // macro definition, and use it to indicate that the stringified token
221
15
  // was generated from that location.
222
15
  const SourceLocation ExpansionLocStartWithinMacro =
223
15
      getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
224
15
  const SourceLocation ExpansionLocEndWithinMacro =
225
15
      getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
226
15
227
15
  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
228
15
      &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
229
15
      ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
230
15
231
15
  if (VCtx.getLeadingSpaceForStringifiedToken())
232
10
    StringifiedVAOPT.setFlag(Token::LeadingSpace);
233
15
234
15
  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
235
15
  // Resize (shrink) the token stream to just capture this stringified token.
236
15
  ResultToks.resize(NumToksPriorToVAOpt + 1);
237
15
  ResultToks.back() = StringifiedVAOPT;
238
15
}
239
240
/// Expand the arguments of a function-like macro so that we can quickly
241
/// return preexpanded tokens from Tokens.
242
4.34M
void TokenLexer::ExpandFunctionArguments() {
243
4.34M
  SmallVector<Token, 128> ResultToks;
244
4.34M
245
4.34M
  // Loop through 'Tokens', expanding them into ResultToks.  Keep
246
4.34M
  // track of whether we change anything.  If not, no need to keep them.  If so,
247
4.34M
  // we install the newly expanded sequence as the new 'Tokens' list.
248
4.34M
  bool MadeChange = false;
249
4.34M
250
4.34M
  Optional<bool> CalledWithVariadicArguments;
251
4.34M
252
4.34M
  VAOptExpansionContext VCtx(PP);
253
4.34M
254
49.9M
  for (unsigned I = 0, E = NumTokens; I != E; 
++I45.6M
) {
255
45.6M
    const Token &CurTok = Tokens[I];
256
45.6M
    // We don't want a space for the next token after a paste
257
45.6M
    // operator.  In valid code, the token will get smooshed onto the
258
45.6M
    // preceding one anyway. In assembler-with-cpp mode, invalid
259
45.6M
    // pastes are allowed through: in this case, we do not want the
260
45.6M
    // extra whitespace to be added.  For example, we want ". ## foo"
261
45.6M
    // -> ".foo" not ". foo".
262
45.6M
    if (I != 0 && 
!Tokens[I-1].is(tok::hashhash)41.2M
&&
CurTok.hasLeadingSpace()40.2M
)
263
8.79M
      NextTokGetsSpace = true;
264
45.6M
265
45.6M
    if (VCtx.isVAOptToken(CurTok)) {
266
47
      MadeChange = true;
267
47
      assert(Tokens[I + 1].is(tok::l_paren) &&
268
47
             "__VA_OPT__ must be followed by '('");
269
47
270
47
      ++I;             // Skip the l_paren
271
47
      VCtx.sawVAOptFollowedByOpeningParens(CurTok.getLocation(),
272
47
                                           ResultToks.size());
273
47
274
47
      continue;
275
47
    }
276
45.6M
277
45.6M
    // We have entered into the __VA_OPT__ context, so handle tokens
278
45.6M
    // appropriately.
279
45.6M
    if (VCtx.isInVAOpt()) {
280
226
      // If we are about to process a token that is either an argument to
281
226
      // __VA_OPT__ or its closing rparen, then:
282
226
      //  1) If the token is the closing rparen that exits us out of __VA_OPT__,
283
226
      //  perform any necessary stringification or placemarker processing,
284
226
      //  and/or skip to the next token.
285
226
      //  2) else if macro was invoked without variadic arguments skip this
286
226
      //  token.
287
226
      //  3) else (macro was invoked with variadic arguments) process the token
288
226
      //  normally.
289
226
290
226
      if (Tokens[I].is(tok::l_paren))
291
2
        VCtx.sawOpeningParen(Tokens[I].getLocation());
292
226
      // Continue skipping tokens within __VA_OPT__ if the macro was not
293
226
      // called with variadic arguments, else let the rest of the loop handle
294
226
      // this token. Note sawClosingParen() returns true only if the r_paren matches
295
226
      // the closing r_paren of the __VA_OPT__.
296
226
      if (!Tokens[I].is(tok::r_paren) || 
!VCtx.sawClosingParen()49
) {
297
179
        // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__.
298
179
        if (!CalledWithVariadicArguments.hasValue()) {
299
35
          CalledWithVariadicArguments =
300
35
              ActualArgs->invokedWithVariadicArgument(Macro, PP);
301
35
        }
302
179
        if (!*CalledWithVariadicArguments) {
303
83
          // Skip this token.
304
83
          continue;
305
83
        }
306
47
        // ... else the macro was called with variadic arguments, and we do not
307
47
        // have a closing rparen - so process this token normally.
308
47
      } else {
309
47
        // Current token is the closing r_paren which marks the end of the
310
47
        // __VA_OPT__ invocation, so handle any place-marker pasting (if
311
47
        // empty) by removing hashhash either before (if exists) or after. And
312
47
        // also stringify the entire contents if VAOPT was preceded by a hash,
313
47
        // but do so only after any token concatenation that needs to occur
314
47
        // within the contents of VAOPT.
315
47
316
47
        if (VCtx.hasStringifyOrCharifyBefore()) {
317
15
          // Replace all the tokens just added from within VAOPT into a single
318
15
          // stringified token. This requires token-pasting to eagerly occur
319
15
          // within these tokens. If either the contents of VAOPT were empty
320
15
          // or the macro wasn't called with any variadic arguments, the result
321
15
          // is a token that represents an empty string.
322
15
          stringifyVAOPTContents(ResultToks, VCtx,
323
15
                                 /*ClosingParenLoc*/ Tokens[I].getLocation());
324
15
325
32
        } else if (/*No tokens within VAOPT*/
326
32
                   ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) {
327
15
          // Treat VAOPT as a placemarker token.  Eat either the '##' before the
328
15
          // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
329
15
          // hashhash was not a placemarker) or the '##'
330
15
          // after VAOPT, but not both.
331
15
332
15
          if (ResultToks.size() && 
ResultToks.back().is(tok::hashhash)11
) {
333
4
            ResultToks.pop_back();
334
11
          } else if ((I + 1 != E) && 
Tokens[I + 1].is(tok::hashhash)9
) {
335
2
            ++I; // Skip the following hashhash.
336
2
          }
337
17
        } else {
338
17
          // If there's a ## before the __VA_OPT__, we might have discovered
339
17
          // that the __VA_OPT__ begins with a placeholder. We delay action on
340
17
          // that to now to avoid messing up our stashed count of tokens before
341
17
          // __VA_OPT__.
342
17
          if (VCtx.beginsWithPlaceholder()) {
343
3
            assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 &&
344
3
                   ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() &&
345
3
                   ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(
346
3
                       tok::hashhash) &&
347
3
                   "no token paste before __VA_OPT__");
348
3
            ResultToks.erase(ResultToks.begin() +
349
3
                             VCtx.getNumberOfTokensPriorToVAOpt() - 1);
350
3
          }
351
17
          // If the expansion of __VA_OPT__ ends with a placeholder, eat any
352
17
          // following '##' token.
353
17
          if (VCtx.endsWithPlaceholder() && 
I + 1 != E3
&&
354
17
              
Tokens[I + 1].is(tok::hashhash)3
) {
355
3
            ++I;
356
3
          }
357
17
        }
358
47
        VCtx.reset();
359
47
        // We processed __VA_OPT__'s closing paren (and the exit out of
360
47
        // __VA_OPT__), so skip to the next token.
361
47
        continue;
362
47
      }
363
45.6M
    }
364
45.6M
365
45.6M
    // If we found the stringify operator, get the argument stringified.  The
366
45.6M
    // preprocessor already verified that the following token is a macro
367
45.6M
    // parameter or __VA_OPT__ when the #define was lexed.
368
45.6M
369
45.6M
    if (CurTok.isOneOf(tok::hash, tok::hashat)) {
370
448k
      int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
371
448k
      assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
372
448k
             "Token following # is not an argument or __VA_OPT__!");
373
448k
374
448k
      if (ArgNo == -1) {
375
15
        // Handle the __VA_OPT__ case.
376
15
        VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
377
15
                                   CurTok.is(tok::hashat));
378
15
        continue;
379
15
      }
380
448k
      // Else handle the simple argument case.
381
448k
      SourceLocation ExpansionLocStart =
382
448k
          getExpansionLocForMacroDefLoc(CurTok.getLocation());
383
448k
      SourceLocation ExpansionLocEnd =
384
448k
          getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
385
448k
386
448k
      Token Res;
387
448k
      if (CurTok.is(tok::hash))  // Stringify
388
448k
        Res = ActualArgs->getStringifiedArgument(ArgNo, PP,
389
448k
                                                 ExpansionLocStart,
390
448k
                                                 ExpansionLocEnd);
391
1
      else {
392
1
        // 'charify': don't bother caching these.
393
1
        Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
394
1
                                           PP, true,
395
1
                                           ExpansionLocStart,
396
1
                                           ExpansionLocEnd);
397
1
      }
398
448k
      Res.setFlag(Token::StringifiedInMacro);
399
448k
400
448k
      // The stringified/charified string leading space flag gets set to match
401
448k
      // the #/#@ operator.
402
448k
      if (NextTokGetsSpace)
403
354k
        Res.setFlag(Token::LeadingSpace);
404
448k
405
448k
      ResultToks.push_back(Res);
406
448k
      MadeChange = true;
407
448k
      ++I;  // Skip arg name.
408
448k
      NextTokGetsSpace = false;
409
448k
      continue;
410
448k
    }
411
45.1M
412
45.1M
    // Find out if there is a paste (##) operator before or after the token.
413
45.1M
    bool NonEmptyPasteBefore =
414
45.1M
      !ResultToks.empty() && 
ResultToks.back().is(tok::hashhash)40.9M
;
415
45.1M
    bool PasteBefore = I != 0 && 
Tokens[I-1].is(tok::hashhash)40.9M
;
416
45.1M
    bool PasteAfter = I+1 != E && 
Tokens[I+1].is(tok::hashhash)40.9M
;
417
45.1M
    bool RParenAfter = I+1 != E && 
Tokens[I+1].is(tok::r_paren)40.9M
;
418
45.1M
419
45.1M
    assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
420
45.1M
           "unexpected ## in ResultToks");
421
45.1M
422
45.1M
    // Otherwise, if this is not an argument token, just add the token to the
423
45.1M
    // output buffer.
424
45.1M
    IdentifierInfo *II = CurTok.getIdentifierInfo();
425
45.1M
    int ArgNo = II ? 
Macro->getParameterNum(II)18.9M
:
-126.1M
;
426
45.1M
    if (ArgNo == -1) {
427
37.8M
      // This isn't an argument, just add it.
428
37.8M
      ResultToks.push_back(CurTok);
429
37.8M
430
37.8M
      if (NextTokGetsSpace) {
431
7.64M
        ResultToks.back().setFlag(Token::LeadingSpace);
432
7.64M
        NextTokGetsSpace = false;
433
30.2M
      } else if (PasteBefore && 
!NonEmptyPasteBefore63.2k
)
434
6
        ResultToks.back().clearFlag(Token::LeadingSpace);
435
37.8M
436
37.8M
      continue;
437
37.8M
    }
438
7.30M
439
7.30M
    // An argument is expanded somehow, the result is different than the
440
7.30M
    // input.
441
7.30M
    MadeChange = true;
442
7.30M
443
7.30M
    // Otherwise, this is a use of the argument.
444
7.30M
445
7.30M
    // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
446
7.30M
    // are no trailing commas if __VA_ARGS__ is empty.
447
7.30M
    if (!PasteBefore && 
ActualArgs->isVarargsElidedUse()6.31M
&&
448
7.30M
        MaybeRemoveCommaBeforeVaArgs(ResultToks,
449
3.26k
                                     /*HasPasteOperator=*/false,
450
3.26k
                                     Macro, ArgNo, PP))
451
3
      continue;
452
7.30M
453
7.30M
    // If it is not the LHS/RHS of a ## operator, we must pre-expand the
454
7.30M
    // argument and substitute the expanded tokens into the result.  This is
455
7.30M
    // C99 6.10.3.1p1.
456
7.30M
    if (!PasteBefore && 
!PasteAfter6.31M
) {
457
5.88M
      const Token *ResultArgToks;
458
5.88M
459
5.88M
      // Only preexpand the argument if it could possibly need it.  This
460
5.88M
      // avoids some work in common cases.
461
5.88M
      const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
462
5.88M
      if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
463
718k
        ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
464
5.16M
      else
465
5.16M
        ResultArgToks = ArgTok;  // Use non-preexpanded tokens.
466
5.88M
467
5.88M
      // If the arg token expanded into anything, append it.
468
5.88M
      if (ResultArgToks->isNot(tok::eof)) {
469
5.25M
        size_t FirstResult = ResultToks.size();
470
5.25M
        unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
471
5.25M
        ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
472
5.25M
473
5.25M
        // In Microsoft-compatibility mode, we follow MSVC's preprocessing
474
5.25M
        // behavior by not considering single commas from nested macro
475
5.25M
        // expansions as argument separators. Set a flag on the token so we can
476
5.25M
        // test for this later when the macro expansion is processed.
477
5.25M
        if (PP.getLangOpts().MSVCCompat && 
NumToks == 110.6k
&&
478
5.25M
            
ResultToks.back().is(tok::comma)7.63k
)
479
4
          ResultToks.back().setFlag(Token::IgnoredComma);
480
5.25M
481
5.25M
        // If the '##' came from expanding an argument, turn it into 'unknown'
482
5.25M
        // to avoid pasting.
483
5.25M
        for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
484
15.4M
                                           ResultToks.end())) {
485
15.4M
          if (Tok.is(tok::hashhash))
486
2
            Tok.setKind(tok::unknown);
487
15.4M
        }
488
5.25M
489
5.25M
        if(ExpandLocStart.isValid()) {
490
5.25M
          updateLocForMacroArgTokens(CurTok.getLocation(),
491
5.25M
                                     ResultToks.begin()+FirstResult,
492
5.25M
                                     ResultToks.end());
493
5.25M
        }
494
5.25M
495
5.25M
        // If any tokens were substituted from the argument, the whitespace
496
5.25M
        // before the first token should match the whitespace of the arg
497
5.25M
        // identifier.
498
5.25M
        ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
499
5.25M
                                             NextTokGetsSpace);
500
5.25M
        ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
501
5.25M
        NextTokGetsSpace = false;
502
5.25M
      } else {
503
627k
        // We're creating a placeholder token. Usually this doesn't matter,
504
627k
        // but it can affect paste behavior when at the start or end of a
505
627k
        // __VA_OPT__.
506
627k
        if (NonEmptyPasteBefore) {
507
1
          // We're imagining a placeholder token is inserted here. If this is
508
1
          // the first token in a __VA_OPT__ after a ##, delete the ##.
509
1
          assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");
510
1
          VCtx.hasPlaceholderAfterHashhashAtStart();
511
1
        }
512
627k
        if (RParenAfter)
513
359k
          VCtx.hasPlaceholderBeforeRParen();
514
627k
      }
515
5.88M
      continue;
516
5.88M
    }
517
1.42M
518
1.42M
    // Okay, we have a token that is either the LHS or RHS of a paste (##)
519
1.42M
    // argument.  It gets substituted as its non-pre-expanded tokens.
520
1.42M
    const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
521
1.42M
    unsigned NumToks = MacroArgs::getArgLength(ArgToks);
522
1.42M
    if (NumToks) {  // Not an empty argument?
523
1.41M
      bool VaArgsPseudoPaste = false;
524
1.41M
      // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
525
1.41M
      // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
526
1.41M
      // the expander tries to paste ',' with the first token of the __VA_ARGS__
527
1.41M
      // expansion.
528
1.41M
      if (NonEmptyPasteBefore && 
ResultToks.size() >= 2978k
&&
529
1.41M
          
ResultToks[ResultToks.size()-2].is(tok::comma)978k
&&
530
1.41M
          
(unsigned)ArgNo == Macro->getNumParams()-1155
&&
531
1.41M
          
Macro->isVariadic()155
) {
532
155
        VaArgsPseudoPaste = true;
533
155
        // Remove the paste operator, report use of the extension.
534
155
        PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
535
155
      }
536
1.41M
537
1.41M
      ResultToks.append(ArgToks, ArgToks+NumToks);
538
1.41M
539
1.41M
      // If the '##' came from expanding an argument, turn it into 'unknown'
540
1.41M
      // to avoid pasting.
541
1.41M
      for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
542
2.46M
                                         ResultToks.end())) {
543
2.46M
        if (Tok.is(tok::hashhash))
544
0
          Tok.setKind(tok::unknown);
545
2.46M
      }
546
1.41M
547
1.41M
      if (ExpandLocStart.isValid()) {
548
1.41M
        updateLocForMacroArgTokens(CurTok.getLocation(),
549
1.41M
                                   ResultToks.end()-NumToks, ResultToks.end());
550
1.41M
      }
551
1.41M
552
1.41M
      // Transfer the leading whitespace information from the token
553
1.41M
      // (the macro argument) onto the first token of the
554
1.41M
      // expansion. Note that we don't do this for the GNU
555
1.41M
      // pseudo-paste extension ", ## __VA_ARGS__".
556
1.41M
      if (!VaArgsPseudoPaste) {
557
1.41M
        ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
558
1.41M
                                                             false);
559
1.41M
        ResultToks[ResultToks.size() - NumToks].setFlagValue(
560
1.41M
            Token::LeadingSpace, NextTokGetsSpace);
561
1.41M
      }
562
1.41M
563
1.41M
      NextTokGetsSpace = false;
564
1.41M
      continue;
565
1.41M
    }
566
9.72k
567
9.72k
    // If an empty argument is on the LHS or RHS of a paste, the standard (C99
568
9.72k
    // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur.  We
569
9.72k
    // implement this by eating ## operators when a LHS or RHS expands to
570
9.72k
    // empty.
571
9.72k
    if (PasteAfter) {
572
78
      // Discard the argument token and skip (don't copy to the expansion
573
78
      // buffer) the paste operator after it.
574
78
      ++I;
575
78
      continue;
576
78
    }
577
9.64k
578
9.64k
    if (RParenAfter)
579
638
      VCtx.hasPlaceholderBeforeRParen();
580
9.64k
581
9.64k
    // If this is on the RHS of a paste operator, we've already copied the
582
9.64k
    // paste operator to the ResultToks list, unless the LHS was empty too.
583
9.64k
    // Remove it.
584
9.64k
    assert(PasteBefore);
585
9.64k
    if (NonEmptyPasteBefore) {
586
9.61k
      assert(ResultToks.back().is(tok::hashhash));
587
9.61k
      // Do not remove the paste operator if it is the one before __VA_OPT__
588
9.61k
      // (and we are still processing tokens within VA_OPT).  We handle the case
589
9.61k
      // of removing the paste operator if __VA_OPT__ reduces to the notional
590
9.61k
      // placemarker above when we encounter the closing paren of VA_OPT.
591
9.61k
      if (!VCtx.isInVAOpt() ||
592
9.61k
          
ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt()2
)
593
9.61k
        ResultToks.pop_back();
594
2
      else
595
2
        VCtx.hasPlaceholderAfterHashhashAtStart();
596
9.61k
    }
597
9.64k
598
9.64k
    // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
599
9.64k
    // and if the macro had at least one real argument, and if the token before
600
9.64k
    // the ## was a comma, remove the comma.  This is a GCC extension which is
601
9.64k
    // disabled when using -std=c99.
602
9.64k
    if (ActualArgs->isVarargsElidedUse())
603
669
      MaybeRemoveCommaBeforeVaArgs(ResultToks,
604
669
                                   /*HasPasteOperator=*/true,
605
669
                                   Macro, ArgNo, PP);
606
9.64k
  }
607
4.34M
608
4.34M
  // If anything changed, install this as the new Tokens list.
609
4.34M
  if (MadeChange) {
610
4.31M
    assert(!OwnsTokens && "This would leak if we already own the token list");
611
4.31M
    // This is deleted in the dtor.
612
4.31M
    NumTokens = ResultToks.size();
613
4.31M
    // The tokens will be added to Preprocessor's cache and will be removed
614
4.31M
    // when this TokenLexer finishes lexing them.
615
4.31M
    Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
616
4.31M
617
4.31M
    // The preprocessor cache of macro expanded tokens owns these tokens,not us.
618
4.31M
    OwnsTokens = false;
619
4.31M
  }
620
4.34M
}
621
622
/// Checks if two tokens form wide string literal.
623
static bool isWideStringLiteralFromMacro(const Token &FirstTok,
624
602k
                                         const Token &SecondTok) {
625
602k
  return FirstTok.is(tok::identifier) &&
626
602k
         
FirstTok.getIdentifierInfo()->isStr("L")147k
&&
SecondTok.isLiteral()2
&&
627
602k
         
SecondTok.stringifiedInMacro()2
;
628
602k
}
629
630
/// Lex - Lex and return a token from this macro stream.
631
157M
bool TokenLexer::Lex(Token &Tok) {
632
157M
  // Lexing off the end of the macro, pop this macro off the expansion stack.
633
157M
  if (isAtEnd()) {
634
14.3M
    // If this is a macro (not a token stream), mark the macro enabled now
635
14.3M
    // that it is no longer being expanded.
636
14.3M
    if (Macro) 
Macro->EnableMacro()12.9M
;
637
14.3M
638
14.3M
    Tok.startToken();
639
14.3M
    Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
640
14.3M
    Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || 
NextTokGetsSpace13.9M
);
641
14.3M
    if (CurTokenIdx == 0)
642
262k
      Tok.setFlag(Token::LeadingEmptyMacro);
643
14.3M
    return PP.HandleEndOfTokenLexer(Tok);
644
14.3M
  }
645
143M
646
143M
  SourceManager &SM = PP.getSourceManager();
647
143M
648
143M
  // If this is the first token of the expanded result, we inherit spacing
649
143M
  // properties later.
650
143M
  bool isFirstToken = CurTokenIdx == 0;
651
143M
652
143M
  // Get the next token to return.
653
143M
  Tok = Tokens[CurTokenIdx++];
654
143M
  if (IsReinject)
655
33.1M
    Tok.setFlag(Token::IsReinjected);
656
143M
657
143M
  bool TokenIsFromPaste = false;
658
143M
659
143M
  // If this token is followed by a token paste (##) operator, paste the tokens!
660
143M
  // Note that ## is a normal token when not expanding a macro.
661
143M
  if (!isAtEnd() && 
Macro129M
&&
662
143M
      
(94.5M
Tokens[CurTokenIdx].is(tok::hashhash)94.5M
||
663
94.5M
       // Special processing of L#x macros in -fms-compatibility mode.
664
94.5M
       // Microsoft compiler is able to form a wide string literal from
665
94.5M
       // 'L#macro_arg' construct in a function-like macro.
666
94.5M
       
(93.5M
PP.getLangOpts().MSVCCompat93.5M
&&
667
93.5M
        
isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])602k
))) {
668
1.01M
    // When handling the microsoft /##/ extension, the final token is
669
1.01M
    // returned by pasteTokens, not the pasted token.
670
1.01M
    if (pasteTokens(Tok))
671
4
      return true;
672
1.01M
673
1.01M
    TokenIsFromPaste = true;
674
1.01M
  }
675
143M
676
143M
  // The token's current location indicate where the token was lexed from.  We
677
143M
  // need this information to compute the spelling of the token, but any
678
143M
  // diagnostics for the expanded token should appear as if they came from
679
143M
  // ExpansionLoc.  Pull this information together into a new SourceLocation
680
143M
  // that captures all of this.
681
143M
  
if (143M
ExpandLocStart.isValid()143M
&& // Don't do this for token streams.
682
143M
      // Check that the token's location was not already set properly.
683
143M
      
SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)106M
) {
684
88.7M
    SourceLocation instLoc;
685
88.7M
    if (Tok.is(tok::comment)) {
686
3
      instLoc = SM.createExpansionLoc(Tok.getLocation(),
687
3
                                      ExpandLocStart,
688
3
                                      ExpandLocEnd,
689
3
                                      Tok.getLength());
690
88.7M
    } else {
691
88.7M
      instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
692
88.7M
    }
693
88.7M
694
88.7M
    Tok.setLocation(instLoc);
695
88.7M
  }
696
143M
697
143M
  // If this is the first token, set the lexical properties of the token to
698
143M
  // match the lexical properties of the macro identifier.
699
143M
  if (isFirstToken) {
700
14.7M
    Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
701
14.7M
    Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
702
128M
  } else {
703
128M
    // If this is not the first token, we may still need to pass through
704
128M
    // leading whitespace if we've expanded a macro.
705
128M
    if (AtStartOfLine) 
Tok.setFlag(Token::StartOfLine)276k
;
706
128M
    if (HasLeadingSpace) 
Tok.setFlag(Token::LeadingSpace)403k
;
707
128M
  }
708
143M
  AtStartOfLine = false;
709
143M
  HasLeadingSpace = false;
710
143M
711
143M
  // Handle recursive expansion!
712
143M
  if (!Tok.isAnnotation() && 
Tok.getIdentifierInfo() != nullptr143M
) {
713
49.2M
    // Change the kind of this identifier to the appropriate token kind, e.g.
714
49.2M
    // turning "for" into a keyword.
715
49.2M
    IdentifierInfo *II = Tok.getIdentifierInfo();
716
49.2M
    Tok.setKind(II->getTokenID());
717
49.2M
718
49.2M
    // If this identifier was poisoned and from a paste, emit an error.  This
719
49.2M
    // won't be handled by Preprocessor::HandleIdentifier because this is coming
720
49.2M
    // from a macro expansion.
721
49.2M
    if (II->isPoisoned() && 
TokenIsFromPaste4
) {
722
2
      PP.HandlePoisonedIdentifier(Tok);
723
2
    }
724
49.2M
725
49.2M
    if (!DisableMacroExpansion && 
II->isHandleIdentifierCase()38.6M
)
726
10.2M
      return PP.HandleIdentifier(Tok);
727
133M
  }
728
133M
729
133M
  // Otherwise, return a normal token.
730
133M
  return true;
731
133M
}
732
733
1.01M
bool TokenLexer::pasteTokens(Token &Tok) {
734
1.01M
  return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
735
1.01M
}
736
737
/// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
738
/// operator.  Read the ## and RHS, and paste the LHS/RHS together.  If there
739
/// are more ## after it, chomp them iteratively.  Return the result as LHSTok.
740
/// If this returns true, the caller should immediately return the token.
741
bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
742
1.01M
                             unsigned int &CurIdx) {
743
1.01M
  assert(CurIdx > 0 && "## can not be the first token within tokens");
744
1.01M
  assert((TokenStream[CurIdx].is(tok::hashhash) ||
745
1.01M
         (PP.getLangOpts().MSVCCompat &&
746
1.01M
          isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
747
1.01M
             "Token at this Index must be ## or part of the MSVC 'L "
748
1.01M
             "#macro-arg' pasting pair");
749
1.01M
750
1.01M
  // MSVC: If previous token was pasted, this must be a recovery from an invalid
751
1.01M
  // paste operation. Ignore spaces before this token to mimic MSVC output.
752
1.01M
  // Required for generating valid UUID strings in some MS headers.
753
1.01M
  if (PP.getLangOpts().MicrosoftExt && 
(CurIdx >= 2)2.29k
&&
754
1.01M
      
TokenStream[CurIdx - 2].is(tok::hashhash)783
)
755
8
    LHSTok.clearFlag(Token::LeadingSpace);
756
1.01M
757
1.01M
  SmallString<128> Buffer;
758
1.01M
  const char *ResultTokStrPtr = nullptr;
759
1.01M
  SourceLocation StartLoc = LHSTok.getLocation();
760
1.01M
  SourceLocation PasteOpLoc;
761
1.01M
762
1.04M
  auto IsAtEnd = [&TokenStream, &CurIdx] {
763
1.04M
    return TokenStream.size() == CurIdx;
764
1.04M
  };
765
1.01M
766
1.04M
  do {
767
1.04M
    // Consume the ## operator if any.
768
1.04M
    PasteOpLoc = TokenStream[CurIdx].getLocation();
769
1.04M
    if (TokenStream[CurIdx].is(tok::hashhash))
770
1.04M
      ++CurIdx;
771
1.04M
    assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
772
1.04M
773
1.04M
    // Get the RHS token.
774
1.04M
    const Token &RHS = TokenStream[CurIdx];
775
1.04M
776
1.04M
    // Allocate space for the result token.  This is guaranteed to be enough for
777
1.04M
    // the two tokens.
778
1.04M
    Buffer.resize(LHSTok.getLength() + RHS.getLength());
779
1.04M
780
1.04M
    // Get the spelling of the LHS token in Buffer.
781
1.04M
    const char *BufPtr = &Buffer[0];
782
1.04M
    bool Invalid = false;
783
1.04M
    unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);
784
1.04M
    if (BufPtr != &Buffer[0])   // Really, we want the chars in Buffer!
785
1.04M
      memcpy(&Buffer[0], BufPtr, LHSLen);
786
1.04M
    if (Invalid)
787
0
      return true;
788
1.04M
789
1.04M
    BufPtr = Buffer.data() + LHSLen;
790
1.04M
    unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
791
1.04M
    if (Invalid)
792
0
      return true;
793
1.04M
    if (RHSLen && 
BufPtr != &Buffer[LHSLen]1.04M
)
794
1.04M
      // Really, we want the chars in Buffer!
795
1.04M
      memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
796
1.04M
797
1.04M
    // Trim excess space.
798
1.04M
    Buffer.resize(LHSLen+RHSLen);
799
1.04M
800
1.04M
    // Plop the pasted result (including the trailing newline and null) into a
801
1.04M
    // scratch buffer where we can lex it.
802
1.04M
    Token ResultTokTmp;
803
1.04M
    ResultTokTmp.startToken();
804
1.04M
805
1.04M
    // Claim that the tmp token is a string_literal so that we can get the
806
1.04M
    // character pointer back from CreateString in getLiteralData().
807
1.04M
    ResultTokTmp.setKind(tok::string_literal);
808
1.04M
    PP.CreateString(Buffer, ResultTokTmp);
809
1.04M
    SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
810
1.04M
    ResultTokStrPtr = ResultTokTmp.getLiteralData();
811
1.04M
812
1.04M
    // Lex the resultant pasted token into Result.
813
1.04M
    Token Result;
814
1.04M
815
1.04M
    if (LHSTok.isAnyIdentifier() && 
RHS.isAnyIdentifier()989k
) {
816
587k
      // Common paste case: identifier+identifier = identifier.  Avoid creating
817
587k
      // a lexer and other overhead.
818
587k
      PP.IncrementPasteCounter(true);
819
587k
      Result.startToken();
820
587k
      Result.setKind(tok::raw_identifier);
821
587k
      Result.setRawIdentifierData(ResultTokStrPtr);
822
587k
      Result.setLocation(ResultTokLoc);
823
587k
      Result.setLength(LHSLen+RHSLen);
824
587k
    } else {
825
454k
      PP.IncrementPasteCounter(false);
826
454k
827
454k
      assert(ResultTokLoc.isFileID() &&
828
454k
             "Should be a raw location into scratch buffer");
829
454k
      SourceManager &SourceMgr = PP.getSourceManager();
830
454k
      FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
831
454k
832
454k
      bool Invalid = false;
833
454k
      const char *ScratchBufStart
834
454k
        = SourceMgr.getBufferData(LocFileID, &Invalid).data();
835
454k
      if (Invalid)
836
0
        return false;
837
454k
838
454k
      // Make a lexer to lex this string from.  Lex just this one token.
839
454k
      // Make a lexer object so that we lex and expand the paste result.
840
454k
      Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
841
454k
               PP.getLangOpts(), ScratchBufStart,
842
454k
               ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
843
454k
844
454k
      // Lex a token in raw mode.  This way it won't look up identifiers
845
454k
      // automatically, lexing off the end will return an eof token, and
846
454k
      // warnings are disabled.  This returns true if the result token is the
847
454k
      // entire buffer.
848
454k
      bool isInvalid = !TL.LexFromRawLexer(Result);
849
454k
850
454k
      // If we got an EOF token, we didn't form even ONE token.  For example, we
851
454k
      // did "/ ## /" to get "//".
852
454k
      isInvalid |= Result.is(tok::eof);
853
454k
854
454k
      // If pasting the two tokens didn't form a full new token, this is an
855
454k
      // error.  This occurs with "x ## +"  and other stuff.  Return with LHSTok
856
454k
      // unmodified and with RHS as the next token to lex.
857
454k
      if (isInvalid) {
858
30
        // Explicitly convert the token location to have proper expansion
859
30
        // information so that the user knows where it came from.
860
30
        SourceManager &SM = PP.getSourceManager();
861
30
        SourceLocation Loc =
862
30
          SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
863
30
864
30
        // Test for the Microsoft extension of /##/ turning into // here on the
865
30
        // error path.
866
30
        if (PP.getLangOpts().MicrosoftExt && 
LHSTok.is(tok::slash)17
&&
867
30
            
RHS.is(tok::slash)4
) {
868
4
          HandleMicrosoftCommentPaste(LHSTok, Loc);
869
4
          return true;
870
4
        }
871
26
872
26
        // Do not emit the error when preprocessing assembler code.
873
26
        if (!PP.getLangOpts().AsmPreprocessor) {
874
17
          // If we're in microsoft extensions mode, downgrade this from a hard
875
17
          // error to an extension that defaults to an error.  This allows
876
17
          // disabling it.
877
17
          PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? 
diag::ext_pp_bad_paste_ms12
878
17
                                                     : 
diag::err_pp_bad_paste5
)
879
17
              << Buffer;
880
17
        }
881
26
882
26
        // An error has occurred so exit loop.
883
26
        break;
884
26
      }
885
454k
886
454k
      // Turn ## into 'unknown' to avoid # ## # from looking like a paste
887
454k
      // operator.
888
454k
      if (Result.is(tok::hashhash))
889
2
        Result.setKind(tok::unknown);
890
454k
    }
891
1.04M
892
1.04M
    // Transfer properties of the LHS over the Result.
893
1.04M
    Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine());
894
1.04M
    Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace());
895
1.04M
896
1.04M
    // Finally, replace LHS with the result, consume the RHS, and iterate.
897
1.04M
    ++CurIdx;
898
1.04M
    LHSTok = Result;
899
1.04M
  } while (!IsAtEnd() && 
TokenStream[CurIdx].is(tok::hashhash)525k
);
900
1.01M
901
1.01M
  SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
902
1.01M
903
1.01M
  // The token's current location indicate where the token was lexed from.  We
904
1.01M
  // need this information to compute the spelling of the token, but any
905
1.01M
  // diagnostics for the expanded token should appear as if the token was
906
1.01M
  // expanded from the full ## expression. Pull this information together into
907
1.01M
  // a new SourceLocation that captures all of this.
908
1.01M
  SourceManager &SM = PP.getSourceManager();
909
1.01M
  if (StartLoc.isFileID())
910
587k
    StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
911
1.01M
  if (EndLoc.isFileID())
912
59.5k
    EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
913
1.01M
  FileID MacroFID = SM.getFileID(MacroExpansionStart);
914
1.44M
  while (SM.getFileID(StartLoc) != MacroFID)
915
428k
    StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();
916
1.97M
  while (SM.getFileID(EndLoc) != MacroFID)
917
956k
    EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();
918
1.01M
919
1.01M
  LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
920
1.01M
                                        LHSTok.getLength()));
921
1.01M
922
1.01M
  // Now that we got the result token, it will be subject to expansion.  Since
923
1.01M
  // token pasting re-lexes the result token in raw mode, identifier information
924
1.01M
  // isn't looked up.  As such, if the result is an identifier, look up id info.
925
1.01M
  if (LHSTok.is(tok::raw_identifier)) {
926
974k
    // Look up the identifier info for the token.  We disabled identifier lookup
927
974k
    // by saying we're skipping contents, so we need to do this manually.
928
974k
    PP.LookUpIdentifierInfo(LHSTok);
929
974k
  }
930
1.01M
  return false;
931
1.01M
}
932
933
/// isNextTokenLParen - If the next token lexed will pop this macro off the
934
/// expansion stack, return 2.  If the next unexpanded token is a '(', return
935
/// 1, otherwise return 0.
936
3.51M
unsigned TokenLexer::isNextTokenLParen() const {
937
3.51M
  // Out of tokens?
938
3.51M
  if (isAtEnd())
939
185k
    return 2;
940
3.32M
  return Tokens[CurTokenIdx].is(tok::l_paren);
941
3.32M
}
942
943
/// isParsingPreprocessorDirective - Return true if we are in the middle of a
944
/// preprocessor directive.
945
197k
bool TokenLexer::isParsingPreprocessorDirective() const {
946
197k
  return Tokens[NumTokens-1].is(tok::eod) && 
!isAtEnd()19
;
947
197k
}
948
949
/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
950
/// together to form a comment that comments out everything in the current
951
/// macro, other active macros, and anything left on the current physical
952
/// source line of the expanded buffer.  Handle this by returning the
953
/// first token on the next line.
954
4
void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
955
4
  PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
956
4
957
4
  // We 'comment out' the rest of this macro by just ignoring the rest of the
958
4
  // tokens that have not been lexed yet, if any.
959
4
960
4
  // Since this must be a macro, mark the macro enabled now that it is no longer
961
4
  // being expanded.
962
4
  assert(Macro && "Token streams can't paste comments");
963
4
  Macro->EnableMacro();
964
4
965
4
  PP.HandleMicrosoftCommentPaste(Tok);
966
4
}
967
968
/// If \arg loc is a file ID and points inside the current macro
969
/// definition, returns the appropriate source location pointing at the
970
/// macro expansion source location entry, otherwise it returns an invalid
971
/// SourceLocation.
972
SourceLocation
973
96.9M
TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
974
96.9M
  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
975
96.9M
         "Not appropriate for token streams");
976
96.9M
  assert(loc.isValid() && loc.isFileID());
977
96.9M
978
96.9M
  SourceManager &SM = PP.getSourceManager();
979
96.9M
  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
980
96.9M
         "Expected loc to come from the macro definition");
981
96.9M
982
96.9M
  unsigned relativeOffset = 0;
983
96.9M
  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
984
96.9M
  return MacroExpansionStart.getLocWithOffset(relativeOffset);
985
96.9M
}
986
987
/// Finds the tokens that are consecutive (from the same FileID)
988
/// creates a single SLocEntry, and assigns SourceLocations to each token that
989
/// point to that SLocEntry. e.g for
990
///   assert(foo == bar);
991
/// There will be a single SLocEntry for the "foo == bar" chunk and locations
992
/// for the 'foo', '==', 'bar' tokens will point inside that chunk.
993
///
994
/// \arg begin_tokens will be updated to a position past all the found
995
/// consecutive tokens.
996
static void updateConsecutiveMacroArgTokens(SourceManager &SM,
997
                                            SourceLocation InstLoc,
998
                                            Token *&begin_tokens,
999
3.14M
                                            Token * end_tokens) {
1000
3.14M
  assert(begin_tokens < end_tokens);
1001
3.14M
1002
3.14M
  SourceLocation FirstLoc = begin_tokens->getLocation();
1003
3.14M
  SourceLocation CurLoc = FirstLoc;
1004
3.14M
1005
3.14M
  // Compare the source location offset of tokens and group together tokens that
1006
3.14M
  // are close, even if their locations point to different FileIDs. e.g.
1007
3.14M
  //
1008
3.14M
  //  |bar    |  foo | cake   |  (3 tokens from 3 consecutive FileIDs)
1009
3.14M
  //  ^                    ^
1010
3.14M
  //  |bar       foo   cake|     (one SLocEntry chunk for all tokens)
1011
3.14M
  //
1012
3.14M
  // we can perform this "merge" since the token's spelling location depends
1013
3.14M
  // on the relative offset.
1014
3.14M
1015
3.14M
  Token *NextTok = begin_tokens + 1;
1016
12.8M
  for (; NextTok < end_tokens; 
++NextTok9.73M
) {
1017
11.2M
    SourceLocation NextLoc = NextTok->getLocation();
1018
11.2M
    if (CurLoc.isFileID() != NextLoc.isFileID())
1019
139k
      break; // Token from different kind of FileID.
1020
11.1M
1021
11.1M
    int RelOffs;
1022
11.1M
    if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
1023
0
      break; // Token from different local/loaded location.
1024
11.1M
    // Check that token is not before the previous token or more than 50
1025
11.1M
    // "characters" away.
1026
11.1M
    if (RelOffs < 0 || 
RelOffs > 5010.5M
)
1027
835k
      break;
1028
10.3M
1029
10.3M
    if (CurLoc.isMacroID() && 
!SM.isWrittenInSameFile(CurLoc, NextLoc)4.74M
)
1030
569k
      break; // Token from a different macro.
1031
9.73M
1032
9.73M
    CurLoc = NextLoc;
1033
9.73M
  }
1034
3.14M
1035
3.14M
  // For the consecutive tokens, find the length of the SLocEntry to contain
1036
3.14M
  // all of them.
1037
3.14M
  Token &LastConsecutiveTok = *(NextTok-1);
1038
3.14M
  int LastRelOffs = 0;
1039
3.14M
  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
1040
3.14M
                           &LastRelOffs);
1041
3.14M
  unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength();
1042
3.14M
1043
3.14M
  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
1044
3.14M
  SourceLocation Expansion =
1045
3.14M
      SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
1046
3.14M
1047
3.14M
  // Change the location of the tokens from the spelling location to the new
1048
3.14M
  // expanded location.
1049
16.0M
  for (; begin_tokens < NextTok; 
++begin_tokens12.8M
) {
1050
12.8M
    Token &Tok = *begin_tokens;
1051
12.8M
    int RelOffs = 0;
1052
12.8M
    SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
1053
12.8M
    Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
1054
12.8M
  }
1055
3.14M
}
1056
1057
/// Creates SLocEntries and updates the locations of macro argument
1058
/// tokens to their new expanded locations.
1059
///
1060
/// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1061
/// definition.
1062
void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1063
                                            Token *begin_tokens,
1064
6.67M
                                            Token *end_tokens) {
1065
6.67M
  SourceManager &SM = PP.getSourceManager();
1066
6.67M
1067
6.67M
  SourceLocation InstLoc =
1068
6.67M
      getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1069
6.67M
1070
9.81M
  while (begin_tokens < end_tokens) {
1071
8.21M
    // If there's only one token just create a SLocEntry for it.
1072
8.21M
    if (end_tokens - begin_tokens == 1) {
1073
5.07M
      Token &Tok = *begin_tokens;
1074
5.07M
      Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(),
1075
5.07M
                                                    InstLoc,
1076
5.07M
                                                    Tok.getLength()));
1077
5.07M
      return;
1078
5.07M
    }
1079
3.14M
1080
3.14M
    updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
1081
3.14M
  }
1082
6.67M
}
1083
1084
8.65M
void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1085
8.65M
  AtStartOfLine = Result.isAtStartOfLine();
1086
8.65M
  HasLeadingSpace = Result.hasLeadingSpace();
1087
8.65M
}