Coverage Report

Created: 2022-01-25 06:29

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Lex/Lexer.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- Lexer.cpp - C Language Family Lexer --------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
//  This file implements the Lexer and Token interfaces.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "clang/Lex/Lexer.h"
14
#include "UnicodeCharSets.h"
15
#include "clang/Basic/CharInfo.h"
16
#include "clang/Basic/Diagnostic.h"
17
#include "clang/Basic/IdentifierTable.h"
18
#include "clang/Basic/LLVM.h"
19
#include "clang/Basic/LangOptions.h"
20
#include "clang/Basic/SourceLocation.h"
21
#include "clang/Basic/SourceManager.h"
22
#include "clang/Basic/TokenKinds.h"
23
#include "clang/Lex/LexDiagnostic.h"
24
#include "clang/Lex/LiteralSupport.h"
25
#include "clang/Lex/MultipleIncludeOpt.h"
26
#include "clang/Lex/Preprocessor.h"
27
#include "clang/Lex/PreprocessorOptions.h"
28
#include "clang/Lex/Token.h"
29
#include "llvm/ADT/None.h"
30
#include "llvm/ADT/Optional.h"
31
#include "llvm/ADT/STLExtras.h"
32
#include "llvm/ADT/StringExtras.h"
33
#include "llvm/ADT/StringRef.h"
34
#include "llvm/ADT/StringSwitch.h"
35
#include "llvm/Support/Compiler.h"
36
#include "llvm/Support/ConvertUTF.h"
37
#include "llvm/Support/MathExtras.h"
38
#include "llvm/Support/MemoryBufferRef.h"
39
#include "llvm/Support/NativeFormatting.h"
40
#include "llvm/Support/UnicodeCharRanges.h"
41
#include <algorithm>
42
#include <cassert>
43
#include <cstddef>
44
#include <cstdint>
45
#include <cstring>
46
#include <string>
47
#include <tuple>
48
#include <utility>
49
50
using namespace clang;
51
52
//===----------------------------------------------------------------------===//
53
// Token Class Implementation
54
//===----------------------------------------------------------------------===//
55
56
/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
57
447k
bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
58
447k
  if (isAnnotation())
59
2
    return false;
60
447k
  if (IdentifierInfo *II = getIdentifierInfo())
61
435k
    return II->getObjCKeywordID() == objcKey;
62
12.5k
  return false;
63
447k
}
64
65
/// getObjCKeywordID - Return the ObjC keyword kind.
66
984k
tok::ObjCKeywordKind Token::getObjCKeywordID() const {
67
984k
  if (isAnnotation())
68
1
    return tok::objc_not_keyword;
69
984k
  IdentifierInfo *specId = getIdentifierInfo();
70
984k
  return specId ? 
specId->getObjCKeywordID()847k
:
tok::objc_not_keyword137k
;
71
984k
}
72
73
//===----------------------------------------------------------------------===//
74
// Lexer Class Implementation
75
//===----------------------------------------------------------------------===//
76
77
0
void Lexer::anchor() {}
78
79
void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
80
52.2M
                      const char *BufEnd) {
81
52.2M
  BufferStart = BufStart;
82
52.2M
  BufferPtr = BufPtr;
83
52.2M
  BufferEnd = BufEnd;
84
85
52.2M
  assert(BufEnd[0] == 0 &&
86
52.2M
         "We assume that the input buffer has a null character at the end"
87
52.2M
         " to simplify lexing!");
88
89
  // Check whether we have a BOM in the beginning of the buffer. If yes - act
90
  // accordingly. Right now we support only UTF-8 with and without BOM, so, just
91
  // skip the UTF-8 BOM if it's present.
92
52.2M
  if (BufferStart == BufferPtr) {
93
    // Determine the size of the BOM.
94
1.27M
    StringRef Buf(BufferStart, BufferEnd - BufferStart);
95
1.27M
    size_t BOMLength = llvm::StringSwitch<size_t>(Buf)
96
1.27M
      .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM
97
1.27M
      .Default(0);
98
99
    // Skip the BOM.
100
1.27M
    BufferPtr += BOMLength;
101
1.27M
  }
102
103
52.2M
  Is_PragmaLexer = false;
104
52.2M
  CurrentConflictMarkerState = CMK_None;
105
106
  // Start of the file is a start of line.
107
52.2M
  IsAtStartOfLine = true;
108
52.2M
  IsAtPhysicalStartOfLine = true;
109
110
52.2M
  HasLeadingSpace = false;
111
52.2M
  HasLeadingEmptyMacro = false;
112
113
  // We are not after parsing a #.
114
52.2M
  ParsingPreprocessorDirective = false;
115
116
  // We are not after parsing #include.
117
52.2M
  ParsingFilename = false;
118
119
  // We are not in raw mode.  Raw mode disables diagnostics and interpretation
120
  // of tokens (e.g. identifiers, thus disabling macro expansion).  It is used
121
  // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
122
  // or otherwise skipping over tokens.
123
52.2M
  LexingRawMode = false;
124
125
  // Default to not keeping comments.
126
52.2M
  ExtendedTokenMode = 0;
127
128
52.2M
  NewLinePtr = nullptr;
129
52.2M
}
130
131
/// Lexer constructor - Create a new lexer object for the specified buffer
132
/// with the specified preprocessor managing the lexing process.  This lexer
133
/// assumes that the associated file buffer and Preprocessor objects will
134
/// outlive it, so it doesn't take ownership of either of them.
135
Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile,
136
             Preprocessor &PP, bool IsFirstIncludeOfFile)
137
    : PreprocessorLexer(&PP, FID),
138
      FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
139
1.12M
      LangOpts(PP.getLangOpts()), IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
140
1.12M
  InitLexer(InputFile.getBufferStart(), InputFile.getBufferStart(),
141
1.12M
            InputFile.getBufferEnd());
142
143
1.12M
  resetExtendedTokenMode();
144
1.12M
}
145
146
/// Lexer constructor - Create a new raw lexer object.  This object is only
147
/// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the text
148
/// range will outlive it, so it doesn't take ownership of it.
149
Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
150
             const char *BufStart, const char *BufPtr, const char *BufEnd,
151
             bool IsFirstIncludeOfFile)
152
    : FileLoc(fileloc), LangOpts(langOpts),
153
51.0M
      IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
154
51.0M
  InitLexer(BufStart, BufPtr, BufEnd);
155
156
  // We *are* in raw mode.
157
51.0M
  LexingRawMode = true;
158
51.0M
}
159
160
/// Lexer constructor - Create a new raw lexer object.  This object is only
161
/// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the text
162
/// range will outlive it, so it doesn't take ownership of it.
163
Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
164
             const SourceManager &SM, const LangOptions &langOpts,
165
             bool IsFirstIncludeOfFile)
166
    : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(),
167
            FromFile.getBufferStart(), FromFile.getBufferEnd(),
168
72.7k
            IsFirstIncludeOfFile) {}
Unexecuted instantiation: clang::Lexer::Lexer(clang::FileID, llvm::MemoryBufferRef const&, clang::SourceManager const&, clang::LangOptions const&, bool)
clang::Lexer::Lexer(clang::FileID, llvm::MemoryBufferRef const&, clang::SourceManager const&, clang::LangOptions const&, bool)
Line
Count
Source
168
72.7k
            IsFirstIncludeOfFile) {}
169
170
85.9M
void Lexer::resetExtendedTokenMode() {
171
85.9M
  assert(PP && "Cannot reset token mode without a preprocessor");
172
85.9M
  if (LangOpts.TraditionalCPP)
173
1.10k
    SetKeepWhitespaceMode(true);
174
85.9M
  else
175
85.9M
    SetCommentRetentionState(PP->getCommentRetentionState());
176
85.9M
}
177
178
/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
179
/// _Pragma expansion.  This has a variety of magic semantics that this method
180
/// sets up.  It returns a new'd Lexer that must be delete'd when done.
181
///
182
/// On entrance to this routine, TokStartLoc is a macro location which has a
183
/// spelling loc that indicates the bytes to be lexed for the token and an
184
/// expansion location that indicates where all lexed tokens should be
185
/// "expanded from".
186
///
187
/// TODO: It would really be nice to make _Pragma just be a wrapper around a
188
/// normal lexer that remaps tokens as they fly by.  This would require making
189
/// Preprocessor::Lex virtual.  Given that, we could just dump in a magic lexer
190
/// interface that could handle this stuff.  This would pull GetMappedTokenLoc
191
/// out of the critical path of the lexer!
192
///
193
Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
194
                                 SourceLocation ExpansionLocStart,
195
                                 SourceLocation ExpansionLocEnd,
196
362k
                                 unsigned TokLen, Preprocessor &PP) {
197
362k
  SourceManager &SM = PP.getSourceManager();
198
199
  // Create the lexer as if we were going to lex the file normally.
200
362k
  FileID SpellingFID = SM.getFileID(SpellingLoc);
201
362k
  llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(SpellingFID);
202
362k
  Lexer *L = new Lexer(SpellingFID, InputFile, PP);
203
204
  // Now that the lexer is created, change the start/end locations so that we
205
  // just lex the subsection of the file that we want.  This is lexing from a
206
  // scratch buffer.
207
362k
  const char *StrData = SM.getCharacterData(SpellingLoc);
208
209
362k
  L->BufferPtr = StrData;
210
362k
  L->BufferEnd = StrData+TokLen;
211
362k
  assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!");
212
213
  // Set the SourceLocation with the remapping information.  This ensures that
214
  // GetMappedTokenLoc will remap the tokens as they are lexed.
215
0
  L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID),
216
362k
                                     ExpansionLocStart,
217
362k
                                     ExpansionLocEnd, TokLen);
218
219
  // Ensure that the lexer thinks it is inside a directive, so that end \n will
220
  // return an EOD token.
221
362k
  L->ParsingPreprocessorDirective = true;
222
223
  // This lexer really is for _Pragma.
224
362k
  L->Is_PragmaLexer = true;
225
362k
  return L;
226
362k
}
227
228
34
bool Lexer::skipOver(unsigned NumBytes) {
229
34
  IsAtPhysicalStartOfLine = true;
230
34
  IsAtStartOfLine = true;
231
34
  if ((BufferPtr + NumBytes) > BufferEnd)
232
0
    return true;
233
34
  BufferPtr += NumBytes;
234
34
  return false;
235
34
}
236
237
473
template <typename T> static void StringifyImpl(T &Str, char Quote) {
238
473
  typename T::size_type i = 0, e = Str.size();
239
38.8k
  while (i < e) {
240
38.3k
    if (Str[i] == '\\' || 
Str[i] == Quote38.3k
) {
241
226
      Str.insert(Str.begin() + i, '\\');
242
226
      i += 2;
243
226
      ++e;
244
38.1k
    } else if (Str[i] == '\n' || 
Str[i] == '\r'38.1k
) {
245
      // Replace '\r\n' and '\n\r' to '\\' followed by 'n'.
246
17
      if ((i < e - 1) && (Str[i + 1] == '\n' || 
Str[i + 1] == '\r'13
) &&
247
17
          
Str[i] != Str[i + 1]4
) {
248
0
        Str[i] = '\\';
249
0
        Str[i + 1] = 'n';
250
17
      } else {
251
        // Replace '\n' and '\r' to '\\' followed by 'n'.
252
17
        Str[i] = '\\';
253
17
        Str.insert(Str.begin() + i + 1, 'n');
254
17
        ++e;
255
17
      }
256
17
      i += 2;
257
17
    } else
258
38.1k
      ++i;
259
38.3k
  }
260
473
}
Lexer.cpp:void StringifyImpl<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, char)
Line
Count
Source
237
111
template <typename T> static void StringifyImpl(T &Str, char Quote) {
238
111
  typename T::size_type i = 0, e = Str.size();
239
2.46k
  while (i < e) {
240
2.35k
    if (Str[i] == '\\' || 
Str[i] == Quote2.34k
) {
241
215
      Str.insert(Str.begin() + i, '\\');
242
215
      i += 2;
243
215
      ++e;
244
2.14k
    } else if (Str[i] == '\n' || 
Str[i] == '\r'2.13k
) {
245
      // Replace '\r\n' and '\n\r' to '\\' followed by 'n'.
246
9
      if ((i < e - 1) && (Str[i + 1] == '\n' || 
Str[i + 1] == '\r'7
) &&
247
9
          
Str[i] != Str[i + 1]2
) {
248
0
        Str[i] = '\\';
249
0
        Str[i + 1] = 'n';
250
9
      } else {
251
        // Replace '\n' and '\r' to '\\' followed by 'n'.
252
9
        Str[i] = '\\';
253
9
        Str.insert(Str.begin() + i + 1, 'n');
254
9
        ++e;
255
9
      }
256
9
      i += 2;
257
9
    } else
258
2.13k
      ++i;
259
2.35k
  }
260
111
}
Lexer.cpp:void StringifyImpl<llvm::SmallVectorImpl<char> >(llvm::SmallVectorImpl<char>&, char)
Line
Count
Source
237
362
template <typename T> static void StringifyImpl(T &Str, char Quote) {
238
362
  typename T::size_type i = 0, e = Str.size();
239
36.3k
  while (i < e) {
240
36.0k
    if (Str[i] == '\\' || 
Str[i] == Quote35.9k
) {
241
11
      Str.insert(Str.begin() + i, '\\');
242
11
      i += 2;
243
11
      ++e;
244
35.9k
    } else if (Str[i] == '\n' || 
Str[i] == '\r'35.9k
) {
245
      // Replace '\r\n' and '\n\r' to '\\' followed by 'n'.
246
8
      if ((i < e - 1) && (Str[i + 1] == '\n' || 
Str[i + 1] == '\r'6
) &&
247
8
          
Str[i] != Str[i + 1]2
) {
248
0
        Str[i] = '\\';
249
0
        Str[i + 1] = 'n';
250
8
      } else {
251
        // Replace '\n' and '\r' to '\\' followed by 'n'.
252
8
        Str[i] = '\\';
253
8
        Str.insert(Str.begin() + i + 1, 'n');
254
8
        ++e;
255
8
      }
256
8
      i += 2;
257
8
    } else
258
35.9k
      ++i;
259
36.0k
  }
260
362
}
261
262
111
std::string Lexer::Stringify(StringRef Str, bool Charify) {
263
111
  std::string Result = std::string(Str);
264
111
  char Quote = Charify ? 
'\''0
: '"';
265
111
  StringifyImpl(Result, Quote);
266
111
  return Result;
267
111
}
268
269
362
void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, '"'); }
270
271
//===----------------------------------------------------------------------===//
272
// Token Spelling
273
//===----------------------------------------------------------------------===//
274
275
/// Slow case of getSpelling. Extract the characters comprising the
276
/// spelling of this token from the provided input buffer.
277
static size_t getSpellingSlow(const Token &Tok, const char *BufPtr,
278
8.72k
                              const LangOptions &LangOpts, char *Spelling) {
279
8.72k
  assert(Tok.needsCleaning() && "getSpellingSlow called on simple token");
280
281
0
  size_t Length = 0;
282
8.72k
  const char *BufEnd = BufPtr + Tok.getLength();
283
284
8.72k
  if (tok::isStringLiteral(Tok.getKind())) {
285
    // Munch the encoding-prefix and opening double-quote.
286
122
    while (BufPtr < BufEnd) {
287
122
      unsigned Size;
288
122
      Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
289
122
      BufPtr += Size;
290
291
122
      if (Spelling[Length - 1] == '"')
292
96
        break;
293
122
    }
294
295
    // Raw string literals need special handling; trigraph expansion and line
296
    // splicing do not occur within their d-char-sequence nor within their
297
    // r-char-sequence.
298
96
    if (Length >= 2 &&
299
96
        
Spelling[Length - 2] == 'R'12
&&
Spelling[Length - 1] == '"'12
) {
300
      // Search backwards from the end of the token to find the matching closing
301
      // quote.
302
12
      const char *RawEnd = BufEnd;
303
30
      do --RawEnd; while (*RawEnd != '"');
304
12
      size_t RawLength = RawEnd - BufPtr + 1;
305
306
      // Everything between the quotes is included verbatim in the spelling.
307
12
      memcpy(Spelling + Length, BufPtr, RawLength);
308
12
      Length += RawLength;
309
12
      BufPtr += RawLength;
310
311
      // The rest of the token is lexed normally.
312
12
    }
313
96
  }
314
315
110k
  while (BufPtr < BufEnd) {
316
101k
    unsigned Size;
317
101k
    Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
318
101k
    BufPtr += Size;
319
101k
  }
320
321
8.72k
  assert(Length < Tok.getLength() &&
322
8.72k
         "NeedsCleaning flag set on token that didn't need cleaning!");
323
0
  return Length;
324
8.72k
}
325
326
/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
327
/// token are the characters used to represent the token in the source file
328
/// after trigraph expansion and escaped-newline folding.  In particular, this
329
/// wants to get the true, uncanonicalized, spelling of things like digraphs
330
/// UCNs, etc.
331
StringRef Lexer::getSpelling(SourceLocation loc,
332
                             SmallVectorImpl<char> &buffer,
333
                             const SourceManager &SM,
334
                             const LangOptions &options,
335
1.47k
                             bool *invalid) {
336
  // Break down the source location.
337
1.47k
  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);
338
339
  // Try to the load the file buffer.
340
1.47k
  bool invalidTemp = false;
341
1.47k
  StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
342
1.47k
  if (invalidTemp) {
343
0
    if (invalid) *invalid = true;
344
0
    return {};
345
0
  }
346
347
1.47k
  const char *tokenBegin = file.data() + locInfo.second;
348
349
  // Lex from the start of the given location.
350
1.47k
  Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options,
351
1.47k
              file.begin(), tokenBegin, file.end());
352
1.47k
  Token token;
353
1.47k
  lexer.LexFromRawLexer(token);
354
355
1.47k
  unsigned length = token.getLength();
356
357
  // Common case:  no need for cleaning.
358
1.47k
  if (!token.needsCleaning())
359
1.47k
    return StringRef(tokenBegin, length);
360
361
  // Hard case, we need to relex the characters into the string.
362
1
  buffer.resize(length);
363
1
  buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data()));
364
1
  return StringRef(buffer.data(), buffer.size());
365
1.47k
}
366
367
/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
368
/// token are the characters used to represent the token in the source file
369
/// after trigraph expansion and escaped-newline folding.  In particular, this
370
/// wants to get the true, uncanonicalized, spelling of things like digraphs
371
/// UCNs, etc.
372
std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
373
2.50M
                               const LangOptions &LangOpts, bool *Invalid) {
374
2.50M
  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
375
376
0
  bool CharDataInvalid = false;
377
2.50M
  const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(),
378
2.50M
                                                    &CharDataInvalid);
379
2.50M
  if (Invalid)
380
107
    *Invalid = CharDataInvalid;
381
2.50M
  if (CharDataInvalid)
382
0
    return {};
383
384
  // If this token contains nothing interesting, return it directly.
385
2.50M
  if (!Tok.needsCleaning())
386
2.50M
    return std::string(TokStart, TokStart + Tok.getLength());
387
388
5
  std::string Result;
389
5
  Result.resize(Tok.getLength());
390
5
  Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin()));
391
5
  return Result;
392
2.50M
}
393
394
/// getSpelling - This method is used to get the spelling of a token into a
395
/// preallocated buffer, instead of as an std::string.  The caller is required
396
/// to allocate enough space for the token, which is guaranteed to be at least
397
/// Tok.getLength() bytes long.  The actual length of the token is returned.
398
///
399
/// Note that this method may do two possible things: it may either fill in
400
/// the buffer specified with characters, or it may *change the input pointer*
401
/// to point to a constant buffer with the data already in it (avoiding a
402
/// copy).  The caller is not allowed to modify the returned buffer pointer
403
/// if an internal buffer is returned.
404
unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
405
                            const SourceManager &SourceMgr,
406
35.8M
                            const LangOptions &LangOpts, bool *Invalid) {
407
35.8M
  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
408
409
0
  const char *TokStart = nullptr;
410
  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
411
35.8M
  if (Tok.is(tok::raw_identifier))
412
983k
    TokStart = Tok.getRawIdentifier().data();
413
34.8M
  else if (!Tok.hasUCN()) {
414
34.8M
    if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
415
      // Just return the string from the identifier table, which is very quick.
416
12.8M
      Buffer = II->getNameStart();
417
12.8M
      return II->getLength();
418
12.8M
    }
419
34.8M
  }
420
421
  // NOTE: this can be checked even after testing for an IdentifierInfo.
422
23.0M
  if (Tok.isLiteral())
423
21.8M
    TokStart = Tok.getLiteralData();
424
425
23.0M
  if (!TokStart) {
426
    // Compute the start of the token in the input lexer buffer.
427
366k
    bool CharDataInvalid = false;
428
366k
    TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
429
366k
    if (Invalid)
430
176k
      *Invalid = CharDataInvalid;
431
366k
    if (CharDataInvalid) {
432
0
      Buffer = "";
433
0
      return 0;
434
0
    }
435
366k
  }
436
437
  // If this token contains nothing interesting, return it directly.
438
23.0M
  if (!Tok.needsCleaning()) {
439
22.9M
    Buffer = TokStart;
440
22.9M
    return Tok.getLength();
441
22.9M
  }
442
443
  // Otherwise, hard case, relex the characters into the string.
444
8.72k
  return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer));
445
23.0M
}
446
447
/// MeasureTokenLength - Relex the token at the specified location and return
448
/// its length in bytes in the input file.  If the token needs cleaning (e.g.
449
/// includes a trigraph or an escaped newline) then this count includes bytes
450
/// that are part of that.
451
unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
452
                                   const SourceManager &SM,
453
50.3M
                                   const LangOptions &LangOpts) {
454
50.3M
  Token TheTok;
455
50.3M
  if (getRawToken(Loc, TheTok, SM, LangOpts))
456
1.56k
    return 0;
457
50.3M
  return TheTok.getLength();
458
50.3M
}
459
460
/// Relex the token at the specified location.
461
/// \returns true if there was a failure, false on success.
462
bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
463
                        const SourceManager &SM,
464
                        const LangOptions &LangOpts,
465
50.3M
                        bool IgnoreWhiteSpace) {
466
  // TODO: this could be special cased for common tokens like identifiers, ')',
467
  // etc to make this faster, if it mattered.  Just look at StrData[0] to handle
468
  // all obviously single-char tokens.  This could use
469
  // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
470
  // something.
471
472
  // If this comes from a macro expansion, we really do want the macro name, not
473
  // the token this macro expanded to.
474
50.3M
  Loc = SM.getExpansionLoc(Loc);
475
50.3M
  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
476
50.3M
  bool Invalid = false;
477
50.3M
  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
478
50.3M
  if (Invalid)
479
16
    return true;
480
481
50.3M
  const char *StrData = Buffer.data()+LocInfo.second;
482
483
50.3M
  if (!IgnoreWhiteSpace && 
isWhitespace(StrData[0])50.3M
)
484
1.55k
    return true;
485
486
  // Create a lexer starting at the beginning of this token.
487
50.3M
  Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
488
50.3M
                 Buffer.begin(), StrData, Buffer.end());
489
50.3M
  TheLexer.SetCommentRetentionState(true);
490
50.3M
  TheLexer.LexFromRawLexer(Result);
491
50.3M
  return false;
492
50.3M
}
493
494
/// Returns the pointer that points to the beginning of line that contains
495
/// the given offset, or null if the offset if invalid.
496
10.7k
static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) {
497
10.7k
  const char *BufStart = Buffer.data();
498
10.7k
  if (Offset >= Buffer.size())
499
6
    return nullptr;
500
501
10.7k
  const char *LexStart = BufStart + Offset;
502
284k
  for (; LexStart != BufStart; 
--LexStart273k
) {
503
283k
    if (isVerticalWhitespace(LexStart[0]) &&
504
283k
        
!Lexer::isNewLineEscaped(BufStart, LexStart)10.3k
) {
505
      // LexStart should point at first character of logical line.
506
10.3k
      ++LexStart;
507
10.3k
      break;
508
10.3k
    }
509
283k
  }
510
10.7k
  return LexStart;
511
10.7k
}
512
513
static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
514
                                              const SourceManager &SM,
515
10.5k
                                              const LangOptions &LangOpts) {
516
10.5k
  assert(Loc.isFileID());
517
0
  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
518
10.5k
  if (LocInfo.first.isInvalid())
519
0
    return Loc;
520
521
10.5k
  bool Invalid = false;
522
10.5k
  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
523
10.5k
  if (Invalid)
524
0
    return Loc;
525
526
  // Back up from the current location until we hit the beginning of a line
527
  // (or the buffer). We'll relex from that point.
528
10.5k
  const char *StrData = Buffer.data() + LocInfo.second;
529
10.5k
  const char *LexStart = findBeginningOfLine(Buffer, LocInfo.second);
530
10.5k
  if (!LexStart || 
LexStart == StrData10.5k
)
531
255
    return Loc;
532
533
  // Create a lexer starting at the beginning of this token.
534
10.2k
  SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second);
535
10.2k
  Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart,
536
10.2k
                 Buffer.end());
537
10.2k
  TheLexer.SetCommentRetentionState(true);
538
539
  // Lex tokens until we find the token that contains the source location.
540
10.2k
  Token TheTok;
541
16.7k
  do {
542
16.7k
    TheLexer.LexFromRawLexer(TheTok);
543
544
16.7k
    if (TheLexer.getBufferLocation() > StrData) {
545
      // Lexing this token has taken the lexer past the source location we're
546
      // looking for. If the current token encompasses our source location,
547
      // return the beginning of that token.
548
10.2k
      if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
549
9.79k
        return TheTok.getLocation();
550
551
      // We ended up skipping over the source location entirely, which means
552
      // that it points into whitespace. We're done here.
553
492
      break;
554
10.2k
    }
555
16.7k
  } while (
TheTok.getKind() != tok::eof6.41k
);
556
557
  // We've passed our source location; just return the original source location.
558
492
  return Loc;
559
10.2k
}
560
561
SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
562
                                          const SourceManager &SM,
563
10.5k
                                          const LangOptions &LangOpts) {
564
10.5k
  if (Loc.isFileID())
565
10.5k
    return getBeginningOfFileToken(Loc, SM, LangOpts);
566
567
20
  if (!SM.isMacroArgExpansion(Loc))
568
0
    return Loc;
569
570
20
  SourceLocation FileLoc = SM.getSpellingLoc(Loc);
571
20
  SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts);
572
20
  std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc);
573
20
  std::pair<FileID, unsigned> BeginFileLocInfo =
574
20
      SM.getDecomposedLoc(BeginFileLoc);
575
20
  assert(FileLocInfo.first == BeginFileLocInfo.first &&
576
20
         FileLocInfo.second >= BeginFileLocInfo.second);
577
0
  return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second);
578
20
}
579
580
namespace {
581
582
enum PreambleDirectiveKind {
583
  PDK_Skipped,
584
  PDK_Unknown
585
};
586
587
} // namespace
588
589
PreambleBounds Lexer::ComputePreamble(StringRef Buffer,
590
                                      const LangOptions &LangOpts,
591
516
                                      unsigned MaxLines) {
592
  // Create a lexer starting at the beginning of the file. Note that we use a
593
  // "fake" file source location at offset 1 so that the lexer will track our
594
  // position within the file.
595
516
  const SourceLocation::UIntTy StartOffset = 1;
596
516
  SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset);
597
516
  Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(),
598
516
                 Buffer.end());
599
516
  TheLexer.SetCommentRetentionState(true);
600
601
516
  bool InPreprocessorDirective = false;
602
516
  Token TheTok;
603
516
  SourceLocation ActiveCommentLoc;
604
605
516
  unsigned MaxLineOffset = 0;
606
516
  if (MaxLines) {
607
84
    const char *CurPtr = Buffer.begin();
608
84
    unsigned CurLine = 0;
609
12.3k
    while (CurPtr != Buffer.end()) {
610
12.3k
      char ch = *CurPtr++;
611
12.3k
      if (ch == '\n') {
612
642
        ++CurLine;
613
642
        if (CurLine == MaxLines)
614
83
          break;
615
642
      }
616
12.3k
    }
617
84
    if (CurPtr != Buffer.end())
618
78
      MaxLineOffset = CurPtr - Buffer.begin();
619
84
  }
620
621
3.95k
  do {
622
3.95k
    TheLexer.LexFromRawLexer(TheTok);
623
624
3.95k
    if (InPreprocessorDirective) {
625
      // If we've hit the end of the file, we're done.
626
2.83k
      if (TheTok.getKind() == tok::eof) {
627
14
        break;
628
14
      }
629
630
      // If we haven't hit the end of the preprocessor directive, skip this
631
      // token.
632
2.82k
      if (!TheTok.isAtStartOfLine())
633
1.90k
        continue;
634
635
      // We've passed the end of the preprocessor directive, and will look
636
      // at this token again below.
637
920
      InPreprocessorDirective = false;
638
920
    }
639
640
    // Keep track of the # of lines in the preamble.
641
2.03k
    if (TheTok.isAtStartOfLine()) {
642
2.01k
      unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset;
643
644
      // If we were asked to limit the number of lines in the preamble,
645
      // and we're about to exceed that limit, we're done.
646
2.01k
      if (MaxLineOffset && 
TokOffset >= MaxLineOffset360
)
647
18
        break;
648
2.01k
    }
649
650
    // Comments are okay; skip over them.
651
2.01k
    if (TheTok.getKind() == tok::comment) {
652
601
      if (ActiveCommentLoc.isInvalid())
653
227
        ActiveCommentLoc = TheTok.getLocation();
654
601
      continue;
655
601
    }
656
657
1.41k
    if (TheTok.isAtStartOfLine() && 
TheTok.getKind() == tok::hash1.39k
) {
658
      // This is the start of a preprocessor directive.
659
934
      Token HashTok = TheTok;
660
934
      InPreprocessorDirective = true;
661
934
      ActiveCommentLoc = SourceLocation();
662
663
      // Figure out which directive this is. Since we're lexing raw tokens,
664
      // we don't have an identifier table available. Instead, just look at
665
      // the raw identifier to recognize and categorize preprocessor directives.
666
934
      TheLexer.LexFromRawLexer(TheTok);
667
934
      if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
668
934
        StringRef Keyword = TheTok.getRawIdentifier();
669
934
        PreambleDirectiveKind PDK
670
934
          = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
671
934
              .Case("include", PDK_Skipped)
672
934
              .Case("__include_macros", PDK_Skipped)
673
934
              .Case("define", PDK_Skipped)
674
934
              .Case("undef", PDK_Skipped)
675
934
              .Case("line", PDK_Skipped)
676
934
              .Case("error", PDK_Skipped)
677
934
              .Case("pragma", PDK_Skipped)
678
934
              .Case("import", PDK_Skipped)
679
934
              .Case("include_next", PDK_Skipped)
680
934
              .Case("warning", PDK_Skipped)
681
934
              .Case("ident", PDK_Skipped)
682
934
              .Case("sccs", PDK_Skipped)
683
934
              .Case("assert", PDK_Skipped)
684
934
              .Case("unassert", PDK_Skipped)
685
934
              .Case("if", PDK_Skipped)
686
934
              .Case("ifdef", PDK_Skipped)
687
934
              .Case("ifndef", PDK_Skipped)
688
934
              .Case("elif", PDK_Skipped)
689
934
              .Case("elifdef", PDK_Skipped)
690
934
              .Case("elifndef", PDK_Skipped)
691
934
              .Case("else", PDK_Skipped)
692
934
              .Case("endif", PDK_Skipped)
693
934
              .Default(PDK_Unknown);
694
695
934
        switch (PDK) {
696
934
        case PDK_Skipped:
697
934
          continue;
698
699
0
        case PDK_Unknown:
700
          // We don't know what this directive is; stop at the '#'.
701
0
          break;
702
934
        }
703
934
      }
704
705
      // We only end up here if we didn't recognize the preprocessor
706
      // directive or it was one that can't occur in the preamble at this
707
      // point. Roll back the current token to the location of the '#'.
708
0
      TheTok = HashTok;
709
0
    }
710
711
    // We hit a token that we don't recognize as being in the
712
    // "preprocessing only" part of the file, so we're no longer in
713
    // the preamble.
714
484
    break;
715
3.43k
  } while (true);
716
717
516
  SourceLocation End;
718
516
  if (ActiveCommentLoc.isValid())
719
80
    End = ActiveCommentLoc; // don't truncate a decl comment.
720
436
  else
721
436
    End = TheTok.getLocation();
722
723
516
  return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(),
724
516
                        TheTok.isAtStartOfLine());
725
516
}
726
727
unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo,
728
                                     const SourceManager &SM,
729
68.5k
                                     const LangOptions &LangOpts) {
730
  // Figure out how many physical characters away the specified expansion
731
  // character is.  This needs to take into consideration newlines and
732
  // trigraphs.
733
68.5k
  bool Invalid = false;
734
68.5k
  const char *TokPtr = SM.getCharacterData(TokStart, &Invalid);
735
736
  // If they request the first char of the token, we're trivially done.
737
68.5k
  if (Invalid || (CharNo == 0 && 
Lexer::isObviouslySimpleCharacter(*TokPtr)2.94k
))
738
2.92k
    return 0;
739
740
65.6k
  unsigned PhysOffset = 0;
741
742
  // The usual case is that tokens don't contain anything interesting.  Skip
743
  // over the uninteresting characters.  If a token only consists of simple
744
  // chars, this method is extremely fast.
745
613k
  while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
746
610k
    if (CharNo == 0)
747
63.4k
      return PhysOffset;
748
547k
    ++TokPtr;
749
547k
    --CharNo;
750
547k
    ++PhysOffset;
751
547k
  }
752
753
  // If we have a character that may be a trigraph or escaped newline, use a
754
  // lexer to parse it correctly.
755
12.0k
  
for (; 2.17k
CharNo;
--CharNo9.88k
) {
756
9.88k
    unsigned Size;
757
9.88k
    Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts);
758
9.88k
    TokPtr += Size;
759
9.88k
    PhysOffset += Size;
760
9.88k
  }
761
762
  // Final detail: if we end up on an escaped newline, we want to return the
763
  // location of the actual byte of the token.  For example foo\<newline>bar
764
  // advanced by 3 should return the location of b, not of \\.  One compounding
765
  // detail of this is that the escape may be made by a trigraph.
766
2.17k
  if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
767
1.57k
    PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
768
769
2.17k
  return PhysOffset;
770
65.6k
}
771
772
/// Computes the source location just past the end of the
773
/// token at this source location.
774
///
775
/// This routine can be used to produce a source location that
776
/// points just past the end of the token referenced by \p Loc, and
777
/// is generally used when a diagnostic needs to point just after a
778
/// token where it expected something different that it received. If
779
/// the returned source location would not be meaningful (e.g., if
780
/// it points into a macro), this routine returns an invalid
781
/// source location.
782
///
783
/// \param Offset an offset from the end of the token, where the source
784
/// location should refer to. The default offset (0) produces a source
785
/// location pointing just past the end of the token; an offset of 1 produces
786
/// a source location pointing to the last character in the token, etc.
787
SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
788
                                          const SourceManager &SM,
789
4.02M
                                          const LangOptions &LangOpts) {
790
4.02M
  if (Loc.isInvalid())
791
91
    return {};
792
793
4.02M
  if (Loc.isMacroID()) {
794
391
    if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc))
795
183
      return {}; // Points inside the macro expansion.
796
391
  }
797
798
4.02M
  unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
799
4.02M
  if (Len > Offset)
800
4.02M
    Len = Len - Offset;
801
424
  else
802
424
    return Loc;
803
804
4.02M
  return Loc.getLocWithOffset(Len);
805
4.02M
}
806
807
/// Returns true if the given MacroID location points at the first
808
/// token of the macro expansion.
809
bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
810
                                      const SourceManager &SM,
811
                                      const LangOptions &LangOpts,
812
50.3M
                                      SourceLocation *MacroBegin) {
813
50.3M
  assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
814
815
0
  SourceLocation expansionLoc;
816
50.3M
  if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc))
817
25.8M
    return false;
818
819
24.4M
  if (expansionLoc.isFileID()) {
820
    // No other macro expansions, this is the first.
821
6.39M
    if (MacroBegin)
822
196
      *MacroBegin = expansionLoc;
823
6.39M
    return true;
824
6.39M
  }
825
826
18.0M
  return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin);
827
24.4M
}
828
829
/// Returns true if the given MacroID location points at the last
830
/// token of the macro expansion.
831
bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
832
                                    const SourceManager &SM,
833
                                    const LangOptions &LangOpts,
834
45.8M
                                    SourceLocation *MacroEnd) {
835
45.8M
  assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
836
837
0
  SourceLocation spellLoc = SM.getSpellingLoc(loc);
838
45.8M
  unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts);
839
45.8M
  if (tokLen == 0)
840
0
    return false;
841
842
45.8M
  SourceLocation afterLoc = loc.getLocWithOffset(tokLen);
843
45.8M
  SourceLocation expansionLoc;
844
45.8M
  if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc))
845
4.34M
    return false;
846
847
41.5M
  if (expansionLoc.isFileID()) {
848
    // No other macro expansions.
849
27.9M
    if (MacroEnd)
850
300
      *MacroEnd = expansionLoc;
851
27.9M
    return true;
852
27.9M
  }
853
854
13.6M
  return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd);
855
41.5M
}
856
857
static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range,
858
                                             const SourceManager &SM,
859
3.90M
                                             const LangOptions &LangOpts) {
860
3.90M
  SourceLocation Begin = Range.getBegin();
861
3.90M
  SourceLocation End = Range.getEnd();
862
3.90M
  assert(Begin.isFileID() && End.isFileID());
863
3.90M
  if (Range.isTokenRange()) {
864
3.90M
    End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts);
865
3.90M
    if (End.isInvalid())
866
0
      return {};
867
3.90M
  }
868
869
  // Break down the source locations.
870
3.90M
  FileID FID;
871
3.90M
  unsigned BeginOffs;
872
3.90M
  std::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin);
873
3.90M
  if (FID.isInvalid())
874
0
    return {};
875
876
3.90M
  unsigned EndOffs;
877
3.90M
  if (!SM.isInFileID(End, FID, &EndOffs) ||
878
3.90M
      BeginOffs > EndOffs)
879
0
    return {};
880
881
3.90M
  return CharSourceRange::getCharRange(Begin, End);
882
3.90M
}
883
884
// Assumes that `Loc` is in an expansion.
885
static bool isInExpansionTokenRange(const SourceLocation Loc,
886
88
                                    const SourceManager &SM) {
887
88
  return SM.getSLocEntry(SM.getFileID(Loc))
888
88
      .getExpansion()
889
88
      .isExpansionTokenRange();
890
88
}
891
892
CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range,
893
                                         const SourceManager &SM,
894
3.90M
                                         const LangOptions &LangOpts) {
895
3.90M
  SourceLocation Begin = Range.getBegin();
896
3.90M
  SourceLocation End = Range.getEnd();
897
3.90M
  if (Begin.isInvalid() || 
End.isInvalid()3.90M
)
898
8
    return {};
899
900
3.90M
  if (Begin.isFileID() && 
End.isFileID()3.90M
)
901
3.90M
    return makeRangeFromFileLocs(Range, SM, LangOpts);
902
903
296
  if (Begin.isMacroID() && 
End.isFileID()288
) {
904
71
    if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin))
905
1
      return {};
906
70
    Range.setBegin(Begin);
907
70
    return makeRangeFromFileLocs(Range, SM, LangOpts);
908
71
  }
909
910
225
  if (Begin.isFileID() && 
End.isMacroID()8
) {
911
8
    if (Range.isTokenRange()) {
912
8
      if (!isAtEndOfMacroExpansion(End, SM, LangOpts, &End))
913
0
        return {};
914
      // Use the *original* end, not the expanded one in `End`.
915
8
      Range.setTokenRange(isInExpansionTokenRange(Range.getEnd(), SM));
916
8
    } else 
if (0
!isAtStartOfMacroExpansion(End, SM, LangOpts, &End)0
)
917
0
      return {};
918
8
    Range.setEnd(End);
919
8
    return makeRangeFromFileLocs(Range, SM, LangOpts);
920
8
  }
921
922
217
  assert(Begin.isMacroID() && End.isMacroID());
923
0
  SourceLocation MacroBegin, MacroEnd;
924
217
  if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) &&
925
217
      
(100
(100
Range.isTokenRange()100
&& isAtEndOfMacroExpansion(End, SM, LangOpts,
926
98
                                                        &MacroEnd)) ||
927
100
       
(20
Range.isCharRange()20
&& isAtStartOfMacroExpansion(End, SM, LangOpts,
928
80
                                                         &MacroEnd)))) {
929
80
    Range.setBegin(MacroBegin);
930
80
    Range.setEnd(MacroEnd);
931
    // Use the *original* `End`, not the expanded one in `MacroEnd`.
932
80
    if (Range.isTokenRange())
933
80
      Range.setTokenRange(isInExpansionTokenRange(End, SM));
934
80
    return makeRangeFromFileLocs(Range, SM, LangOpts);
935
80
  }
936
937
137
  bool Invalid = false;
938
137
  const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin),
939
137
                                                        &Invalid);
940
137
  if (Invalid)
941
0
    return {};
942
943
137
  if (BeginEntry.getExpansion().isMacroArgExpansion()) {
944
116
    const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End),
945
116
                                                        &Invalid);
946
116
    if (Invalid)
947
0
      return {};
948
949
116
    if (EndEntry.getExpansion().isMacroArgExpansion() &&
950
116
        BeginEntry.getExpansion().getExpansionLocStart() ==
951
115
            EndEntry.getExpansion().getExpansionLocStart()) {
952
114
      Range.setBegin(SM.getImmediateSpellingLoc(Begin));
953
114
      Range.setEnd(SM.getImmediateSpellingLoc(End));
954
114
      return makeFileCharRange(Range, SM, LangOpts);
955
114
    }
956
116
  }
957
958
23
  return {};
959
137
}
960
961
StringRef Lexer::getSourceText(CharSourceRange Range,
962
                               const SourceManager &SM,
963
                               const LangOptions &LangOpts,
964
3.89M
                               bool *Invalid) {
965
3.89M
  Range = makeFileCharRange(Range, SM, LangOpts);
966
3.89M
  if (Range.isInvalid()) {
967
9
    if (Invalid) 
*Invalid = true2
;
968
9
    return {};
969
9
  }
970
971
  // Break down the source location.
972
3.89M
  std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin());
973
3.89M
  if (beginInfo.first.isInvalid()) {
974
0
    if (Invalid) *Invalid = true;
975
0
    return {};
976
0
  }
977
978
3.89M
  unsigned EndOffs;
979
3.89M
  if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) ||
980
3.89M
      beginInfo.second > EndOffs) {
981
0
    if (Invalid) *Invalid = true;
982
0
    return {};
983
0
  }
984
985
  // Try to the load the file buffer.
986
3.89M
  bool invalidTemp = false;
987
3.89M
  StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp);
988
3.89M
  if (invalidTemp) {
989
0
    if (Invalid) *Invalid = true;
990
0
    return {};
991
0
  }
992
993
3.89M
  if (Invalid) 
*Invalid = false264
;
994
3.89M
  return file.substr(beginInfo.second, EndOffs - beginInfo.second);
995
3.89M
}
996
997
StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
998
                                       const SourceManager &SM,
999
6.35k
                                       const LangOptions &LangOpts) {
1000
6.35k
  assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1001
1002
  // Find the location of the immediate macro expansion.
1003
20.6k
  while (true) {
1004
20.6k
    FileID FID = SM.getFileID(Loc);
1005
20.6k
    const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
1006
20.6k
    const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
1007
20.6k
    Loc = Expansion.getExpansionLocStart();
1008
20.6k
    if (!Expansion.isMacroArgExpansion())
1009
4.19k
      break;
1010
1011
    // For macro arguments we need to check that the argument did not come
1012
    // from an inner macro, e.g: "MAC1( MAC2(foo) )"
1013
1014
    // Loc points to the argument id of the macro definition, move to the
1015
    // macro expansion.
1016
16.4k
    Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1017
16.4k
    SourceLocation SpellLoc = Expansion.getSpellingLoc();
1018
16.4k
    if (SpellLoc.isFileID())
1019
2.16k
      break; // No inner macro.
1020
1021
    // If spelling location resides in the same FileID as macro expansion
1022
    // location, it means there is no inner macro.
1023
14.3k
    FileID MacroFID = SM.getFileID(Loc);
1024
14.3k
    if (SM.isInFileID(SpellLoc, MacroFID))
1025
1
      break;
1026
1027
    // Argument came from inner macro.
1028
14.3k
    Loc = SpellLoc;
1029
14.3k
  }
1030
1031
  // Find the spelling location of the start of the non-argument expansion
1032
  // range. This is where the macro name was spelled in order to begin
1033
  // expanding this macro.
1034
6.35k
  Loc = SM.getSpellingLoc(Loc);
1035
1036
  // Dig out the buffer where the macro name was spelled and the extents of the
1037
  // name so that we can render it into the expansion note.
1038
6.35k
  std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc);
1039
6.35k
  unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1040
6.35k
  StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first);
1041
6.35k
  return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength);
1042
6.35k
}
1043
1044
StringRef Lexer::getImmediateMacroNameForDiagnostics(
1045
635
    SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) {
1046
635
  assert(Loc.isMacroID() && "Only reasonable to call this on macros");
1047
  // Walk past macro argument expansions.
1048
635
  while (SM.isMacroArgExpansion(Loc))
1049
0
    Loc = SM.getImmediateExpansionRange(Loc).getBegin();
1050
1051
  // If the macro's spelling has no FileID, then it's actually a token paste
1052
  // or stringization (or similar) and not a macro at all.
1053
635
  if (!SM.getFileEntryForID(SM.getFileID(SM.getSpellingLoc(Loc))))
1054
50
    return {};
1055
1056
  // Find the spelling location of the start of the non-argument expansion
1057
  // range. This is where the macro name was spelled in order to begin
1058
  // expanding this macro.
1059
585
  Loc = SM.getSpellingLoc(SM.getImmediateExpansionRange(Loc).getBegin());
1060
1061
  // Dig out the buffer where the macro name was spelled and the extents of the
1062
  // name so that we can render it into the expansion note.
1063
585
  std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc);
1064
585
  unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1065
585
  StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first);
1066
585
  return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength);
1067
635
}
1068
1069
1.38k
bool Lexer::isAsciiIdentifierContinueChar(char c, const LangOptions &LangOpts) {
1070
1.38k
  return isAsciiIdentifierContinue(c, LangOpts.DollarIdents);
1071
1.38k
}
1072
1073
10.4k
bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) {
1074
10.4k
  assert(isVerticalWhitespace(Str[0]));
1075
10.4k
  if (Str - 1 < BufferStart)
1076
2
    return false;
1077
1078
10.4k
  if ((Str[0] == '\n' && 
Str[-1] == '\r'10.3k
) ||
1079
10.4k
      
(10.3k
Str[0] == '\r'10.3k
&&
Str[-1] == '\n'18
)) {
1080
18
    if (Str - 2 < BufferStart)
1081
2
      return false;
1082
16
    --Str;
1083
16
  }
1084
10.4k
  --Str;
1085
1086
  // Rewind to first non-space character:
1087
10.5k
  while (Str > BufferStart && 
isHorizontalWhitespace(*Str)10.5k
)
1088
106
    --Str;
1089
1090
10.4k
  return *Str == '\\';
1091
10.4k
}
1092
1093
StringRef Lexer::getIndentationForLine(SourceLocation Loc,
1094
176
                                       const SourceManager &SM) {
1095
176
  if (Loc.isInvalid() || Loc.isMacroID())
1096
0
    return {};
1097
176
  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
1098
176
  if (LocInfo.first.isInvalid())
1099
0
    return {};
1100
176
  bool Invalid = false;
1101
176
  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
1102
176
  if (Invalid)
1103
0
    return {};
1104
176
  const char *Line = findBeginningOfLine(Buffer, LocInfo.second);
1105
176
  if (!Line)
1106
0
    return {};
1107
176
  StringRef Rest = Buffer.substr(Line - Buffer.data());
1108
176
  size_t NumWhitespaceChars = Rest.find_first_not_of(" \t");
1109
176
  return NumWhitespaceChars == StringRef::npos
1110
176
             ? 
""0
1111
176
             : Rest.take_front(NumWhitespaceChars);
1112
176
}
1113
1114
//===----------------------------------------------------------------------===//
1115
// Diagnostics forwarding code.
1116
//===----------------------------------------------------------------------===//
1117
1118
/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
1119
/// lexer buffer was all expanded at a single point, perform the mapping.
1120
/// This is currently only used for _Pragma implementation, so it is the slow
1121
/// path of the hot getSourceLocation method.  Do not allow it to be inlined.
1122
static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
1123
    Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
1124
static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
1125
                                        SourceLocation FileLoc,
1126
1.95M
                                        unsigned CharNo, unsigned TokLen) {
1127
1.95M
  assert(FileLoc.isMacroID() && "Must be a macro expansion");
1128
1129
  // Otherwise, we're lexing "mapped tokens".  This is used for things like
1130
  // _Pragma handling.  Combine the expansion location of FileLoc with the
1131
  // spelling location.
1132
0
  SourceManager &SM = PP.getSourceManager();
1133
1134
  // Create a new SLoc which is expanded from Expansion(FileLoc) but whose
1135
  // characters come from spelling(FileLoc)+Offset.
1136
1.95M
  SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc);
1137
1.95M
  SpellingLoc = SpellingLoc.getLocWithOffset(CharNo);
1138
1139
  // Figure out the expansion loc range, which is the range covered by the
1140
  // original _Pragma(...) sequence.
1141
1.95M
  CharSourceRange II = SM.getImmediateExpansionRange(FileLoc);
1142
1143
1.95M
  return SM.createExpansionLoc(SpellingLoc, II.getBegin(), II.getEnd(), TokLen);
1144
1.95M
}
1145
1146
/// getSourceLocation - Return a source location identifier for the specified
1147
/// offset in the current file.
1148
SourceLocation Lexer::getSourceLocation(const char *Loc,
1149
1.82G
                                        unsigned TokLen) const {
1150
1.82G
  assert(Loc >= BufferStart && Loc <= BufferEnd &&
1151
1.82G
         "Location out of range for this buffer!");
1152
1153
  // In the normal case, we're just lexing from a simple file buffer, return
1154
  // the file id from FileLoc with the offset specified.
1155
0
  unsigned CharNo = Loc-BufferStart;
1156
1.82G
  if (FileLoc.isFileID())
1157
1.81G
    return FileLoc.getLocWithOffset(CharNo);
1158
1159
  // Otherwise, this is the _Pragma lexer case, which pretends that all of the
1160
  // tokens are lexed from where the _Pragma was defined.
1161
1.95M
  assert(PP && "This doesn't work on raw lexers");
1162
0
  return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen);
1163
1.82G
}
1164
1165
/// Diag - Forwarding function for diagnostics.  This translate a source
1166
/// position in the current buffer into a SourceLocation object for rendering.
1167
72.2k
DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
1168
72.2k
  return PP->Diag(getSourceLocation(Loc), DiagID);
1169
72.2k
}
1170
1171
//===----------------------------------------------------------------------===//
1172
// Trigraph and Escaped Newline Handling Code.
1173
//===----------------------------------------------------------------------===//
1174
1175
/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
1176
/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
1177
276
static char GetTrigraphCharForLetter(char Letter) {
1178
276
  switch (Letter) {
1179
123
  default:   return 0;
1180
41
  case '=':  return '#';
1181
31
  case ')':  return ']';
1182
30
  case '(':  return '[';
1183
6
  case '!':  return '|';
1184
1
  case '\'': return '^';
1185
4
  case '>':  return '}';
1186
34
  case '/':  return '\\';
1187
3
  case '<':  return '{';
1188
3
  case '-':  return '~';
1189
276
  }
1190
276
}
1191
1192
/// DecodeTrigraphChar - If the specified character is a legal trigraph when
1193
/// prefixed with ??, emit a trigraph warning.  If trigraphs are enabled,
1194
/// return the result character.  Finally, emit a warning about trigraph use
1195
/// whether trigraphs are enabled or not.
1196
248
static char DecodeTrigraphChar(const char *CP, Lexer *L) {
1197
248
  char Res = GetTrigraphCharForLetter(*CP);
1198
248
  if (!Res || 
!L125
)
return Res140
;
1199
1200
108
  if (!L->getLangOpts().Trigraphs) {
1201
51
    if (!L->isLexingRawMode())
1202
26
      L->Diag(CP-2, diag::trigraph_ignored);
1203
51
    return 0;
1204
51
  }
1205
1206
57
  if (!L->isLexingRawMode())
1207
46
    L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1);
1208
57
  return Res;
1209
108
}
1210
1211
/// getEscapedNewLineSize - Return the size of the specified escaped newline,
1212
/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a
1213
/// trigraph equivalent on entry to this function.
1214
6.52M
unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
1215
6.52M
  unsigned Size = 0;
1216
6.52M
  while (isWhitespace(Ptr[Size])) {
1217
6.52M
    ++Size;
1218
1219
6.52M
    if (Ptr[Size-1] != '\n' && 
Ptr[Size-1] != '\r'266
)
1220
205
      continue;
1221
1222
    // If this is a \r\n or \n\r, skip the other half.
1223
6.52M
    if ((Ptr[Size] == '\r' || 
Ptr[Size] == '\n'6.52M
) &&
1224
6.52M
        
Ptr[Size-1] != Ptr[Size]97
)
1225
61
      ++Size;
1226
1227
6.52M
    return Size;
1228
6.52M
  }
1229
1230
  // Not an escaped newline, must be a \t or something else.
1231
1.58k
  return 0;
1232
6.52M
}
1233
1234
/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
1235
/// them), skip over them and return the first non-escaped-newline found,
1236
/// otherwise return P.
1237
1.57k
const char *Lexer::SkipEscapedNewLines(const char *P) {
1238
1.59k
  while (true) {
1239
1.59k
    const char *AfterEscape;
1240
1.59k
    if (*P == '\\') {
1241
1.57k
      AfterEscape = P+1;
1242
1.57k
    } else 
if (21
*P == '?'21
) {
1243
      // If not a trigraph for escape, bail out.
1244
1
      if (P[1] != '?' || 
P[2] != '/'0
)
1245
1
        return P;
1246
      // FIXME: Take LangOpts into account; the language might not
1247
      // support trigraphs.
1248
0
      AfterEscape = P+3;
1249
20
    } else {
1250
20
      return P;
1251
20
    }
1252
1253
1.57k
    unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape);
1254
1.57k
    if (NewLineSize == 0) 
return P1.55k
;
1255
20
    P = AfterEscape+NewLineSize;
1256
20
  }
1257
1.57k
}
1258
1259
Optional<Token> Lexer::findNextToken(SourceLocation Loc,
1260
                                     const SourceManager &SM,
1261
680
                                     const LangOptions &LangOpts) {
1262
680
  if (Loc.isMacroID()) {
1263
2
    if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc))
1264
2
      return None;
1265
2
  }
1266
678
  Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts);
1267
1268
  // Break down the source location.
1269
678
  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
1270
1271
  // Try to load the file buffer.
1272
678
  bool InvalidTemp = false;
1273
678
  StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp);
1274
678
  if (InvalidTemp)
1275
0
    return None;
1276
1277
678
  const char *TokenBegin = File.data() + LocInfo.second;
1278
1279
  // Lex from the start of the given location.
1280
678
  Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(),
1281
678
                                      TokenBegin, File.end());
1282
  // Find the token.
1283
678
  Token Tok;
1284
678
  lexer.LexFromRawLexer(Tok);
1285
678
  return Tok;
1286
678
}
1287
1288
/// Checks that the given token is the first token that occurs after the
1289
/// given location (this excludes comments and whitespace). Returns the location
1290
/// immediately after the specified token. If the token is not found or the
1291
/// location is inside a macro, the returned source location will be invalid.
1292
SourceLocation Lexer::findLocationAfterToken(
1293
    SourceLocation Loc, tok::TokenKind TKind, const SourceManager &SM,
1294
485
    const LangOptions &LangOpts, bool SkipTrailingWhitespaceAndNewLine) {
1295
485
  Optional<Token> Tok = findNextToken(Loc, SM, LangOpts);
1296
485
  if (!Tok || 
Tok->isNot(TKind)483
)
1297
60
    return {};
1298
425
  SourceLocation TokenLoc = Tok->getLocation();
1299
1300
  // Calculate how much whitespace needs to be skipped if any.
1301
425
  unsigned NumWhitespaceChars = 0;
1302
425
  if (SkipTrailingWhitespaceAndNewLine) {
1303
235
    const char *TokenEnd = SM.getCharacterData(TokenLoc) + Tok->getLength();
1304
235
    unsigned char C = *TokenEnd;
1305
463
    while (isHorizontalWhitespace(C)) {
1306
228
      C = *(++TokenEnd);
1307
228
      NumWhitespaceChars++;
1308
228
    }
1309
1310
    // Skip \r, \n, \r\n, or \n\r
1311
235
    if (C == '\n' || 
C == '\r'132
) {
1312
104
      char PrevC = C;
1313
104
      C = *(++TokenEnd);
1314
104
      NumWhitespaceChars++;
1315
104
      if ((C == '\n' || 
C == '\r'103
) &&
C != PrevC1
)
1316
1
        NumWhitespaceChars++;
1317
104
    }
1318
235
  }
1319
1320
425
  return TokenLoc.getLocWithOffset(Tok->getLength() + NumWhitespaceChars);
1321
485
}
1322
1323
/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
1324
/// get its size, and return it.  This is tricky in several cases:
1325
///   1. If currently at the start of a trigraph, we warn about the trigraph,
1326
///      then either return the trigraph (skipping 3 chars) or the '?',
1327
///      depending on whether trigraphs are enabled or not.
1328
///   2. If this is an escaped newline (potentially with whitespace between
1329
///      the backslash and newline), implicitly skip the newline and return
1330
///      the char after it.
1331
///
1332
/// This handles the slow/uncommon case of the getCharAndSize method.  Here we
1333
/// know that we can accumulate into Size, and that we have already incremented
1334
/// Ptr by Size bytes.
1335
///
1336
/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
1337
/// be updated to match.
1338
char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
1339
13.4M
                               Token *Tok) {
1340
  // If we have a slash, look for an escaped newline.
1341
13.4M
  if (Ptr[0] == '\\') {
1342
6.76M
    ++Size;
1343
6.76M
    ++Ptr;
1344
6.76M
Slash:
1345
    // Common case, backslash-char where the char is not whitespace.
1346
6.76M
    if (!isWhitespace(Ptr[0])) 
return '\\'255k
;
1347
1348
    // See if we have optional whitespace characters between the slash and
1349
    // newline.
1350
6.51M
    if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1351
      // Remember that this token needs to be cleaned.
1352
6.51M
      if (Tok) 
Tok->setFlag(Token::NeedsCleaning)6.50M
;
1353
1354
      // Warn if there was whitespace between the backslash and newline.
1355
6.51M
      if (Ptr[0] != '\n' && 
Ptr[0] != '\r'82
&&
Tok23
&&
!isLexingRawMode()16
)
1356
8
        Diag(Ptr, diag::backslash_newline_space);
1357
1358
      // Found backslash<whitespace><newline>.  Parse the char after it.
1359
6.51M
      Size += EscapedNewLineSize;
1360
6.51M
      Ptr  += EscapedNewLineSize;
1361
1362
      // Use slow version to accumulate a correct size field.
1363
6.51M
      return getCharAndSizeSlow(Ptr, Size, Tok);
1364
6.51M
    }
1365
1366
    // Otherwise, this is not an escaped newline, just return the slash.
1367
29
    return '\\';
1368
6.51M
  }
1369
1370
  // If this is a trigraph, process it.
1371
6.72M
  if (Ptr[0] == '?' && 
Ptr[1] == '?'215k
) {
1372
    // If this is actually a legal trigraph (not something like "??x"), emit
1373
    // a trigraph warning.  If so, and if trigraphs are enabled, return it.
1374
248
    if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : nullptr)) {
1375
      // Remember that this token needs to be cleaned.
1376
74
      if (Tok) 
Tok->setFlag(Token::NeedsCleaning)57
;
1377
1378
74
      Ptr += 3;
1379
74
      Size += 3;
1380
74
      if (C == '\\') 
goto Slash18
;
1381
56
      return C;
1382
74
    }
1383
248
  }
1384
1385
  // If this is neither, return a single character.
1386
6.72M
  ++Size;
1387
6.72M
  return *Ptr;
1388
6.72M
}
1389
1390
/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
1391
/// getCharAndSizeNoWarn method.  Here we know that we can accumulate into Size,
1392
/// and that we have already incremented Ptr by Size bytes.
1393
///
1394
/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
1395
/// be updated to match.
1396
char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
1397
19.3k
                                     const LangOptions &LangOpts) {
1398
  // If we have a slash, look for an escaped newline.
1399
19.3k
  if (Ptr[0] == '\\') {
1400
10.5k
    ++Size;
1401
10.5k
    ++Ptr;
1402
10.5k
Slash:
1403
    // Common case, backslash-char where the char is not whitespace.
1404
10.5k
    if (!isWhitespace(Ptr[0])) 
return '\\'1.76k
;
1405
1406
    // See if we have optional whitespace characters followed by a newline.
1407
8.76k
    if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1408
      // Found backslash<whitespace><newline>.  Parse the char after it.
1409
8.76k
      Size += EscapedNewLineSize;
1410
8.76k
      Ptr  += EscapedNewLineSize;
1411
1412
      // Use slow version to accumulate a correct size field.
1413
8.76k
      return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts);
1414
8.76k
    }
1415
1416
    // Otherwise, this is not an escaped newline, just return the slash.
1417
0
    return '\\';
1418
8.76k
  }
1419
1420
  // If this is a trigraph, process it.
1421
8.83k
  if (LangOpts.Trigraphs && 
Ptr[0] == '?'3.22k
&&
Ptr[1] == '?'28
) {
1422
    // If this is actually a legal trigraph (not something like "??x"), return
1423
    // it.
1424
28
    if (char C = GetTrigraphCharForLetter(Ptr[2])) {
1425
28
      Ptr += 3;
1426
28
      Size += 3;
1427
28
      if (C == '\\') 
goto Slash8
;
1428
20
      return C;
1429
28
    }
1430
28
  }
1431
1432
  // If this is neither, return a single character.
1433
8.80k
  ++Size;
1434
8.80k
  return *Ptr;
1435
8.83k
}
1436
1437
//===----------------------------------------------------------------------===//
1438
// Helper methods for lexing.
1439
//===----------------------------------------------------------------------===//
1440
1441
/// Routine that indiscriminately sets the offset into the source file.
1442
420
void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) {
1443
420
  BufferPtr = BufferStart + Offset;
1444
420
  if (BufferPtr > BufferEnd)
1445
0
    BufferPtr = BufferEnd;
1446
  // FIXME: What exactly does the StartOfLine bit mean?  There are two
1447
  // possible meanings for the "start" of the line: the first token on the
1448
  // unexpanded line, or the first token on the expanded line.
1449
420
  IsAtStartOfLine = StartOfLine;
1450
420
  IsAtPhysicalStartOfLine = StartOfLine;
1451
420
}
1452
1453
221
static bool isUnicodeWhitespace(uint32_t Codepoint) {
1454
221
  static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars(
1455
221
      UnicodeWhitespaceCharRanges);
1456
221
  return UnicodeWhitespaceChars.contains(Codepoint);
1457
221
}
1458
1459
557
static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) {
1460
557
  if (LangOpts.AsmPreprocessor) {
1461
2
    return false;
1462
555
  } else if (LangOpts.DollarIdents && '$' == C) {
1463
3
    return true;
1464
552
  } else if (LangOpts.CPlusPlus) {
1465
    // A non-leading codepoint must have the XID_Continue property.
1466
    // XIDContinueRanges doesn't contains characters also in XIDStartRanges,
1467
    // so we need to check both tables.
1468
    // '_' doesn't have the XID_Continue property but is allowed in C++.
1469
171
    static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
1470
171
    static const llvm::sys::UnicodeCharSet XIDContinueChars(XIDContinueRanges);
1471
171
    return C == '_' || XIDStartChars.contains(C) ||
1472
171
           
XIDContinueChars.contains(C)61
;
1473
381
  } else if (LangOpts.C11) {
1474
232
    static const llvm::sys::UnicodeCharSet C11AllowedIDChars(
1475
232
        C11AllowedIDCharRanges);
1476
232
    return C11AllowedIDChars.contains(C);
1477
232
  } else {
1478
149
    static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1479
149
        C99AllowedIDCharRanges);
1480
149
    return C99AllowedIDChars.contains(C);
1481
149
  }
1482
557
}
1483
1484
269
static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) {
1485
269
  if (LangOpts.AsmPreprocessor) {
1486
0
    return false;
1487
0
  }
1488
269
  if (LangOpts.CPlusPlus) {
1489
110
    static const llvm::sys::UnicodeCharSet XIDStartChars(XIDStartRanges);
1490
    // '_' doesn't have the XID_Start property but is allowed in C++.
1491
110
    return C == '_' || XIDStartChars.contains(C);
1492
110
  }
1493
159
  if (!isAllowedIDChar(C, LangOpts))
1494
64
    return false;
1495
95
  if (LangOpts.C11) {
1496
72
    static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars(
1497
72
        C11DisallowedInitialIDCharRanges);
1498
72
    return !C11DisallowedInitialIDChars.contains(C);
1499
72
  }
1500
23
  static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1501
23
      C99DisallowedInitialIDCharRanges);
1502
23
  return !C99DisallowedInitialIDChars.contains(C);
1503
95
}
1504
1505
static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin,
1506
592
                                            const char *End) {
1507
592
  return CharSourceRange::getCharRange(L.getSourceLocation(Begin),
1508
592
                                       L.getSourceLocation(End));
1509
592
}
1510
1511
static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C,
1512
330
                                      CharSourceRange Range, bool IsFirst) {
1513
  // Check C99 compatibility.
1514
330
  if (!Diags.isIgnored(diag::warn_c99_compat_unicode_id, Range.getBegin())) {
1515
12
    enum {
1516
12
      CannotAppearInIdentifier = 0,
1517
12
      CannotStartIdentifier
1518
12
    };
1519
1520
12
    static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
1521
12
        C99AllowedIDCharRanges);
1522
12
    static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
1523
12
        C99DisallowedInitialIDCharRanges);
1524
12
    if (!C99AllowedIDChars.contains(C)) {
1525
5
      Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id)
1526
5
        << Range
1527
5
        << CannotAppearInIdentifier;
1528
7
    } else if (IsFirst && 
C99DisallowedInitialIDChars.contains(C)3
) {
1529
2
      Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id)
1530
2
        << Range
1531
2
        << CannotStartIdentifier;
1532
2
    }
1533
12
  }
1534
330
}
1535
1536
/// After encountering UTF-8 character C and interpreting it as an identifier
1537
/// character, check whether it's a homoglyph for a common non-identifier
1538
/// source character that is unlikely to be an intentional identifier
1539
/// character and warn if so.
1540
static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C,
1541
216
                                       CharSourceRange Range) {
1542
  // FIXME: Handle Unicode quotation marks (smart quotes, fullwidth quotes).
1543
216
  struct HomoglyphPair {
1544
216
    uint32_t Character;
1545
216
    char LooksLike;
1546
1.27k
    bool operator<(HomoglyphPair R) const { return Character < R.Character; }
1547
216
  };
1548
216
  static constexpr HomoglyphPair SortedHomoglyphs[] = {
1549
216
    {U'\u00ad', 0},   // SOFT HYPHEN
1550
216
    {U'\u01c3', '!'}, // LATIN LETTER RETROFLEX CLICK
1551
216
    {U'\u037e', ';'}, // GREEK QUESTION MARK
1552
216
    {U'\u200b', 0},   // ZERO WIDTH SPACE
1553
216
    {U'\u200c', 0},   // ZERO WIDTH NON-JOINER
1554
216
    {U'\u200d', 0},   // ZERO WIDTH JOINER
1555
216
    {U'\u2060', 0},   // WORD JOINER
1556
216
    {U'\u2061', 0},   // FUNCTION APPLICATION
1557
216
    {U'\u2062', 0},   // INVISIBLE TIMES
1558
216
    {U'\u2063', 0},   // INVISIBLE SEPARATOR
1559
216
    {U'\u2064', 0},   // INVISIBLE PLUS
1560
216
    {U'\u2212', '-'}, // MINUS SIGN
1561
216
    {U'\u2215', '/'}, // DIVISION SLASH
1562
216
    {U'\u2216', '\\'}, // SET MINUS
1563
216
    {U'\u2217', '*'}, // ASTERISK OPERATOR
1564
216
    {U'\u2223', '|'}, // DIVIDES
1565
216
    {U'\u2227', '^'}, // LOGICAL AND
1566
216
    {U'\u2236', ':'}, // RATIO
1567
216
    {U'\u223c', '~'}, // TILDE OPERATOR
1568
216
    {U'\ua789', ':'}, // MODIFIER LETTER COLON
1569
216
    {U'\ufeff', 0},   // ZERO WIDTH NO-BREAK SPACE
1570
216
    {U'\uff01', '!'}, // FULLWIDTH EXCLAMATION MARK
1571
216
    {U'\uff03', '#'}, // FULLWIDTH NUMBER SIGN
1572
216
    {U'\uff04', '$'}, // FULLWIDTH DOLLAR SIGN
1573
216
    {U'\uff05', '%'}, // FULLWIDTH PERCENT SIGN
1574
216
    {U'\uff06', '&'}, // FULLWIDTH AMPERSAND
1575
216
    {U'\uff08', '('}, // FULLWIDTH LEFT PARENTHESIS
1576
216
    {U'\uff09', ')'}, // FULLWIDTH RIGHT PARENTHESIS
1577
216
    {U'\uff0a', '*'}, // FULLWIDTH ASTERISK
1578
216
    {U'\uff0b', '+'}, // FULLWIDTH ASTERISK
1579
216
    {U'\uff0c', ','}, // FULLWIDTH COMMA
1580
216
    {U'\uff0d', '-'}, // FULLWIDTH HYPHEN-MINUS
1581
216
    {U'\uff0e', '.'}, // FULLWIDTH FULL STOP
1582
216
    {U'\uff0f', '/'}, // FULLWIDTH SOLIDUS
1583
216
    {U'\uff1a', ':'}, // FULLWIDTH COLON
1584
216
    {U'\uff1b', ';'}, // FULLWIDTH SEMICOLON
1585
216
    {U'\uff1c', '<'}, // FULLWIDTH LESS-THAN SIGN
1586
216
    {U'\uff1d', '='}, // FULLWIDTH EQUALS SIGN
1587
216
    {U'\uff1e', '>'}, // FULLWIDTH GREATER-THAN SIGN
1588
216
    {U'\uff1f', '?'}, // FULLWIDTH QUESTION MARK
1589
216
    {U'\uff20', '@'}, // FULLWIDTH COMMERCIAL AT
1590
216
    {U'\uff3b', '['}, // FULLWIDTH LEFT SQUARE BRACKET
1591
216
    {U'\uff3c', '\\'}, // FULLWIDTH REVERSE SOLIDUS
1592
216
    {U'\uff3d', ']'}, // FULLWIDTH RIGHT SQUARE BRACKET
1593
216
    {U'\uff3e', '^'}, // FULLWIDTH CIRCUMFLEX ACCENT
1594
216
    {U'\uff5b', '{'}, // FULLWIDTH LEFT CURLY BRACKET
1595
216
    {U'\uff5c', '|'}, // FULLWIDTH VERTICAL LINE
1596
216
    {U'\uff5d', '}'}, // FULLWIDTH RIGHT CURLY BRACKET
1597
216
    {U'\uff5e', '~'}, // FULLWIDTH TILDE
1598
216
    {0, 0}
1599
216
  };
1600
216
  auto Homoglyph =
1601
216
      std::lower_bound(std::begin(SortedHomoglyphs),
1602
216
                       std::end(SortedHomoglyphs) - 1, HomoglyphPair{C, '\0'});
1603
216
  if (Homoglyph->Character == C) {
1604
39
    llvm::SmallString<5> CharBuf;
1605
39
    {
1606
39
      llvm::raw_svector_ostream CharOS(CharBuf);
1607
39
      llvm::write_hex(CharOS, C, llvm::HexPrintStyle::Upper, 4);
1608
39
    }
1609
39
    if (Homoglyph->LooksLike) {
1610
32
      const char LooksLikeStr[] = {Homoglyph->LooksLike, 0};
1611
32
      Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_homoglyph)
1612
32
          << Range << CharBuf << LooksLikeStr;
1613
32
    } else {
1614
7
      Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_zero_width)
1615
7
          << Range << CharBuf;
1616
7
    }
1617
39
  }
1618
216
}
1619
1620
static void diagnoseInvalidUnicodeCodepointInIdentifier(
1621
    DiagnosticsEngine &Diags, const LangOptions &LangOpts, uint32_t CodePoint,
1622
37
    CharSourceRange Range, bool IsFirst) {
1623
37
  if (isASCII(CodePoint))
1624
0
    return;
1625
1626
37
  bool IsIDStart = isAllowedInitiallyIDChar(CodePoint, LangOpts);
1627
37
  bool IsIDContinue = IsIDStart || isAllowedIDChar(CodePoint, LangOpts);
1628
1629
37
  if ((IsFirst && 
IsIDStart13
) || (!IsFirst &&
IsIDContinue24
))
1630
0
    return;
1631
1632
37
  bool InvalidOnlyAtStart = IsFirst && 
!IsIDStart13
&&
IsIDContinue13
;
1633
1634
37
  llvm::SmallString<5> CharBuf;
1635
37
  llvm::raw_svector_ostream CharOS(CharBuf);
1636
37
  llvm::write_hex(CharOS, CodePoint, llvm::HexPrintStyle::Upper, 4);
1637
1638
37
  if (!IsFirst || 
InvalidOnlyAtStart13
) {
1639
30
    Diags.Report(Range.getBegin(), diag::err_character_not_allowed_identifier)
1640
30
        << Range << CharBuf << int(InvalidOnlyAtStart)
1641
30
        << FixItHint::CreateRemoval(Range);
1642
30
  } else {
1643
7
    Diags.Report(Range.getBegin(), diag::err_character_not_allowed)
1644
7
        << Range << CharBuf << FixItHint::CreateRemoval(Range);
1645
7
  }
1646
37
}
1647
1648
bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
1649
156
                                    Token &Result) {
1650
156
  const char *UCNPtr = CurPtr + Size;
1651
156
  uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/nullptr);
1652
156
  if (CodePoint == 0) {
1653
29
    return false;
1654
29
  }
1655
1656
127
  if (!isAllowedIDChar(CodePoint, LangOpts)) {
1657
11
    if (isASCII(CodePoint) || 
isUnicodeWhitespace(CodePoint)9
)
1658
2
      return false;
1659
9
    if (!isLexingRawMode() && 
!ParsingPreprocessorDirective8
&&
1660
9
        
!PP->isPreprocessedOutput()8
)
1661
8
      diagnoseInvalidUnicodeCodepointInIdentifier(
1662
8
          PP->getDiagnostics(), LangOpts, CodePoint,
1663
8
          makeCharRange(*this, CurPtr, UCNPtr),
1664
8
          /*IsFirst=*/false);
1665
1666
    // We got a unicode codepoint that is neither a space nor a
1667
    // a valid identifier part.
1668
    // Carry on as if the codepoint was valid for recovery purposes.
1669
116
  } else if (!isLexingRawMode())
1670
114
    maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint,
1671
114
                              makeCharRange(*this, CurPtr, UCNPtr),
1672
114
                              /*IsFirst=*/false);
1673
1674
125
  Result.setFlag(Token::HasUCN);
1675
125
  if ((UCNPtr - CurPtr ==  6 && 
CurPtr[1] == 'u'94
) ||
1676
125
      
(31
UCNPtr - CurPtr == 1031
&&
CurPtr[1] == 'U'21
))
1677
109
    CurPtr = UCNPtr;
1678
16
  else
1679
114
    
while (16
CurPtr != UCNPtr)
1680
98
      (void)getAndAdvanceChar(CurPtr, Result);
1681
125
  return true;
1682
127
}
1683
1684
249
bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr) {
1685
249
  const char *UnicodePtr = CurPtr;
1686
249
  llvm::UTF32 CodePoint;
1687
249
  llvm::ConversionResult Result =
1688
249
      llvm::convertUTF8Sequence((const llvm::UTF8 **)&UnicodePtr,
1689
249
                                (const llvm::UTF8 *)BufferEnd,
1690
249
                                &CodePoint,
1691
249
                                llvm::strictConversion);
1692
249
  if (Result != llvm::conversionOK)
1693
15
    return false;
1694
1695
234
  if (!isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) {
1696
67
    if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint))
1697
8
      return false;
1698
1699
59
    if (!isLexingRawMode() && 
!ParsingPreprocessorDirective35
&&
1700
59
        
!PP->isPreprocessedOutput()35
)
1701
16
      diagnoseInvalidUnicodeCodepointInIdentifier(
1702
16
          PP->getDiagnostics(), LangOpts, CodePoint,
1703
16
          makeCharRange(*this, CurPtr, UnicodePtr), /*IsFirst=*/false);
1704
    // We got a unicode codepoint that is neither a space nor a
1705
    // a valid identifier part. Carry on as if the codepoint was
1706
    // valid for recovery purposes.
1707
167
  } else if (!isLexingRawMode()) {
1708
139
    maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint,
1709
139
                              makeCharRange(*this, CurPtr, UnicodePtr),
1710
139
                              /*IsFirst=*/false);
1711
139
    maybeDiagnoseUTF8Homoglyph(PP->getDiagnostics(), CodePoint,
1712
139
                               makeCharRange(*this, CurPtr, UnicodePtr));
1713
139
  }
1714
1715
226
  CurPtr = UnicodePtr;
1716
226
  return true;
1717
234
}
1718
1719
bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
1720
219
                                      const char *CurPtr) {
1721
219
  if (isAllowedInitiallyIDChar(C, LangOpts)) {
1722
134
    if (!isLexingRawMode() && 
!ParsingPreprocessorDirective101
&&
1723
134
        
!PP->isPreprocessedOutput()86
) {
1724
77
      maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C,
1725
77
                                makeCharRange(*this, BufferPtr, CurPtr),
1726
77
                                /*IsFirst=*/true);
1727
77
      maybeDiagnoseUTF8Homoglyph(PP->getDiagnostics(), C,
1728
77
                                 makeCharRange(*this, BufferPtr, CurPtr));
1729
77
    }
1730
1731
134
    MIOpt.ReadToken();
1732
134
    return LexIdentifierContinue(Result, CurPtr);
1733
134
  }
1734
1735
85
  if (!isLexingRawMode() && 
!ParsingPreprocessorDirective53
&&
1736
85
      
!PP->isPreprocessedOutput()38
&&
!isASCII(*BufferPtr)21
&&
1737
85
      
!isAllowedInitiallyIDChar(C, LangOpts)13
&&
!isUnicodeWhitespace(C)13
) {
1738
    // Non-ASCII characters tend to creep into source code unintentionally.
1739
    // Instead of letting the parser complain about the unknown token,
1740
    // just drop the character.
1741
    // Note that we can /only/ do this when the non-ASCII character is actually
1742
    // spelled as Unicode, not written as a UCN. The standard requires that
1743
    // we not throw away any possible preprocessor tokens, but there's a
1744
    // loophole in the mapping of Unicode characters to basic character set
1745
    // characters that allows us to map these particular characters to, say,
1746
    // whitespace.
1747
13
    diagnoseInvalidUnicodeCodepointInIdentifier(
1748
13
        PP->getDiagnostics(), LangOpts, C,
1749
13
        makeCharRange(*this, BufferPtr, CurPtr), /*IsStart*/ true);
1750
13
    BufferPtr = CurPtr;
1751
13
    return false;
1752
13
  }
1753
1754
  // Otherwise, we have an explicit UCN or a character that's unlikely to show
1755
  // up by accident.
1756
72
  MIOpt.ReadToken();
1757
72
  FormTokenWithChars(Result, CurPtr, tok::unknown);
1758
72
  return true;
1759
85
}
1760
1761
715M
bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
1762
  // Match [_A-Za-z0-9]*, we have already matched an identifier start.
1763
8.00G
  while (true) {
1764
8.00G
    unsigned char C = *CurPtr;
1765
    // Fast path.
1766
8.00G
    if (isAsciiIdentifierContinue(C)) {
1767
7.29G
      ++CurPtr;
1768
7.29G
      continue;
1769
7.29G
    }
1770
1771
715M
    unsigned Size;
1772
    // Slow path: handle trigraph, unicode codepoints, UCNs.
1773
715M
    C = getCharAndSize(CurPtr, Size);
1774
715M
    if (isAsciiIdentifierContinue(C)) {
1775
82
      CurPtr = ConsumeChar(CurPtr, Size, Result);
1776
82
      continue;
1777
82
    }
1778
715M
    if (C == '$') {
1779
      // If we hit a $ and they are not supported in identifiers, we are done.
1780
2.94k
      if (!LangOpts.DollarIdents)
1781
2
        break;
1782
      // Otherwise, emit a diagnostic and continue.
1783
2.94k
      if (!isLexingRawMode())
1784
2.94k
        Diag(CurPtr, diag::ext_dollar_in_identifier);
1785
2.94k
      CurPtr = ConsumeChar(CurPtr, Size, Result);
1786
2.94k
      continue;
1787
2.94k
    }
1788
715M
    if (C == '\\' && 
tryConsumeIdentifierUCN(CurPtr, Size, Result)124
)
1789
101
      continue;
1790
715M
    if (!isASCII(C) && 
tryConsumeIdentifierUTF8Char(CurPtr)220
)
1791
197
      continue;
1792
    // Neither an expected Unicode codepoint nor a UCN.
1793
715M
    break;
1794
715M
  }
1795
1796
715M
  const char *IdStart = BufferPtr;
1797
715M
  FormTokenWithChars(Result, CurPtr, tok::raw_identifier);
1798
715M
  Result.setRawIdentifierData(IdStart);
1799
1800
  // If we are in raw mode, return this identifier raw.  There is no need to
1801
  // look up identifier information or attempt to macro expand it.
1802
715M
  if (LexingRawMode)
1803
229M
    return true;
1804
1805
  // Fill in Result.IdentifierInfo and update the token kind,
1806
  // looking up the identifier in the identifier table.
1807
485M
  IdentifierInfo *II = PP->LookUpIdentifierInfo(Result);
1808
  // Note that we have to call PP->LookUpIdentifierInfo() even for code
1809
  // completion, it writes IdentifierInfo into Result, and callers rely on it.
1810
1811
  // If the completion point is at the end of an identifier, we want to treat
1812
  // the identifier as incomplete even if it resolves to a macro or a keyword.
1813
  // This allows e.g. 'class^' to complete to 'classifier'.
1814
485M
  if (isCodeCompletionPoint(CurPtr)) {
1815
    // Return the code-completion token.
1816
85
    Result.setKind(tok::code_completion);
1817
    // Skip the code-completion char and all immediate identifier characters.
1818
    // This ensures we get consistent behavior when completing at any point in
1819
    // an identifier (i.e. at the start, in the middle, at the end). Note that
1820
    // only simple cases (i.e. [a-zA-Z0-9_]) are supported to keep the code
1821
    // simpler.
1822
85
    assert(*CurPtr == 0 && "Completion character must be 0");
1823
0
    ++CurPtr;
1824
    // Note that code completion token is not added as a separate character
1825
    // when the completion point is at the end of the buffer. Therefore, we need
1826
    // to check if the buffer has ended.
1827
85
    if (CurPtr < BufferEnd) {
1828
131
      while (isAsciiIdentifierContinue(*CurPtr))
1829
47
        ++CurPtr;
1830
84
    }
1831
85
    BufferPtr = CurPtr;
1832
85
    return true;
1833
85
  }
1834
1835
  // Finally, now that we know we have an identifier, pass this off to the
1836
  // preprocessor, which may macro expand it or something.
1837
485M
  if (II->isHandleIdentifierCase())
1838
41.6M
    return PP->HandleIdentifier(Result);
1839
1840
443M
  return true;
1841
485M
}
1842
1843
/// isHexaLiteral - Return true if Start points to a hex constant.
1844
/// in microsoft mode (where this is supposed to be several different tokens).
1845
141k
bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
1846
141k
  unsigned Size;
1847
141k
  char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts);
1848
141k
  if (C1 != '0')
1849
134k
    return false;
1850
7.36k
  char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts);
1851
7.36k
  return (C2 == 'x' || 
C2 == 'X'15
);
1852
141k
}
1853
1854
/// LexNumericConstant - Lex the remainder of a integer or floating point
1855
/// constant. From[-1] is the first character lexed.  Return the end of the
1856
/// constant.
1857
61.4M
bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
1858
61.4M
  unsigned Size;
1859
61.4M
  char C = getCharAndSize(CurPtr, Size);
1860
61.4M
  char PrevCh = 0;
1861
200M
  while (isPreprocessingNumberBody(C)) {
1862
139M
    CurPtr = ConsumeChar(CurPtr, Size, Result);
1863
139M
    PrevCh = C;
1864
139M
    C = getCharAndSize(CurPtr, Size);
1865
139M
  }
1866
1867
  // If we fell out, check for a sign, due to 1e+12.  If we have one, continue.
1868
61.4M
  if ((C == '-' || 
C == '+'60.6M
) &&
(1.15M
PrevCh == 'E'1.15M
||
PrevCh == 'e'1.15M
)) {
1869
    // If we are in Microsoft mode, don't continue if the constant is hex.
1870
    // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
1871
1.03M
    if (!LangOpts.MicrosoftExt || 
!isHexaLiteral(BufferPtr, LangOpts)134k
)
1872
1.03M
      return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
1873
1.03M
  }
1874
1875
  // If we have a hex FP constant, continue.
1876
60.4M
  if ((C == '-' || 
C == '+'60.4M
) &&
(114k
PrevCh == 'P'114k
||
PrevCh == 'p'114k
)) {
1877
    // Outside C99 and C++17, we accept hexadecimal floating point numbers as a
1878
    // not-quite-conforming extension. Only do so if this looks like it's
1879
    // actually meant to be a hexfloat, and not if it has a ud-suffix.
1880
11.3k
    bool IsHexFloat = true;
1881
11.3k
    if (!LangOpts.C99) {
1882
7.34k
      if (!isHexaLiteral(BufferPtr, LangOpts))
1883
9
        IsHexFloat = false;
1884
7.33k
      else if (!getLangOpts().CPlusPlus17 &&
1885
7.33k
               
std::find(BufferPtr, CurPtr, '_') != CurPtr6.90k
)
1886
3
        IsHexFloat = false;
1887
7.34k
    }
1888
11.3k
    if (IsHexFloat)
1889
11.3k
      return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
1890
11.3k
  }
1891
1892
  // If we have a digit separator, continue.
1893
60.4M
  if (C == '\'' && 
(1.46k
getLangOpts().CPlusPlus141.46k
||
getLangOpts().C2x624
)) {
1894
892
    unsigned NextSize;
1895
892
    char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts());
1896
892
    if (isAsciiIdentifierContinue(Next)) {
1897
874
      if (!isLexingRawMode())
1898
452
        Diag(CurPtr, getLangOpts().CPlusPlus
1899
452
                         ? 
diag::warn_cxx11_compat_digit_separator410
1900
452
                         : 
diag::warn_c2x_compat_digit_separator42
);
1901
874
      CurPtr = ConsumeChar(CurPtr, Size, Result);
1902
874
      CurPtr = ConsumeChar(CurPtr, NextSize, Result);
1903
874
      return LexNumericConstant(Result, CurPtr);
1904
874
    }
1905
892
  }
1906
1907
  // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue.
1908
60.4M
  if (C == '\\' && 
tryConsumeIdentifierUCN(CurPtr, Size, Result)6
)
1909
5
    return LexNumericConstant(Result, CurPtr);
1910
60.4M
  if (!isASCII(C) && 
tryConsumeIdentifierUTF8Char(CurPtr)13
)
1911
13
    return LexNumericConstant(Result, CurPtr);
1912
1913
  // Update the location of token as well as BufferPtr.
1914
60.4M
  const char *TokStart = BufferPtr;
1915
60.4M
  FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
1916
60.4M
  Result.setLiteralData(TokStart);
1917
60.4M
  return true;
1918
60.4M
}
1919
1920
/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
1921
/// in C++11, or warn on a ud-suffix in C++98.
1922
const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
1923
7.97M
                               bool IsStringLiteral) {
1924
7.97M
  assert(getLangOpts().CPlusPlus);
1925
1926
  // Maximally munch an identifier.
1927
0
  unsigned Size;
1928
7.97M
  char C = getCharAndSize(CurPtr, Size);
1929
7.97M
  bool Consumed = false;
1930
1931
7.97M
  if (!isAsciiIdentifierStart(C)) {
1932
7.97M
    if (C == '\\' && 
tryConsumeIdentifierUCN(CurPtr, Size, Result)7
)
1933
0
      Consumed = true;
1934
7.97M
    else if (!isASCII(C) && 
tryConsumeIdentifierUTF8Char(CurPtr)0
)
1935
0
      Consumed = true;
1936
7.97M
    else
1937
7.97M
      return CurPtr;
1938
7.97M
  }
1939
1940
7.72k
  if (!getLangOpts().CPlusPlus11) {
1941
19
    if (!isLexingRawMode())
1942
8
      Diag(CurPtr,
1943
8
           C == '_' ? 
diag::warn_cxx11_compat_user_defined_literal3
1944
8
                    : 
diag::warn_cxx11_compat_reserved_user_defined_literal5
)
1945
8
        << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " ");
1946
19
    return CurPtr;
1947
19
  }
1948
1949
  // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix
1950
  // that does not start with an underscore is ill-formed. As a conforming
1951
  // extension, we treat all such suffixes as if they had whitespace before
1952
  // them. We assume a suffix beginning with a UCN or UTF-8 character is more
1953
  // likely to be a ud-suffix than a macro, however, and accept that.
1954
7.71k
  
if (7.70k
!Consumed7.70k
) {
1955
7.71k
    bool IsUDSuffix = false;
1956
7.71k
    if (C == '_')
1957
376
      IsUDSuffix = true;
1958
7.33k
    else if (IsStringLiteral && 
getLangOpts().CPlusPlus147.05k
) {
1959
      // In C++1y, we need to look ahead a few characters to see if this is a
1960
      // valid suffix for a string literal or a numeric literal (this could be
1961
      // the 'operator""if' defining a numeric literal operator).
1962
573
      const unsigned MaxStandardSuffixLength = 3;
1963
573
      char Buffer[MaxStandardSuffixLength] = { C };
1964
573
      unsigned Consumed = Size;
1965
573
      unsigned Chars = 1;
1966
1.00k
      while (true) {
1967
1.00k
        unsigned NextSize;
1968
1.00k
        char Next = getCharAndSizeNoWarn(CurPtr + Consumed, NextSize,
1969
1.00k
                                         getLangOpts());
1970
1.00k
        if (!isAsciiIdentifierContinue(Next)) {
1971
          // End of suffix. Check whether this is on the allowed list.
1972
569
          const StringRef CompleteSuffix(Buffer, Chars);
1973
569
          IsUDSuffix = StringLiteralParser::isValidUDSuffix(getLangOpts(),
1974
569
                                                            CompleteSuffix);
1975
569
          break;
1976
569
        }
1977
1978
440
        if (Chars == MaxStandardSuffixLength)
1979
          // Too long: can't be a standard suffix.
1980
4
          break;
1981
1982
436
        Buffer[Chars++] = Next;
1983
436
        Consumed += NextSize;
1984
436
      }
1985
573
    }
1986
1987
7.71k
    if (!IsUDSuffix) {
1988
6.81k
      if (!isLexingRawMode())
1989
12
        Diag(CurPtr, getLangOpts().MSVCCompat
1990
12
                         ? 
diag::ext_ms_reserved_user_defined_literal0
1991
12
                         : diag::ext_reserved_user_defined_literal)
1992
12
          << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " ");
1993
6.81k
      return CurPtr;
1994
6.81k
    }
1995
1996
905
    CurPtr = ConsumeChar(CurPtr, Size, Result);
1997
905
  }
1998
1999
895
  Result.setFlag(Token::HasUDSuffix);
2000
2.15k
  while (
true2.14k
) {
2001
2.15k
    C = getCharAndSize(CurPtr, Size);
2002
2.15k
    if (isAsciiIdentifierContinue(C)) {
2003
1.21k
      CurPtr = ConsumeChar(CurPtr, Size, Result);
2004
1.21k
    } else 
if (940
C == '\\'940
&&
tryConsumeIdentifierUCN(CurPtr, Size, Result)19
) {
2005
921
    } else if (!isASCII(C) && 
tryConsumeIdentifierUTF8Char(CurPtr)16
) {
2006
16
    } else
2007
905
      break;
2008
2.15k
  }
2009
2010
895
  return CurPtr;
2011
7.70k
}
2012
2013
/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
2014
/// either " or L" or u8" or u" or U".
2015
bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
2016
12.1M
                             tok::TokenKind Kind) {
2017
12.1M
  const char *AfterQuote = CurPtr;
2018
  // Does this string contain the \0 character?
2019
12.1M
  const char *NulCharacter = nullptr;
2020
2021
12.1M
  if (!isLexingRawMode() &&
2022
12.1M
      
(9.97M
Kind == tok::utf8_string_literal9.97M
||
2023
9.97M
       
Kind == tok::utf16_string_literal9.97M
||
2024
9.97M
       
Kind == tok::utf32_string_literal9.97M
))
2025
587
    Diag(BufferPtr, getLangOpts().CPlusPlus
2026
587
           ? 
diag::warn_cxx98_compat_unicode_literal507
2027
587
           : 
diag::warn_c99_compat_unicode_literal80
);
2028
2029
12.1M
  char C = getAndAdvanceChar(CurPtr, Result);
2030
123M
  while (C != '"') {
2031
    // Skip escaped characters.  Escaped newlines will already be processed by
2032
    // getAndAdvanceChar.
2033
111M
    if (C == '\\')
2034
240k
      C = getAndAdvanceChar(CurPtr, Result);
2035
2036
111M
    if (C == '\n' || 
C == '\r'111M
|| // Newline.
2037
111M
        
(111M
C == 0111M
&&
CurPtr-1 == BufferEnd43
)) { // End of file.
2038
71
      if (!isLexingRawMode() && 
!LangOpts.AsmPreprocessor7
)
2039
5
        Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1;
2040
71
      FormTokenWithChars(Result, CurPtr-1, tok::unknown);
2041
71
      return true;
2042
71
    }
2043
2044
111M
    if (C == 0) {
2045
21
      if (isCodeCompletionPoint(CurPtr-1)) {
2046
12
        if (ParsingFilename)
2047
5
          codeCompleteIncludedFile(AfterQuote, CurPtr - 1, /*IsAngled=*/false);
2048
7
        else
2049
7
          PP->CodeCompleteNaturalLanguage();
2050
12
        FormTokenWithChars(Result, CurPtr - 1, tok::unknown);
2051
12
        cutOffLexing();
2052
12
        return true;
2053
12
      }
2054
2055
9
      NulCharacter = CurPtr-1;
2056
9
    }
2057
111M
    C = getAndAdvanceChar(CurPtr, Result);
2058
111M
  }
2059
2060
  // If we are in C++11, lex the optional ud-suffix.
2061
12.1M
  if (getLangOpts().CPlusPlus)
2062
7.79M
    CurPtr = LexUDSuffix(Result, CurPtr, true);
2063
2064
  // If a nul character existed in the string, warn about it.
2065
12.1M
  if (NulCharacter && 
!isLexingRawMode()8
)
2066
2
    Diag(NulCharacter, diag::null_in_char_or_string) << 1;
2067
2068
  // Update the location of the token as well as the BufferPtr instance var.
2069
12.1M
  const char *TokStart = BufferPtr;
2070
12.1M
  FormTokenWithChars(Result, CurPtr, Kind);
2071
12.1M
  Result.setLiteralData(TokStart);
2072
12.1M
  return true;
2073
12.1M
}
2074
2075
/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
2076
/// having lexed R", LR", u8R", uR", or UR".
2077
bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
2078
569
                                tok::TokenKind Kind) {
2079
  // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
2080
  //  Between the initial and final double quote characters of the raw string,
2081
  //  any transformations performed in phases 1 and 2 (trigraphs,
2082
  //  universal-character-names, and line splicing) are reverted.
2083
2084
569
  if (!isLexingRawMode())
2085
130
    Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal);
2086
2087
569
  unsigned PrefixLen = 0;
2088
2089
1.79k
  while (PrefixLen != 16 && 
isRawStringDelimBody(CurPtr[PrefixLen])1.79k
)
2090
1.22k
    ++PrefixLen;
2091
2092
  // If the last character was not a '(', then we didn't lex a valid delimiter.
2093
569
  if (CurPtr[PrefixLen] != '(') {
2094
4
    if (!isLexingRawMode()) {
2095
1
      const char *PrefixEnd = &CurPtr[PrefixLen];
2096
1
      if (PrefixLen == 16) {
2097
1
        Diag(PrefixEnd, diag::err_raw_delim_too_long);
2098
1
      } else {
2099
0
        Diag(PrefixEnd, diag::err_invalid_char_raw_delim)
2100
0
          << StringRef(PrefixEnd, 1);
2101
0
      }
2102
1
    }
2103
2104
    // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
2105
    // it's possible the '"' was intended to be part of the raw string, but
2106
    // there's not much we can do about that.
2107
347
    while (true) {
2108
347
      char C = *CurPtr++;
2109
2110
347
      if (C == '"')
2111
4
        break;
2112
343
      if (C == 0 && 
CurPtr-1 == BufferEnd0
) {
2113
0
        --CurPtr;
2114
0
        break;
2115
0
      }
2116
343
    }
2117
2118
4
    FormTokenWithChars(Result, CurPtr, tok::unknown);
2119
4
    return true;
2120
4
  }
2121
2122
  // Save prefix and move CurPtr past it
2123
565
  const char *Prefix = CurPtr;
2124
565
  CurPtr += PrefixLen + 1; // skip over prefix and '('
2125
2126
9.96k
  while (true) {
2127
9.96k
    char C = *CurPtr++;
2128
2129
9.96k
    if (C == ')') {
2130
      // Check for prefix match and closing quote.
2131
569
      if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && 
CurPtr[PrefixLen] == '"'561
) {
2132
561
        CurPtr += PrefixLen + 1; // skip over prefix and '"'
2133
561
        break;
2134
561
      }
2135
9.39k
    } else if (C == 0 && 
CurPtr-1 == BufferEnd4
) { // End of file.
2136
4
      if (!isLexingRawMode())
2137
1
        Diag(BufferPtr, diag::err_unterminated_raw_string)
2138
1
          << StringRef(Prefix, PrefixLen);
2139
4
      FormTokenWithChars(Result, CurPtr-1, tok::unknown);
2140
4
      return true;
2141
4
    }
2142
9.96k
  }
2143
2144
  // If we are in C++11, lex the optional ud-suffix.
2145
561
  if (getLangOpts().CPlusPlus)
2146
561
    CurPtr = LexUDSuffix(Result, CurPtr, true);
2147
2148
  // Update the location of token as well as BufferPtr.
2149
561
  const char *TokStart = BufferPtr;
2150
561
  FormTokenWithChars(Result, CurPtr, Kind);
2151
561
  Result.setLiteralData(TokStart);
2152
561
  return true;
2153
565
}
2154
2155
/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
2156
/// after having lexed the '<' character.  This is used for #include filenames.
2157
1.62M
bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
2158
  // Does this string contain the \0 character?
2159
1.62M
  const char *NulCharacter = nullptr;
2160
1.62M
  const char *AfterLessPos = CurPtr;
2161
1.62M
  char C = getAndAdvanceChar(CurPtr, Result);
2162
30.4M
  while (C != '>') {
2163
    // Skip escaped characters.  Escaped newlines will already be processed by
2164
    // getAndAdvanceChar.
2165
28.8M
    if (C == '\\')
2166
6
      C = getAndAdvanceChar(CurPtr, Result);
2167
2168
28.8M
    if (isVerticalWhitespace(C) ||               // Newline.
2169
28.8M
        
(28.8M
C == 028.8M
&&
(CurPtr - 1 == BufferEnd)13
)) { // End of file.
2170
      // If the filename is unterminated, then it must just be a lone <
2171
      // character.  Return this as such.
2172
10
      FormTokenWithChars(Result, AfterLessPos, tok::less);
2173
10
      return true;
2174
10
    }
2175
2176
28.8M
    if (C == 0) {
2177
10
      if (isCodeCompletionPoint(CurPtr - 1)) {
2178
9
        codeCompleteIncludedFile(AfterLessPos, CurPtr - 1, /*IsAngled=*/true);
2179
9
        cutOffLexing();
2180
9
        FormTokenWithChars(Result, CurPtr - 1, tok::unknown);
2181
9
        return true;
2182
9
      }
2183
1
      NulCharacter = CurPtr-1;
2184
1
    }
2185
28.8M
    C = getAndAdvanceChar(CurPtr, Result);
2186
28.8M
  }
2187
2188
  // If a nul character existed in the string, warn about it.
2189
1.62M
  if (NulCharacter && 
!isLexingRawMode()1
)
2190
1
    Diag(NulCharacter, diag::null_in_char_or_string) << 1;
2191
2192
  // Update the location of token as well as BufferPtr.
2193
1.62M
  const char *TokStart = BufferPtr;
2194
1.62M
  FormTokenWithChars(Result, CurPtr, tok::header_name);
2195
1.62M
  Result.setLiteralData(TokStart);
2196
1.62M
  return true;
2197
1.62M
}
2198
2199
void Lexer::codeCompleteIncludedFile(const char *PathStart,
2200
                                     const char *CompletionPoint,
2201
14
                                     bool IsAngled) {
2202
  // Completion only applies to the filename, after the last slash.
2203
14
  StringRef PartialPath(PathStart, CompletionPoint - PathStart);
2204
14
  llvm::StringRef SlashChars = LangOpts.MSVCCompat ? 
"/\\"4
:
"/"10
;
2205
14
  auto Slash = PartialPath.find_last_of(SlashChars);
2206
14
  StringRef Dir =
2207
14
      (Slash == StringRef::npos) ? 
""10
:
PartialPath.take_front(Slash)4
;
2208
14
  const char *StartOfFilename =
2209
14
      (Slash == StringRef::npos) ? 
PathStart10
:
PathStart + Slash + 14
;
2210
  // Code completion filter range is the filename only, up to completion point.
2211
14
  PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get(
2212
14
      StringRef(StartOfFilename, CompletionPoint - StartOfFilename)));
2213
  // We should replace the characters up to the closing quote or closest slash,
2214
  // if any.
2215
85
  while (CompletionPoint < BufferEnd) {
2216
85
    char Next = *(CompletionPoint + 1);
2217
85
    if (Next == 0 || Next == '\r' || Next == '\n')
2218
0
      break;
2219
85
    ++CompletionPoint;
2220
85
    if (Next == (IsAngled ? 
'>'60
:
'"'25
))
2221
13
      break;
2222
72
    if (llvm::is_contained(SlashChars, Next))
2223
1
      break;
2224
72
  }
2225
2226
14
  PP->setCodeCompletionTokenRange(
2227
14
      FileLoc.getLocWithOffset(StartOfFilename - BufferStart),
2228
14
      FileLoc.getLocWithOffset(CompletionPoint - BufferStart));
2229
14
  PP->CodeCompleteIncludedFile(Dir, IsAngled);
2230
14
}
2231
2232
/// LexCharConstant - Lex the remainder of a character constant, after having
2233
/// lexed either ' or L' or u8' or u' or U'.
2234
bool Lexer::LexCharConstant(Token &Result, const char *CurPtr,
2235
653k
                            tok::TokenKind Kind) {
2236
  // Does this character contain the \0 character?
2237
653k
  const char *NulCharacter = nullptr;
2238
2239
653k
  if (!isLexingRawMode()) {
2240
609k
    if (Kind == tok::utf16_char_constant || 
Kind == tok::utf32_char_constant609k
)
2241
204
      Diag(BufferPtr, getLangOpts().CPlusPlus
2242
204
                          ? 
diag::warn_cxx98_compat_unicode_literal178
2243
204
                          : 
diag::warn_c99_compat_unicode_literal26
);
2244
609k
    else if (Kind == tok::utf8_char_constant)
2245
151
      Diag(BufferPtr, diag::warn_cxx14_compat_u8_character_literal);
2246
609k
  }
2247
2248
653k
  char C = getAndAdvanceChar(CurPtr, Result);
2249
653k
  if (C == '\'') {
2250
28
    if (!isLexingRawMode() && 
!LangOpts.AsmPreprocessor2
)
2251
0
      Diag(BufferPtr, diag::ext_empty_character);
2252
28
    FormTokenWithChars(Result, CurPtr, tok::unknown);
2253
28
    return true;
2254
28
  }
2255
2256
2.89M
  
while (653k
C != '\'') {
2257
    // Skip escaped characters.
2258
2.24M
    if (C == '\\')
2259
11.4k
      C = getAndAdvanceChar(CurPtr, Result);
2260
2261
2.24M
    if (C == '\n' || 
C == '\r'2.24M
|| // Newline.
2262
2.24M
        
(2.24M
C == 02.24M
&&
CurPtr-1 == BufferEnd19
)) { // End of file.
2263
326
      if (!isLexingRawMode() && 
!LangOpts.AsmPreprocessor7
)
2264
4
        Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 0;
2265
326
      FormTokenWithChars(Result, CurPtr-1, tok::unknown);
2266
326
      return true;
2267
326
    }
2268
2269
2.24M
    if (C == 0) {
2270
14
      if (isCodeCompletionPoint(CurPtr-1)) {
2271
6
        PP->CodeCompleteNaturalLanguage();
2272
6
        FormTokenWithChars(Result, CurPtr-1, tok::unknown);
2273
6
        cutOffLexing();
2274
6
        return true;
2275
6
      }
2276
2277
8
      NulCharacter = CurPtr-1;
2278
8
    }
2279
2.24M
    C = getAndAdvanceChar(CurPtr, Result);
2280
2.24M
  }
2281
2282
  // If we are in C++11, lex the optional ud-suffix.
2283
653k
  if (getLangOpts().CPlusPlus)
2284
179k
    CurPtr = LexUDSuffix(Result, CurPtr, false);
2285
2286
  // If a nul character existed in the character, warn about it.
2287
653k
  if (NulCharacter && 
!isLexingRawMode()8
)
2288
2
    Diag(NulCharacter, diag::null_in_char_or_string) << 0;
2289
2290
  // Update the location of token as well as BufferPtr.
2291
653k
  const char *TokStart = BufferPtr;
2292
653k
  FormTokenWithChars(Result, CurPtr, Kind);
2293
653k
  Result.setLiteralData(TokStart);
2294
653k
  return true;
2295
653k
}
2296
2297
/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
2298
/// Update BufferPtr to point to the next non-whitespace character and return.
2299
///
2300
/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
2301
bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr,
2302
179M
                           bool &TokAtPhysicalStartOfLine) {
2303
  // Whitespace - Skip it, then return the token after the whitespace.
2304
179M
  bool SawNewline = isVerticalWhitespace(CurPtr[-1]);
2305
2306
179M
  unsigned char Char = *CurPtr;
2307
2308
179M
  const char *lastNewLine = nullptr;
2309
188M
  auto setLastNewLine = [&](const char *Ptr) {
2310
188M
    lastNewLine = Ptr;
2311
188M
    if (!NewLinePtr)
2312
164M
      NewLinePtr = Ptr;
2313
188M
  };
2314
179M
  if (SawNewline)
2315
173M
    setLastNewLine(CurPtr - 1);
2316
2317
  // Skip consecutive spaces efficiently.
2318
194M
  while (true) {
2319
    // Skip horizontal whitespace very aggressively.
2320
493M
    while (isHorizontalWhitespace(Char))
2321
298M
      Char = *++CurPtr;
2322
2323
    // Otherwise if we have something other than whitespace, we're done.
2324
194M
    if (!isVerticalWhitespace(Char))
2325
179M
      break;
2326
2327
15.0M
    if (ParsingPreprocessorDirective) {
2328
      // End of preprocessor directive line, let LexTokenInternal handle this.
2329
1.39k
      BufferPtr = CurPtr;
2330
1.39k
      return false;
2331
1.39k
    }
2332
2333
    // OK, but handle newline.
2334
15.0M
    if (*CurPtr == '\n')
2335
15.0M
      setLastNewLine(CurPtr);
2336
15.0M
    SawNewline = true;
2337
15.0M
    Char = *++CurPtr;
2338
15.0M
  }
2339
2340
  // If the client wants us to return whitespace, return it now.
2341
179M
  if (isKeepWhitespaceMode()) {
2342
77.8k
    FormTokenWithChars(Result, CurPtr, tok::unknown);
2343
77.8k
    if (SawNewline) {
2344
77.0k
      IsAtStartOfLine = true;
2345
77.0k
      IsAtPhysicalStartOfLine = true;
2346
77.0k
    }
2347
    // FIXME: The next token will not have LeadingSpace set.
2348
77.8k
    return true;
2349
77.8k
  }
2350
2351
  // If this isn't immediately after a newline, there is leading space.
2352
179M
  char PrevChar = CurPtr[-1];
2353
179M
  bool HasLeadingSpace = !isVerticalWhitespace(PrevChar);
2354
2355
179M
  Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
2356
179M
  if (SawNewline) {
2357
173M
    Result.setFlag(Token::StartOfLine);
2358
173M
    TokAtPhysicalStartOfLine = true;
2359
2360
173M
    if (NewLinePtr && 
lastNewLine173M
&& NewLinePtr != lastNewLine &&
PP20.6M
) {
2361
19.8M
      if (auto *Handler = PP->getEmptylineHandler())
2362
184
        Handler->HandleEmptyline(SourceRange(getSourceLocation(NewLinePtr + 1),
2363
184
                                             getSourceLocation(lastNewLine)));
2364
19.8M
    }
2365
173M
  }
2366
2367
179M
  BufferPtr = CurPtr;
2368
179M
  return false;
2369
179M
}
2370
2371
/// We have just read the // characters from input.  Skip until we find the
2372
/// newline character that terminates the comment.  Then update BufferPtr and
2373
/// return.
2374
///
2375
/// If we're in KeepCommentMode or any CommentHandler has inserted
2376
/// some tokens, this will store the first token and return true.
2377
bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
2378
45.5M
                            bool &TokAtPhysicalStartOfLine) {
2379
  // If Line comments aren't explicitly enabled for this language, emit an
2380
  // extension warning.
2381
45.5M
  if (!LangOpts.LineComment && 
!isLexingRawMode()7.54k
) {
2382
4.01k
    Diag(BufferPtr, diag::ext_line_comment);
2383
2384
    // Mark them enabled so we only emit one warning for this translation
2385
    // unit.
2386
4.01k
    LangOpts.LineComment = true;
2387
4.01k
  }
2388
2389
  // Scan over the body of the comment.  The common case, when scanning, is that
2390
  // the comment contains normal ascii characters with nothing interesting in
2391
  // them.  As such, optimize for this case with the inner loop.
2392
  //
2393
  // This loop terminates with CurPtr pointing at the newline (or end of buffer)
2394
  // character that ends the line comment.
2395
45.5M
  char C;
2396
45.6M
  while (true) {
2397
45.6M
    C = *CurPtr;
2398
    // Skip over characters in the fast loop.
2399
2.91G
    while (C != 0 &&                // Potentially EOF.
2400
2.91G
           
C != '\n'2.91G
&&
C != '\r'2.86G
) // Newline or DOS-style newline.
2401
2.86G
      C = *++CurPtr;
2402
2403
45.6M
    const char *NextLine = CurPtr;
2404
45.6M
    if (C != 0) {
2405
      // We found a newline, see if it's escaped.
2406
45.6M
      const char *EscapePtr = CurPtr-1;
2407
45.6M
      bool HasSpace = false;
2408
45.6M
      while (isHorizontalWhitespace(*EscapePtr)) { // Skip whitespace.
2409
17.8k
        --EscapePtr;
2410
17.8k
        HasSpace = true;
2411
17.8k
      }
2412
2413
45.6M
      if (*EscapePtr == '\\')
2414
        // Escaped newline.
2415
54.2k
        CurPtr = EscapePtr;
2416
45.5M
      else if (EscapePtr[0] == '/' && 
EscapePtr[-1] == '?'3.65M
&&
2417
45.5M
               
EscapePtr[-2] == '?'8
&&
LangOpts.Trigraphs8
)
2418
        // Trigraph-escaped newline.
2419
3
        CurPtr = EscapePtr-2;
2420
45.5M
      else
2421
45.5M
        break; // This is a newline, we're done.
2422
2423
      // If there was space between the backslash and newline, warn about it.
2424
54.2k
      if (HasSpace && 
!isLexingRawMode()8
)
2425
6
        Diag(EscapePtr, diag::backslash_newline_space);
2426
54.2k
    }
2427
2428
    // Otherwise, this is a hard case.  Fall back on getAndAdvanceChar to
2429
    // properly decode the character.  Read it in raw mode to avoid emitting
2430
    // diagnostics about things like trigraphs.  If we see an escaped newline,
2431
    // we'll handle it below.
2432
58.0k
    const char *OldPtr = CurPtr;
2433
58.0k
    bool OldRawMode = isLexingRawMode();
2434
58.0k
    LexingRawMode = true;
2435
58.0k
    C = getAndAdvanceChar(CurPtr, Result);
2436
58.0k
    LexingRawMode = OldRawMode;
2437
2438
    // If we only read only one character, then no special handling is needed.
2439
    // We're done and can skip forward to the newline.
2440
58.0k
    if (C != 0 && 
CurPtr == OldPtr+154.2k
) {
2441
0
      CurPtr = NextLine;
2442
0
      break;
2443
0
    }
2444
2445
    // If we read multiple characters, and one of those characters was a \r or
2446
    // \n, then we had an escaped newline within the comment.  Emit diagnostic
2447
    // unless the next line is also a // comment.
2448
58.0k
    if (CurPtr != OldPtr + 1 && 
C != '/'54.2k
&&
2449
58.0k
        
(5.13k
CurPtr == BufferEnd + 15.13k
||
CurPtr[0] != '/'5.13k
)) {
2450
10.2k
      for (; OldPtr != CurPtr; 
++OldPtr5.13k
)
2451
10.2k
        if (OldPtr[0] == '\n' || 
OldPtr[0] == '\r'5.13k
) {
2452
          // Okay, we found a // comment that ends in a newline, if the next
2453
          // line is also a // comment, but has spaces, don't emit a diagnostic.
2454
5.12k
          if (isWhitespace(C)) {
2455
4.80k
            const char *ForwardPtr = CurPtr;
2456
62.8k
            while (isWhitespace(*ForwardPtr))  // Skip whitespace.
2457
58.0k
              ++ForwardPtr;
2458
4.80k
            if (ForwardPtr[0] == '/' && 
ForwardPtr[1] == '/'4.14k
)
2459
4.14k
              break;
2460
4.80k
          }
2461
2462
975
          if (!isLexingRawMode())
2463
817
            Diag(OldPtr-1, diag::ext_multi_line_line_comment);
2464
975
          break;
2465
5.12k
        }
2466
5.12k
    }
2467
2468
58.0k
    if (
C == '\r'58.0k
|| C == '\n' ||
CurPtr == BufferEnd + 157.9k
) {
2469
3.82k
      --CurPtr;
2470
3.82k
      break;
2471
3.82k
    }
2472
2473
54.1k
    if (C == '\0' && 
isCodeCompletionPoint(CurPtr-1)9
) {
2474
8
      PP->CodeCompleteNaturalLanguage();
2475
8
      cutOffLexing();
2476
8
      return false;
2477
8
    }
2478
54.1k
  }
2479
2480
  // Found but did not consume the newline.  Notify comment handlers about the
2481
  // comment unless we're in a #if 0 block.
2482
45.5M
  if (PP && 
!isLexingRawMode()45.4M
&&
2483
45.5M
      PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
2484
41.7M
                                            getSourceLocation(CurPtr)))) {
2485
0
    BufferPtr = CurPtr;
2486
0
    return true; // A token has to be returned.
2487
0
  }
2488
2489
  // If we are returning comments as tokens, return this comment as a token.
2490
45.5M
  if (inKeepCommentMode())
2491
44.5k
    return SaveLineComment(Result, CurPtr);
2492
2493
  // If we are inside a preprocessor directive and we see the end of line,
2494
  // return immediately, so that the lexer can return this as an EOD token.
2495
45.5M
  if (ParsingPreprocessorDirective || 
CurPtr == BufferEnd44.7M
) {
2496
789k
    BufferPtr = CurPtr;
2497
789k
    return false;
2498
789k
  }
2499
2500
  // Otherwise, eat the \n character.  We don't care if this is a \n\r or
2501
  // \r\n sequence.  This is an efficiency hack (because we know the \n can't
2502
  // contribute to another token), it isn't needed for correctness.  Note that
2503
  // this is ok even in KeepWhitespaceMode, because we would have returned the
2504
  /// comment above in that mode.
2505
44.7M
  NewLinePtr = CurPtr++;
2506
2507
  // The next returned token is at the start of the line.
2508
44.7M
  Result.setFlag(Token::StartOfLine);
2509
44.7M
  TokAtPhysicalStartOfLine = true;
2510
  // No leading whitespace seen so far.
2511
44.7M
  Result.clearFlag(Token::LeadingSpace);
2512
44.7M
  BufferPtr = CurPtr;
2513
44.7M
  return false;
2514
45.5M
}
2515
2516
/// If in save-comment mode, package up this Line comment in an appropriate
2517
/// way and return it.
2518
44.5k
bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) {
2519
  // If we're not in a preprocessor directive, just return the // comment
2520
  // directly.
2521
44.5k
  FormTokenWithChars(Result, CurPtr, tok::comment);
2522
2523
44.5k
  if (!ParsingPreprocessorDirective || 
LexingRawMode2
)
2524
44.5k
    return true;
2525
2526
  // If this Line-style comment is in a macro definition, transmogrify it into
2527
  // a C-style block comment.
2528
2
  bool Invalid = false;
2529
2
  std::string Spelling = PP->getSpelling(Result, &Invalid);
2530
2
  if (Invalid)
2531
0
    return true;
2532
2533
2
  assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?");
2534
0
  Spelling[1] = '*';   // Change prefix to "/*".
2535
2
  Spelling += "*/";    // add suffix.
2536
2537
2
  Result.setKind(tok::comment);
2538
2
  PP->CreateString(Spelling, Result,
2539
2
                   Result.getLocation(), Result.getLocation());
2540
2
  return true;
2541
2
}
2542
2543
/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
2544
/// character (either \\n or \\r) is part of an escaped newline sequence.  Issue
2545
/// a diagnostic if so.  We know that the newline is inside of a block comment.
2546
static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr,
2547
62.5k
                                                  Lexer *L) {
2548
62.5k
  assert(CurPtr[0] == '\n' || CurPtr[0] == '\r');
2549
2550
  // Position of the first trigraph in the ending sequence.
2551
0
  const char *TrigraphPos = nullptr;
2552
  // Position of the first whitespace after a '\' in the ending sequence.
2553
62.5k
  const char *SpacePos = nullptr;
2554
2555
62.5k
  while (true) {
2556
    // Back up off the newline.
2557
62.5k
    --CurPtr;
2558
2559
    // If this is a two-character newline sequence, skip the other character.
2560
62.5k
    if (CurPtr[0] == '\n' || 
CurPtr[0] == '\r'6.19k
) {
2561
      // \n\n or \r\r -> not escaped newline.
2562
56.3k
      if (CurPtr[0] == CurPtr[1])
2563
56.3k
        return false;
2564
      // \n\r or \r\n -> skip the newline.
2565
0
      --CurPtr;
2566
0
    }
2567
2568
    // If we have horizontal whitespace, skip over it.  We allow whitespace
2569
    // between the slash and newline.
2570
6.24k
    
while (6.19k
isHorizontalWhitespace(*CurPtr) ||
*CurPtr == 06.19k
) {
2571
53
      SpacePos = CurPtr;
2572
53
      --CurPtr;
2573
53
    }
2574
2575
    // If we have a slash, this is an escaped newline.
2576
6.19k
    if (*CurPtr == '\\') {
2577
18
      --CurPtr;
2578
6.17k
    } else if (CurPtr[0] == '/' && 
CurPtr[-1] == '?'672
&&
CurPtr[-2] == '?'15
) {
2579
      // This is a trigraph encoding of a slash.
2580
15
      TrigraphPos = CurPtr - 2;
2581
15
      CurPtr -= 3;
2582
6.16k
    } else {
2583
6.16k
      return false;
2584
6.16k
    }
2585
2586
    // If the character preceding the escaped newline is a '*', then after line
2587
    // splicing we have a '*/' ending the comment.
2588
33
    if (*CurPtr == '*')
2589
18
      break;
2590
2591
15
    if (*CurPtr != '\n' && 
*CurPtr != '\r'0
)
2592
0
      return false;
2593
15
  }
2594
2595
18
  if (TrigraphPos) {
2596
    // If no trigraphs are enabled, warn that we ignored this trigraph and
2597
    // ignore this * character.
2598
10
    if (!L->getLangOpts().Trigraphs) {
2599
0
      if (!L->isLexingRawMode())
2600
0
        L->Diag(TrigraphPos, diag::trigraph_ignored_block_comment);
2601
0
      return false;
2602
0
    }
2603
10
    if (!L->isLexingRawMode())
2604
10
      L->Diag(TrigraphPos, diag::trigraph_ends_block_comment);
2605
10
  }
2606
2607
  // Warn about having an escaped newline between the */ characters.
2608
18
  if (!L->isLexingRawMode())
2609
15
    L->Diag(CurPtr + 1, diag::escaped_newline_block_comment_end);
2610
2611
  // If there was space between the backslash and newline, warn about it.
2612
18
  if (SpacePos && !L->isLexingRawMode())
2613
15
    L->Diag(SpacePos, diag::backslash_newline_space);
2614
2615
18
  return true;
2616
18
}
2617
2618
#ifdef __SSE2__
2619
#include <emmintrin.h>
2620
#elif __ALTIVEC__
2621
#include <altivec.h>
2622
#undef bool
2623
#endif
2624
2625
/// We have just read from input the / and * characters that started a comment.
2626
/// Read until we find the * and / characters that terminate the comment.
2627
/// Note that we don't bother decoding trigraphs or escaped newlines in block
2628
/// comments, because they cannot cause the comment to end.  The only thing
2629
/// that can happen is the comment could end with an escaped newline between
2630
/// the terminating * and /.
2631
///
2632
/// If we're in KeepCommentMode or any CommentHandler has inserted
2633
/// some tokens, this will store the first token and return true.
2634
bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr,
2635
32.8M
                             bool &TokAtPhysicalStartOfLine) {
2636
  // Scan one character past where we should, looking for a '/' character.  Once
2637
  // we find it, check to see if it was preceded by a *.  This common
2638
  // optimization helps people who like to put a lot of * characters in their
2639
  // comments.
2640
2641
  // The first character we get with newlines and trigraphs skipped to handle
2642
  // the degenerate /*/ case below correctly if the * has an escaped newline
2643
  // after it.
2644
32.8M
  unsigned CharSize;
2645
32.8M
  unsigned char C = getCharAndSize(CurPtr, CharSize);
2646
32.8M
  CurPtr += CharSize;
2647
32.8M
  if (C == 0 && 
CurPtr == BufferEnd+14
) {
2648
3
    if (!isLexingRawMode())
2649
0
      Diag(BufferPtr, diag::err_unterminated_block_comment);
2650
3
    --CurPtr;
2651
2652
    // KeepWhitespaceMode should return this broken comment as a token.  Since
2653
    // it isn't a well formed comment, just return it as an 'unknown' token.
2654
3
    if (isKeepWhitespaceMode()) {
2655
2
      FormTokenWithChars(Result, CurPtr, tok::unknown);
2656
2
      return true;
2657
2
    }
2658
2659
1
    BufferPtr = CurPtr;
2660
1
    return false;
2661
3
  }
2662
2663
  // Check to see if the first character after the '/*' is another /.  If so,
2664
  // then this slash does not end the block comment, it is part of it.
2665
32.8M
  if (C == '/')
2666
230
    C = *CurPtr++;
2667
2668
37.9M
  while (true) {
2669
    // Skip over all non-interesting characters until we find end of buffer or a
2670
    // (probably ending) '/' character.
2671
37.9M
    if (CurPtr + 24 < BufferEnd &&
2672
        // If there is a code-completion point avoid the fast scan because it
2673
        // doesn't check for '\0'.
2674
37.9M
        
!(37.6M
PP37.6M
&&
PP->getCodeCompletionFileLoc() == FileLoc37.6M
)) {
2675
      // While not aligned to a 16-byte boundary.
2676
295M
      while (C != '/' && 
((intptr_t)CurPtr & 0x0F) != 0293M
)
2677
258M
        C = *CurPtr++;
2678
2679
37.6M
      if (C == '/') 
goto FoundSlash2.57M
;
2680
2681
35.1M
#ifdef __SSE2__
2682
35.1M
      __m128i Slashes = _mm_set1_epi8('/');
2683
265M
      while (CurPtr+16 <= BufferEnd) {
2684
265M
        int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr,
2685
265M
                                    Slashes));
2686
265M
        if (cmp != 0) {
2687
          // Adjust the pointer to point directly after the first slash. It's
2688
          // not necessary to set C here, it will be overwritten at the end of
2689
          // the outer loop.
2690
35.0M
          CurPtr += llvm::countTrailingZeros<unsigned>(cmp) + 1;
2691
35.0M
          goto FoundSlash;
2692
35.0M
        }
2693
230M
        CurPtr += 16;
2694
230M
      }
2695
#elif __ALTIVEC__
2696
      __vector unsigned char Slashes = {
2697
        '/', '/', '/', '/',  '/', '/', '/', '/',
2698
        '/', '/', '/', '/',  '/', '/', '/', '/'
2699
      };
2700
      while (CurPtr + 16 <= BufferEnd &&
2701
             !vec_any_eq(*(const __vector unsigned char *)CurPtr, Slashes))
2702
        CurPtr += 16;
2703
#else
2704
      // Scan for '/' quickly.  Many block comments are very large.
2705
      while (CurPtr[0] != '/' &&
2706
             CurPtr[1] != '/' &&
2707
             CurPtr[2] != '/' &&
2708
             CurPtr[3] != '/' &&
2709
             CurPtr+4 < BufferEnd) {
2710
        CurPtr += 4;
2711
      }
2712
#endif
2713
2714
      // It has to be one of the bytes scanned, increment to it and read one.
2715
44.8k
      C = *CurPtr++;
2716
44.8k
    }
2717
2718
    // Loop to scan the remainder.
2719
4.32M
    
while (289k
C != '/' &&
C != '\0'4.03M
)
2720
4.03M
      C = *CurPtr++;
2721
2722
289k
    if (C == '/') {
2723
37.9M
  FoundSlash:
2724
37.9M
      if (CurPtr[-2] == '*')  // We found the final */.  We're done!
2725
32.8M
        break;
2726
2727
5.08M
      if ((CurPtr[-2] == '\n' || 
CurPtr[-2] == '\r'5.02M
)) {
2728
62.5k
        if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) {
2729
          // We found the final */, though it had an escaped newline between the
2730
          // * and /.  We're done!
2731
18
          break;
2732
18
        }
2733
62.5k
      }
2734
5.08M
      if (CurPtr[0] == '*' && 
CurPtr[1] != '/'229
) {
2735
        // If this is a /* inside of the comment, emit a warning.  Don't do this
2736
        // if this is a /*/, which will end the comment.  This misses cases with
2737
        // embedded escaped newlines, but oh well.
2738
10
        if (!isLexingRawMode())
2739
9
          Diag(CurPtr-1, diag::warn_nested_block_comment);
2740
10
      }
2741
5.08M
    } else 
if (14
C == 014
&&
CurPtr == BufferEnd+114
) {
2742
2
      if (!isLexingRawMode())
2743
2
        Diag(BufferPtr, diag::err_unterminated_block_comment);
2744
      // Note: the user probably forgot a */.  We could continue immediately
2745
      // after the /*, but this would involve lexing a lot of what really is the
2746
      // comment, which surely would confuse the parser.
2747
2
      --CurPtr;
2748
2749
      // KeepWhitespaceMode should return this broken comment as a token.  Since
2750
      // it isn't a well formed comment, just return it as an 'unknown' token.
2751
2
      if (isKeepWhitespaceMode()) {
2752
0
        FormTokenWithChars(Result, CurPtr, tok::unknown);
2753
0
        return true;
2754
0
      }
2755
2756
2
      BufferPtr = CurPtr;
2757
2
      return false;
2758
12
    } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
2759
9
      PP->CodeCompleteNaturalLanguage();
2760
9
      cutOffLexing();
2761
9
      return false;
2762
9
    }
2763
2764
5.08M
    C = *CurPtr++;
2765
5.08M
  }
2766
2767
  // Notify comment handlers about the comment unless we're in a #if 0 block.
2768
32.8M
  if (PP && 
!isLexingRawMode()32.8M
&&
2769
32.8M
      PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
2770
30.2M
                                            getSourceLocation(CurPtr)))) {
2771
0
    BufferPtr = CurPtr;
2772
0
    return true; // A token has to be returned.
2773
0
  }
2774
2775
  // If we are returning comments as tokens, return this comment as a token.
2776
32.8M
  if (inKeepCommentMode()) {
2777
3.35k
    FormTokenWithChars(Result, CurPtr, tok::comment);
2778
3.35k
    return true;
2779
3.35k
  }
2780
2781
  // It is common for the tokens immediately after a /**/ comment to be
2782
  // whitespace.  Instead of going through the big switch, handle it
2783
  // efficiently now.  This is safe even in KeepWhitespaceMode because we would
2784
  // have already returned above with the comment as a token.
2785
32.8M
  if (isHorizontalWhitespace(*CurPtr)) {
2786
79.6k
    SkipWhitespace(Result, CurPtr+1, TokAtPhysicalStartOfLine);
2787
79.6k
    return false;
2788
79.6k
  }
2789
2790
  // Otherwise, just return so that the next character will be lexed as a token.
2791
32.7M
  BufferPtr = CurPtr;
2792
32.7M
  Result.setFlag(Token::LeadingSpace);
2793
32.7M
  return false;
2794
32.8M
}
2795
2796
//===----------------------------------------------------------------------===//
2797
// Primary Lexing Entry Points
2798
//===----------------------------------------------------------------------===//
2799
2800
/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
2801
/// uninterpreted string.  This switches the lexer out of directive mode.
2802
37.7k
void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) {
2803
37.7k
  assert(ParsingPreprocessorDirective && ParsingFilename == false &&
2804
37.7k
         "Must be in a preprocessing directive!");
2805
0
  Token Tmp;
2806
37.7k
  Tmp.startToken();
2807
2808
  // CurPtr - Cache BufferPtr in an automatic variable.
2809
37.7k
  const char *CurPtr = BufferPtr;
2810
731k
  while (true) {
2811
731k
    char Char = getAndAdvanceChar(CurPtr, Tmp);
2812
731k
    switch (Char) {
2813
694k
    default:
2814
694k
      if (Result)
2815
694k
        Result->push_back(Char);
2816
694k
      break;
2817
18
    case 0:  // Null.
2818
      // Found end of file?
2819
18
      if (CurPtr-1 != BufferEnd) {
2820
18
        if (isCodeCompletionPoint(CurPtr-1)) {
2821
18
          PP->CodeCompleteNaturalLanguage();
2822
18
          cutOffLexing();
2823
18
          return;
2824
18
        }
2825
2826
        // Nope, normal character, continue.
2827
0
        if (Result)
2828
0
          Result->push_back(Char);
2829
0
        break;
2830
18
      }
2831
      // FALL THROUGH.
2832
18
      
LLVM_FALLTHROUGH0
;0
2833
2
    case '\r':
2834
37.7k
    case '\n':
2835
      // Okay, we found the end of the line. First, back up past the \0, \r, \n.
2836
37.7k
      assert(CurPtr[-1] == Char && "Trigraphs for newline?");
2837
0
      BufferPtr = CurPtr-1;
2838
2839
      // Next, lex the character, which should handle the EOD transition.
2840
37.7k
      Lex(Tmp);
2841
37.7k
      if (Tmp.is(tok::code_completion)) {
2842
0
        if (PP)
2843
0
          PP->CodeCompleteNaturalLanguage();
2844
0
        Lex(Tmp);
2845
0
      }
2846
37.7k
      assert(Tmp.is(tok::eod) && "Unexpected token!");
2847
2848
      // Finally, we're done;
2849
0
      return;
2850
731k
    }
2851
731k
  }
2852
37.7k
}
2853
2854
/// LexEndOfFile - CurPtr points to the end of this file.  Handle this
2855
/// condition, reporting diagnostics and handling other edge cases as required.
2856
/// This returns true if Result contains a token, false if PP.Lex should be
2857
/// called again.
2858
1.25M
bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
2859
  // If we hit the end of the file while parsing a preprocessor directive,
2860
  // end the preprocessor directive first.  The next token returned will
2861
  // then be the end of file.
2862
1.25M
  if (ParsingPreprocessorDirective) {
2863
    // Done parsing the "line".
2864
1.12k
    ParsingPreprocessorDirective = false;
2865
    // Update the location of token as well as BufferPtr.
2866
1.12k
    FormTokenWithChars(Result, CurPtr, tok::eod);
2867
2868
    // Restore comment saving mode, in case it was disabled for directive.
2869
1.12k
    if (PP)
2870
1.12k
      resetExtendedTokenMode();
2871
1.12k
    return true;  // Have a token.
2872
1.12k
  }
2873
2874
  // If we are in raw mode, return this event as an EOF token.  Let the caller
2875
  // that put us in raw mode handle the event.
2876
1.25M
  if (isLexingRawMode()) {
2877
121k
    Result.startToken();
2878
121k
    BufferPtr = BufferEnd;
2879
121k
    FormTokenWithChars(Result, BufferEnd, tok::eof);
2880
121k
    return true;
2881
121k
  }
2882
2883
1.12M
  if (PP->isRecordingPreamble() && 
PP->isInPrimaryFile()257
) {
2884
91
    PP->setRecordedPreambleConditionalStack(ConditionalStack);
2885
    // If the preamble cuts off the end of a header guard, consider it guarded.
2886
    // The guard is valid for the preamble content itself, and for tools the
2887
    // most useful answer is "yes, this file has a header guard".
2888
91
    if (!ConditionalStack.empty())
2889
7
      MIOpt.ExitTopLevelConditional();
2890
91
    ConditionalStack.clear();
2891
91
  }
2892
2893
  // Issue diagnostics for unterminated #if and missing newline.
2894
2895
  // If we are in a #if directive, emit an error.
2896
1.12M
  while (!ConditionalStack.empty()) {
2897
8
    if (PP->getCodeCompletionFileLoc() != FileLoc)
2898
8
      PP->Diag(ConditionalStack.back().IfLoc,
2899
8
               diag::err_pp_unterminated_conditional);
2900
8
    ConditionalStack.pop_back();
2901
8
  }
2902
2903
1.12M
  SourceLocation EndLoc = getSourceLocation(BufferEnd);
2904
  // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue
2905
  // a pedwarn.
2906
1.12M
  if (CurPtr != BufferStart && 
(1.12M
CurPtr[-1] != '\n'1.12M
&&
CurPtr[-1] != '\r'26.3k
)) {
2907
26.3k
    DiagnosticsEngine &Diags = PP->getDiagnostics();
2908
26.3k
    unsigned DiagID;
2909
2910
26.3k
    if (LangOpts.CPlusPlus11) {
2911
      // C++11 [lex.phases] 2.2 p2
2912
      // Prefer the C++98 pedantic compatibility warning over the generic,
2913
      // non-extension, user-requested "missing newline at EOF" warning.
2914
19.4k
      if (!Diags.isIgnored(diag::warn_cxx98_compat_no_newline_eof, EndLoc)) {
2915
2
        DiagID = diag::warn_cxx98_compat_no_newline_eof;
2916
19.4k
      } else {
2917
19.4k
        DiagID = diag::warn_no_newline_eof;
2918
19.4k
      }
2919
19.4k
    } else {
2920
6.96k
      DiagID = diag::ext_no_newline_eof;
2921
6.96k
    }
2922
2923
26.3k
    Diag(BufferEnd, DiagID)
2924
26.3k
      << FixItHint::CreateInsertion(EndLoc, "\n");
2925
26.3k
  }
2926
2927
1.12M
  BufferPtr = CurPtr;
2928
2929
  // Finally, let the preprocessor handle this.
2930
1.12M
  return PP->HandleEndOfFile(Result, EndLoc, isPragmaLexer());
2931
1.25M
}
2932
2933
/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
2934
/// the specified lexer will return a tok::l_paren token, 0 if it is something
2935
/// else and 2 if there are no more tokens in the buffer controlled by the
2936
/// lexer.
2937
3.00M
unsigned Lexer::isNextPPTokenLParen() {
2938
3.00M
  assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?");
2939
2940
  // Switch to 'skipping' mode.  This will ensure that we can lex a token
2941
  // without emitting diagnostics, disables macro expansion, and will cause EOF
2942
  // to return an EOF token instead of popping the include stack.
2943
0
  LexingRawMode = true;
2944
2945
  // Save state that can be changed while lexing so that we can restore it.
2946
3.00M
  const char *TmpBufferPtr = BufferPtr;
2947
3.00M
  bool inPPDirectiveMode = ParsingPreprocessorDirective;
2948
3.00M
  bool atStartOfLine = IsAtStartOfLine;
2949
3.00M
  bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
2950
3.00M
  bool leadingSpace = HasLeadingSpace;
2951
2952
3.00M
  Token Tok;
2953
3.00M
  Lex(Tok);
2954
2955
  // Restore state that may have changed.
2956
3.00M
  BufferPtr = TmpBufferPtr;
2957
3.00M
  ParsingPreprocessorDirective = inPPDirectiveMode;
2958
3.00M
  HasLeadingSpace = leadingSpace;
2959
3.00M
  IsAtStartOfLine = atStartOfLine;
2960
3.00M
  IsAtPhysicalStartOfLine = atPhysicalStartOfLine;
2961
2962
  // Restore the lexer back to non-skipping mode.
2963
3.00M
  LexingRawMode = false;
2964
2965
3.00M
  if (Tok.is(tok::eof))
2966
3
    return 2;
2967
3.00M
  return Tok.is(tok::l_paren);
2968
3.00M
}
2969
2970
/// Find the end of a version control conflict marker.
2971
static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd,
2972
10
                                   ConflictMarkerKind CMK) {
2973
10
  const char *Terminator = CMK == CMK_Perforce ? 
"<<<<\n"5
:
">>>>>>>"5
;
2974
10
  size_t TermLen = CMK == CMK_Perforce ? 
55
:
75
;
2975
10
  auto RestOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(TermLen);
2976
10
  size_t Pos = RestOfBuffer.find(Terminator);
2977
11
  while (Pos != StringRef::npos) {
2978
    // Must occur at start of line.
2979
8
    if (Pos == 0 ||
2980
8
        
(7
RestOfBuffer[Pos - 1] != '\r'7
&&
RestOfBuffer[Pos - 1] != '\n'7
)) {
2981
1
      RestOfBuffer = RestOfBuffer.substr(Pos+TermLen);
2982
1
      Pos = RestOfBuffer.find(Terminator);
2983
1
      continue;
2984
1
    }
2985
7
    return RestOfBuffer.data()+Pos;
2986
8
  }
2987
3
  return nullptr;
2988
10
}
2989
2990
/// IsStartOfConflictMarker - If the specified pointer is the start of a version
2991
/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
2992
/// and recover nicely.  This returns true if it is a conflict marker and false
2993
/// if not.
2994
19.9k
bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
2995
  // Only a conflict marker if it starts at the beginning of a line.
2996
19.9k
  if (CurPtr != BufferStart &&
2997
19.9k
      CurPtr[-1] != '\n' && 
CurPtr[-1] != '\r'19.9k
)
2998
19.9k
    return false;
2999
3000
  // Check to see if we have <<<<<<< or >>>>.
3001
28
  if (!StringRef(CurPtr, BufferEnd - CurPtr).startswith("<<<<<<<") &&
3002
28
      
!StringRef(CurPtr, BufferEnd - CurPtr).startswith(">>>> ")19
)
3003
14
    return false;
3004
3005
  // If we have a situation where we don't care about conflict markers, ignore
3006
  // it.
3007
14
  if (CurrentConflictMarkerState || isLexingRawMode())
3008
9
    return false;
3009
3010
5
  ConflictMarkerKind Kind = *CurPtr == '<' ? 
CMK_Normal3
:
CMK_Perforce2
;
3011
3012
  // Check to see if there is an ending marker somewhere in the buffer at the
3013
  // start of a line to terminate this conflict marker.
3014
5
  if (FindConflictEnd(CurPtr, BufferEnd, Kind)) {
3015
    // We found a match.  We are really in a conflict marker.
3016
    // Diagnose this, and ignore to the end of line.
3017
4
    Diag(CurPtr, diag::err_conflict_marker);
3018
4
    CurrentConflictMarkerState = Kind;
3019
3020
    // Skip ahead to the end of line.  We know this exists because the
3021
    // end-of-conflict marker starts with \r or \n.
3022
76
    while (*CurPtr != '\r' && *CurPtr != '\n') {
3023
72
      assert(CurPtr != BufferEnd && "Didn't find end of line");
3024
0
      ++CurPtr;
3025
72
    }
3026
4
    BufferPtr = CurPtr;
3027
4
    return true;
3028
4
  }
3029
3030
  // No end of conflict marker found.
3031
1
  return false;
3032
5
}
3033
3034
/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if
3035
/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it
3036
/// is the end of a conflict marker.  Handle it by ignoring up until the end of
3037
/// the line.  This returns true if it is a conflict marker and false if not.
3038
20.0k
bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
3039
  // Only a conflict marker if it starts at the beginning of a line.
3040
20.0k
  if (CurPtr != BufferStart &&
3041
20.0k
      
CurPtr[-1] != '\n'20.0k
&&
CurPtr[-1] != '\r'19.9k
)
3042
19.9k
    return false;
3043
3044
  // If we have a situation where we don't care about conflict markers, ignore
3045
  // it.
3046
52
  if (!CurrentConflictMarkerState || 
isLexingRawMode()5
)
3047
47
    return false;
3048
3049
  // Check to see if we have the marker (4 characters in a row).
3050
20
  
for (unsigned i = 1; 5
i != 4;
++i15
)
3051
15
    if (CurPtr[i] != CurPtr[0])
3052
0
      return false;
3053
3054
  // If we do have it, search for the end of the conflict marker.  This could
3055
  // fail if it got skipped with a '#if 0' or something.  Note that CurPtr might
3056
  // be the end of conflict marker.
3057
5
  if (const char *End = FindConflictEnd(CurPtr, BufferEnd,
3058
5
                                        CurrentConflictMarkerState)) {
3059
3
    CurPtr = End;
3060
3061
    // Skip ahead to the end of line.
3062
37
    while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
3063
34
      ++CurPtr;
3064
3065
3
    BufferPtr = CurPtr;
3066
3067
    // No longer in the conflict marker.
3068
3
    CurrentConflictMarkerState = CMK_None;
3069
3
    return true;
3070
3
  }
3071
3072
2
  return false;
3073
5
}
3074
3075
static const char *findPlaceholderEnd(const char *CurPtr,
3076
43
                                      const char *BufferEnd) {
3077
43
  if (CurPtr == BufferEnd)
3078
0
    return nullptr;
3079
43
  BufferEnd -= 1; // Scan until the second last character.
3080
404
  for (; CurPtr != BufferEnd; 
++CurPtr361
) {
3081
404
    if (CurPtr[0] == '#' && 
CurPtr[1] == '>'43
)
3082
43
      return CurPtr + 2;
3083
404
  }
3084
0
  return nullptr;
3085
43
}
3086
3087
45
bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) {
3088
45
  assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!");
3089
45
  if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || 
LexingRawMode44
)
3090
2
    return false;
3091
43
  const char *End = findPlaceholderEnd(CurPtr + 1, BufferEnd);
3092
43
  if (!End)
3093
0
    return false;
3094
43
  const char *Start = CurPtr - 1;
3095
43
  if (!LangOpts.AllowEditorPlaceholders)
3096
22
    Diag(Start, diag::err_placeholder_in_source);
3097
43
  Result.startToken();
3098
43
  FormTokenWithChars(Result, End, tok::raw_identifier);
3099
43
  Result.setRawIdentifierData(Start);
3100
43
  PP->LookUpIdentifierInfo(Result);
3101
43
  Result.setFlag(Token::IsEditorPlaceholder);
3102
43
  BufferPtr = End;
3103
43
  return true;
3104
43
}
3105
3106
485M
bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
3107
485M
  if (
PP485M
&& PP->isCodeCompletionEnabled()) {
3108
1.11M
    SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart);
3109
1.11M
    return Loc == PP->getCodeCompletionLoc();
3110
1.11M
  }
3111
3112
484M
  return false;
3113
485M
}
3114
3115
uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc,
3116
1.33k
                           Token *Result) {
3117
1.33k
  unsigned CharSize;
3118
1.33k
  char Kind = getCharAndSize(StartPtr, CharSize);
3119
1.33k
  bool Delimited = false;
3120
1.33k
  bool FoundEndDelimiter = false;
3121
1.33k
  unsigned Count = 0;
3122
1.33k
  bool Diagnose = Result && 
!isLexingRawMode()1.17k
;
3123
3124
1.33k
  unsigned NumHexDigits;
3125
1.33k
  if (Kind == 'u')
3126
243
    NumHexDigits = 4;
3127
1.09k
  else if (Kind == 'U')
3128
48
    NumHexDigits = 8;
3129
1.04k
  else
3130
1.04k
    return 0;
3131
3132
291
  if (!LangOpts.CPlusPlus && 
!LangOpts.C99150
) {
3133
5
    if (Diagnose)
3134
3
      Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89);
3135
5
    return 0;
3136
5
  }
3137
3138
286
  const char *CurPtr = StartPtr + CharSize;
3139
286
  const char *KindLoc = &CurPtr[-1];
3140
3141
286
  uint32_t CodePoint = 0;
3142
1.54k
  while (Count != NumHexDigits || 
Delimited251
) {
3143
1.29k
    char C = getCharAndSize(CurPtr, CharSize);
3144
1.29k
    if (!Delimited && 
C == '{'1.22k
) {
3145
21
      Delimited = true;
3146
21
      CurPtr += CharSize;
3147
21
      continue;
3148
21
    }
3149
3150
1.27k
    if (Delimited && 
C == '}'74
) {
3151
12
      CurPtr += CharSize;
3152
12
      FoundEndDelimiter = true;
3153
12
      break;
3154
12
    }
3155
3156
1.26k
    unsigned Value = llvm::hexDigitValue(C);
3157
1.26k
    if (Value == -1U) {
3158
27
      if (!Delimited)
3159
21
        break;
3160
6
      if (Diagnose)
3161
6
        Diag(BufferPtr, diag::warn_delimited_ucn_incomplete)
3162
6
            << StringRef(&C, 1);
3163
6
      return 0;
3164
27
    }
3165
3166
1.23k
    if (CodePoint & 0xF000'0000) {
3167
3
      if (Diagnose)
3168
3
        Diag(KindLoc, diag::err_escape_too_large) << 0;
3169
3
      return 0;
3170
3
    }
3171
3172
1.23k
    CodePoint <<= 4;
3173
1.23k
    CodePoint |= Value;
3174
1.23k
    CurPtr += CharSize;
3175
1.23k
    Count++;
3176
1.23k
  }
3177
3178
277
  if (Count == 0) {
3179
9
    if (Diagnose)
3180
9
      Diag(StartPtr, FoundEndDelimiter ? 
diag::warn_delimited_ucn_empty3
3181
9
                                       : 
diag::warn_ucn_escape_no_digits6
)
3182
9
          << StringRef(KindLoc, 1);
3183
9
    return 0;
3184
9
  }
3185
3186
268
  if (!Delimited && 
Count != NumHexDigits259
) {
3187
15
    if (Diagnose) {
3188
12
      Diag(BufferPtr, diag::warn_ucn_escape_incomplete);
3189
      // If the user wrote \U1234, suggest a fixit to \u.
3190
12
      if (Count == 4 && 
NumHexDigits == 83
) {
3191
3
        CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1);
3192
3
        Diag(KindLoc, diag::note_ucn_four_not_eight)
3193
3
            << FixItHint::CreateReplacement(URange, "u");
3194
3
      }
3195
12
    }
3196
15
    return 0;
3197
15
  }
3198
3199
253
  if (Delimited && 
PP9
) {
3200
9
    Diag(BufferPtr, diag::ext_delimited_escape_sequence);
3201
9
  }
3202
3203
253
  if (Result) {
3204
125
    Result->setFlag(Token::HasUCN);
3205
125
    if (CurPtr - StartPtr == (ptrdiff_t)(Count + 2 + (Delimited ? 
25
:
0120
)))
3206
0
      StartPtr = CurPtr;
3207
125
    else
3208
852
      
while (125
StartPtr != CurPtr)
3209
727
        (void)getAndAdvanceChar(StartPtr, *Result);
3210
128
  } else {
3211
128
    StartPtr = CurPtr;
3212
128
  }
3213
3214
  // Don't apply C family restrictions to UCNs in assembly mode
3215
253
  if (LangOpts.AsmPreprocessor)
3216
2
    return CodePoint;
3217
3218
  // C99 6.4.3p2: A universal character name shall not specify a character whose
3219
  //   short identifier is less than 00A0 other than 0024 ($), 0040 (@), or
3220
  //   0060 (`), nor one in the range D800 through DFFF inclusive.)
3221
  // C++11 [lex.charset]p2: If the hexadecimal value for a
3222
  //   universal-character-name corresponds to a surrogate code point (in the
3223
  //   range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally,
3224
  //   if the hexadecimal value for a universal-character-name outside the
3225
  //   c-char-sequence, s-char-sequence, or r-char-sequence of a character or
3226
  //   string literal corresponds to a control character (in either of the
3227
  //   ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the
3228
  //   basic source character set, the program is ill-formed.
3229
251
  if (CodePoint < 0xA0) {
3230
27
    if (CodePoint == 0x24 || 
CodePoint == 0x4024
||
CodePoint == 0x6024
)
3231
3
      return CodePoint;
3232
3233
    // We don't use isLexingRawMode() here because we need to warn about bad
3234
    // UCNs even when skipping preprocessing tokens in a #if block.
3235
24
    if (Result && 
PP23
) {
3236
23
      if (CodePoint < 0x20 || 
CodePoint >= 0x7F15
)
3237
16
        Diag(BufferPtr, diag::err_ucn_control_character);
3238
7
      else {
3239
7
        char C = static_cast<char>(CodePoint);
3240
7
        Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1);
3241
7
      }
3242
23
    }
3243
3244
24
    return 0;
3245
224
  } else if (CodePoint >= 0xD800 && 
CodePoint <= 0xDFFF37
) {
3246
    // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't.
3247
    // We don't use isLexingRawMode() here because we need to diagnose bad
3248
    // UCNs even when skipping preprocessing tokens in a #if block.
3249
4
    if (Result && PP) {
3250
4
      if (LangOpts.CPlusPlus && 
!LangOpts.CPlusPlus112
)
3251
1
        Diag(BufferPtr, diag::warn_ucn_escape_surrogate);
3252
3
      else
3253
3
        Diag(BufferPtr, diag::err_ucn_escape_invalid);
3254
4
    }
3255
4
    return 0;
3256
4
  }
3257
3258
220
  return CodePoint;
3259
251
}
3260
3261
bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C,
3262
225
                                   const char *CurPtr) {
3263
225
  if (!isLexingRawMode() && 
!PP->isPreprocessedOutput()160
&&
3264
225
      
isUnicodeWhitespace(C)132
) {
3265
6
    Diag(BufferPtr, diag::ext_unicode_whitespace)
3266
6
      << makeCharRange(*this, BufferPtr, CurPtr);
3267
3268
6
    Result.setFlag(Token::LeadingSpace);
3269
6
    return true;
3270
6
  }
3271
219
  return false;
3272
225
}
3273
3274
32.7M
void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
3275
32.7M
  IsAtStartOfLine = Result.isAtStartOfLine();
3276
32.7M
  HasLeadingSpace = Result.hasLeadingSpace();
3277
32.7M
  HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro();
3278
  // Note that this doesn't affect IsAtPhysicalStartOfLine.
3279
32.7M
}
3280
3281
1.67G
bool Lexer::Lex(Token &Result) {
3282
  // Start a new token.
3283
1.67G
  Result.startToken();
3284
3285
  // Set up misc whitespace flags for LexTokenInternal.
3286
1.67G
  if (IsAtStartOfLine) {
3287
121M
    Result.setFlag(Token::StartOfLine);
3288
121M
    IsAtStartOfLine = false;
3289
121M
  }
3290
3291
1.67G
  if (HasLeadingSpace) {
3292
576k
    Result.setFlag(Token::LeadingSpace);
3293
576k
    HasLeadingSpace = false;
3294
576k
  }
3295
3296
1.67G
  if (HasLeadingEmptyMacro) {
3297
791k
    Result.setFlag(Token::LeadingEmptyMacro);
3298
791k
    HasLeadingEmptyMacro = false;
3299
791k
  }
3300
3301
1.67G
  bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
3302
1.67G
  IsAtPhysicalStartOfLine = false;
3303
1.67G
  bool isRawLex = isLexingRawMode();
3304
1.67G
  (void) isRawLex;
3305
1.67G
  bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine);
3306
  // (After the LexTokenInternal call, the lexer might be destroyed.)
3307
1.67G
  assert((returnedToken || !isRawLex) && "Raw lex must succeed");
3308
0
  return returnedToken;
3309
1.67G
}
3310
3311
/// LexTokenInternal - This implements a simple C family lexer.  It is an
3312
/// extremely performance critical piece of code.  This assumes that the buffer
3313
/// has a null character at the end of the file.  This returns a preprocessing
3314
/// token, not a normal token, as such, it is an internal interface.  It assumes
3315
/// that the Flags of result have been cleared before calling this.
3316
1.67G
bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
3317
1.88G
LexNextToken:
3318
  // New token, can't need cleaning yet.
3319
1.88G
  Result.clearFlag(Token::NeedsCleaning);
3320
1.88G
  Result.setIdentifierInfo(nullptr);
3321
3322
  // CurPtr - Cache BufferPtr in an automatic variable.
3323
1.88G
  const char *CurPtr = BufferPtr;
3324
3325
  // Small amounts of horizontal whitespace is very common between tokens.
3326
1.88G
  if (isHorizontalWhitespace(*CurPtr)) {
3327
704M
    do {
3328
704M
      ++CurPtr;
3329
704M
    } while (isHorizontalWhitespace(*CurPtr));
3330
3331
    // If we are keeping whitespace and other tokens, just return what we just
3332
    // skipped.  The next lexer invocation will return the token after the
3333
    // whitespace.
3334
474M
    if (isKeepWhitespaceMode()) {
3335
327k
      FormTokenWithChars(Result, CurPtr, tok::unknown);
3336
      // FIXME: The next token will not have LeadingSpace set.
3337
327k
      return true;
3338
327k
    }
3339
3340
473M
    BufferPtr = CurPtr;
3341
473M
    Result.setFlag(Token::LeadingSpace);
3342
473M
  }
3343
3344
1.88G
  unsigned SizeTmp, SizeTmp2;   // Temporaries for use in cases below.
3345
3346
  // Read a character, advancing over it.
3347
1.88G
  char Char = getAndAdvanceChar(CurPtr, Result);
3348
1.88G
  tok::TokenKind Kind;
3349
3350
1.88G
  if (!isVerticalWhitespace(Char))
3351
1.64G
    NewLinePtr = nullptr;
3352
3353
1.88G
  switch (Char) {
3354
1.25M
  case 0:  // Null.
3355
    // Found end of file?
3356
1.25M
    if (CurPtr-1 == BufferEnd)
3357
1.25M
      return LexEndOfFile(Result, CurPtr-1);
3358
3359
    // Check if we are performing code completion.
3360
1.14k
    if (isCodeCompletionPoint(CurPtr-1)) {
3361
      // Return the code-completion token.
3362
1.14k
      Result.startToken();
3363
1.14k
      FormTokenWithChars(Result, CurPtr, tok::code_completion);
3364
1.14k
      return true;
3365
1.14k
    }
3366
3367
5
    if (!isLexingRawMode())
3368
2
      Diag(CurPtr-1, diag::null_in_file);
3369
5
    Result.setFlag(Token::LeadingSpace);
3370
5
    if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
3371
0
      return true; // KeepWhitespaceMode
3372
3373
    // We know the lexer hasn't changed, so just try again with this lexer.
3374
    // (We manually eliminate the tail call to avoid recursion.)
3375
5
    goto LexNextToken;
3376
3377
5
  case 26:  // DOS & CP/M EOF: "^Z".
3378
    // If we're in Microsoft extensions mode, treat this as end of file.
3379
1
    if (LangOpts.MicrosoftExt) {
3380
1
      if (!isLexingRawMode())
3381
1
        Diag(CurPtr-1, diag::ext_ctrl_z_eof_microsoft);
3382
1
      return LexEndOfFile(Result, CurPtr-1);
3383
1
    }
3384
3385
    // If Microsoft extensions are disabled, this is just random garbage.
3386
0
    Kind = tok::unknown;
3387
0
    break;
3388
3389
1.51k
  case '\r':
3390
1.51k
    if (CurPtr[0] == '\n')
3391
1.51k
      (void)getAndAdvanceChar(CurPtr, Result);
3392
1.51k
    LLVM_FALLTHROUGH;
3393
241M
  case '\n':
3394
    // If we are inside a preprocessor directive and we see the end of line,
3395
    // we know we are done with the directive, so return an EOD token.
3396
241M
    if (ParsingPreprocessorDirective) {
3397
      // Done parsing the "line".
3398
69.5M
      ParsingPreprocessorDirective = false;
3399
3400
      // Restore comment saving mode, in case it was disabled for directive.
3401
69.5M
      if (PP)
3402
69.5M
        resetExtendedTokenMode();
3403
3404
      // Since we consumed a newline, we are back at the start of a line.
3405
69.5M
      IsAtStartOfLine = true;
3406
69.5M
      IsAtPhysicalStartOfLine = true;
3407
69.5M
      NewLinePtr = CurPtr - 1;
3408
3409
69.5M
      Kind = tok::eod;
3410
69.5M
      break;
3411
69.5M
    }
3412
3413
    // No leading whitespace seen so far.
3414
171M
    Result.clearFlag(Token::LeadingSpace);
3415
3416
171M
    if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
3417
77.0k
      return true; // KeepWhitespaceMode
3418
3419
    // We only saw whitespace, so just try again with this lexer.
3420
    // (We manually eliminate the tail call to avoid recursion.)
3421
171M
    goto LexNextToken;
3422
171M
  case ' ':
3423
5.68M
  case '\t':
3424
5.68M
  case '\f':
3425
5.68M
  case '\v':
3426
7.33M
  SkipHorizontalWhitespace:
3427
7.33M
    Result.setFlag(Token::LeadingSpace);
3428
7.33M
    if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
3429
746
      return true; // KeepWhitespaceMode
3430
3431
52.8M
  SkipIgnoredUnits:
3432
52.8M
    CurPtr = BufferPtr;
3433
3434
    // If the next token is obviously a // or /* */ comment, skip it efficiently
3435
    // too (without going through the big switch stmt).
3436
52.8M
    if (CurPtr[0] == '/' && 
CurPtr[1] == '/'41.2M
&&
!inKeepCommentMode()41.2M
&&
3437
52.8M
        
LangOpts.LineComment41.2M
&&
3438
52.8M
        
(41.2M
LangOpts.CPlusPlus41.2M
||
!LangOpts.TraditionalCPP7.47M
)) {
3439
41.2M
      if (SkipLineComment(Result, CurPtr+2, TokAtPhysicalStartOfLine))
3440
0
        return true; // There is a token to return.
3441
41.2M
      goto SkipIgnoredUnits;
3442
41.2M
    } else 
if (11.5M
CurPtr[0] == '/'11.5M
&&
CurPtr[1] == '*'7.50k
&&
!inKeepCommentMode()4.70k
) {
3443
4.70k
      if (SkipBlockComment(Result, CurPtr+2, TokAtPhysicalStartOfLine))
3444
0
        return true; // There is a token to return.
3445
4.70k
      goto SkipIgnoredUnits;
3446
11.5M
    } else if (isHorizontalWhitespace(*CurPtr)) {
3447
1.65M
      goto SkipHorizontalWhitespace;
3448
1.65M
    }
3449
    // We only saw whitespace, so just try again with this lexer.
3450
    // (We manually eliminate the tail call to avoid recursion.)
3451
9.92M
    goto LexNextToken;
3452
3453
  // C99 6.4.4.1: Integer Constants.
3454
  // C99 6.4.4.2: Floating Constants.
3455
49.1M
  
case '0': 7.86M
case '1': 30.3M
case '2': 40.0M
case '3': 45.2M
case '4':
3456
60.4M
  
case '5': 51.0M
case '6': 54.0M
case '7': 55.4M
case '8': 57.7M
case '9':
3457
    // Notify MIOpt that we read a non-whitespace/non-comment token.
3458
60.4M
    MIOpt.ReadToken();
3459
60.4M
    return LexNumericConstant(Result, CurPtr);
3460
3461
18.2M
  case 'u':   // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal
3462
    // Notify MIOpt that we read a non-whitespace/non-comment token.
3463
18.2M
    MIOpt.ReadToken();
3464
3465
18.2M
    if (LangOpts.CPlusPlus11 || 
LangOpts.C1111.3M
) {
3466
17.5M
      Char = getCharAndSize(CurPtr, SizeTmp);
3467
3468
      // UTF-16 string literal
3469
17.5M
      if (Char == '"')
3470
163
        return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
3471
163
                                tok::utf16_string_literal);
3472
3473
      // UTF-16 character constant
3474
17.5M
      if (Char == '\'')
3475
130
        return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
3476
130
                               tok::utf16_char_constant);
3477
3478
      // UTF-16 raw string literal
3479
17.5M
      if (Char == 'R' && 
LangOpts.CPlusPlus1166
&&
3480
17.5M
          
getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"'58
)
3481
53
        return LexRawStringLiteral(Result,
3482
53
                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3483
53
                                           SizeTmp2, Result),
3484
53
                               tok::utf16_string_literal);
3485
3486
17.5M
      if (Char == '8') {
3487
5.68k
        char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2);
3488
3489
        // UTF-8 string literal
3490
5.68k
        if (Char2 == '"')
3491
336
          return LexStringLiteral(Result,
3492
336
                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3493
336
                                           SizeTmp2, Result),
3494
336
                               tok::utf8_string_literal);
3495
5.35k
        if (Char2 == '\'' && 
LangOpts.CPlusPlus17178
)
3496
166
          return LexCharConstant(
3497
166
              Result, ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3498
166
                                  SizeTmp2, Result),
3499
166
              tok::utf8_char_constant);
3500
3501
5.18k
        if (Char2 == 'R' && 
LangOpts.CPlusPlus1138
) {
3502
30
          unsigned SizeTmp3;
3503
30
          char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
3504
          // UTF-8 raw string literal
3505
30
          if (Char3 == '"') {
3506
25
            return LexRawStringLiteral(Result,
3507
25
                   ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3508
25
                                           SizeTmp2, Result),
3509
25
                               SizeTmp3, Result),
3510
25
                   tok::utf8_string_literal);
3511
25
          }
3512
30
        }
3513
5.18k
      }
3514
17.5M
    }
3515
3516
    // treat u like the start of an identifier.
3517
18.2M
    return LexIdentifierContinue(Result, CurPtr);
3518
3519
2.31M
  case 'U':   // Identifier (Uber) or C11/C++11 UTF-32 string literal
3520
    // Notify MIOpt that we read a non-whitespace/non-comment token.
3521
2.31M
    MIOpt.ReadToken();
3522
3523
2.31M
    if (LangOpts.CPlusPlus11 || 
LangOpts.C111.49M
) {
3524
2.24M
      Char = getCharAndSize(CurPtr, SizeTmp);
3525
3526
      // UTF-32 string literal
3527
2.24M
      if (Char == '"')
3528
160
        return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
3529
160
                                tok::utf32_string_literal);
3530
3531
      // UTF-32 character constant
3532
2.24M
      if (Char == '\'')
3533
148
        return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
3534
148
                               tok::utf32_char_constant);
3535
3536
      // UTF-32 raw string literal
3537
2.24M
      if (Char == 'R' && 
LangOpts.CPlusPlus1131.3k
&&
3538
2.24M
          
getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"'3.76k
)
3539
27
        return LexRawStringLiteral(Result,
3540
27
                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3541
27
                                           SizeTmp2, Result),
3542
27
                               tok::utf32_string_literal);
3543
2.24M
    }
3544
3545
    // treat U like the start of an identifier.
3546
2.31M
    return LexIdentifierContinue(Result, CurPtr);
3547
3548
700k
  case 'R': // Identifier or C++0x raw string literal
3549
    // Notify MIOpt that we read a non-whitespace/non-comment token.
3550
700k
    MIOpt.ReadToken();
3551
3552
700k
    if (LangOpts.CPlusPlus11) {
3553
187k
      Char = getCharAndSize(CurPtr, SizeTmp);
3554
3555
187k
      if (Char == '"')
3556
445
        return LexRawStringLiteral(Result,
3557
445
                                   ConsumeChar(CurPtr, SizeTmp, Result),
3558
445
                                   tok::string_literal);
3559
187k
    }
3560
3561
    // treat R like the start of an identifier.
3562
700k
    return LexIdentifierContinue(Result, CurPtr);
3563
3564
652k
  case 'L':   // Identifier (Loony) or wide literal (L'x' or L"xyz").
3565
    // Notify MIOpt that we read a non-whitespace/non-comment token.
3566
652k
    MIOpt.ReadToken();
3567
652k
    Char = getCharAndSize(CurPtr, SizeTmp);
3568
3569
    // Wide string literal.
3570
652k
    if (Char == '"')
3571
1.61k
      return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
3572
1.61k
                              tok::wide_string_literal);
3573
3574
    // Wide raw string literal.
3575
650k
    if (LangOpts.CPlusPlus11 && 
Char == 'R'343k
&&
3576
650k
        
getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"'91
)
3577
19
      return LexRawStringLiteral(Result,
3578
19
                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3579
19
                                           SizeTmp2, Result),
3580
19
                               tok::wide_string_literal);
3581
3582
    // Wide character constant.
3583
650k
    if (Char == '\'')
3584
1.49k
      return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
3585
1.49k
                             tok::wide_char_constant);
3586
    // FALL THROUGH, treating L like the start of an identifier.
3587
650k
    
LLVM_FALLTHROUGH648k
;648k
3588
3589
  // C99 6.4.2: Identifiers.
3590
17.4M
  
case 'A': 4.29M
case 'B': 5.41M
case 'C': 13.6M
case 'D': 15.2M
case 'E': 15.8M
case 'F': 16.9M
case 'G':
3591
26.5M
  
case 'H': 17.9M
case 'I': 20.3M
case 'J': 20.3M
case 'K': /*'L'*/20.8M
case 'M': 22.4M
case 'N':
3592
32.3M
  
case 'O': 27.8M
case 'P': 28.7M
case 'Q': /*'R'*/28.8M
case 'S': 31.0M
case 'T': /*'U'*/
3593
33.5M
  
case 'V': 32.8M
case 'W': 33.1M
case 'X': 33.4M
case 'Y': 33.5M
case 'Z':
3594
154M
  
case 'a': 42.2M
case 'b': 46.2M
case 'c': 66.4M
case 'd': 125M
case 'e': 145M
case 'f': 153M
case 'g':
3595
224M
  
case 'h': 157M
case 'i': 203M
case 'j': 203M
case 'k': 208M
case 'l': 212M
case 'm': 222M
case 'n':
3596
385M
  
case 'o': 227M
case 'p': 233M
case 'q': 233M
case 'r': 241M
case 's': 370M
case 't': /*'u'*/
3597
402M
  
case 'v': 398M
case 'w': 400M
case 'x': 402M
case 'y': 402M
case 'z':
3598
694M
  case '_':
3599
    // Notify MIOpt that we read a non-whitespace/non-comment token.
3600
694M
    MIOpt.ReadToken();
3601
694M
    return LexIdentifierContinue(Result, CurPtr);
3602
3603
36.3k
  case '$':   // $ in identifiers.
3604
36.3k
    if (LangOpts.DollarIdents) {
3605
36.2k
      if (!isLexingRawMode())
3606
35.3k
        Diag(CurPtr-1, diag::ext_dollar_in_identifier);
3607
      // Notify MIOpt that we read a non-whitespace/non-comment token.
3608
36.2k
      MIOpt.ReadToken();
3609
36.2k
      return LexIdentifierContinue(Result, CurPtr);
3610
36.2k
    }
3611
3612
13
    Kind = tok::unknown;
3613
13
    break;
3614
3615
  // C99 6.4.4: Character Constants.
3616
651k
  case '\'':
3617
    // Notify MIOpt that we read a non-whitespace/non-comment token.
3618
651k
    MIOpt.ReadToken();
3619
651k
    return LexCharConstant(Result, CurPtr, tok::char_constant);
3620
3621
  // C99 6.4.5: String Literals.
3622
12.1M
  case '"':
3623
    // Notify MIOpt that we read a non-whitespace/non-comment token.
3624
12.1M
    MIOpt.ReadToken();
3625
12.1M
    return LexStringLiteral(Result, CurPtr,
3626
12.1M
                            ParsingFilename ? 
tok::header_name68.6k
3627
12.1M
                                            : 
tok::string_literal12.0M
);
3628
3629
  // C99 6.4.6: Punctuators.
3630
212k
  case '?':
3631
212k
    Kind = tok::question;
3632
212k
    break;
3633
2.71M
  case '[':
3634
2.71M
    Kind = tok::l_square;
3635
2.71M
    break;
3636
2.71M
  case ']':
3637
2.71M
    Kind = tok::r_square;
3638
2.71M
    break;
3639
196M
  case '(':
3640
196M
    Kind = tok::l_paren;
3641
196M
    break;
3642
236M
  case ')':
3643
236M
    Kind = tok::r_paren;
3644
236M
    break;
3645
12.3M
  case '{':
3646
12.3M
    Kind = tok::l_brace;
3647
12.3M
    break;
3648
12.3M
  case '}':
3649
12.3M
    Kind = tok::r_brace;
3650
12.3M
    break;
3651
6.08M
  case '.':
3652
6.08M
    Char = getCharAndSize(CurPtr, SizeTmp);
3653
6.08M
    if (Char >= '0' && 
Char <= '9'3.73M
) {
3654
      // Notify MIOpt that we read a non-whitespace/non-comment token.
3655
1.55k
      MIOpt.ReadToken();
3656
3657
1.55k
      return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result));
3658
6.08M
    } else if (LangOpts.CPlusPlus && 
Char == '*'3.91M
) {
3659
90.3k
      Kind = tok::periodstar;
3660
90.3k
      CurPtr += SizeTmp;
3661
5.99M
    } else if (Char == '.' &&
3662
5.99M
               
getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.'2.18M
) {
3663
2.18M
      Kind = tok::ellipsis;
3664
2.18M
      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3665
2.18M
                           SizeTmp2, Result);
3666
3.80M
    } else {
3667
3.80M
      Kind = tok::period;
3668
3.80M
    }
3669
6.08M
    break;
3670
6.08M
  case '&':
3671
4.60M
    Char = getCharAndSize(CurPtr, SizeTmp);
3672
4.60M
    if (Char == '&') {
3673
1.80M
      Kind = tok::ampamp;
3674
1.80M
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3675
2.80M
    } else if (Char == '=') {
3676
19.8k
      Kind = tok::ampequal;
3677
19.8k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3678
2.78M
    } else {
3679
2.78M
      Kind = tok::amp;
3680
2.78M
    }
3681
4.60M
    break;
3682
12.5M
  case '*':
3683
12.5M
    if (getCharAndSize(CurPtr, SizeTmp) == '=') {
3684
8.81k
      Kind = tok::starequal;
3685
8.81k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3686
12.5M
    } else {
3687
12.5M
      Kind = tok::star;
3688
12.5M
    }
3689
12.5M
    break;
3690
1.65M
  case '+':
3691
1.65M
    Char = getCharAndSize(CurPtr, SizeTmp);
3692
1.65M
    if (Char == '+') {
3693
507k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3694
507k
      Kind = tok::plusplus;
3695
1.14M
    } else if (Char == '=') {
3696
117k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3697
117k
      Kind = tok::plusequal;
3698
1.02M
    } else {
3699
1.02M
      Kind = tok::plus;
3700
1.02M
    }
3701
1.65M
    break;
3702
4.03M
  case '-':
3703
4.03M
    Char = getCharAndSize(CurPtr, SizeTmp);
3704
4.03M
    if (Char == '-') {      // --
3705
118k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3706
118k
      Kind = tok::minusminus;
3707
3.91M
    } else if (Char == '>' && 
LangOpts.CPlusPlus897k
&&
3708
3.91M
               
getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*'794k
) { // C++ ->*
3709
2.17k
      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3710
2.17k
                           SizeTmp2, Result);
3711
2.17k
      Kind = tok::arrowstar;
3712
3.91M
    } else if (Char == '>') {   // ->
3713
895k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3714
895k
      Kind = tok::arrow;
3715
3.01M
    } else if (Char == '=') {   // -=
3716
56.9k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3717
56.9k
      Kind = tok::minusequal;
3718
2.95M
    } else {
3719
2.95M
      Kind = tok::minus;
3720
2.95M
    }
3721
4.03M
    break;
3722
211k
  case '~':
3723
211k
    Kind = tok::tilde;
3724
211k
    break;
3725
1.59M
  case '!':
3726
1.59M
    if (getCharAndSize(CurPtr, SizeTmp) == '=') {
3727
312k
      Kind = tok::exclaimequal;
3728
312k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3729
1.27M
    } else {
3730
1.27M
      Kind = tok::exclaim;
3731
1.27M
    }
3732
1.59M
    break;
3733
37.7M
  case '/':
3734
    // 6.4.9: Comments
3735
37.7M
    Char = getCharAndSize(CurPtr, SizeTmp);
3736
37.7M
    if (Char == '/') {         // Line comment.
3737
      // Even if Line comments are disabled (e.g. in C89 mode), we generally
3738
      // want to lex this as a comment.  There is one problem with this though,
3739
      // that in one particular corner case, this can change the behavior of the
3740
      // resultant program.  For example, In  "foo //**/ bar", C89 would lex
3741
      // this as "foo / bar" and languages with Line comments would lex it as
3742
      // "foo".  Check to see if the character after the second slash is a '*'.
3743
      // If so, we will lex that as a "/" instead of the start of a comment.
3744
      // However, we never do this if we are just preprocessing.
3745
4.28M
      bool TreatAsComment = LangOpts.LineComment &&
3746
4.28M
                            
(4.27M
LangOpts.CPlusPlus4.27M
||
!LangOpts.TraditionalCPP1.45M
);
3747
4.28M
      if (!TreatAsComment)
3748
7.63k
        if (!(PP && 
PP->isPreprocessedOutput()7.35k
))
3749
7.54k
          TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*';
3750
3751
4.28M
      if (TreatAsComment) {
3752
4.28M
        if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result),
3753
4.28M
                            TokAtPhysicalStartOfLine))
3754
44.5k
          return true; // There is a token to return.
3755
3756
        // It is common for the tokens immediately after a // comment to be
3757
        // whitespace (indentation for the next line).  Instead of going through
3758
        // the big switch, handle it efficiently now.
3759
4.23M
        goto SkipIgnoredUnits;
3760
4.28M
      }
3761
4.28M
    }
3762
3763
33.4M
    if (Char == '*') {  // /**/ comment.
3764
32.8M
      if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result),
3765
32.8M
                           TokAtPhysicalStartOfLine))
3766
3.36k
        return true; // There is a token to return.
3767
3768
      // We only saw whitespace, so just try again with this lexer.
3769
      // (We manually eliminate the tail call to avoid recursion.)
3770
32.8M
      goto LexNextToken;
3771
32.8M
    }
3772
3773
608k
    if (Char == '=') {
3774
6.49k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3775
6.49k
      Kind = tok::slashequal;
3776
602k
    } else {
3777
602k
      Kind = tok::slash;
3778
602k
    }
3779
608k
    break;
3780
36.6k
  case '%':
3781
36.6k
    Char = getCharAndSize(CurPtr, SizeTmp);
3782
36.6k
    if (Char == '=') {
3783
3.16k
      Kind = tok::percentequal;
3784
3.16k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3785
33.5k
    } else if (LangOpts.Digraphs && 
Char == '>'29.4k
) {
3786
10
      Kind = tok::r_brace;                             // '%>' -> '}'
3787
10
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3788
33.5k
    } else if (LangOpts.Digraphs && 
Char == ':'29.4k
) {
3789
15
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3790
15
      Char = getCharAndSize(CurPtr, SizeTmp);
3791
15
      if (Char == '%' && 
getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':'0
) {
3792
0
        Kind = tok::hashhash;                          // '%:%:' -> '##'
3793
0
        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3794
0
                             SizeTmp2, Result);
3795
15
      } else if (Char == '@' && 
LangOpts.MicrosoftExt0
) {// %:@ -> #@ -> Charize
3796
0
        CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3797
0
        if (!isLexingRawMode())
3798
0
          Diag(BufferPtr, diag::ext_charize_microsoft);
3799
0
        Kind = tok::hashat;
3800
15
      } else {                                         // '%:' -> '#'
3801
        // We parsed a # character.  If this occurs at the start of the line,
3802
        // it's actually the start of a preprocessing directive.  Callback to
3803
        // the preprocessor to handle it.
3804
        // TODO: -fpreprocessed mode??
3805
15
        if (TokAtPhysicalStartOfLine && !LexingRawMode && 
!Is_PragmaLexer12
)
3806
12
          goto HandleDirective;
3807
3808
3
        Kind = tok::hash;
3809
3
      }
3810
33.4k
    } else {
3811
33.4k
      Kind = tok::percent;
3812
33.4k
    }
3813
36.6k
    break;
3814
15.0M
  case '<':
3815
15.0M
    Char = getCharAndSize(CurPtr, SizeTmp);
3816
15.0M
    if (ParsingFilename) {
3817
1.62M
      return LexAngledStringLiteral(Result, CurPtr);
3818
13.4M
    } else if (Char == '<') {
3819
508k
      char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
3820
508k
      if (After == '=') {
3821
3.89k
        Kind = tok::lesslessequal;
3822
3.89k
        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3823
3.89k
                             SizeTmp2, Result);
3824
504k
      } else if (After == '<' && 
IsStartOfConflictMarker(CurPtr-1)282
) {
3825
        // If this is actually a '<<<<<<<' version control conflict marker,
3826
        // recognize it as such and recover nicely.
3827
2
        goto LexNextToken;
3828
504k
      } else if (After == '<' && 
HandleEndOfConflictMarker(CurPtr-1)280
) {
3829
        // If this is '<<<<' and we're in a Perforce-style conflict marker,
3830
        // ignore it.
3831
0
        goto LexNextToken;
3832
504k
      } else if (LangOpts.CUDA && 
After == '<'182
) {
3833
173
        Kind = tok::lesslessless;
3834
173
        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3835
173
                             SizeTmp2, Result);
3836
504k
      } else {
3837
504k
        CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3838
504k
        Kind = tok::lessless;
3839
504k
      }
3840
12.9M
    } else if (Char == '=') {
3841
183k
      char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
3842
183k
      if (After == '>') {
3843
15.3k
        if (getLangOpts().CPlusPlus20) {
3844
927
          if (!isLexingRawMode())
3845
877
            Diag(BufferPtr, diag::warn_cxx17_compat_spaceship);
3846
927
          CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3847
927
                               SizeTmp2, Result);
3848
927
          Kind = tok::spaceship;
3849
927
          break;
3850
927
        }
3851
        // Suggest adding a space between the '<=' and the '>' to avoid a
3852
        // change in semantics if this turns up in C++ <=17 mode.
3853
14.4k
        if (getLangOpts().CPlusPlus && 
!isLexingRawMode()14.4k
) {
3854
12
          Diag(BufferPtr, diag::warn_cxx20_compat_spaceship)
3855
12
            << FixItHint::CreateInsertion(
3856
12
                   getSourceLocation(CurPtr + SizeTmp, SizeTmp2), " ");
3857
12
        }
3858
14.4k
      }
3859
182k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3860
182k
      Kind = tok::lessequal;
3861
12.7M
    } else if (LangOpts.Digraphs && 
Char == ':'12.2M
) { // '<:' -> '['
3862
106
      if (LangOpts.CPlusPlus11 &&
3863
106
          
getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':'75
) {
3864
        // C++0x [lex.pptoken]p3:
3865
        //  Otherwise, if the next three characters are <:: and the subsequent
3866
        //  character is neither : nor >, the < is treated as a preprocessor
3867
        //  token by itself and not as the first character of the alternative
3868
        //  token <:.
3869
60
        unsigned SizeTmp3;
3870
60
        char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
3871
60
        if (After != ':' && 
After != '>'59
) {
3872
58
          Kind = tok::less;
3873
58
          if (!isLexingRawMode())
3874
55
            Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon);
3875
58
          break;
3876
58
        }
3877
60
      }
3878
3879
48
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3880
48
      Kind = tok::l_square;
3881
12.7M
    } else if (LangOpts.Digraphs && 
Char == '%'12.2M
) { // '<%' -> '{'
3882
9
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3883
9
      Kind = tok::l_brace;
3884
12.7M
    } else if (Char == '#' && /*Not a trigraph*/ 
SizeTmp == 147
&&
3885
12.7M
               
lexEditorPlaceholder(Result, CurPtr)45
) {
3886
43
      return true;
3887
12.7M
    } else {
3888
12.7M
      Kind = tok::less;
3889
12.7M
    }
3890
13.4M
    break;
3891
13.4M
  case '>':
3892
13.3M
    Char = getCharAndSize(CurPtr, SizeTmp);
3893
13.3M
    if (Char == '=') {
3894
605k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3895
605k
      Kind = tok::greaterequal;
3896
12.7M
    } else if (Char == '>') {
3897
331k
      char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
3898
331k
      if (After == '=') {
3899
3.38k
        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3900
3.38k
                             SizeTmp2, Result);
3901
3.38k
        Kind = tok::greatergreaterequal;
3902
328k
      } else if (After == '>' && 
IsStartOfConflictMarker(CurPtr-1)19.6k
) {
3903
        // If this is actually a '>>>>' conflict marker, recognize it as such
3904
        // and recover nicely.
3905
2
        goto LexNextToken;
3906
328k
      } else if (After == '>' && 
HandleEndOfConflictMarker(CurPtr-1)19.6k
) {
3907
        // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
3908
0
        goto LexNextToken;
3909
328k
      } else if (LangOpts.CUDA && 
After == '>'187
) {
3910
185
        Kind = tok::greatergreatergreater;
3911
185
        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3912
185
                             SizeTmp2, Result);
3913
328k
      } else {
3914
328k
        CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3915
328k
        Kind = tok::greatergreater;
3916
328k
      }
3917
12.4M
    } else {
3918
12.4M
      Kind = tok::greater;
3919
12.4M
    }
3920
13.3M
    break;
3921
13.3M
  case '^':
3922
116k
    Char = getCharAndSize(CurPtr, SizeTmp);
3923
116k
    if (Char == '=') {
3924
10.4k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3925
10.4k
      Kind = tok::caretequal;
3926
105k
    } else if (LangOpts.OpenCL && 
Char == '^'383
) {
3927
2
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3928
2
      Kind = tok::caretcaret;
3929
105k
    } else {
3930
105k
      Kind = tok::caret;
3931
105k
    }
3932
116k
    break;
3933
1.04M
  case '|':
3934
1.04M
    Char = getCharAndSize(CurPtr, SizeTmp);
3935
1.04M
    if (Char == '=') {
3936
34.5k
      Kind = tok::pipeequal;
3937
34.5k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3938
1.00M
    } else if (Char == '|') {
3939
      // If this is '|||||||' and we're in a conflict marker, ignore it.
3940
702k
      if (CurPtr[1] == '|' && 
HandleEndOfConflictMarker(CurPtr-1)19
)
3941
1
        goto LexNextToken;
3942
702k
      Kind = tok::pipepipe;
3943
702k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3944
702k
    } else {
3945
303k
      Kind = tok::pipe;
3946
303k
    }
3947
1.04M
    break;
3948
8.45M
  case ':':
3949
8.45M
    Char = getCharAndSize(CurPtr, SizeTmp);
3950
8.45M
    if (LangOpts.Digraphs && 
Char == '>'8.14M
) {
3951
21
      Kind = tok::r_square; // ':>' -> ']'
3952
21
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3953
8.45M
    } else if ((LangOpts.CPlusPlus ||
3954
8.45M
                
LangOpts.DoubleSquareBracketAttributes1.08M
) &&
3955
8.45M
               
Char == ':'7.37M
) {
3956
5.42M
      Kind = tok::coloncolon;
3957
5.42M
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3958
5.42M
    } else {
3959
3.03M
      Kind = tok::colon;
3960
3.03M
    }
3961
8.45M
    break;
3962
63.1M
  case ';':
3963
63.1M
    Kind = tok::semi;
3964
63.1M
    break;
3965
21.2M
  case '=':
3966
21.2M
    Char = getCharAndSize(CurPtr, SizeTmp);
3967
21.2M
    if (Char == '=') {
3968
      // If this is '====' and we're in a conflict marker, ignore it.
3969
675k
      if (CurPtr[1] == '=' && 
HandleEndOfConflictMarker(CurPtr-1)45
)
3970
2
        goto LexNextToken;
3971
3972
675k
      Kind = tok::equalequal;
3973
675k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3974
20.5M
    } else {
3975
20.5M
      Kind = tok::equal;
3976
20.5M
    }
3977
21.2M
    break;
3978
118M
  case ',':
3979
118M
    Kind = tok::comma;
3980
118M
    break;
3981
78.4M
  case '#':
3982
78.4M
    Char = getCharAndSize(CurPtr, SizeTmp);
3983
78.4M
    if (Char == '#') {
3984
242k
      Kind = tok::hashhash;
3985
242k
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3986
78.2M
    } else if (Char == '@' && 
LangOpts.MicrosoftExt3
) { // #@ -> Charize
3987
3
      Kind = tok::hashat;
3988
3
      if (!isLexingRawMode())
3989
3
        Diag(BufferPtr, diag::ext_charize_microsoft);
3990
3
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3991
78.2M
    } else {
3992
      // We parsed a # character.  If this occurs at the start of the line,
3993
      // it's actually the start of a preprocessing directive.  Callback to
3994
      // the preprocessor to handle it.
3995
      // TODO: -fpreprocessed mode??
3996
78.2M
      if (TokAtPhysicalStartOfLine && 
!LexingRawMode78.1M
&&
!Is_PragmaLexer60.1M
)
3997
60.1M
        goto HandleDirective;
3998
3999
18.1M
      Kind = tok::hash;
4000
18.1M
    }
4001
18.3M
    break;
4002
4003
18.3M
  case '@':
4004
    // Objective C support.
4005
747k
    if (CurPtr[-1] == '@' && LangOpts.ObjC)
4006
746k
      Kind = tok::at;
4007
1.13k
    else
4008
1.13k
      Kind = tok::unknown;
4009
747k
    break;
4010
4011
  // UCNs (C99 6.4.3, C++11 [lex.charset]p2)
4012
1.18k
  case '\\':
4013
1.18k
    if (!LangOpts.AsmPreprocessor) {
4014
1.17k
      if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) {
4015
98
        if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) {
4016
0
          if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
4017
0
            return true; // KeepWhitespaceMode
4018
4019
          // We only saw whitespace, so just try again with this lexer.
4020
          // (We manually eliminate the tail call to avoid recursion.)
4021
0
          goto LexNextToken;
4022
0
        }
4023
4024
98
        return LexUnicodeIdentifierStart(Result, CodePoint, CurPtr);
4025
98
      }
4026
1.17k
    }
4027
4028
1.08k
    Kind = tok::unknown;
4029
1.08k
    break;
4030
4031
358
  default: {
4032
358
    if (isASCII(Char)) {
4033
165
      Kind = tok::unknown;
4034
165
      break;
4035
165
    }
4036
4037
193
    llvm::UTF32 CodePoint;
4038
4039
    // We can't just reset CurPtr to BufferPtr because BufferPtr may point to
4040
    // an escaped newline.
4041
193
    --CurPtr;
4042
193
    llvm::ConversionResult Status =
4043
193
        llvm::convertUTF8Sequence((const llvm::UTF8 **)&CurPtr,
4044
193
                                  (const llvm::UTF8 *)BufferEnd,
4045
193
                                  &CodePoint,
4046
193
                                  llvm::strictConversion);
4047
193
    if (Status == llvm::conversionOK) {
4048
127
      if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) {
4049
6
        if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
4050
0
          return true; // KeepWhitespaceMode
4051
4052
        // We only saw whitespace, so just try again with this lexer.
4053
        // (We manually eliminate the tail call to avoid recursion.)
4054
6
        goto LexNextToken;
4055
6
      }
4056
121
      return LexUnicodeIdentifierStart(Result, CodePoint, CurPtr);
4057
127
    }
4058
4059
66
    if (isLexingRawMode() || 
ParsingPreprocessorDirective4
||
4060
66
        
PP->isPreprocessedOutput()2
) {
4061
65
      ++CurPtr;
4062
65
      Kind = tok::unknown;
4063
65
      break;
4064
65
    }
4065
4066
    // Non-ASCII characters tend to creep into source code unintentionally.
4067
    // Instead of letting the parser complain about the unknown token,
4068
    // just diagnose the invalid UTF-8, then drop the character.
4069
1
    Diag(CurPtr, diag::err_invalid_utf8);
4070
4071
1
    BufferPtr = CurPtr+1;
4072
    // We're pretending the character didn't exist, so just try again with
4073
    // this lexer.
4074
    // (We manually eliminate the tail call to avoid recursion.)
4075
1
    goto LexNextToken;
4076
66
  }
4077
1.88G
  }
4078
4079
  // Notify MIOpt that we read a non-whitespace/non-comment token.
4080
822M
  MIOpt.ReadToken();
4081
4082
  // Update the location of token as well as BufferPtr.
4083
822M
  FormTokenWithChars(Result, CurPtr, Kind);
4084
822M
  return true;
4085
4086
60.1M
HandleDirective:
4087
  // We parsed a # character and it's the start of a preprocessing directive.
4088
4089
60.1M
  FormTokenWithChars(Result, CurPtr, tok::hash);
4090
60.1M
  PP->HandleDirective(Result);
4091
4092
60.1M
  if (PP->hadModuleLoaderFatalFailure()) {
4093
    // With a fatal failure in the module loader, we abort parsing.
4094
2
    assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof");
4095
0
    return true;
4096
2
  }
4097
4098
  // We parsed the directive; lex a token with the new state.
4099
60.1M
  return false;
4100
60.1M
}