Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/lib/AST/CommentLexer.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CommentLexer.cpp -------------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/AST/CommentLexer.h"
10
#include "clang/AST/CommentCommandTraits.h"
11
#include "clang/AST/CommentDiagnostic.h"
12
#include "clang/Basic/CharInfo.h"
13
#include "llvm/ADT/StringExtras.h"
14
#include "llvm/ADT/StringSwitch.h"
15
#include "llvm/Support/ConvertUTF.h"
16
#include "llvm/Support/ErrorHandling.h"
17
18
namespace clang {
19
namespace comments {
20
21
0
void Token::dump(const Lexer &L, const SourceManager &SM) const {
22
0
  llvm::errs() << "comments::Token Kind=" << Kind << " ";
23
0
  Loc.print(llvm::errs(), SM);
24
0
  llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
25
0
}
26
27
341
static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
28
341
  return isLetter(C);
29
341
}
30
31
115
static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
32
115
  return isDigit(C);
33
115
}
34
35
63
static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
36
63
  return isHexDigit(C);
37
63
}
38
39
static inline StringRef convertCodePointToUTF8(
40
                                      llvm::BumpPtrAllocator &Allocator,
41
36
                                      unsigned CodePoint) {
42
36
  char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
43
36
  char *ResolvedPtr = Resolved;
44
36
  if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
45
36
    return StringRef(Resolved, ResolvedPtr - Resolved);
46
0
  else
47
0
    return StringRef();
48
36
}
49
50
namespace {
51
52
#include "clang/AST/CommentHTMLTags.inc"
53
#include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
54
55
} // end anonymous namespace
56
57
52
StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
58
52
  // Fast path, first check a few most widely used named character references.
59
52
  return llvm::StringSwitch<StringRef>(Name)
60
52
      .Case("amp", "&")
61
52
      .Case("lt", "<")
62
52
      .Case("gt", ">")
63
52
      .Case("quot", "\"")
64
52
      .Case("apos", "\'")
65
52
      // Slow path.
66
52
      .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
67
52
}
68
69
17
StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
70
17
  unsigned CodePoint = 0;
71
67
  for (unsigned i = 0, e = Name.size(); i != e; 
++i50
) {
72
50
    assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]));
73
50
    CodePoint *= 10;
74
50
    CodePoint += Name[i] - '0';
75
50
  }
76
17
  return convertCodePointToUTF8(Allocator, CodePoint);
77
17
}
78
79
19
StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
80
19
  unsigned CodePoint = 0;
81
57
  for (unsigned i = 0, e = Name.size(); i != e; 
++i38
) {
82
38
    CodePoint *= 16;
83
38
    const char C = Name[i];
84
38
    assert(isHTMLHexCharacterReferenceCharacter(C));
85
38
    CodePoint += llvm::hexDigitValue(C);
86
38
  }
87
19
  return convertCodePointToUTF8(Allocator, CodePoint);
88
19
}
89
90
1.55k
void Lexer::skipLineStartingDecorations() {
91
1.55k
  // This function should be called only for C comments
92
1.55k
  assert(CommentState == LCS_InsideCComment);
93
1.55k
94
1.55k
  if (BufferPtr == CommentEnd)
95
356
    return;
96
1.19k
97
1.19k
  switch (*BufferPtr) {
98
1.19k
  case ' ':
99
862
  case '\t':
100
862
  case '\f':
101
862
  case '\v': {
102
862
    const char *NewBufferPtr = BufferPtr;
103
862
    NewBufferPtr++;
104
862
    if (NewBufferPtr == CommentEnd)
105
99
      return;
106
763
107
763
    char C = *NewBufferPtr;
108
1.06k
    while (isHorizontalWhitespace(C)) {
109
318
      NewBufferPtr++;
110
318
      if (NewBufferPtr == CommentEnd)
111
14
        return;
112
304
      C = *NewBufferPtr;
113
304
    }
114
763
    
if (749
C == '*'749
)
115
624
      BufferPtr = NewBufferPtr + 1;
116
749
    break;
117
763
  }
118
763
  case '*':
119
6
    BufferPtr++;
120
6
    break;
121
1.19k
  }
122
1.19k
}
123
124
namespace {
125
/// Returns pointer to the first newline character in the string.
126
240
const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
127
4.07k
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr3.83k
) {
128
3.93k
    if (isVerticalWhitespace(*BufferPtr))
129
95
      return BufferPtr;
130
3.93k
  }
131
240
  
return BufferEnd145
;
132
240
}
133
134
1.58k
const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
135
1.58k
  if (BufferPtr == BufferEnd)
136
26
    return BufferPtr;
137
1.55k
138
1.55k
  if (*BufferPtr == '\n')
139
1.54k
    BufferPtr++;
140
15
  else {
141
15
    assert(*BufferPtr == '\r');
142
15
    BufferPtr++;
143
15
    if (BufferPtr != BufferEnd && 
*BufferPtr == '\n'13
)
144
8
      BufferPtr++;
145
15
  }
146
1.55k
  return BufferPtr;
147
1.55k
}
148
149
const char *skipNamedCharacterReference(const char *BufferPtr,
150
54
                                        const char *BufferEnd) {
151
243
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr189
) {
152
242
    if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
153
53
      return BufferPtr;
154
242
  }
155
54
  
return BufferEnd1
;
156
54
}
157
158
const char *skipDecimalCharacterReference(const char *BufferPtr,
159
19
                                          const char *BufferEnd) {
160
73
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr54
) {
161
72
    if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
162
18
      return BufferPtr;
163
72
  }
164
19
  
return BufferEnd1
;
165
19
}
166
167
const char *skipHexCharacterReference(const char *BufferPtr,
168
23
                                      const char *BufferEnd) {
169
65
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr42
) {
170
63
    if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
171
21
      return BufferPtr;
172
63
  }
173
23
  
return BufferEnd2
;
174
23
}
175
176
517
bool isHTMLIdentifierStartingCharacter(char C) {
177
517
  return isLetter(C);
178
517
}
179
180
1.24k
bool isHTMLIdentifierCharacter(char C) {
181
1.24k
  return isAlphanumeric(C);
182
1.24k
}
183
184
335
const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
185
946
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr611
) {
186
936
    if (!isHTMLIdentifierCharacter(*BufferPtr))
187
325
      return BufferPtr;
188
936
  }
189
335
  
return BufferEnd10
;
190
335
}
191
192
/// Skip HTML string quoted in single or double quotes.  Escaping quotes inside
193
/// string allowed.
194
///
195
/// Returns pointer to closing quote.
196
const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
197
44
{
198
44
  const char Quote = *BufferPtr;
199
44
  assert(Quote == '\"' || Quote == '\'');
200
44
201
44
  BufferPtr++;
202
424
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr380
) {
203
418
    const char C = *BufferPtr;
204
418
    if (C == Quote && 
BufferPtr[-1] != '\\'44
)
205
38
      return BufferPtr;
206
418
  }
207
44
  
return BufferEnd6
;
208
44
}
209
210
551
const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
211
696
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr145
) {
212
652
    if (!isWhitespace(*BufferPtr))
213
507
      return BufferPtr;
214
652
  }
215
551
  
return BufferEnd44
;
216
551
}
217
218
34
bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
219
34
  return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
220
34
}
221
222
2.44k
bool isCommandNameStartCharacter(char C) {
223
2.44k
  return isLetter(C);
224
2.44k
}
225
226
14.8k
bool isCommandNameCharacter(char C) {
227
14.8k
  return isAlphanumeric(C);
228
14.8k
}
229
230
2.43k
const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
231
14.9k
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr12.5k
) {
232
14.8k
    if (!isCommandNameCharacter(*BufferPtr))
233
2.30k
      return BufferPtr;
234
14.8k
  }
235
2.43k
  
return BufferEnd132
;
236
2.43k
}
237
238
/// Return the one past end pointer for BCPL comments.
239
/// Handles newlines escaped with backslash or trigraph for backslahs.
240
2.64k
const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
241
2.64k
  const char *CurPtr = BufferPtr;
242
2.65k
  while (CurPtr != BufferEnd) {
243
46.0k
    while (!isVerticalWhitespace(*CurPtr)) {
244
44.9k
      CurPtr++;
245
44.9k
      if (CurPtr == BufferEnd)
246
1.52k
        return BufferEnd;
247
44.9k
    }
248
2.62k
    // We found a newline, check if it is escaped.
249
2.62k
    const char *EscapePtr = CurPtr - 1;
250
1.11k
    while(isHorizontalWhitespace(*EscapePtr))
251
11
      EscapePtr--;
252
1.10k
253
1.10k
    if (*EscapePtr == '\\' ||
254
1.10k
        
(1.10k
EscapePtr - 2 >= BufferPtr1.10k
&&
EscapePtr[0] == '/'955
&&
255
1.10k
         
EscapePtr[-1] == '?'3
&&
EscapePtr[-2] == '?'3
)) {
256
9
      // We found an escaped newline.
257
9
      CurPtr = skipNewline(CurPtr, BufferEnd);
258
9
    } else
259
1.09k
      return CurPtr; // Not an escaped newline.
260
1.10k
  }
261
2.64k
  
return BufferEnd22
;
262
2.64k
}
263
264
/// Return the one past end pointer for C comments.
265
/// Very dumb, does not handle escaped newlines or trigraphs.
266
648
const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
267
27.2k
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr26.5k
) {
268
27.2k
    if (*BufferPtr == '*') {
269
1.29k
      assert(BufferPtr + 1 != BufferEnd);
270
1.29k
      if (*(BufferPtr + 1) == '/')
271
648
        return BufferPtr;
272
1.29k
    }
273
27.2k
  }
274
648
  
llvm_unreachable0
("buffer end hit before '*/' was seen");
275
648
}
276
277
} // end anonymous namespace
278
279
void Lexer::formTokenWithChars(Token &Result, const char *TokEnd,
280
17.1k
                               tok::TokenKind Kind) {
281
17.1k
  const unsigned TokLen = TokEnd - BufferPtr;
282
17.1k
  Result.setLocation(getSourceLocation(BufferPtr));
283
17.1k
  Result.setKind(Kind);
284
17.1k
  Result.setLength(TokLen);
285
#ifndef NDEBUG
286
  Result.TextPtr = "<UNSET>";
287
  Result.IntVal = 7;
288
#endif
289
  BufferPtr = TokEnd;
290
17.1k
}
291
292
10.3k
void Lexer::lexCommentText(Token &T) {
293
10.3k
  assert(CommentState == LCS_InsideBCPLComment ||
294
10.3k
         CommentState == LCS_InsideCComment);
295
10.3k
296
10.3k
  // Handles lexing non-command text, i.e. text and newline.
297
10.3k
  auto HandleNonCommandToken = [&]() -> void {
298
6.77k
    assert(State == LS_Normal);
299
6.77k
300
6.77k
    const char *TokenPtr = BufferPtr;
301
6.77k
    assert(TokenPtr < CommentEnd);
302
6.77k
    switch (*TokenPtr) {
303
6.77k
      case '\n':
304
1.51k
      case '\r':
305
1.51k
          TokenPtr = skipNewline(TokenPtr, CommentEnd);
306
1.51k
          formTokenWithChars(T, TokenPtr, tok::newline);
307
1.51k
308
1.51k
          if (CommentState == LCS_InsideCComment)
309
1.50k
            skipLineStartingDecorations();
310
1.51k
          return;
311
1.51k
312
5.25k
      default: {
313
5.25k
          StringRef TokStartSymbols = ParseCommands ? 
"\n\r\\@&<"5.22k
:
"\n\r"28
;
314
5.25k
          size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr)
315
5.25k
                           .find_first_of(TokStartSymbols);
316
5.25k
          if (End != StringRef::npos)
317
2.98k
            TokenPtr += End;
318
2.27k
          else
319
2.27k
            TokenPtr = CommentEnd;
320
5.25k
          formTextToken(T, TokenPtr);
321
5.25k
          return;
322
1.51k
      }
323
6.77k
    }
324
6.77k
  };
325
10.3k
326
10.3k
  if (!ParseCommands)
327
36
    return HandleNonCommandToken();
328
10.2k
329
10.2k
  switch (State) {
330
10.2k
  case LS_Normal:
331
9.64k
    break;
332
10.2k
  case LS_VerbatimBlockFirstLine:
333
41
    lexVerbatimBlockFirstLine(T);
334
41
    return;
335
10.2k
  case LS_VerbatimBlockBody:
336
69
    lexVerbatimBlockBody(T);
337
69
    return;
338
10.2k
  case LS_VerbatimLineText:
339
116
    lexVerbatimLineText(T);
340
116
    return;
341
10.2k
  case LS_HTMLStartTag:
342
309
    lexHTMLStartTag(T);
343
309
    return;
344
10.2k
  case LS_HTMLEndTag:
345
81
    lexHTMLEndTag(T);
346
81
    return;
347
9.64k
  }
348
9.64k
349
9.64k
  assert(State == LS_Normal);
350
9.64k
  const char *TokenPtr = BufferPtr;
351
9.64k
  assert(TokenPtr < CommentEnd);
352
9.64k
  switch(*TokenPtr) {
353
9.64k
    case '\\':
354
2.52k
    case '@': {
355
2.52k
      // Commands that start with a backslash and commands that start with
356
2.52k
      // 'at' have equivalent semantics.  But we keep information about the
357
2.52k
      // exact syntax in AST for comments.
358
2.52k
      tok::TokenKind CommandKind =
359
2.52k
          (*TokenPtr == '@') ? 
tok::at_command481
:
tok::backslash_command2.04k
;
360
2.52k
      TokenPtr++;
361
2.52k
      if (TokenPtr == CommentEnd) {
362
12
        formTextToken(T, TokenPtr);
363
12
        return;
364
12
      }
365
2.51k
      char C = *TokenPtr;
366
2.51k
      switch (C) {
367
2.51k
      default:
368
2.44k
        break;
369
2.51k
370
2.51k
      
case '\\': 68
case '@': 68
case '&': 68
case '$':
371
68
      case '#':  case '<': case '>': case '%':
372
68
      case '\"': case '.': case ':':
373
68
        // This is one of \\ \@ \& \$ etc escape sequences.
374
68
        TokenPtr++;
375
68
        if (C == ':' && 
TokenPtr != CommentEnd6
&&
*TokenPtr == ':'6
) {
376
6
          // This is the \:: escape sequence.
377
6
          TokenPtr++;
378
6
        }
379
68
        StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
380
68
        formTokenWithChars(T, TokenPtr, tok::text);
381
68
        T.setText(UnescapedText);
382
68
        return;
383
2.44k
      }
384
2.44k
385
2.44k
      // Don't make zero-length commands.
386
2.44k
      if (!isCommandNameStartCharacter(*TokenPtr)) {
387
8
        formTextToken(T, TokenPtr);
388
8
        return;
389
8
      }
390
2.43k
391
2.43k
      TokenPtr = skipCommandName(TokenPtr, CommentEnd);
392
2.43k
      unsigned Length = TokenPtr - (BufferPtr + 1);
393
2.43k
394
2.43k
      // Hardcoded support for lexing LaTeX formula commands
395
2.43k
      // \f$ \f[ \f] \f{ \f} as a single command.
396
2.43k
      if (Length == 1 && 
TokenPtr[-1] == 'f'111
&&
TokenPtr != CommentEnd14
) {
397
14
        C = *TokenPtr;
398
14
        if (C == '$' || 
C == '['4
||
C == ']'3
||
C == '{'3
||
C == '}'2
) {
399
12
          TokenPtr++;
400
12
          Length++;
401
12
        }
402
14
      }
403
2.43k
404
2.43k
      StringRef CommandName(BufferPtr + 1, Length);
405
2.43k
406
2.43k
      const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
407
2.43k
      if (!Info) {
408
309
        if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
409
11
          StringRef CorrectedName = Info->Name;
410
11
          SourceLocation Loc = getSourceLocation(BufferPtr);
411
11
          SourceLocation EndLoc = getSourceLocation(TokenPtr);
412
11
          SourceRange FullRange = SourceRange(Loc, EndLoc);
413
11
          SourceRange CommandRange(Loc.getLocWithOffset(1), EndLoc);
414
11
          Diag(Loc, diag::warn_correct_comment_command_name)
415
11
            << FullRange << CommandName << CorrectedName
416
11
            << FixItHint::CreateReplacement(CommandRange, CorrectedName);
417
298
        } else {
418
298
          formTokenWithChars(T, TokenPtr, tok::unknown_command);
419
298
          T.setUnknownCommandName(CommandName);
420
298
          Diag(T.getLocation(), diag::warn_unknown_comment_command_name)
421
298
              << SourceRange(T.getLocation(), T.getEndLocation());
422
298
          return;
423
298
        }
424
2.14k
      }
425
2.14k
      if (Info->IsVerbatimBlockCommand) {
426
52
        setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
427
52
        return;
428
52
      }
429
2.08k
      if (Info->IsVerbatimLineCommand) {
430
120
        setupAndLexVerbatimLine(T, TokenPtr, Info);
431
120
        return;
432
120
      }
433
1.96k
      formTokenWithChars(T, TokenPtr, CommandKind);
434
1.96k
      T.setCommandID(Info->getID());
435
1.96k
      return;
436
1.96k
    }
437
1.96k
438
1.96k
    case '&':
439
100
      lexHTMLCharacterReference(T);
440
100
      return;
441
1.96k
442
1.96k
    case '<': {
443
283
      TokenPtr++;
444
283
      if (TokenPtr == CommentEnd) {
445
1
        formTextToken(T, TokenPtr);
446
1
        return;
447
1
      }
448
282
      const char C = *TokenPtr;
449
282
      if (isHTMLIdentifierStartingCharacter(C))
450
174
        setupAndLexHTMLStartTag(T);
451
108
      else if (C == '/')
452
86
        setupAndLexHTMLEndTag(T);
453
22
      else
454
22
        formTextToken(T, TokenPtr);
455
282
      return;
456
282
    }
457
282
458
6.73k
    default:
459
6.73k
      return HandleNonCommandToken();
460
9.64k
  }
461
9.64k
}
462
463
void Lexer::setupAndLexVerbatimBlock(Token &T,
464
                                     const char *TextBegin,
465
52
                                     char Marker, const CommandInfo *Info) {
466
52
  assert(Info->IsVerbatimBlockCommand);
467
52
468
52
  VerbatimBlockEndCommandName.clear();
469
52
  VerbatimBlockEndCommandName.append(Marker == '\\' ? 
"\\"47
:
"@"5
);
470
52
  VerbatimBlockEndCommandName.append(Info->EndCommandName);
471
52
472
52
  formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
473
52
  T.setVerbatimBlockID(Info->getID());
474
52
475
52
  // If there is a newline following the verbatim opening command, skip the
476
52
  // newline so that we don't create an tok::verbatim_block_line with empty
477
52
  // text content.
478
52
  if (BufferPtr != CommentEnd &&
479
52
      
isVerticalWhitespace(*BufferPtr)36
) {
480
9
    BufferPtr = skipNewline(BufferPtr, CommentEnd);
481
9
    State = LS_VerbatimBlockBody;
482
9
    return;
483
9
  }
484
43
485
43
  State = LS_VerbatimBlockFirstLine;
486
43
}
487
488
107
void Lexer::lexVerbatimBlockFirstLine(Token &T) {
489
124
again:
490
124
  assert(BufferPtr < CommentEnd);
491
124
492
124
  // FIXME: It would be better to scan the text once, finding either the block
493
124
  // end command or newline.
494
124
  //
495
124
  // Extract current line.
496
124
  const char *Newline = findNewline(BufferPtr, CommentEnd);
497
124
  StringRef Line(BufferPtr, Newline - BufferPtr);
498
124
499
124
  // Look for end command in current line.
500
124
  size_t Pos = Line.find(VerbatimBlockEndCommandName);
501
124
  const char *TextEnd;
502
124
  const char *NextLine;
503
124
  if (Pos == StringRef::npos) {
504
49
    // Current line is completely verbatim.
505
49
    TextEnd = Newline;
506
49
    NextLine = skipNewline(Newline, CommentEnd);
507
75
  } else if (Pos == 0) {
508
41
    // Current line contains just an end command.
509
41
    const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
510
41
    StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
511
41
    formTokenWithChars(T, End, tok::verbatim_block_end);
512
41
    T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
513
41
    State = LS_Normal;
514
41
    return;
515
41
  } else {
516
34
    // There is some text, followed by end command.  Extract text first.
517
34
    TextEnd = BufferPtr + Pos;
518
34
    NextLine = TextEnd;
519
34
    // If there is only whitespace before end command, skip whitespace.
520
34
    if (isWhitespace(BufferPtr, TextEnd)) {
521
17
      BufferPtr = TextEnd;
522
17
      goto again;
523
17
    }
524
66
  }
525
66
526
66
  StringRef Text(BufferPtr, TextEnd - BufferPtr);
527
66
  formTokenWithChars(T, NextLine, tok::verbatim_block_line);
528
66
  T.setVerbatimBlockText(Text);
529
66
530
66
  State = LS_VerbatimBlockBody;
531
66
}
532
533
69
void Lexer::lexVerbatimBlockBody(Token &T) {
534
69
  assert(State == LS_VerbatimBlockBody);
535
69
536
69
  if (CommentState == LCS_InsideCComment)
537
44
    skipLineStartingDecorations();
538
69
539
69
  if (BufferPtr == CommentEnd) {
540
3
    formTokenWithChars(T, BufferPtr, tok::verbatim_block_line);
541
3
    T.setVerbatimBlockText("");
542
3
    return;
543
3
  }
544
66
545
66
  lexVerbatimBlockFirstLine(T);
546
66
}
547
548
void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
549
120
                                    const CommandInfo *Info) {
550
120
  assert(Info->IsVerbatimLineCommand);
551
120
  formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
552
120
  T.setVerbatimLineID(Info->getID());
553
120
554
120
  State = LS_VerbatimLineText;
555
120
}
556
557
116
void Lexer::lexVerbatimLineText(Token &T) {
558
116
  assert(State == LS_VerbatimLineText);
559
116
560
116
  // Extract current line.
561
116
  const char *Newline = findNewline(BufferPtr, CommentEnd);
562
116
  StringRef Text(BufferPtr, Newline - BufferPtr);
563
116
  formTokenWithChars(T, Newline, tok::verbatim_line_text);
564
116
  T.setVerbatimLineText(Text);
565
116
566
116
  State = LS_Normal;
567
116
}
568
569
100
void Lexer::lexHTMLCharacterReference(Token &T) {
570
100
  const char *TokenPtr = BufferPtr;
571
100
  assert(*TokenPtr == '&');
572
100
  TokenPtr++;
573
100
  if (TokenPtr == CommentEnd) {
574
1
    formTextToken(T, TokenPtr);
575
1
    return;
576
1
  }
577
99
  const char *NamePtr;
578
99
  bool isNamed = false;
579
99
  bool isDecimal = false;
580
99
  char C = *TokenPtr;
581
99
  if (isHTMLNamedCharacterReferenceCharacter(C)) {
582
54
    NamePtr = TokenPtr;
583
54
    TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
584
54
    isNamed = true;
585
54
  } else 
if (45
C == '#'45
) {
586
44
    TokenPtr++;
587
44
    if (TokenPtr == CommentEnd) {
588
1
      formTextToken(T, TokenPtr);
589
1
      return;
590
1
    }
591
43
    C = *TokenPtr;
592
43
    if (isHTMLDecimalCharacterReferenceCharacter(C)) {
593
19
      NamePtr = TokenPtr;
594
19
      TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
595
19
      isDecimal = true;
596
24
    } else if (C == 'x' || 
C == 'X'7
) {
597
23
      TokenPtr++;
598
23
      NamePtr = TokenPtr;
599
23
      TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
600
23
    } else {
601
1
      formTextToken(T, TokenPtr);
602
1
      return;
603
1
    }
604
1
  } else {
605
1
    formTextToken(T, TokenPtr);
606
1
    return;
607
1
  }
608
96
  if (NamePtr == TokenPtr || 
TokenPtr == CommentEnd94
||
609
96
      
*TokenPtr != ';'91
) {
610
8
    formTextToken(T, TokenPtr);
611
8
    return;
612
8
  }
613
88
  StringRef Name(NamePtr, TokenPtr - NamePtr);
614
88
  TokenPtr++; // Skip semicolon.
615
88
  StringRef Resolved;
616
88
  if (isNamed)
617
52
    Resolved = resolveHTMLNamedCharacterReference(Name);
618
36
  else if (isDecimal)
619
17
    Resolved = resolveHTMLDecimalCharacterReference(Name);
620
19
  else
621
19
    Resolved = resolveHTMLHexCharacterReference(Name);
622
88
623
88
  if (Resolved.empty()) {
624
0
    formTextToken(T, TokenPtr);
625
0
    return;
626
0
  }
627
88
  formTokenWithChars(T, TokenPtr, tok::text);
628
88
  T.setText(Resolved);
629
88
}
630
631
174
void Lexer::setupAndLexHTMLStartTag(Token &T) {
632
174
  assert(BufferPtr[0] == '<' &&
633
174
         isHTMLIdentifierStartingCharacter(BufferPtr[1]));
634
174
  const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
635
174
  StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
636
174
  if (!isHTMLTagName(Name)) {
637
8
    formTextToken(T, TagNameEnd);
638
8
    return;
639
8
  }
640
166
641
166
  formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
642
166
  T.setHTMLTagStartName(Name);
643
166
644
166
  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
645
166
646
166
  const char C = *BufferPtr;
647
166
  if (BufferPtr != CommentEnd &&
648
166
      
(163
C == '>'163
||
C == '/'88
||
isHTMLIdentifierStartingCharacter(C)74
))
649
155
    State = LS_HTMLStartTag;
650
166
}
651
652
309
void Lexer::lexHTMLStartTag(Token &T) {
653
309
  assert(State == LS_HTMLStartTag);
654
309
655
309
  const char *TokenPtr = BufferPtr;
656
309
  char C = *TokenPtr;
657
309
  if (isHTMLIdentifierCharacter(C)) {
658
75
    TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
659
75
    StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
660
75
    formTokenWithChars(T, TokenPtr, tok::html_ident);
661
75
    T.setHTMLIdent(Ident);
662
234
  } else {
663
234
    switch (C) {
664
234
    case '=':
665
63
      TokenPtr++;
666
63
      formTokenWithChars(T, TokenPtr, tok::html_equals);
667
63
      break;
668
234
    case '\"':
669
44
    case '\'': {
670
44
      const char *OpenQuote = TokenPtr;
671
44
      TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
672
44
      const char *ClosingQuote = TokenPtr;
673
44
      if (TokenPtr != CommentEnd) // Skip closing quote.
674
38
        TokenPtr++;
675
44
      formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
676
44
      T.setHTMLQuotedString(StringRef(OpenQuote + 1,
677
44
                                      ClosingQuote - (OpenQuote + 1)));
678
44
      break;
679
44
    }
680
113
    case '>':
681
113
      TokenPtr++;
682
113
      formTokenWithChars(T, TokenPtr, tok::html_greater);
683
113
      State = LS_Normal;
684
113
      return;
685
44
    case '/':
686
14
      TokenPtr++;
687
14
      if (TokenPtr != CommentEnd && *TokenPtr == '>') {
688
12
        TokenPtr++;
689
12
        formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
690
12
      } else
691
2
        formTextToken(T, TokenPtr);
692
14
693
14
      State = LS_Normal;
694
14
      return;
695
182
    }
696
182
  }
697
182
698
182
  // Now look ahead and return to normal state if we don't see any HTML tokens
699
182
  // ahead.
700
182
  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
701
182
  if (BufferPtr == CommentEnd) {
702
21
    State = LS_Normal;
703
21
    return;
704
21
  }
705
161
706
161
  C = *BufferPtr;
707
161
  if (!isHTMLIdentifierStartingCharacter(C) &&
708
161
      
C != '='152
&&
C != '\"'89
&&
C != '\''50
&&
C != '>'45
) {
709
7
    State = LS_Normal;
710
7
    return;
711
7
  }
712
161
}
713
714
86
void Lexer::setupAndLexHTMLEndTag(Token &T) {
715
86
  assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
716
86
717
86
  const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
718
86
  const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
719
86
  StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
720
86
  if (!isHTMLTagName(Name)) {
721
3
    formTextToken(T, TagNameEnd);
722
3
    return;
723
3
  }
724
83
725
83
  const char *End = skipWhitespace(TagNameEnd, CommentEnd);
726
83
727
83
  formTokenWithChars(T, End, tok::html_end_tag);
728
83
  T.setHTMLTagEndName(Name);
729
83
730
83
  if (BufferPtr != CommentEnd && 
*BufferPtr == '>'81
)
731
81
    State = LS_HTMLEndTag;
732
83
}
733
734
81
void Lexer::lexHTMLEndTag(Token &T) {
735
81
  assert(BufferPtr != CommentEnd && *BufferPtr == '>');
736
81
737
81
  formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
738
81
  State = LS_Normal;
739
81
}
740
741
Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
742
             const CommandTraits &Traits, SourceLocation FileLoc,
743
             const char *BufferStart, const char *BufferEnd,
744
             bool ParseCommands)
745
    : Allocator(Allocator), Diags(Diags), Traits(Traits),
746
      BufferStart(BufferStart), BufferEnd(BufferEnd), FileLoc(FileLoc),
747
      BufferPtr(BufferStart), CommentState(LCS_BeforeComment), State(LS_Normal),
748
2.24k
      ParseCommands(ParseCommands) {}
749
750
17.1k
void Lexer::lex(Token &T) {
751
23.0k
again:
752
23.0k
  switch (CommentState) {
753
23.0k
  case LCS_BeforeComment:
754
6.28k
    if (BufferPtr == BufferEnd) {
755
2.99k
      formTokenWithChars(T, BufferPtr, tok::eof);
756
2.99k
      return;
757
2.99k
    }
758
3.28k
759
3.28k
    assert(*BufferPtr == '/');
760
3.28k
    BufferPtr++; // Skip first slash.
761
3.28k
    switch(*BufferPtr) {
762
3.28k
    case '/': { // BCPL comment.
763
2.64k
      BufferPtr++; // Skip second slash.
764
2.64k
765
2.64k
      if (BufferPtr != BufferEnd) {
766
2.63k
        // Skip Doxygen magic marker, if it is present.
767
2.63k
        // It might be missing because of a typo //< or /*<, or because we
768
2.63k
        // merged this non-Doxygen comment into a bunch of Doxygen comments
769
2.63k
        // around it: /** ... */ /* ... */ /** ... */
770
2.63k
        const char C = *BufferPtr;
771
2.63k
        if (C == '/' || 
C == '!'302
)
772
2.37k
          BufferPtr++;
773
2.63k
      }
774
2.64k
775
2.64k
      // Skip less-than symbol that marks trailing comments.
776
2.64k
      // Skip it even if the comment is not a Doxygen one, because //< and /*<
777
2.64k
      // are frequent typos.
778
2.64k
      if (BufferPtr != BufferEnd && 
*BufferPtr == '<'2.62k
)
779
100
        BufferPtr++;
780
2.64k
781
2.64k
      CommentState = LCS_InsideBCPLComment;
782
2.64k
      if (State != LS_VerbatimBlockBody && 
State != LS_VerbatimBlockFirstLine2.61k
)
783
2.60k
        State = LS_Normal;
784
2.64k
      CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
785
2.64k
      goto again;
786
3.28k
    }
787
3.28k
    case '*': { // C comment.
788
648
      BufferPtr++; // Skip star.
789
648
790
648
      // Skip Doxygen magic marker.
791
648
      const char C = *BufferPtr;
792
648
      if ((C == '*' && 
*(BufferPtr + 1) != '/'529
) ||
C == '!'120
)
793
623
        BufferPtr++;
794
648
795
648
      // Skip less-than symbol that marks trailing comments.
796
648
      if (BufferPtr != BufferEnd && *BufferPtr == '<')
797
40
        BufferPtr++;
798
648
799
648
      CommentState = LCS_InsideCComment;
800
648
      State = LS_Normal;
801
648
      CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
802
648
      goto again;
803
3.28k
    }
804
3.28k
    default:
805
0
      llvm_unreachable("second character of comment should be '/' or '*'");
806
0
    }
807
0
808
3.21k
  case LCS_BetweenComments: {
809
3.21k
    // Consecutive comments are extracted only if there is only whitespace
810
3.21k
    // between them.  So we can search for the start of the next comment.
811
3.21k
    const char *EndWhitespace = BufferPtr;
812
4.55k
    while(EndWhitespace != BufferEnd && 
*EndWhitespace != '/'2.38k
)
813
1.34k
      EndWhitespace++;
814
3.21k
815
3.21k
    // Turn any whitespace between comments (and there is only whitespace
816
3.21k
    // between them -- guaranteed by comment extraction) into a newline.  We
817
3.21k
    // have two newlines between C comments in total (first one was synthesized
818
3.21k
    // after a comment).
819
3.21k
    formTokenWithChars(T, EndWhitespace, tok::newline);
820
3.21k
821
3.21k
    CommentState = LCS_BeforeComment;
822
3.21k
    break;
823
0
  }
824
0
825
13.5k
  case LCS_InsideBCPLComment:
826
13.5k
  case LCS_InsideCComment:
827
13.5k
    if (BufferPtr != CommentEnd) {
828
10.3k
      lexCommentText(T);
829
10.3k
      break;
830
10.3k
    } else {
831
3.21k
      // Skip C comment closing sequence.
832
3.21k
      if (CommentState == LCS_InsideCComment) {
833
645
        assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
834
645
        BufferPtr += 2;
835
645
        assert(BufferPtr <= BufferEnd);
836
645
837
645
        // Synthenize newline just after the C comment, regardless if there is
838
645
        // actually a newline.
839
645
        formTokenWithChars(T, BufferPtr, tok::newline);
840
645
841
645
        CommentState = LCS_BetweenComments;
842
645
        break;
843
2.56k
      } else {
844
2.56k
        // Don't synthesized a newline after BCPL comment.
845
2.56k
        CommentState = LCS_BetweenComments;
846
2.56k
        goto again;
847
2.56k
      }
848
3.21k
    }
849
23.0k
  }
850
23.0k
}
851
852
StringRef Lexer::getSpelling(const Token &Tok,
853
                             const SourceManager &SourceMgr,
854
28
                             bool *Invalid) const {
855
28
  SourceLocation Loc = Tok.getLocation();
856
28
  std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
857
28
858
28
  bool InvalidTemp = false;
859
28
  StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
860
28
  if (InvalidTemp) {
861
0
    *Invalid = true;
862
0
    return StringRef();
863
0
  }
864
28
865
28
  const char *Begin = File.data() + LocInfo.second;
866
28
  return StringRef(Begin, Tok.getLength());
867
28
}
868
869
} // end namespace comments
870
} // end namespace clang