Coverage Report

Created: 2020-02-15 09:57

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/AST/CommentLexer.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CommentLexer.cpp -------------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/AST/CommentLexer.h"
10
#include "clang/AST/CommentCommandTraits.h"
11
#include "clang/AST/CommentDiagnostic.h"
12
#include "clang/Basic/CharInfo.h"
13
#include "llvm/ADT/StringExtras.h"
14
#include "llvm/ADT/StringSwitch.h"
15
#include "llvm/Support/ConvertUTF.h"
16
#include "llvm/Support/ErrorHandling.h"
17
18
namespace clang {
19
namespace comments {
20
21
0
void Token::dump(const Lexer &L, const SourceManager &SM) const {
22
0
  llvm::errs() << "comments::Token Kind=" << Kind << " ";
23
0
  Loc.print(llvm::errs(), SM);
24
0
  llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
25
0
}
26
27
341
static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
28
341
  return isLetter(C);
29
341
}
30
31
165
static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
32
165
  return isDigit(C);
33
165
}
34
35
101
static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
36
101
  return isHexDigit(C);
37
101
}
38
39
static inline StringRef convertCodePointToUTF8(
40
                                      llvm::BumpPtrAllocator &Allocator,
41
36
                                      unsigned CodePoint) {
42
36
  char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
43
36
  char *ResolvedPtr = Resolved;
44
36
  if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
45
36
    return StringRef(Resolved, ResolvedPtr - Resolved);
46
0
  else
47
0
    return StringRef();
48
36
}
49
50
namespace {
51
52
#include "clang/AST/CommentHTMLTags.inc"
53
#include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
54
55
} // end anonymous namespace
56
57
52
StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
58
52
  // Fast path, first check a few most widely used named character references.
59
52
  return llvm::StringSwitch<StringRef>(Name)
60
52
      .Case("amp", "&")
61
52
      .Case("lt", "<")
62
52
      .Case("gt", ">")
63
52
      .Case("quot", "\"")
64
52
      .Case("apos", "\'")
65
52
      // Slow path.
66
52
      .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
67
52
}
68
69
17
StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
70
17
  unsigned CodePoint = 0;
71
67
  for (unsigned i = 0, e = Name.size(); i != e; 
++i50
) {
72
50
    assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]));
73
50
    CodePoint *= 10;
74
50
    CodePoint += Name[i] - '0';
75
50
  }
76
17
  return convertCodePointToUTF8(Allocator, CodePoint);
77
17
}
78
79
19
StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
80
19
  unsigned CodePoint = 0;
81
57
  for (unsigned i = 0, e = Name.size(); i != e; 
++i38
) {
82
38
    CodePoint *= 16;
83
38
    const char C = Name[i];
84
38
    assert(isHTMLHexCharacterReferenceCharacter(C));
85
38
    CodePoint += llvm::hexDigitValue(C);
86
38
  }
87
19
  return convertCodePointToUTF8(Allocator, CodePoint);
88
19
}
89
90
1.64k
void Lexer::skipLineStartingDecorations() {
91
1.64k
  // This function should be called only for C comments
92
1.64k
  assert(CommentState == LCS_InsideCComment);
93
1.64k
94
1.64k
  if (BufferPtr == CommentEnd)
95
367
    return;
96
1.27k
97
1.27k
  switch (*BufferPtr) {
98
933
  case ' ':
99
933
  case '\t':
100
933
  case '\f':
101
933
  case '\v': {
102
933
    const char *NewBufferPtr = BufferPtr;
103
933
    NewBufferPtr++;
104
933
    if (NewBufferPtr == CommentEnd)
105
111
      return;
106
822
107
822
    char C = *NewBufferPtr;
108
1.15k
    while (isHorizontalWhitespace(C)) {
109
351
      NewBufferPtr++;
110
351
      if (NewBufferPtr == CommentEnd)
111
17
        return;
112
334
      C = *NewBufferPtr;
113
334
    }
114
822
    
if (805
C == '*'805
)
115
672
      BufferPtr = NewBufferPtr + 1;
116
805
    break;
117
822
  }
118
822
  case '*':
119
8
    BufferPtr++;
120
8
    break;
121
1.27k
  }
122
1.27k
}
123
124
namespace {
125
/// Returns pointer to the first newline character in the string.
126
265
const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
127
4.59k
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr4.32k
) {
128
4.44k
    if (isVerticalWhitespace(*BufferPtr))
129
112
      return BufferPtr;
130
4.44k
  }
131
265
  
return BufferEnd153
;
132
265
}
133
134
1.67k
const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
135
1.67k
  if (BufferPtr == BufferEnd)
136
27
    return BufferPtr;
137
1.64k
138
1.64k
  if (*BufferPtr == '\n')
139
1.63k
    BufferPtr++;
140
15
  else {
141
15
    assert(*BufferPtr == '\r');
142
15
    BufferPtr++;
143
15
    if (BufferPtr != BufferEnd && 
*BufferPtr == '\n'13
)
144
8
      BufferPtr++;
145
15
  }
146
1.64k
  return BufferPtr;
147
1.64k
}
148
149
const char *skipNamedCharacterReference(const char *BufferPtr,
150
54
                                        const char *BufferEnd) {
151
243
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr189
) {
152
242
    if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
153
53
      return BufferPtr;
154
242
  }
155
54
  
return BufferEnd1
;
156
54
}
157
158
const char *skipDecimalCharacterReference(const char *BufferPtr,
159
19
                                          const char *BufferEnd) {
160
73
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr54
) {
161
72
    if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
162
18
      return BufferPtr;
163
72
  }
164
19
  
return BufferEnd1
;
165
19
}
166
167
const char *skipHexCharacterReference(const char *BufferPtr,
168
23
                                      const char *BufferEnd) {
169
65
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr42
) {
170
63
    if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
171
21
      return BufferPtr;
172
63
  }
173
23
  
return BufferEnd2
;
174
23
}
175
176
784
bool isHTMLIdentifierStartingCharacter(char C) {
177
784
  return isLetter(C);
178
784
}
179
180
1.40k
bool isHTMLIdentifierCharacter(char C) {
181
1.40k
  return isAlphanumeric(C);
182
1.40k
}
183
184
382
const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
185
1.06k
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr679
) {
186
1.05k
    if (!isHTMLIdentifierCharacter(*BufferPtr))
187
371
      return BufferPtr;
188
1.05k
  }
189
382
  
return BufferEnd11
;
190
382
}
191
192
/// Skip HTML string quoted in single or double quotes.  Escaping quotes inside
193
/// string allowed.
194
///
195
/// Returns pointer to closing quote.
196
const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
197
48
{
198
48
  const char Quote = *BufferPtr;
199
48
  assert(Quote == '\"' || Quote == '\'');
200
48
201
48
  BufferPtr++;
202
434
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr386
) {
203
428
    const char C = *BufferPtr;
204
428
    if (C == Quote && 
BufferPtr[-1] != '\\'48
)
205
42
      return BufferPtr;
206
428
  }
207
48
  
return BufferEnd6
;
208
48
}
209
210
625
const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
211
790
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr165
) {
212
744
    if (!isWhitespace(*BufferPtr))
213
579
      return BufferPtr;
214
744
  }
215
625
  
return BufferEnd46
;
216
625
}
217
218
37
bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
219
37
  return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
220
37
}
221
222
2.96k
bool isCommandNameStartCharacter(char C) {
223
2.96k
  return isLetter(C);
224
2.96k
}
225
226
18.2k
bool isCommandNameCharacter(char C) {
227
18.2k
  return isAlphanumeric(C);
228
18.2k
}
229
230
2.96k
const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
231
18.4k
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr15.5k
) {
232
18.2k
    if (!isCommandNameCharacter(*BufferPtr))
233
2.74k
      return BufferPtr;
234
18.2k
  }
235
2.96k
  
return BufferEnd220
;
236
2.96k
}
237
238
/// Return the one past end pointer for BCPL comments.
239
/// Handles newlines escaped with backslash or trigraph for backslahs.
240
3.13k
const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
241
3.13k
  const char *CurPtr = BufferPtr;
242
3.14k
  while (CurPtr != BufferEnd) {
243
52.8k
    while (!isVerticalWhitespace(*CurPtr)) {
244
51.5k
      CurPtr++;
245
51.5k
      if (CurPtr == BufferEnd)
246
1.86k
        return BufferEnd;
247
51.5k
    }
248
3.11k
    // We found a newline, check if it is escaped.
249
3.11k
    const char *EscapePtr = CurPtr - 1;
250
1.26k
    while(isHorizontalWhitespace(*EscapePtr))
251
12
      EscapePtr--;
252
1.25k
253
1.25k
    if (*EscapePtr == '\\' ||
254
1.25k
        
(1.24k
EscapePtr - 2 >= BufferPtr1.24k
&&
EscapePtr[0] == '/'1.08k
&&
255
1.24k
         
EscapePtr[-1] == '?'3
&&
EscapePtr[-2] == '?'3
)) {
256
9
      // We found an escaped newline.
257
9
      CurPtr = skipNewline(CurPtr, BufferEnd);
258
9
    } else
259
1.24k
      return CurPtr; // Not an escaped newline.
260
1.25k
  }
261
3.13k
  
return BufferEnd23
;
262
3.13k
}
263
264
/// Return the one past end pointer for C comments.
265
/// Very dumb, does not handle escaped newlines or trigraphs.
266
678
const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
267
28.6k
  for ( ; BufferPtr != BufferEnd; 
++BufferPtr27.9k
) {
268
28.6k
    if (*BufferPtr == '*') {
269
1.37k
      assert(BufferPtr + 1 != BufferEnd);
270
1.37k
      if (*(BufferPtr + 1) == '/')
271
678
        return BufferPtr;
272
1.37k
    }
273
28.6k
  }
274
678
  
llvm_unreachable0
("buffer end hit before '*/' was seen");
275
678
}
276
277
} // end anonymous namespace
278
279
void Lexer::formTokenWithChars(Token &Result, const char *TokEnd,
280
20.0k
                               tok::TokenKind Kind) {
281
20.0k
  const unsigned TokLen = TokEnd - BufferPtr;
282
20.0k
  Result.setLocation(getSourceLocation(BufferPtr));
283
20.0k
  Result.setKind(Kind);
284
20.0k
  Result.setLength(TokLen);
285
20.0k
#ifndef NDEBUG
286
20.0k
  Result.TextPtr = "<UNSET>";
287
20.0k
  Result.IntVal = 7;
288
20.0k
#endif
289
20.0k
  BufferPtr = TokEnd;
290
20.0k
}
291
292
11.9k
void Lexer::lexCommentText(Token &T) {
293
11.9k
  assert(CommentState == LCS_InsideBCPLComment ||
294
11.9k
         CommentState == LCS_InsideCComment);
295
11.9k
296
11.9k
  // Handles lexing non-command text, i.e. text and newline.
297
11.9k
  auto HandleNonCommandToken = [&]() -> void {
298
7.76k
    assert(State == LS_Normal);
299
7.76k
300
7.76k
    const char *TokenPtr = BufferPtr;
301
7.76k
    assert(TokenPtr < CommentEnd);
302
7.76k
    switch (*TokenPtr) {
303
1.60k
      case '\n':
304
1.60k
      case '\r':
305
1.60k
          TokenPtr = skipNewline(TokenPtr, CommentEnd);
306
1.60k
          formTokenWithChars(T, TokenPtr, tok::newline);
307
1.60k
308
1.60k
          if (CommentState == LCS_InsideCComment)
309
1.59k
            skipLineStartingDecorations();
310
1.60k
          return;
311
1.60k
312
6.16k
      default: {
313
6.16k
          StringRef TokStartSymbols = ParseCommands ? 
"\n\r\\@&<"6.13k
:
"\n\r"28
;
314
6.16k
          size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr)
315
6.16k
                           .find_first_of(TokStartSymbols);
316
6.16k
          if (End != StringRef::npos)
317
3.50k
            TokenPtr += End;
318
2.65k
          else
319
2.65k
            TokenPtr = CommentEnd;
320
6.16k
          formTextToken(T, TokenPtr);
321
6.16k
          return;
322
1.60k
      }
323
7.76k
    }
324
7.76k
  };
325
11.9k
326
11.9k
  if (!ParseCommands)
327
36
    return HandleNonCommandToken();
328
11.8k
329
11.8k
  switch (State) {
330
11.1k
  case LS_Normal:
331
11.1k
    break;
332
45
  case LS_VerbatimBlockFirstLine:
333
45
    lexVerbatimBlockFirstLine(T);
334
45
    return;
335
75
  case LS_VerbatimBlockBody:
336
75
    lexVerbatimBlockBody(T);
337
75
    return;
338
132
  case LS_VerbatimLineText:
339
132
    lexVerbatimLineText(T);
340
132
    return;
341
352
  case LS_HTMLStartTag:
342
352
    lexHTMLStartTag(T);
343
352
    return;
344
92
  case LS_HTMLEndTag:
345
92
    lexHTMLEndTag(T);
346
92
    return;
347
11.1k
  }
348
11.1k
349
11.1k
  assert(State == LS_Normal);
350
11.1k
  const char *TokenPtr = BufferPtr;
351
11.1k
  assert(TokenPtr < CommentEnd);
352
11.1k
  switch(*TokenPtr) {
353
3.04k
    case '\\':
354
3.04k
    case '@': {
355
3.04k
      // Commands that start with a backslash and commands that start with
356
3.04k
      // 'at' have equivalent semantics.  But we keep information about the
357
3.04k
      // exact syntax in AST for comments.
358
3.04k
      tok::TokenKind CommandKind =
359
3.04k
          (*TokenPtr == '@') ? 
tok::at_command535
:
tok::backslash_command2.51k
;
360
3.04k
      TokenPtr++;
361
3.04k
      if (TokenPtr == CommentEnd) {
362
12
        formTextToken(T, TokenPtr);
363
12
        return;
364
12
      }
365
3.03k
      char C = *TokenPtr;
366
3.03k
      switch (C) {
367
2.96k
      default:
368
2.96k
        break;
369
0
370
68
      case '\\': case '@': case '&': case '$':
371
68
      case '#':  case '<': case '>': case '%':
372
68
      case '\"': case '.': case ':':
373
68
        // This is one of \\ \@ \& \$ etc escape sequences.
374
68
        TokenPtr++;
375
68
        if (C == ':' && 
TokenPtr != CommentEnd6
&&
*TokenPtr == ':'6
) {
376
6
          // This is the \:: escape sequence.
377
6
          TokenPtr++;
378
6
        }
379
68
        StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
380
68
        formTokenWithChars(T, TokenPtr, tok::text);
381
68
        T.setText(UnescapedText);
382
68
        return;
383
2.96k
      }
384
2.96k
385
2.96k
      // Don't make zero-length commands.
386
2.96k
      if (!isCommandNameStartCharacter(*TokenPtr)) {
387
8
        formTextToken(T, TokenPtr);
388
8
        return;
389
8
      }
390
2.96k
391
2.96k
      TokenPtr = skipCommandName(TokenPtr, CommentEnd);
392
2.96k
      unsigned Length = TokenPtr - (BufferPtr + 1);
393
2.96k
394
2.96k
      // Hardcoded support for lexing LaTeX formula commands
395
2.96k
      // \f$ \f[ \f] \f{ \f} as a single command.
396
2.96k
      if (Length == 1 && 
TokenPtr[-1] == 'f'155
&&
TokenPtr != CommentEnd17
) {
397
17
        C = *TokenPtr;
398
17
        if (C == '$' || 
C == '['4
||
C == ']'3
||
C == '{'3
||
C == '}'2
) {
399
15
          TokenPtr++;
400
15
          Length++;
401
15
        }
402
17
      }
403
2.96k
404
2.96k
      StringRef CommandName(BufferPtr + 1, Length);
405
2.96k
406
2.96k
      const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
407
2.96k
      if (!Info) {
408
310
        if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
409
11
          StringRef CorrectedName = Info->Name;
410
11
          SourceLocation Loc = getSourceLocation(BufferPtr);
411
11
          SourceLocation EndLoc = getSourceLocation(TokenPtr);
412
11
          SourceRange FullRange = SourceRange(Loc, EndLoc);
413
11
          SourceRange CommandRange(Loc.getLocWithOffset(1), EndLoc);
414
11
          Diag(Loc, diag::warn_correct_comment_command_name)
415
11
            << FullRange << CommandName << CorrectedName
416
11
            << FixItHint::CreateReplacement(CommandRange, CorrectedName);
417
299
        } else {
418
299
          formTokenWithChars(T, TokenPtr, tok::unknown_command);
419
299
          T.setUnknownCommandName(CommandName);
420
299
          Diag(T.getLocation(), diag::warn_unknown_comment_command_name)
421
299
              << SourceRange(T.getLocation(), T.getEndLocation());
422
299
          return;
423
299
        }
424
2.66k
      }
425
2.66k
      if (Info->IsVerbatimBlockCommand) {
426
57
        setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
427
57
        return;
428
57
      }
429
2.60k
      if (Info->IsVerbatimLineCommand) {
430
136
        setupAndLexVerbatimLine(T, TokenPtr, Info);
431
136
        return;
432
136
      }
433
2.46k
      formTokenWithChars(T, TokenPtr, CommandKind);
434
2.46k
      T.setCommandID(Info->getID());
435
2.46k
      return;
436
2.46k
    }
437
2.46k
438
2.46k
    case '&':
439
100
      lexHTMLCharacterReference(T);
440
100
      return;
441
2.46k
442
2.46k
    case '<': {
443
319
      TokenPtr++;
444
319
      if (TokenPtr == CommentEnd) {
445
1
        formTextToken(T, TokenPtr);
446
1
        return;
447
1
      }
448
318
      const char C = *TokenPtr;
449
318
      if (isHTMLIdentifierStartingCharacter(C))
450
199
        setupAndLexHTMLStartTag(T);
451
119
      else if (C == '/')
452
97
        setupAndLexHTMLEndTag(T);
453
22
      else
454
22
        formTextToken(T, TokenPtr);
455
318
      return;
456
318
    }
457
318
458
7.72k
    default:
459
7.72k
      return HandleNonCommandToken();
460
11.1k
  }
461
11.1k
}
462
463
void Lexer::setupAndLexVerbatimBlock(Token &T,
464
                                     const char *TextBegin,
465
57
                                     char Marker, const CommandInfo *Info) {
466
57
  assert(Info->IsVerbatimBlockCommand);
467
57
468
57
  VerbatimBlockEndCommandName.clear();
469
57
  VerbatimBlockEndCommandName.append(Marker == '\\' ? 
"\\"51
:
"@"6
);
470
57
  VerbatimBlockEndCommandName.append(Info->EndCommandName);
471
57
472
57
  formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
473
57
  T.setVerbatimBlockID(Info->getID());
474
57
475
57
  // If there is a newline following the verbatim opening command, skip the
476
57
  // newline so that we don't create an tok::verbatim_block_line with empty
477
57
  // text content.
478
57
  if (BufferPtr != CommentEnd &&
479
57
      
isVerticalWhitespace(*BufferPtr)41
) {
480
10
    BufferPtr = skipNewline(BufferPtr, CommentEnd);
481
10
    State = LS_VerbatimBlockBody;
482
10
    return;
483
10
  }
484
47
485
47
  State = LS_VerbatimBlockFirstLine;
486
47
}
487
488
116
void Lexer::lexVerbatimBlockFirstLine(Token &T) {
489
133
again:
490
133
  assert(BufferPtr < CommentEnd);
491
133
492
133
  // FIXME: It would be better to scan the text once, finding either the block
493
133
  // end command or newline.
494
133
  //
495
133
  // Extract current line.
496
133
  const char *Newline = findNewline(BufferPtr, CommentEnd);
497
133
  StringRef Line(BufferPtr, Newline - BufferPtr);
498
133
499
133
  // Look for end command in current line.
500
133
  size_t Pos = Line.find(VerbatimBlockEndCommandName);
501
133
  const char *TextEnd;
502
133
  const char *NextLine;
503
133
  if (Pos == StringRef::npos) {
504
52
    // Current line is completely verbatim.
505
52
    TextEnd = Newline;
506
52
    NextLine = skipNewline(Newline, CommentEnd);
507
81
  } else if (Pos == 0) {
508
44
    // Current line contains just an end command.
509
44
    const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
510
44
    StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
511
44
    formTokenWithChars(T, End, tok::verbatim_block_end);
512
44
    T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
513
44
    State = LS_Normal;
514
44
    return;
515
44
  } else {
516
37
    // There is some text, followed by end command.  Extract text first.
517
37
    TextEnd = BufferPtr + Pos;
518
37
    NextLine = TextEnd;
519
37
    // If there is only whitespace before end command, skip whitespace.
520
37
    if (isWhitespace(BufferPtr, TextEnd)) {
521
17
      BufferPtr = TextEnd;
522
17
      goto again;
523
17
    }
524
72
  }
525
72
526
72
  StringRef Text(BufferPtr, TextEnd - BufferPtr);
527
72
  formTokenWithChars(T, NextLine, tok::verbatim_block_line);
528
72
  T.setVerbatimBlockText(Text);
529
72
530
72
  State = LS_VerbatimBlockBody;
531
72
}
532
533
75
void Lexer::lexVerbatimBlockBody(Token &T) {
534
75
  assert(State == LS_VerbatimBlockBody);
535
75
536
75
  if (CommentState == LCS_InsideCComment)
537
50
    skipLineStartingDecorations();
538
75
539
75
  if (BufferPtr == CommentEnd) {
540
4
    formTokenWithChars(T, BufferPtr, tok::verbatim_block_line);
541
4
    T.setVerbatimBlockText("");
542
4
    return;
543
4
  }
544
71
545
71
  lexVerbatimBlockFirstLine(T);
546
71
}
547
548
void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
549
136
                                    const CommandInfo *Info) {
550
136
  assert(Info->IsVerbatimLineCommand);
551
136
  formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
552
136
  T.setVerbatimLineID(Info->getID());
553
136
554
136
  State = LS_VerbatimLineText;
555
136
}
556
557
132
void Lexer::lexVerbatimLineText(Token &T) {
558
132
  assert(State == LS_VerbatimLineText);
559
132
560
132
  // Extract current line.
561
132
  const char *Newline = findNewline(BufferPtr, CommentEnd);
562
132
  StringRef Text(BufferPtr, Newline - BufferPtr);
563
132
  formTokenWithChars(T, Newline, tok::verbatim_line_text);
564
132
  T.setVerbatimLineText(Text);
565
132
566
132
  State = LS_Normal;
567
132
}
568
569
100
void Lexer::lexHTMLCharacterReference(Token &T) {
570
100
  const char *TokenPtr = BufferPtr;
571
100
  assert(*TokenPtr == '&');
572
100
  TokenPtr++;
573
100
  if (TokenPtr == CommentEnd) {
574
1
    formTextToken(T, TokenPtr);
575
1
    return;
576
1
  }
577
99
  const char *NamePtr;
578
99
  bool isNamed = false;
579
99
  bool isDecimal = false;
580
99
  char C = *TokenPtr;
581
99
  if (isHTMLNamedCharacterReferenceCharacter(C)) {
582
54
    NamePtr = TokenPtr;
583
54
    TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
584
54
    isNamed = true;
585
54
  } else 
if (45
C == '#'45
) {
586
44
    TokenPtr++;
587
44
    if (TokenPtr == CommentEnd) {
588
1
      formTextToken(T, TokenPtr);
589
1
      return;
590
1
    }
591
43
    C = *TokenPtr;
592
43
    if (isHTMLDecimalCharacterReferenceCharacter(C)) {
593
19
      NamePtr = TokenPtr;
594
19
      TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
595
19
      isDecimal = true;
596
24
    } else if (C == 'x' || 
C == 'X'7
) {
597
23
      TokenPtr++;
598
23
      NamePtr = TokenPtr;
599
23
      TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
600
23
    } else {
601
1
      formTextToken(T, TokenPtr);
602
1
      return;
603
1
    }
604
1
  } else {
605
1
    formTextToken(T, TokenPtr);
606
1
    return;
607
1
  }
608
96
  if (NamePtr == TokenPtr || 
TokenPtr == CommentEnd94
||
609
96
      
*TokenPtr != ';'91
) {
610
8
    formTextToken(T, TokenPtr);
611
8
    return;
612
8
  }
613
88
  StringRef Name(NamePtr, TokenPtr - NamePtr);
614
88
  TokenPtr++; // Skip semicolon.
615
88
  StringRef Resolved;
616
88
  if (isNamed)
617
52
    Resolved = resolveHTMLNamedCharacterReference(Name);
618
36
  else if (isDecimal)
619
17
    Resolved = resolveHTMLDecimalCharacterReference(Name);
620
19
  else
621
19
    Resolved = resolveHTMLHexCharacterReference(Name);
622
88
623
88
  if (Resolved.empty()) {
624
0
    formTextToken(T, TokenPtr);
625
0
    return;
626
0
  }
627
88
  formTokenWithChars(T, TokenPtr, tok::text);
628
88
  T.setText(Resolved);
629
88
}
630
631
199
void Lexer::setupAndLexHTMLStartTag(Token &T) {
632
199
  assert(BufferPtr[0] == '<' &&
633
199
         isHTMLIdentifierStartingCharacter(BufferPtr[1]));
634
199
  const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
635
199
  StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
636
199
  if (!isHTMLTagName(Name)) {
637
8
    formTextToken(T, TagNameEnd);
638
8
    return;
639
8
  }
640
191
641
191
  formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
642
191
  T.setHTMLTagStartName(Name);
643
191
644
191
  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
645
191
646
191
  const char C = *BufferPtr;
647
191
  if (BufferPtr != CommentEnd &&
648
191
      
(188
C == '>'188
||
C == '/'99
||
isHTMLIdentifierStartingCharacter(C)84
))
649
178
    State = LS_HTMLStartTag;
650
191
}
651
652
352
void Lexer::lexHTMLStartTag(Token &T) {
653
352
  assert(State == LS_HTMLStartTag);
654
352
655
352
  const char *TokenPtr = BufferPtr;
656
352
  char C = *TokenPtr;
657
352
  if (isHTMLIdentifierCharacter(C)) {
658
86
    TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
659
86
    StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
660
86
    formTokenWithChars(T, TokenPtr, tok::html_ident);
661
86
    T.setHTMLIdent(Ident);
662
266
  } else {
663
266
    switch (C) {
664
72
    case '=':
665
72
      TokenPtr++;
666
72
      formTokenWithChars(T, TokenPtr, tok::html_equals);
667
72
      break;
668
48
    case '\"':
669
48
    case '\'': {
670
48
      const char *OpenQuote = TokenPtr;
671
48
      TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
672
48
      const char *ClosingQuote = TokenPtr;
673
48
      if (TokenPtr != CommentEnd) // Skip closing quote.
674
42
        TokenPtr++;
675
48
      formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
676
48
      T.setHTMLQuotedString(StringRef(OpenQuote + 1,
677
48
                                      ClosingQuote - (OpenQuote + 1)));
678
48
      break;
679
48
    }
680
131
    case '>':
681
131
      TokenPtr++;
682
131
      formTokenWithChars(T, TokenPtr, tok::html_greater);
683
131
      State = LS_Normal;
684
131
      return;
685
48
    case '/':
686
15
      TokenPtr++;
687
15
      if (TokenPtr != CommentEnd && *TokenPtr == '>') {
688
13
        TokenPtr++;
689
13
        formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
690
13
      } else
691
2
        formTextToken(T, TokenPtr);
692
15
693
15
      State = LS_Normal;
694
15
      return;
695
206
    }
696
206
  }
697
206
698
206
  // Now look ahead and return to normal state if we don't see any HTML tokens
699
206
  // ahead.
700
206
  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
701
206
  if (BufferPtr == CommentEnd) {
702
23
    State = LS_Normal;
703
23
    return;
704
23
  }
705
183
706
183
  C = *BufferPtr;
707
183
  if (!isHTMLIdentifierStartingCharacter(C) &&
708
183
      
C != '='171
&&
C != '\"'99
&&
C != '\''56
&&
C != '>'51
) {
709
9
    State = LS_Normal;
710
9
    return;
711
9
  }
712
183
}
713
714
97
void Lexer::setupAndLexHTMLEndTag(Token &T) {
715
97
  assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
716
97
717
97
  const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
718
97
  const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
719
97
  StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
720
97
  if (!isHTMLTagName(Name)) {
721
3
    formTextToken(T, TagNameEnd);
722
3
    return;
723
3
  }
724
94
725
94
  const char *End = skipWhitespace(TagNameEnd, CommentEnd);
726
94
727
94
  formTokenWithChars(T, End, tok::html_end_tag);
728
94
  T.setHTMLTagEndName(Name);
729
94
730
94
  if (BufferPtr != CommentEnd && 
*BufferPtr == '>'92
)
731
92
    State = LS_HTMLEndTag;
732
94
}
733
734
92
void Lexer::lexHTMLEndTag(Token &T) {
735
92
  assert(BufferPtr != CommentEnd && *BufferPtr == '>');
736
92
737
92
  formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
738
92
  State = LS_Normal;
739
92
}
740
741
Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
742
             const CommandTraits &Traits, SourceLocation FileLoc,
743
             const char *BufferStart, const char *BufferEnd,
744
             bool ParseCommands)
745
    : Allocator(Allocator), Diags(Diags), Traits(Traits),
746
      BufferStart(BufferStart), BufferEnd(BufferEnd), FileLoc(FileLoc),
747
      BufferPtr(BufferStart), CommentState(LCS_BeforeComment), State(LS_Normal),
748
2.61k
      ParseCommands(ParseCommands) {}
749
750
20.0k
void Lexer::lex(Token &T) {
751
26.8k
again:
752
26.8k
  switch (CommentState) {
753
7.49k
  case LCS_BeforeComment:
754
7.49k
    if (BufferPtr == BufferEnd) {
755
3.68k
      formTokenWithChars(T, BufferPtr, tok::eof);
756
3.68k
      return;
757
3.68k
    }
758
3.81k
759
3.81k
    assert(*BufferPtr == '/');
760
3.81k
    BufferPtr++; // Skip first slash.
761
3.81k
    switch(*BufferPtr) {
762
3.13k
    case '/': { // BCPL comment.
763
3.13k
      BufferPtr++; // Skip second slash.
764
3.13k
765
3.13k
      if (BufferPtr != BufferEnd) {
766
3.12k
        // Skip Doxygen magic marker, if it is present.
767
3.12k
        // It might be missing because of a typo //< or /*<, or because we
768
3.12k
        // merged this non-Doxygen comment into a bunch of Doxygen comments
769
3.12k
        // around it: /** ... */ /* ... */ /** ... */
770
3.12k
        const char C = *BufferPtr;
771
3.12k
        if (C == '/' || 
C == '!'302
)
772
2.86k
          BufferPtr++;
773
3.12k
      }
774
3.13k
775
3.13k
      // Skip less-than symbol that marks trailing comments.
776
3.13k
      // Skip it even if the comment is not a Doxygen one, because //< and /*<
777
3.13k
      // are frequent typos.
778
3.13k
      if (BufferPtr != BufferEnd && 
*BufferPtr == '<'3.11k
)
779
134
        BufferPtr++;
780
3.13k
781
3.13k
      CommentState = LCS_InsideBCPLComment;
782
3.13k
      if (State != LS_VerbatimBlockBody && 
State != LS_VerbatimBlockFirstLine3.11k
)
783
3.09k
        State = LS_Normal;
784
3.13k
      CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
785
3.13k
      goto again;
786
0
    }
787
678
    case '*': { // C comment.
788
678
      BufferPtr++; // Skip star.
789
678
790
678
      // Skip Doxygen magic marker.
791
678
      const char C = *BufferPtr;
792
678
      if ((C == '*' && 
*(BufferPtr + 1) != '/'546
) ||
C == '!'133
)
793
653
        BufferPtr++;
794
678
795
678
      // Skip less-than symbol that marks trailing comments.
796
678
      if (BufferPtr != BufferEnd && *BufferPtr == '<')
797
40
        BufferPtr++;
798
678
799
678
      CommentState = LCS_InsideCComment;
800
678
      State = LS_Normal;
801
678
      CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
802
678
      goto again;
803
0
    }
804
0
    default:
805
0
      llvm_unreachable("second character of comment should be '/' or '*'");
806
0
    }
807
0
808
3.73k
  case LCS_BetweenComments: {
809
3.73k
    // Consecutive comments are extracted only if there is only whitespace
810
3.73k
    // between them.  So we can search for the start of the next comment.
811
3.73k
    const char *EndWhitespace = BufferPtr;
812
5.23k
    while(EndWhitespace != BufferEnd && 
*EndWhitespace != '/'2.69k
)
813
1.50k
      EndWhitespace++;
814
3.73k
815
3.73k
    // Turn any whitespace between comments (and there is only whitespace
816
3.73k
    // between them -- guaranteed by comment extraction) into a newline.  We
817
3.73k
    // have two newlines between C comments in total (first one was synthesized
818
3.73k
    // after a comment).
819
3.73k
    formTokenWithChars(T, EndWhitespace, tok::newline);
820
3.73k
821
3.73k
    CommentState = LCS_BeforeComment;
822
3.73k
    break;
823
0
  }
824
0
825
15.6k
  case LCS_InsideBCPLComment:
826
15.6k
  case LCS_InsideCComment:
827
15.6k
    if (BufferPtr != CommentEnd) {
828
11.9k
      lexCommentText(T);
829
11.9k
      break;
830
11.9k
    } else {
831
3.73k
      // Skip C comment closing sequence.
832
3.73k
      if (CommentState == LCS_InsideCComment) {
833
675
        assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
834
675
        BufferPtr += 2;
835
675
        assert(BufferPtr <= BufferEnd);
836
675
837
675
        // Synthenize newline just after the C comment, regardless if there is
838
675
        // actually a newline.
839
675
        formTokenWithChars(T, BufferPtr, tok::newline);
840
675
841
675
        CommentState = LCS_BetweenComments;
842
675
        break;
843
3.05k
      } else {
844
3.05k
        // Don't synthesized a newline after BCPL comment.
845
3.05k
        CommentState = LCS_BetweenComments;
846
3.05k
        goto again;
847
3.05k
      }
848
3.73k
    }
849
26.8k
  }
850
26.8k
}
851
852
StringRef Lexer::getSpelling(const Token &Tok,
853
28
                             const SourceManager &SourceMgr) const {
854
28
  SourceLocation Loc = Tok.getLocation();
855
28
  std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
856
28
857
28
  bool InvalidTemp = false;
858
28
  StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
859
28
  if (InvalidTemp)
860
0
    return StringRef();
861
28
862
28
  const char *Begin = File.data() + LocInfo.second;
863
28
  return StringRef(Begin, Tok.getLength());
864
28
}
865
866
} // end namespace comments
867
} // end namespace clang