Coverage Report

Created: 2022-07-16 07:03

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/AST/CommentParser.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CommentParser.cpp - Doxygen comment parser -----------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/AST/CommentParser.h"
10
#include "clang/AST/CommentCommandTraits.h"
11
#include "clang/AST/CommentDiagnostic.h"
12
#include "clang/AST/CommentSema.h"
13
#include "clang/Basic/CharInfo.h"
14
#include "clang/Basic/SourceManager.h"
15
#include "llvm/Support/ErrorHandling.h"
16
17
namespace clang {
18
19
8.88k
static inline bool isWhitespace(llvm::StringRef S) {
20
34.5k
  for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; 
++I25.6k
) {
21
31.1k
    if (!isWhitespace(*I))
22
5.50k
      return false;
23
31.1k
  }
24
3.38k
  return true;
25
8.88k
}
26
27
namespace comments {
28
29
/// Re-lexes a sequence of tok::text tokens.
30
class TextTokenRetokenizer {
31
  llvm::BumpPtrAllocator &Allocator;
32
  Parser &P;
33
34
  /// This flag is set when there are no more tokens we can fetch from lexer.
35
  bool NoMoreInterestingTokens;
36
37
  /// Token buffer: tokens we have processed and lookahead.
38
  SmallVector<Token, 16> Toks;
39
40
  /// A position in \c Toks.
41
  struct Position {
42
    const char *BufferStart;
43
    const char *BufferEnd;
44
    const char *BufferPtr;
45
    SourceLocation BufferStartLoc;
46
    unsigned CurToken;
47
  };
48
49
  /// Current position in Toks.
50
  Position Pos;
51
52
129k
  bool isEnd() const {
53
129k
    return Pos.CurToken >= Toks.size();
54
129k
  }
55
56
  /// Sets up the buffer pointers to point to current token.
57
8.21k
  void setupBuffer() {
58
8.21k
    assert(!isEnd());
59
0
    const Token &Tok = Toks[Pos.CurToken];
60
61
8.21k
    Pos.BufferStart = Tok.getText().begin();
62
8.21k
    Pos.BufferEnd = Tok.getText().end();
63
8.21k
    Pos.BufferPtr = Pos.BufferStart;
64
8.21k
    Pos.BufferStartLoc = Tok.getLocation();
65
8.21k
  }
66
67
9.02k
  SourceLocation getSourceLocation() const {
68
9.02k
    const unsigned CharNo = Pos.BufferPtr - Pos.BufferStart;
69
9.02k
    return Pos.BufferStartLoc.getLocWithOffset(CharNo);
70
9.02k
  }
71
72
38.9k
  char peek() const {
73
38.9k
    assert(!isEnd());
74
0
    assert(Pos.BufferPtr != Pos.BufferEnd);
75
0
    return *Pos.BufferPtr;
76
38.9k
  }
77
78
23.3k
  void consumeChar() {
79
23.3k
    assert(!isEnd());
80
0
    assert(Pos.BufferPtr != Pos.BufferEnd);
81
0
    Pos.BufferPtr++;
82
23.3k
    if (Pos.BufferPtr == Pos.BufferEnd) {
83
3.38k
      Pos.CurToken++;
84
3.38k
      if (isEnd() && 
!addToken()3.38k
)
85
160
        return;
86
87
3.22k
      assert(!isEnd());
88
0
      setupBuffer();
89
3.22k
    }
90
23.3k
  }
91
92
  /// Add a token.
93
  /// Returns true on success, false if there are no interesting tokens to
94
  /// fetch from lexer.
95
8.40k
  bool addToken() {
96
8.40k
    if (NoMoreInterestingTokens)
97
3
      return false;
98
99
8.39k
    if (P.Tok.is(tok::newline)) {
100
      // If we see a single newline token between text tokens, skip it.
101
3.93k
      Token Newline = P.Tok;
102
3.93k
      P.consumeToken();
103
3.93k
      if (P.Tok.isNot(tok::text)) {
104
172
        P.putBack(Newline);
105
172
        NoMoreInterestingTokens = true;
106
172
        return false;
107
172
      }
108
3.93k
    }
109
8.22k
    if (P.Tok.isNot(tok::text)) {
110
13
      NoMoreInterestingTokens = true;
111
13
      return false;
112
13
    }
113
114
8.21k
    Toks.push_back(P.Tok);
115
8.21k
    P.consumeToken();
116
8.21k
    if (Toks.size() == 1)
117
4.98k
      setupBuffer();
118
8.21k
    return true;
119
8.22k
  }
120
121
7.93k
  void consumeWhitespace() {
122
15.9k
    while (!isEnd()) {
123
15.8k
      if (isWhitespace(peek()))
124
7.96k
        consumeChar();
125
7.91k
      else
126
7.91k
        break;
127
15.8k
    }
128
7.93k
  }
129
130
  void formTokenWithChars(Token &Result,
131
                          SourceLocation Loc,
132
                          const char *TokBegin,
133
                          unsigned TokLength,
134
5.57k
                          StringRef Text) {
135
5.57k
    Result.setLocation(Loc);
136
5.57k
    Result.setKind(tok::text);
137
5.57k
    Result.setLength(TokLength);
138
5.57k
#ifndef NDEBUG
139
5.57k
    Result.TextPtr = "<UNSET>";
140
5.57k
    Result.IntVal = 7;
141
5.57k
#endif
142
5.57k
    Result.setText(Text);
143
5.57k
  }
144
145
public:
146
  TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
147
5.01k
      Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
148
5.01k
    Pos.CurToken = 0;
149
5.01k
    addToken();
150
5.01k
  }
151
152
  /// Extract a word -- sequence of non-whitespace characters.
153
4.47k
  bool lexWord(Token &Tok) {
154
4.47k
    if (isEnd())
155
24
      return false;
156
157
4.44k
    Position SavedPos = Pos;
158
159
4.44k
    consumeWhitespace();
160
4.44k
    SmallString<32> WordText;
161
4.44k
    const char *WordBegin = Pos.BufferPtr;
162
4.44k
    SourceLocation Loc = getSourceLocation();
163
19.5k
    while (!isEnd()) {
164
19.3k
      const char C = peek();
165
19.3k
      if (!isWhitespace(C)) {
166
15.0k
        WordText.push_back(C);
167
15.0k
        consumeChar();
168
15.0k
      } else
169
4.28k
        break;
170
19.3k
    }
171
4.44k
    const unsigned Length = WordText.size();
172
4.44k
    if (Length == 0) {
173
11
      Pos = SavedPos;
174
11
      return false;
175
11
    }
176
177
4.43k
    char *TextPtr = Allocator.Allocate<char>(Length + 1);
178
179
4.43k
    memcpy(TextPtr, WordText.c_str(), Length + 1);
180
4.43k
    StringRef Text = StringRef(TextPtr, Length);
181
182
4.43k
    formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
183
4.43k
    return true;
184
4.44k
  }
185
186
3.48k
  bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) {
187
3.48k
    if (isEnd())
188
2
      return false;
189
190
3.48k
    Position SavedPos = Pos;
191
192
3.48k
    consumeWhitespace();
193
3.48k
    SmallString<32> WordText;
194
3.48k
    const char *WordBegin = Pos.BufferPtr;
195
3.48k
    SourceLocation Loc = getSourceLocation();
196
3.48k
    bool Error = false;
197
3.48k
    if (!isEnd()) {
198
3.48k
      const char C = peek();
199
3.48k
      if (C == OpenDelim) {
200
54
        WordText.push_back(C);
201
54
        consumeChar();
202
54
      } else
203
3.43k
        Error = true;
204
3.48k
    }
205
3.48k
    char C = '\0';
206
3.70k
    while (!Error && 
!isEnd()270
) {
207
267
      C = peek();
208
267
      WordText.push_back(C);
209
267
      consumeChar();
210
267
      if (C == CloseDelim)
211
54
        break;
212
267
    }
213
3.48k
    if (!Error && 
C != CloseDelim57
)
214
3
      Error = true;
215
216
3.48k
    if (Error) {
217
3.43k
      Pos = SavedPos;
218
3.43k
      return false;
219
3.43k
    }
220
221
54
    const unsigned Length = WordText.size();
222
54
    char *TextPtr = Allocator.Allocate<char>(Length + 1);
223
224
54
    memcpy(TextPtr, WordText.c_str(), Length + 1);
225
54
    StringRef Text = StringRef(TextPtr, Length);
226
227
54
    formTokenWithChars(Tok, Loc, WordBegin,
228
54
                       Pos.BufferPtr - WordBegin, Text);
229
54
    return true;
230
3.48k
  }
231
232
  /// Put back tokens that we didn't consume.
233
5.01k
  void putBackLeftoverTokens() {
234
5.01k
    if (isEnd())
235
174
      return;
236
237
4.84k
    bool HavePartialTok = false;
238
4.84k
    Token PartialTok;
239
4.84k
    if (Pos.BufferPtr != Pos.BufferStart) {
240
1.08k
      formTokenWithChars(PartialTok, getSourceLocation(),
241
1.08k
                         Pos.BufferPtr, Pos.BufferEnd - Pos.BufferPtr,
242
1.08k
                         StringRef(Pos.BufferPtr,
243
1.08k
                                   Pos.BufferEnd - Pos.BufferPtr));
244
1.08k
      HavePartialTok = true;
245
1.08k
      Pos.CurToken++;
246
1.08k
    }
247
248
4.84k
    P.putBack(llvm::makeArrayRef(Toks.begin() + Pos.CurToken, Toks.end()));
249
4.84k
    Pos.CurToken = Toks.size();
250
251
4.84k
    if (HavePartialTok)
252
1.08k
      P.putBack(PartialTok);
253
4.84k
  }
254
};
255
256
Parser::Parser(Lexer &L, Sema &S, llvm::BumpPtrAllocator &Allocator,
257
               const SourceManager &SourceMgr, DiagnosticsEngine &Diags,
258
               const CommandTraits &Traits):
259
    L(L), S(S), Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags),
260
3.29k
    Traits(Traits) {
261
3.29k
  consumeToken();
262
3.29k
}
263
264
void Parser::parseParamCommandArgs(ParamCommandComment *PC,
265
3.48k
                                   TextTokenRetokenizer &Retokenizer) {
266
3.48k
  Token Arg;
267
  // Check if argument looks like direction specification: [dir]
268
  // e.g., [in], [out], [in,out]
269
3.48k
  if (Retokenizer.lexDelimitedSeq(Arg, '[', ']'))
270
54
    S.actOnParamCommandDirectionArg(PC,
271
54
                                    Arg.getLocation(),
272
54
                                    Arg.getEndLocation(),
273
54
                                    Arg.getText());
274
275
3.48k
  if (Retokenizer.lexWord(Arg))
276
3.48k
    S.actOnParamCommandParamNameArg(PC,
277
3.48k
                                    Arg.getLocation(),
278
3.48k
                                    Arg.getEndLocation(),
279
3.48k
                                    Arg.getText());
280
3.48k
}
281
282
void Parser::parseTParamCommandArgs(TParamCommandComment *TPC,
283
206
                                    TextTokenRetokenizer &Retokenizer) {
284
206
  Token Arg;
285
206
  if (Retokenizer.lexWord(Arg))
286
199
    S.actOnTParamCommandParamNameArg(TPC,
287
199
                                     Arg.getLocation(),
288
199
                                     Arg.getEndLocation(),
289
199
                                     Arg.getText());
290
206
}
291
292
ArrayRef<Comment::Argument>
293
1.32k
Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs) {
294
1.32k
  auto *Args = new (Allocator.Allocate<Comment::Argument>(NumArgs))
295
1.32k
      Comment::Argument[NumArgs];
296
1.32k
  unsigned ParsedArgs = 0;
297
1.32k
  Token Arg;
298
2.07k
  while (ParsedArgs < NumArgs && 
Retokenizer.lexWord(Arg)775
) {
299
752
    Args[ParsedArgs] = Comment::Argument{
300
752
        SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
301
752
    ParsedArgs++;
302
752
  }
303
304
1.32k
  return llvm::makeArrayRef(Args, ParsedArgs);
305
1.32k
}
306
307
7.64k
BlockCommandComment *Parser::parseBlockCommand() {
308
7.64k
  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
309
310
0
  ParamCommandComment *PC = nullptr;
311
7.64k
  TParamCommandComment *TPC = nullptr;
312
7.64k
  BlockCommandComment *BC = nullptr;
313
7.64k
  const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
314
7.64k
  CommandMarkerKind CommandMarker =
315
7.64k
      Tok.is(tok::backslash_command) ? 
CMK_Backslash7.48k
:
CMK_At155
;
316
7.64k
  if (Info->IsParamCommand) {
317
3.49k
    PC = S.actOnParamCommandStart(Tok.getLocation(),
318
3.49k
                                  Tok.getEndLocation(),
319
3.49k
                                  Tok.getCommandID(),
320
3.49k
                                  CommandMarker);
321
4.15k
  } else if (Info->IsTParamCommand) {
322
207
    TPC = S.actOnTParamCommandStart(Tok.getLocation(),
323
207
                                    Tok.getEndLocation(),
324
207
                                    Tok.getCommandID(),
325
207
                                    CommandMarker);
326
3.94k
  } else {
327
3.94k
    BC = S.actOnBlockCommandStart(Tok.getLocation(),
328
3.94k
                                  Tok.getEndLocation(),
329
3.94k
                                  Tok.getCommandID(),
330
3.94k
                                  CommandMarker);
331
3.94k
  }
332
7.64k
  consumeToken();
333
334
7.64k
  if (isTokBlockCommand()) {
335
    // Block command ahead.  We can't nest block commands, so pretend that this
336
    // command has an empty argument.
337
177
    ParagraphComment *Paragraph = S.actOnParagraphComment(None);
338
177
    if (PC) {
339
4
      S.actOnParamCommandFinish(PC, Paragraph);
340
4
      return PC;
341
173
    } else if (TPC) {
342
1
      S.actOnTParamCommandFinish(TPC, Paragraph);
343
1
      return TPC;
344
172
    } else {
345
172
      S.actOnBlockCommandFinish(BC, Paragraph);
346
172
      return BC;
347
172
    }
348
177
  }
349
350
7.46k
  if (PC || 
TPC3.97k
||
Info->NumArgs > 03.77k
) {
351
    // In order to parse command arguments we need to retokenize a few
352
    // following text tokens.
353
3.71k
    TextTokenRetokenizer Retokenizer(Allocator, *this);
354
355
3.71k
    if (PC)
356
3.48k
      parseParamCommandArgs(PC, Retokenizer);
357
222
    else if (TPC)
358
206
      parseTParamCommandArgs(TPC, Retokenizer);
359
16
    else
360
16
      S.actOnBlockCommandArgs(BC, parseCommandArgs(Retokenizer, Info->NumArgs));
361
362
3.71k
    Retokenizer.putBackLeftoverTokens();
363
3.71k
  }
364
365
  // If there's a block command ahead, we will attach an empty paragraph to
366
  // this command.
367
7.46k
  bool EmptyParagraph = false;
368
7.46k
  if (isTokBlockCommand())
369
3
    EmptyParagraph = true;
370
7.46k
  else if (Tok.is(tok::newline)) {
371
143
    Token PrevTok = Tok;
372
143
    consumeToken();
373
143
    EmptyParagraph = isTokBlockCommand();
374
143
    putBack(PrevTok);
375
143
  }
376
377
7.46k
  ParagraphComment *Paragraph;
378
7.46k
  if (EmptyParagraph)
379
12
    Paragraph = S.actOnParagraphComment(None);
380
7.45k
  else {
381
7.45k
    BlockContentComment *Block = parseParagraphOrBlockCommand();
382
    // Since we have checked for a block command, we should have parsed a
383
    // paragraph.
384
7.45k
    Paragraph = cast<ParagraphComment>(Block);
385
7.45k
  }
386
387
7.46k
  if (PC) {
388
3.48k
    S.actOnParamCommandFinish(PC, Paragraph);
389
3.48k
    return PC;
390
3.97k
  } else if (TPC) {
391
206
    S.actOnTParamCommandFinish(TPC, Paragraph);
392
206
    return TPC;
393
3.77k
  } else {
394
3.77k
    S.actOnBlockCommandFinish(BC, Paragraph);
395
3.77k
    return BC;
396
3.77k
  }
397
7.46k
}
398
399
1.30k
InlineCommandComment *Parser::parseInlineCommand() {
400
1.30k
  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
401
0
  const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
402
403
1.30k
  const Token CommandTok = Tok;
404
1.30k
  consumeToken();
405
406
1.30k
  TextTokenRetokenizer Retokenizer(Allocator, *this);
407
1.30k
  ArrayRef<Comment::Argument> Args =
408
1.30k
      parseCommandArgs(Retokenizer, Info->NumArgs);
409
410
1.30k
  InlineCommandComment *IC = S.actOnInlineCommand(
411
1.30k
      CommandTok.getLocation(), CommandTok.getEndLocation(),
412
1.30k
      CommandTok.getCommandID(), Args);
413
414
1.30k
  if (Args.size() < Info->NumArgs) {
415
23
    Diag(CommandTok.getEndLocation().getLocWithOffset(1),
416
23
         diag::warn_doc_inline_command_not_enough_arguments)
417
23
        << CommandTok.is(tok::at_command) << Info->Name << Args.size()
418
23
        << Info->NumArgs
419
23
        << SourceRange(CommandTok.getLocation(), CommandTok.getEndLocation());
420
23
  }
421
422
1.30k
  Retokenizer.putBackLeftoverTokens();
423
424
1.30k
  return IC;
425
1.30k
}
426
427
147
HTMLStartTagComment *Parser::parseHTMLStartTag() {
428
147
  assert(Tok.is(tok::html_start_tag));
429
0
  HTMLStartTagComment *HST =
430
147
      S.actOnHTMLStartTagStart(Tok.getLocation(),
431
147
                               Tok.getHTMLTagStartName());
432
147
  consumeToken();
433
434
147
  SmallVector<HTMLStartTagComment::Attribute, 2> Attrs;
435
202
  while (true) {
436
202
    switch (Tok.getKind()) {
437
49
    case tok::html_ident: {
438
49
      Token Ident = Tok;
439
49
      consumeToken();
440
49
      if (Tok.isNot(tok::html_equals)) {
441
19
        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
442
19
                                                       Ident.getHTMLIdent()));
443
19
        continue;
444
19
      }
445
30
      Token Equals = Tok;
446
30
      consumeToken();
447
30
      if (Tok.isNot(tok::html_quoted_string)) {
448
9
        Diag(Tok.getLocation(),
449
9
             diag::warn_doc_html_start_tag_expected_quoted_string)
450
9
          << SourceRange(Equals.getLocation());
451
9
        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
452
9
                                                       Ident.getHTMLIdent()));
453
12
        while (Tok.is(tok::html_equals) ||
454
12
               
Tok.is(tok::html_quoted_string)9
)
455
3
          consumeToken();
456
9
        continue;
457
9
      }
458
21
      Attrs.push_back(HTMLStartTagComment::Attribute(
459
21
                              Ident.getLocation(),
460
21
                              Ident.getHTMLIdent(),
461
21
                              Equals.getLocation(),
462
21
                              SourceRange(Tok.getLocation(),
463
21
                                          Tok.getEndLocation()),
464
21
                              Tok.getHTMLQuotedString()));
465
21
      consumeToken();
466
21
      continue;
467
30
    }
468
469
113
    case tok::html_greater:
470
113
      S.actOnHTMLStartTagFinish(HST,
471
113
                                S.copyArray(llvm::makeArrayRef(Attrs)),
472
113
                                Tok.getLocation(),
473
113
                                /* IsSelfClosing = */ false);
474
113
      consumeToken();
475
113
      return HST;
476
477
12
    case tok::html_slash_greater:
478
12
      S.actOnHTMLStartTagFinish(HST,
479
12
                                S.copyArray(llvm::makeArrayRef(Attrs)),
480
12
                                Tok.getLocation(),
481
12
                                /* IsSelfClosing = */ true);
482
12
      consumeToken();
483
12
      return HST;
484
485
6
    case tok::html_equals:
486
9
    case tok::html_quoted_string:
487
9
      Diag(Tok.getLocation(),
488
9
           diag::warn_doc_html_start_tag_expected_ident_or_greater);
489
18
      while (Tok.is(tok::html_equals) ||
490
18
             
Tok.is(tok::html_quoted_string)12
)
491
9
        consumeToken();
492
9
      if (Tok.is(tok::html_ident) ||
493
9
          Tok.is(tok::html_greater) ||
494
9
          
Tok.is(tok::html_slash_greater)3
)
495
6
        continue;
496
497
3
      S.actOnHTMLStartTagFinish(HST,
498
3
                                S.copyArray(llvm::makeArrayRef(Attrs)),
499
3
                                SourceLocation(),
500
3
                                /* IsSelfClosing = */ false);
501
3
      return HST;
502
503
19
    default:
504
      // Not a token from an HTML start tag.  Thus HTML tag prematurely ended.
505
19
      S.actOnHTMLStartTagFinish(HST,
506
19
                                S.copyArray(llvm::makeArrayRef(Attrs)),
507
19
                                SourceLocation(),
508
19
                                /* IsSelfClosing = */ false);
509
19
      bool StartLineInvalid;
510
19
      const unsigned StartLine = SourceMgr.getPresumedLineNumber(
511
19
                                                  HST->getLocation(),
512
19
                                                  &StartLineInvalid);
513
19
      bool EndLineInvalid;
514
19
      const unsigned EndLine = SourceMgr.getPresumedLineNumber(
515
19
                                                  Tok.getLocation(),
516
19
                                                  &EndLineInvalid);
517
19
      if (StartLineInvalid || EndLineInvalid || StartLine == EndLine)
518
16
        Diag(Tok.getLocation(),
519
16
             diag::warn_doc_html_start_tag_expected_ident_or_greater)
520
16
          << HST->getSourceRange();
521
3
      else {
522
3
        Diag(Tok.getLocation(),
523
3
             diag::warn_doc_html_start_tag_expected_ident_or_greater);
524
3
        Diag(HST->getLocation(), diag::note_doc_html_tag_started_here)
525
3
          << HST->getSourceRange();
526
3
      }
527
19
      return HST;
528
202
    }
529
202
  }
530
147
}
531
532
88
HTMLEndTagComment *Parser::parseHTMLEndTag() {
533
88
  assert(Tok.is(tok::html_end_tag));
534
0
  Token TokEndTag = Tok;
535
88
  consumeToken();
536
88
  SourceLocation Loc;
537
88
  if (Tok.is(tok::html_greater)) {
538
87
    Loc = Tok.getLocation();
539
87
    consumeToken();
540
87
  }
541
542
88
  return S.actOnHTMLEndTag(TokEndTag.getLocation(),
543
88
                           Loc,
544
88
                           TokEndTag.getHTMLTagEndName());
545
88
}
546
547
23.4k
BlockContentComment *Parser::parseParagraphOrBlockCommand() {
548
23.4k
  SmallVector<InlineContentComment *, 8> Content;
549
550
68.3k
  while (true) {
551
68.3k
    switch (Tok.getKind()) {
552
97
    case tok::verbatim_block_begin:
553
200
    case tok::verbatim_line_name:
554
200
    case tok::eof:
555
200
      break; // Block content or EOF ahead, finish this parapgaph.
556
557
150
    case tok::unknown_command:
558
150
      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
559
150
                                              Tok.getEndLocation(),
560
150
                                              Tok.getUnknownCommandName()));
561
150
      consumeToken();
562
150
      continue;
563
564
16.1k
    case tok::backslash_command:
565
16.3k
    case tok::at_command: {
566
16.3k
      const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
567
16.3k
      if (Info->IsBlockCommand) {
568
15.0k
        if (Content.size() == 0)
569
7.64k
          return parseBlockCommand();
570
7.43k
        break; // Block command ahead, finish this parapgaph.
571
15.0k
      }
572
1.32k
      if (Info->IsVerbatimBlockEndCommand) {
573
11
        Diag(Tok.getLocation(),
574
11
             diag::warn_verbatim_block_end_without_start)
575
11
          << Tok.is(tok::at_command)
576
11
          << Info->Name
577
11
          << SourceRange(Tok.getLocation(), Tok.getEndLocation());
578
11
        consumeToken();
579
11
        continue;
580
11
      }
581
1.30k
      if (Info->IsUnknownCommand) {
582
4
        Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
583
4
                                                Tok.getEndLocation(),
584
4
                                                Info->getID()));
585
4
        consumeToken();
586
4
        continue;
587
4
      }
588
1.30k
      assert(Info->IsInlineCommand);
589
0
      Content.push_back(parseInlineCommand());
590
1.30k
      continue;
591
1.30k
    }
592
593
17.1k
    case tok::newline: {
594
17.1k
      consumeToken();
595
17.1k
      if (Tok.is(tok::newline) || 
Tok.is(tok::eof)11.8k
) {
596
8.07k
        consumeToken();
597
8.07k
        break; // Two newlines -- end of paragraph.
598
8.07k
      }
599
      // Also allow [tok::newline, tok::text, tok::newline] if the middle
600
      // tok::text is just whitespace.
601
9.03k
      if (Tok.is(tok::text) && 
isWhitespace(Tok.getText())8.88k
) {
602
3.38k
        Token WhitespaceTok = Tok;
603
3.38k
        consumeToken();
604
3.38k
        if (Tok.is(tok::newline) || 
Tok.is(tok::eof)3.25k
) {
605
132
          consumeToken();
606
132
          break;
607
132
        }
608
        // We have [tok::newline, tok::text, non-newline].  Put back tok::text.
609
3.25k
        putBack(WhitespaceTok);
610
3.25k
      }
611
8.90k
      if (Content.size() > 0)
612
8.87k
        Content.back()->addTrailingNewline();
613
8.90k
      continue;
614
9.03k
    }
615
616
    // Don't deal with HTML tag soup now.
617
147
    case tok::html_start_tag:
618
147
      Content.push_back(parseHTMLStartTag());
619
147
      continue;
620
621
88
    case tok::html_end_tag:
622
88
      Content.push_back(parseHTMLEndTag());
623
88
      continue;
624
625
34.2k
    case tok::text:
626
34.2k
      Content.push_back(S.actOnText(Tok.getLocation(),
627
34.2k
                                    Tok.getEndLocation(),
628
34.2k
                                    Tok.getText()));
629
34.2k
      consumeToken();
630
34.2k
      continue;
631
632
0
    case tok::verbatim_block_line:
633
0
    case tok::verbatim_block_end:
634
0
    case tok::verbatim_line_text:
635
0
    case tok::html_ident:
636
0
    case tok::html_equals:
637
0
    case tok::html_quoted_string:
638
0
    case tok::html_greater:
639
0
    case tok::html_slash_greater:
640
0
      llvm_unreachable("should not see this token");
641
68.3k
    }
642
15.8k
    break;
643
68.3k
  }
644
645
15.8k
  return S.actOnParagraphComment(S.copyArray(llvm::makeArrayRef(Content)));
646
23.4k
}
647
648
103
VerbatimBlockComment *Parser::parseVerbatimBlock() {
649
103
  assert(Tok.is(tok::verbatim_block_begin));
650
651
0
  VerbatimBlockComment *VB =
652
103
      S.actOnVerbatimBlockStart(Tok.getLocation(),
653
103
                                Tok.getVerbatimBlockID());
654
103
  consumeToken();
655
656
  // Don't create an empty line if verbatim opening command is followed
657
  // by a newline.
658
103
  if (Tok.is(tok::newline))
659
19
    consumeToken();
660
661
103
  SmallVector<VerbatimBlockLineComment *, 8> Lines;
662
999
  while (Tok.is(tok::verbatim_block_line) ||
663
999
         
Tok.is(tok::newline)110
) {
664
896
    VerbatimBlockLineComment *Line;
665
896
    if (Tok.is(tok::verbatim_block_line)) {
666
889
      Line = S.actOnVerbatimBlockLine(Tok.getLocation(),
667
889
                                      Tok.getVerbatimBlockText());
668
889
      consumeToken();
669
889
      if (Tok.is(tok::newline)) {
670
846
        consumeToken();
671
846
      }
672
889
    } else {
673
      // Empty line, just a tok::newline.
674
7
      Line = S.actOnVerbatimBlockLine(Tok.getLocation(), "");
675
7
      consumeToken();
676
7
    }
677
896
    Lines.push_back(Line);
678
896
  }
679
680
103
  if (Tok.is(tok::verbatim_block_end)) {
681
96
    const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
682
96
    S.actOnVerbatimBlockFinish(VB, Tok.getLocation(),
683
96
                               Info->Name,
684
96
                               S.copyArray(llvm::makeArrayRef(Lines)));
685
96
    consumeToken();
686
96
  } else {
687
    // Unterminated \\verbatim block
688
7
    S.actOnVerbatimBlockFinish(VB, SourceLocation(), "",
689
7
                               S.copyArray(llvm::makeArrayRef(Lines)));
690
7
  }
691
692
103
  return VB;
693
103
}
694
695
111
VerbatimLineComment *Parser::parseVerbatimLine() {
696
111
  assert(Tok.is(tok::verbatim_line_name));
697
698
0
  Token NameTok = Tok;
699
111
  consumeToken();
700
701
111
  SourceLocation TextBegin;
702
111
  StringRef Text;
703
  // Next token might not be a tok::verbatim_line_text if verbatim line
704
  // starting command comes just before a newline or comment end.
705
111
  if (Tok.is(tok::verbatim_line_text)) {
706
109
    TextBegin = Tok.getLocation();
707
109
    Text = Tok.getVerbatimLineText();
708
109
  } else {
709
2
    TextBegin = NameTok.getEndLocation();
710
2
    Text = "";
711
2
  }
712
713
111
  VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
714
111
                                                NameTok.getVerbatimLineID(),
715
111
                                                TextBegin,
716
111
                                                Text);
717
111
  consumeToken();
718
111
  return VL;
719
111
}
720
721
16.2k
BlockContentComment *Parser::parseBlockContent() {
722
16.2k
  switch (Tok.getKind()) {
723
8.37k
  case tok::text:
724
8.38k
  case tok::unknown_command:
725
15.8k
  case tok::backslash_command:
726
16.0k
  case tok::at_command:
727
16.0k
  case tok::html_start_tag:
728
16.0k
  case tok::html_end_tag:
729
16.0k
    return parseParagraphOrBlockCommand();
730
731
103
  case tok::verbatim_block_begin:
732
103
    return parseVerbatimBlock();
733
734
111
  case tok::verbatim_line_name:
735
111
    return parseVerbatimLine();
736
737
0
  case tok::eof:
738
0
  case tok::newline:
739
0
  case tok::verbatim_block_line:
740
0
  case tok::verbatim_block_end:
741
0
  case tok::verbatim_line_text:
742
0
  case tok::html_ident:
743
0
  case tok::html_equals:
744
0
  case tok::html_quoted_string:
745
0
  case tok::html_greater:
746
0
  case tok::html_slash_greater:
747
0
    llvm_unreachable("should not see this token");
748
16.2k
  }
749
0
  llvm_unreachable("bogus token kind");
750
0
}
751
752
3.29k
FullComment *Parser::parseFullComment() {
753
  // Skip newlines at the beginning of the comment.
754
3.61k
  while (Tok.is(tok::newline))
755
325
    consumeToken();
756
757
3.29k
  SmallVector<BlockContentComment *, 8> Blocks;
758
19.5k
  while (Tok.isNot(tok::eof)) {
759
16.2k
    Blocks.push_back(parseBlockContent());
760
761
    // Skip extra newlines after paragraph end.
762
16.8k
    while (Tok.is(tok::newline))
763
618
      consumeToken();
764
16.2k
  }
765
3.29k
  return S.actOnFullComment(S.copyArray(llvm::makeArrayRef(Blocks)));
766
3.29k
}
767
768
} // end namespace comments
769
} // end namespace clang