Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/lib/AST/CommentParser.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CommentParser.cpp - Doxygen comment parser -----------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/AST/CommentParser.h"
10
#include "clang/AST/CommentCommandTraits.h"
11
#include "clang/AST/CommentDiagnostic.h"
12
#include "clang/AST/CommentSema.h"
13
#include "clang/Basic/CharInfo.h"
14
#include "clang/Basic/SourceManager.h"
15
#include "llvm/Support/ErrorHandling.h"
16
17
namespace clang {
18
19
501
static inline bool isWhitespace(llvm::StringRef S) {
20
1.10k
  for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; 
++I600
) {
21
695
    if (!isWhitespace(*I))
22
95
      return false;
23
695
  }
24
501
  
return true406
;
25
501
}
26
27
namespace comments {
28
29
/// Re-lexes a sequence of tok::text tokens.
30
class TextTokenRetokenizer {
31
  llvm::BumpPtrAllocator &Allocator;
32
  Parser &P;
33
34
  /// This flag is set when there are no more tokens we can fetch from lexer.
35
  bool NoMoreInterestingTokens;
36
37
  /// Token buffer: tokens we have processed and lookahead.
38
  SmallVector<Token, 16> Toks;
39
40
  /// A position in \c Toks.
41
  struct Position {
42
    const char *BufferStart;
43
    const char *BufferEnd;
44
    const char *BufferPtr;
45
    SourceLocation BufferStartLoc;
46
    unsigned CurToken;
47
  };
48
49
  /// Current position in Toks.
50
  Position Pos;
51
52
5.69k
  bool isEnd() const {
53
5.69k
    return Pos.CurToken >= Toks.size();
54
5.69k
  }
55
56
  /// Sets up the buffer pointers to point to current token.
57
582
  void setupBuffer() {
58
582
    assert(!isEnd());
59
582
    const Token &Tok = Toks[Pos.CurToken];
60
582
61
582
    Pos.BufferStart = Tok.getText().begin();
62
582
    Pos.BufferEnd = Tok.getText().end();
63
582
    Pos.BufferPtr = Pos.BufferStart;
64
582
    Pos.BufferStartLoc = Tok.getLocation();
65
582
  }
66
67
1.36k
  SourceLocation getSourceLocation() const {
68
1.36k
    const unsigned CharNo = Pos.BufferPtr - Pos.BufferStart;
69
1.36k
    return Pos.BufferStartLoc.getLocWithOffset(CharNo);
70
1.36k
  }
71
72
4.12k
  char peek() const {
73
4.12k
    assert(!isEnd());
74
4.12k
    assert(Pos.BufferPtr != Pos.BufferEnd);
75
4.12k
    return *Pos.BufferPtr;
76
4.12k
  }
77
78
2.44k
  void consumeChar() {
79
2.44k
    assert(!isEnd());
80
2.44k
    assert(Pos.BufferPtr != Pos.BufferEnd);
81
2.44k
    Pos.BufferPtr++;
82
2.44k
    if (Pos.BufferPtr == Pos.BufferEnd) {
83
90
      Pos.CurToken++;
84
90
      if (isEnd() && 
!addToken()89
)
85
43
        return;
86
47
87
47
      assert(!isEnd());
88
47
      setupBuffer();
89
47
    }
90
2.44k
  }
91
92
  /// Add a token.
93
  /// Returns true on success, false if there are no interesting tokens to
94
  /// fetch from lexer.
95
627
  bool addToken() {
96
627
    if (NoMoreInterestingTokens)
97
2
      return false;
98
625
99
625
    if (P.Tok.is(tok::newline)) {
100
72
      // If we see a single newline token between text tokens, skip it.
101
72
      Token Newline = P.Tok;
102
72
      P.consumeToken();
103
72
      if (P.Tok.isNot(tok::text)) {
104
34
        P.putBack(Newline);
105
34
        NoMoreInterestingTokens = true;
106
34
        return false;
107
34
      }
108
591
    }
109
591
    if (P.Tok.isNot(tok::text)) {
110
10
      NoMoreInterestingTokens = true;
111
10
      return false;
112
10
    }
113
581
114
581
    Toks.push_back(P.Tok);
115
581
    P.consumeToken();
116
581
    if (Toks.size() == 1)
117
535
      setupBuffer();
118
581
    return true;
119
581
  }
120
121
886
  void consumeWhitespace() {
122
1.80k
    while (!isEnd()) {
123
1.79k
      if (isWhitespace(peek()))
124
922
        consumeChar();
125
875
      else
126
875
        break;
127
1.79k
    }
128
886
  }
129
130
  void formTokenWithChars(Token &Result,
131
                          SourceLocation Loc,
132
                          const char *TokBegin,
133
                          unsigned TokLength,
134
1.04k
                          StringRef Text) {
135
1.04k
    Result.setLocation(Loc);
136
1.04k
    Result.setKind(tok::text);
137
1.04k
    Result.setLength(TokLength);
138
#ifndef NDEBUG
139
    Result.TextPtr = "<UNSET>";
140
    Result.IntVal = 7;
141
#endif
142
    Result.setText(Text);
143
1.04k
  }
144
145
public:
146
  TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
147
538
      Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
148
538
    Pos.CurToken = 0;
149
538
    addToken();
150
538
  }
151
152
  /// Extract a word -- sequence of non-whitespace characters.
153
538
  bool lexWord(Token &Tok) {
154
538
    if (isEnd())
155
3
      return false;
156
535
157
535
    Position SavedPos = Pos;
158
535
159
535
    consumeWhitespace();
160
535
    SmallString<32> WordText;
161
535
    const char *WordBegin = Pos.BufferPtr;
162
535
    SourceLocation Loc = getSourceLocation();
163
1.81k
    while (!isEnd()) {
164
1.77k
      const char C = peek();
165
1.77k
      if (!isWhitespace(C)) {
166
1.27k
        WordText.push_back(C);
167
1.27k
        consumeChar();
168
1.27k
      } else
169
494
        break;
170
1.77k
    }
171
535
    const unsigned Length = WordText.size();
172
535
    if (Length == 0) {
173
9
      Pos = SavedPos;
174
9
      return false;
175
9
    }
176
526
177
526
    char *TextPtr = Allocator.Allocate<char>(Length + 1);
178
526
179
526
    memcpy(TextPtr, WordText.c_str(), Length + 1);
180
526
    StringRef Text = StringRef(TextPtr, Length);
181
526
182
526
    formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
183
526
    return true;
184
526
  }
185
186
353
  bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) {
187
353
    if (isEnd())
188
2
      return false;
189
351
190
351
    Position SavedPos = Pos;
191
351
192
351
    consumeWhitespace();
193
351
    SmallString<32> WordText;
194
351
    const char *WordBegin = Pos.BufferPtr;
195
351
    SourceLocation Loc = getSourceLocation();
196
351
    bool Error = false;
197
351
    if (!isEnd()) {
198
349
      const char C = peek();
199
349
      if (C == OpenDelim) {
200
40
        WordText.push_back(C);
201
40
        consumeChar();
202
40
      } else
203
309
        Error = true;
204
349
    }
205
351
    char C = '\0';
206
513
    while (!Error && 
!isEnd()204
) {
207
202
      C = peek();
208
202
      WordText.push_back(C);
209
202
      consumeChar();
210
202
      if (C == CloseDelim)
211
40
        break;
212
202
    }
213
351
    if (!Error && 
C != CloseDelim42
)
214
2
      Error = true;
215
351
216
351
    if (Error) {
217
311
      Pos = SavedPos;
218
311
      return false;
219
311
    }
220
40
221
40
    const unsigned Length = WordText.size();
222
40
    char *TextPtr = Allocator.Allocate<char>(Length + 1);
223
40
224
40
    memcpy(TextPtr, WordText.c_str(), Length + 1);
225
40
    StringRef Text = StringRef(TextPtr, Length);
226
40
227
40
    formTokenWithChars(Tok, Loc, WordBegin,
228
40
                       Pos.BufferPtr - WordBegin, Text);
229
40
    return true;
230
40
  }
231
232
  /// Put back tokens that we didn't consume.
233
538
  void putBackLeftoverTokens() {
234
538
    if (isEnd())
235
35
      return;
236
503
237
503
    bool HavePartialTok = false;
238
503
    Token PartialTok;
239
503
    if (Pos.BufferPtr != Pos.BufferStart) {
240
474
      formTokenWithChars(PartialTok, getSourceLocation(),
241
474
                         Pos.BufferPtr, Pos.BufferEnd - Pos.BufferPtr,
242
474
                         StringRef(Pos.BufferPtr,
243
474
                                   Pos.BufferEnd - Pos.BufferPtr));
244
474
      HavePartialTok = true;
245
474
      Pos.CurToken++;
246
474
    }
247
503
248
503
    P.putBack(llvm::makeArrayRef(Toks.begin() + Pos.CurToken, Toks.end()));
249
503
    Pos.CurToken = Toks.size();
250
503
251
503
    if (HavePartialTok)
252
474
      P.putBack(PartialTok);
253
503
  }
254
};
255
256
Parser::Parser(Lexer &L, Sema &S, llvm::BumpPtrAllocator &Allocator,
257
               const SourceManager &SourceMgr, DiagnosticsEngine &Diags,
258
               const CommandTraits &Traits):
259
    L(L), S(S), Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags),
260
1.26k
    Traits(Traits) {
261
1.26k
  consumeToken();
262
1.26k
}
263
264
void Parser::parseParamCommandArgs(ParamCommandComment *PC,
265
353
                                   TextTokenRetokenizer &Retokenizer) {
266
353
  Token Arg;
267
353
  // Check if argument looks like direction specification: [dir]
268
353
  // e.g., [in], [out], [in,out]
269
353
  if (Retokenizer.lexDelimitedSeq(Arg, '[', ']'))
270
40
    S.actOnParamCommandDirectionArg(PC,
271
40
                                    Arg.getLocation(),
272
40
                                    Arg.getEndLocation(),
273
40
                                    Arg.getText());
274
353
275
353
  if (Retokenizer.lexWord(Arg))
276
349
    S.actOnParamCommandParamNameArg(PC,
277
349
                                    Arg.getLocation(),
278
349
                                    Arg.getEndLocation(),
279
349
                                    Arg.getText());
280
353
}
281
282
void Parser::parseTParamCommandArgs(TParamCommandComment *TPC,
283
155
                                    TextTokenRetokenizer &Retokenizer) {
284
155
  Token Arg;
285
155
  if (Retokenizer.lexWord(Arg))
286
149
    S.actOnTParamCommandParamNameArg(TPC,
287
149
                                     Arg.getLocation(),
288
149
                                     Arg.getEndLocation(),
289
149
                                     Arg.getText());
290
155
}
291
292
void Parser::parseBlockCommandArgs(BlockCommandComment *BC,
293
                                   TextTokenRetokenizer &Retokenizer,
294
0
                                   unsigned NumArgs) {
295
0
  typedef BlockCommandComment::Argument Argument;
296
0
  Argument *Args =
297
0
      new (Allocator.Allocate<Argument>(NumArgs)) Argument[NumArgs];
298
0
  unsigned ParsedArgs = 0;
299
0
  Token Arg;
300
0
  while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) {
301
0
    Args[ParsedArgs] = Argument(SourceRange(Arg.getLocation(),
302
0
                                            Arg.getEndLocation()),
303
0
                                Arg.getText());
304
0
    ParsedArgs++;
305
0
  }
306
0
307
0
  S.actOnBlockCommandArgs(BC, llvm::makeArrayRef(Args, ParsedArgs));
308
0
}
309
310
1.22k
BlockCommandComment *Parser::parseBlockCommand() {
311
1.22k
  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
312
1.22k
313
1.22k
  ParamCommandComment *PC = nullptr;
314
1.22k
  TParamCommandComment *TPC = nullptr;
315
1.22k
  BlockCommandComment *BC = nullptr;
316
1.22k
  const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
317
1.22k
  CommandMarkerKind CommandMarker =
318
1.22k
      Tok.is(tok::backslash_command) ? 
CMK_Backslash1.12k
:
CMK_At92
;
319
1.22k
  if (Info->IsParamCommand) {
320
356
    PC = S.actOnParamCommandStart(Tok.getLocation(),
321
356
                                  Tok.getEndLocation(),
322
356
                                  Tok.getCommandID(),
323
356
                                  CommandMarker);
324
864
  } else if (Info->IsTParamCommand) {
325
156
    TPC = S.actOnTParamCommandStart(Tok.getLocation(),
326
156
                                    Tok.getEndLocation(),
327
156
                                    Tok.getCommandID(),
328
156
                                    CommandMarker);
329
708
  } else {
330
708
    BC = S.actOnBlockCommandStart(Tok.getLocation(),
331
708
                                  Tok.getEndLocation(),
332
708
                                  Tok.getCommandID(),
333
708
                                  CommandMarker);
334
708
  }
335
1.22k
  consumeToken();
336
1.22k
337
1.22k
  if (isTokBlockCommand()) {
338
111
    // Block command ahead.  We can't nest block commands, so pretend that this
339
111
    // command has an empty argument.
340
111
    ParagraphComment *Paragraph = S.actOnParagraphComment(None);
341
111
    if (PC) {
342
3
      S.actOnParamCommandFinish(PC, Paragraph);
343
3
      return PC;
344
108
    } else if (TPC) {
345
1
      S.actOnTParamCommandFinish(TPC, Paragraph);
346
1
      return TPC;
347
107
    } else {
348
107
      S.actOnBlockCommandFinish(BC, Paragraph);
349
107
      return BC;
350
107
    }
351
1.10k
  }
352
1.10k
353
1.10k
  if (PC || 
TPC756
||
Info->NumArgs > 0601
) {
354
508
    // In order to parse command arguments we need to retokenize a few
355
508
    // following text tokens.
356
508
    TextTokenRetokenizer Retokenizer(Allocator, *this);
357
508
358
508
    if (PC)
359
353
      parseParamCommandArgs(PC, Retokenizer);
360
155
    else if (TPC)
361
155
      parseTParamCommandArgs(TPC, Retokenizer);
362
0
    else
363
0
      parseBlockCommandArgs(BC, Retokenizer, Info->NumArgs);
364
508
365
508
    Retokenizer.putBackLeftoverTokens();
366
508
  }
367
1.10k
368
1.10k
  // If there's a block command ahead, we will attach an empty paragraph to
369
1.10k
  // this command.
370
1.10k
  bool EmptyParagraph = false;
371
1.10k
  if (isTokBlockCommand())
372
2
    EmptyParagraph = true;
373
1.10k
  else if (Tok.is(tok::newline)) {
374
77
    Token PrevTok = Tok;
375
77
    consumeToken();
376
77
    EmptyParagraph = isTokBlockCommand();
377
77
    putBack(PrevTok);
378
77
  }
379
1.10k
380
1.10k
  ParagraphComment *Paragraph;
381
1.10k
  if (EmptyParagraph)
382
8
    Paragraph = S.actOnParagraphComment(None);
383
1.10k
  else {
384
1.10k
    BlockContentComment *Block = parseParagraphOrBlockCommand();
385
1.10k
    // Since we have checked for a block command, we should have parsed a
386
1.10k
    // paragraph.
387
1.10k
    Paragraph = cast<ParagraphComment>(Block);
388
1.10k
  }
389
1.10k
390
1.10k
  if (PC) {
391
353
    S.actOnParamCommandFinish(PC, Paragraph);
392
353
    return PC;
393
756
  } else if (TPC) {
394
155
    S.actOnTParamCommandFinish(TPC, Paragraph);
395
155
    return TPC;
396
601
  } else {
397
601
    S.actOnBlockCommandFinish(BC, Paragraph);
398
601
    return BC;
399
601
  }
400
1.10k
}
401
402
30
InlineCommandComment *Parser::parseInlineCommand() {
403
30
  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
404
30
405
30
  const Token CommandTok = Tok;
406
30
  consumeToken();
407
30
408
30
  TextTokenRetokenizer Retokenizer(Allocator, *this);
409
30
410
30
  Token ArgTok;
411
30
  bool ArgTokValid = Retokenizer.lexWord(ArgTok);
412
30
413
30
  InlineCommandComment *IC;
414
30
  if (ArgTokValid) {
415
28
    IC = S.actOnInlineCommand(CommandTok.getLocation(),
416
28
                              CommandTok.getEndLocation(),
417
28
                              CommandTok.getCommandID(),
418
28
                              ArgTok.getLocation(),
419
28
                              ArgTok.getEndLocation(),
420
28
                              ArgTok.getText());
421
28
  } else {
422
2
    IC = S.actOnInlineCommand(CommandTok.getLocation(),
423
2
                              CommandTok.getEndLocation(),
424
2
                              CommandTok.getCommandID());
425
2
  }
426
30
427
30
  Retokenizer.putBackLeftoverTokens();
428
30
429
30
  return IC;
430
30
}
431
432
94
HTMLStartTagComment *Parser::parseHTMLStartTag() {
433
94
  assert(Tok.is(tok::html_start_tag));
434
94
  HTMLStartTagComment *HST =
435
94
      S.actOnHTMLStartTagStart(Tok.getLocation(),
436
94
                               Tok.getHTMLTagStartName());
437
94
  consumeToken();
438
94
439
94
  SmallVector<HTMLStartTagComment::Attribute, 2> Attrs;
440
136
  while (true) {
441
136
    switch (Tok.getKind()) {
442
136
    case tok::html_ident: {
443
38
      Token Ident = Tok;
444
38
      consumeToken();
445
38
      if (Tok.isNot(tok::html_equals)) {
446
14
        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
447
14
                                                       Ident.getHTMLIdent()));
448
14
        continue;
449
14
      }
450
24
      Token Equals = Tok;
451
24
      consumeToken();
452
24
      if (Tok.isNot(tok::html_quoted_string)) {
453
6
        Diag(Tok.getLocation(),
454
6
             diag::warn_doc_html_start_tag_expected_quoted_string)
455
6
          << SourceRange(Equals.getLocation());
456
6
        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
457
6
                                                       Ident.getHTMLIdent()));
458
8
        while (Tok.is(tok::html_equals) ||
459
8
               
Tok.is(tok::html_quoted_string)6
)
460
2
          consumeToken();
461
6
        continue;
462
6
      }
463
18
      Attrs.push_back(HTMLStartTagComment::Attribute(
464
18
                              Ident.getLocation(),
465
18
                              Ident.getHTMLIdent(),
466
18
                              Equals.getLocation(),
467
18
                              SourceRange(Tok.getLocation(),
468
18
                                          Tok.getEndLocation()),
469
18
                              Tok.getHTMLQuotedString()));
470
18
      consumeToken();
471
18
      continue;
472
18
    }
473
18
474
69
    case tok::html_greater:
475
69
      S.actOnHTMLStartTagFinish(HST,
476
69
                                S.copyArray(llvm::makeArrayRef(Attrs)),
477
69
                                Tok.getLocation(),
478
69
                                /* IsSelfClosing = */ false);
479
69
      consumeToken();
480
69
      return HST;
481
18
482
18
    case tok::html_slash_greater:
483
9
      S.actOnHTMLStartTagFinish(HST,
484
9
                                S.copyArray(llvm::makeArrayRef(Attrs)),
485
9
                                Tok.getLocation(),
486
9
                                /* IsSelfClosing = */ true);
487
9
      consumeToken();
488
9
      return HST;
489
18
490
18
    case tok::html_equals:
491
6
    case tok::html_quoted_string:
492
6
      Diag(Tok.getLocation(),
493
6
           diag::warn_doc_html_start_tag_expected_ident_or_greater);
494
12
      while (Tok.is(tok::html_equals) ||
495
12
             
Tok.is(tok::html_quoted_string)8
)
496
6
        consumeToken();
497
6
      if (Tok.is(tok::html_ident) ||
498
6
          Tok.is(tok::html_greater) ||
499
6
          
Tok.is(tok::html_slash_greater)2
)
500
4
        continue;
501
2
502
2
      S.actOnHTMLStartTagFinish(HST,
503
2
                                S.copyArray(llvm::makeArrayRef(Attrs)),
504
2
                                SourceLocation(),
505
2
                                /* IsSelfClosing = */ false);
506
2
      return HST;
507
2
508
14
    default:
509
14
      // Not a token from an HTML start tag.  Thus HTML tag prematurely ended.
510
14
      S.actOnHTMLStartTagFinish(HST,
511
14
                                S.copyArray(llvm::makeArrayRef(Attrs)),
512
14
                                SourceLocation(),
513
14
                                /* IsSelfClosing = */ false);
514
14
      bool StartLineInvalid;
515
14
      const unsigned StartLine = SourceMgr.getPresumedLineNumber(
516
14
                                                  HST->getLocation(),
517
14
                                                  &StartLineInvalid);
518
14
      bool EndLineInvalid;
519
14
      const unsigned EndLine = SourceMgr.getPresumedLineNumber(
520
14
                                                  Tok.getLocation(),
521
14
                                                  &EndLineInvalid);
522
14
      if (StartLineInvalid || EndLineInvalid || StartLine == EndLine)
523
12
        Diag(Tok.getLocation(),
524
12
             diag::warn_doc_html_start_tag_expected_ident_or_greater)
525
12
          << HST->getSourceRange();
526
2
      else {
527
2
        Diag(Tok.getLocation(),
528
2
             diag::warn_doc_html_start_tag_expected_ident_or_greater);
529
2
        Diag(HST->getLocation(), diag::note_doc_html_tag_started_here)
530
2
          << HST->getSourceRange();
531
2
      }
532
14
      return HST;
533
136
    }
534
136
  }
535
94
}
536
537
51
HTMLEndTagComment *Parser::parseHTMLEndTag() {
538
51
  assert(Tok.is(tok::html_end_tag));
539
51
  Token TokEndTag = Tok;
540
51
  consumeToken();
541
51
  SourceLocation Loc;
542
51
  if (Tok.is(tok::html_greater)) {
543
50
    Loc = Tok.getLocation();
544
50
    consumeToken();
545
50
  }
546
51
547
51
  return S.actOnHTMLEndTag(TokEndTag.getLocation(),
548
51
                           Loc,
549
51
                           TokEndTag.getHTMLTagEndName());
550
51
}
551
552
3.74k
BlockContentComment *Parser::parseParagraphOrBlockCommand() {
553
3.74k
  SmallVector<InlineContentComment *, 8> Content;
554
3.74k
555
7.72k
  while (true) {
556
7.72k
    switch (Tok.getKind()) {
557
7.72k
    case tok::verbatim_block_begin:
558
95
    case tok::verbatim_line_name:
559
95
    case tok::eof:
560
95
      break; // Block content or EOF ahead, finish this parapgaph.
561
95
562
150
    case tok::unknown_command:
563
150
      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
564
150
                                              Tok.getEndLocation(),
565
150
                                              Tok.getUnknownCommandName()));
566
150
      consumeToken();
567
150
      continue;
568
95
569
2.34k
    case tok::backslash_command:
570
2.34k
    case tok::at_command: {
571
2.34k
      const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
572
2.34k
      if (Info->IsBlockCommand) {
573
2.30k
        if (Content.size() == 0)
574
1.22k
          return parseBlockCommand();
575
1.08k
        break; // Block command ahead, finish this parapgaph.
576
1.08k
      }
577
41
      if (Info->IsVerbatimBlockEndCommand) {
578
8
        Diag(Tok.getLocation(),
579
8
             diag::warn_verbatim_block_end_without_start)
580
8
          << Tok.is(tok::at_command)
581
8
          << Info->Name
582
8
          << SourceRange(Tok.getLocation(), Tok.getEndLocation());
583
8
        consumeToken();
584
8
        continue;
585
8
      }
586
33
      if (Info->IsUnknownCommand) {
587
3
        Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
588
3
                                                Tok.getEndLocation(),
589
3
                                                Info->getID()));
590
3
        consumeToken();
591
3
        continue;
592
3
      }
593
30
      assert(Info->IsInlineCommand);
594
30
      Content.push_back(parseInlineCommand());
595
30
      continue;
596
30
    }
597
30
598
1.91k
    case tok::newline: {
599
1.91k
      consumeToken();
600
1.91k
      if (Tok.is(tok::newline) || 
Tok.is(tok::eof)1.54k
) {
601
1.27k
        consumeToken();
602
1.27k
        break; // Two newlines -- end of paragraph.
603
1.27k
      }
604
646
      // Also allow [tok::newline, tok::text, tok::newline] if the middle
605
646
      // tok::text is just whitespace.
606
646
      if (Tok.is(tok::text) && 
isWhitespace(Tok.getText())501
) {
607
406
        Token WhitespaceTok = Tok;
608
406
        consumeToken();
609
406
        if (Tok.is(tok::newline) || 
Tok.is(tok::eof)330
) {
610
76
          consumeToken();
611
76
          break;
612
76
        }
613
330
        // We have [tok::newline, tok::text, non-newline].  Put back tok::text.
614
330
        putBack(WhitespaceTok);
615
330
      }
616
646
      
if (570
Content.size() > 0570
)
617
556
        Content.back()->addTrailingNewline();
618
570
      continue;
619
646
    }
620
646
621
646
    // Don't deal with HTML tag soup now.
622
646
    case tok::html_start_tag:
623
94
      Content.push_back(parseHTMLStartTag());
624
94
      continue;
625
646
626
646
    case tok::html_end_tag:
627
51
      Content.push_back(parseHTMLEndTag());
628
51
      continue;
629
646
630
3.07k
    case tok::text:
631
3.07k
      Content.push_back(S.actOnText(Tok.getLocation(),
632
3.07k
                                    Tok.getEndLocation(),
633
3.07k
                                    Tok.getText()));
634
3.07k
      consumeToken();
635
3.07k
      continue;
636
646
637
646
    case tok::verbatim_block_line:
638
0
    case tok::verbatim_block_end:
639
0
    case tok::verbatim_line_text:
640
0
    case tok::html_ident:
641
0
    case tok::html_equals:
642
0
    case tok::html_quoted_string:
643
0
    case tok::html_greater:
644
0
    case tok::html_slash_greater:
645
0
      llvm_unreachable("should not see this token");
646
2.52k
    }
647
2.52k
    break;
648
2.52k
  }
649
3.74k
650
3.74k
  
return S.actOnParagraphComment(S.copyArray(llvm::makeArrayRef(Content)))2.52k
;
651
3.74k
}
652
653
30
VerbatimBlockComment *Parser::parseVerbatimBlock() {
654
30
  assert(Tok.is(tok::verbatim_block_begin));
655
30
656
30
  VerbatimBlockComment *VB =
657
30
      S.actOnVerbatimBlockStart(Tok.getLocation(),
658
30
                                Tok.getVerbatimBlockID());
659
30
  consumeToken();
660
30
661
30
  // Don't create an empty line if verbatim opening command is followed
662
30
  // by a newline.
663
30
  if (Tok.is(tok::newline))
664
11
    consumeToken();
665
30
666
30
  SmallVector<VerbatimBlockLineComment *, 8> Lines;
667
76
  while (Tok.is(tok::verbatim_block_line) ||
668
76
         
Tok.is(tok::newline)35
) {
669
46
    VerbatimBlockLineComment *Line;
670
46
    if (Tok.is(tok::verbatim_block_line)) {
671
41
      Line = S.actOnVerbatimBlockLine(Tok.getLocation(),
672
41
                                      Tok.getVerbatimBlockText());
673
41
      consumeToken();
674
41
      if (Tok.is(tok::newline)) {
675
19
        consumeToken();
676
19
      }
677
41
    } else {
678
5
      // Empty line, just a tok::newline.
679
5
      Line = S.actOnVerbatimBlockLine(Tok.getLocation(), "");
680
5
      consumeToken();
681
5
    }
682
46
    Lines.push_back(Line);
683
46
  }
684
30
685
30
  if (Tok.is(tok::verbatim_block_end)) {
686
25
    const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
687
25
    S.actOnVerbatimBlockFinish(VB, Tok.getLocation(),
688
25
                               Info->Name,
689
25
                               S.copyArray(llvm::makeArrayRef(Lines)));
690
25
    consumeToken();
691
25
  } else {
692
5
    // Unterminated \\verbatim block
693
5
    S.actOnVerbatimBlockFinish(VB, SourceLocation(), "",
694
5
                               S.copyArray(llvm::makeArrayRef(Lines)));
695
5
  }
696
30
697
30
  return VB;
698
30
}
699
700
73
VerbatimLineComment *Parser::parseVerbatimLine() {
701
73
  assert(Tok.is(tok::verbatim_line_name));
702
73
703
73
  Token NameTok = Tok;
704
73
  consumeToken();
705
73
706
73
  SourceLocation TextBegin;
707
73
  StringRef Text;
708
73
  // Next token might not be a tok::verbatim_line_text if verbatim line
709
73
  // starting command comes just before a newline or comment end.
710
73
  if (Tok.is(tok::verbatim_line_text)) {
711
71
    TextBegin = Tok.getLocation();
712
71
    Text = Tok.getVerbatimLineText();
713
71
  } else {
714
2
    TextBegin = NameTok.getEndLocation();
715
2
    Text = "";
716
2
  }
717
73
718
73
  VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
719
73
                                                NameTok.getVerbatimLineID(),
720
73
                                                TextBegin,
721
73
                                                Text);
722
73
  consumeToken();
723
73
  return VL;
724
73
}
725
726
2.74k
BlockContentComment *Parser::parseBlockContent() {
727
2.74k
  switch (Tok.getKind()) {
728
2.74k
  case tok::text:
729
2.64k
  case tok::unknown_command:
730
2.64k
  case tok::backslash_command:
731
2.64k
  case tok::at_command:
732
2.64k
  case tok::html_start_tag:
733
2.64k
  case tok::html_end_tag:
734
2.64k
    return parseParagraphOrBlockCommand();
735
2.64k
736
2.64k
  case tok::verbatim_block_begin:
737
30
    return parseVerbatimBlock();
738
2.64k
739
2.64k
  case tok::verbatim_line_name:
740
73
    return parseVerbatimLine();
741
2.64k
742
2.64k
  case tok::eof:
743
0
  case tok::newline:
744
0
  case tok::verbatim_block_line:
745
0
  case tok::verbatim_block_end:
746
0
  case tok::verbatim_line_text:
747
0
  case tok::html_ident:
748
0
  case tok::html_equals:
749
0
  case tok::html_quoted_string:
750
0
  case tok::html_greater:
751
0
  case tok::html_slash_greater:
752
0
    llvm_unreachable("should not see this token");
753
0
  }
754
0
  llvm_unreachable("bogus token kind");
755
0
}
756
757
1.26k
FullComment *Parser::parseFullComment() {
758
1.26k
  // Skip newlines at the beginning of the comment.
759
1.53k
  while (Tok.is(tok::newline))
760
269
    consumeToken();
761
1.26k
762
1.26k
  SmallVector<BlockContentComment *, 8> Blocks;
763
4.01k
  while (Tok.isNot(tok::eof)) {
764
2.74k
    Blocks.push_back(parseBlockContent());
765
2.74k
766
2.74k
    // Skip extra newlines after paragraph end.
767
3.14k
    while (Tok.is(tok::newline))
768
398
      consumeToken();
769
2.74k
  }
770
1.26k
  return S.actOnFullComment(S.copyArray(llvm::makeArrayRef(Blocks)));
771
1.26k
}
772
773
} // end namespace comments
774
} // end namespace clang