Coverage Report

Created: 2023-09-30 09:22

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Format/BreakableToken.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- BreakableToken.cpp - Format C++ code -----------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
///
9
/// \file
10
/// Contains implementation of BreakableToken class and classes derived
11
/// from it.
12
///
13
//===----------------------------------------------------------------------===//
14
15
#include "BreakableToken.h"
16
#include "ContinuationIndenter.h"
17
#include "clang/Basic/CharInfo.h"
18
#include "clang/Format/Format.h"
19
#include "llvm/ADT/STLExtras.h"
20
#include "llvm/Support/Debug.h"
21
#include <algorithm>
22
23
#define DEBUG_TYPE "format-token-breaker"
24
25
namespace clang {
26
namespace format {
27
28
static constexpr StringRef Blanks = " \t\v\f\r";
29
28.2k
static bool IsBlank(char C) {
30
28.2k
  switch (C) {
31
3.92k
  case ' ':
32
3.93k
  case '\t':
33
3.93k
  case '\v':
34
3.93k
  case '\f':
35
3.93k
  case '\r':
36
3.93k
    return true;
37
24.2k
  default:
38
24.2k
    return false;
39
28.2k
  }
40
28.2k
}
41
42
static StringRef getLineCommentIndentPrefix(StringRef Comment,
43
26.1k
                                            const FormatStyle &Style) {
44
26.1k
  static constexpr StringRef KnownCStylePrefixes[] = {"///<", "//!<", "///",
45
26.1k
                                                      "//!",  "//:",  "//"};
46
26.1k
  static constexpr StringRef KnownTextProtoPrefixes[] = {"####", "###", "##",
47
26.1k
                                                         "//", "#"};
48
26.1k
  ArrayRef<StringRef> KnownPrefixes(KnownCStylePrefixes);
49
26.1k
  if (Style.Language == FormatStyle::LK_TextProto)
50
296
    KnownPrefixes = KnownTextProtoPrefixes;
51
52
26.1k
  assert(
53
26.1k
      llvm::is_sorted(KnownPrefixes, [](StringRef Lhs, StringRef Rhs) noexcept {
54
26.1k
        return Lhs.size() > Rhs.size();
55
26.1k
      }));
56
57
155k
  
for (StringRef KnownPrefix : KnownPrefixes)26.1k
{
58
155k
    if (Comment.startswith(KnownPrefix)) {
59
26.1k
      const auto PrefixLength =
60
26.1k
          Comment.find_first_not_of(' ', KnownPrefix.size());
61
26.1k
      return Comment.substr(0, PrefixLength);
62
26.1k
    }
63
155k
  }
64
0
  return {};
65
26.1k
}
66
67
static BreakableToken::Split
68
getCommentSplit(StringRef Text, unsigned ContentStartColumn,
69
                unsigned ColumnLimit, unsigned TabWidth,
70
                encoding::Encoding Encoding, const FormatStyle &Style,
71
4.72k
                bool DecorationEndsWithStar = false) {
72
4.72k
  LLVM_DEBUG(llvm::dbgs() << "Comment split: \"" << Text
73
4.72k
                          << "\", Column limit: " << ColumnLimit
74
4.72k
                          << ", Content start: " << ContentStartColumn << "\n");
75
4.72k
  if (ColumnLimit <= ContentStartColumn + 1)
76
912
    return BreakableToken::Split(StringRef::npos, 0);
77
78
3.81k
  unsigned MaxSplit = ColumnLimit - ContentStartColumn + 1;
79
3.81k
  unsigned MaxSplitBytes = 0;
80
81
3.81k
  for (unsigned NumChars = 0;
82
77.4k
       NumChars < MaxSplit && 
MaxSplitBytes < Text.size()73.7k
;) {
83
73.6k
    unsigned BytesInChar =
84
73.6k
        encoding::getCodePointNumBytes(Text[MaxSplitBytes], Encoding);
85
73.6k
    NumChars += encoding::columnWidthWithTabs(
86
73.6k
        Text.substr(MaxSplitBytes, BytesInChar), ContentStartColumn + NumChars,
87
73.6k
        TabWidth, Encoding);
88
73.6k
    MaxSplitBytes += BytesInChar;
89
73.6k
  }
90
91
  // In JavaScript, some @tags can be followed by {, and machinery that parses
92
  // these comments will fail to understand the comment if followed by a line
93
  // break. So avoid ever breaking before a {.
94
3.81k
  if (Style.isJavaScript()) {
95
117
    StringRef::size_type SpaceOffset =
96
117
        Text.find_first_of(Blanks, MaxSplitBytes);
97
117
    if (SpaceOffset != StringRef::npos && 
SpaceOffset + 1 < Text.size()55
&&
98
117
        
Text[SpaceOffset + 1] == '{'40
) {
99
18
      MaxSplitBytes = SpaceOffset + 1;
100
18
    }
101
117
  }
102
103
3.81k
  StringRef::size_type SpaceOffset = Text.find_last_of(Blanks, MaxSplitBytes);
104
105
3.81k
  static const auto kNumberedListRegexp = llvm::Regex("^[1-9][0-9]?\\.");
106
  // Some spaces are unacceptable to break on, rewind past them.
107
3.93k
  while (SpaceOffset != StringRef::npos) {
108
    // If a line-comment ends with `\`, the next line continues the comment,
109
    // whether or not it starts with `//`. This is confusing and triggers
110
    // -Wcomment.
111
    // Avoid introducing multiline comments by not allowing a break right
112
    // after '\'.
113
2.42k
    if (Style.isCpp()) {
114
2.25k
      StringRef::size_type LastNonBlank =
115
2.25k
          Text.find_last_not_of(Blanks, SpaceOffset);
116
2.25k
      if (LastNonBlank != StringRef::npos && 
Text[LastNonBlank] == '\\'2.21k
) {
117
38
        SpaceOffset = Text.find_last_of(Blanks, LastNonBlank);
118
38
        continue;
119
38
      }
120
2.25k
    }
121
122
    // Do not split before a number followed by a dot: this would be interpreted
123
    // as a numbered list, which would prevent re-flowing in subsequent passes.
124
2.39k
    if (kNumberedListRegexp.match(Text.substr(SpaceOffset).ltrim(Blanks))) {
125
4
      SpaceOffset = Text.find_last_of(Blanks, SpaceOffset);
126
4
      continue;
127
4
    }
128
129
    // Avoid ever breaking before a @tag or a { in JavaScript.
130
2.38k
    if (Style.isJavaScript() && 
SpaceOffset + 1 < Text.size()129
&&
131
2.38k
        
(119
Text[SpaceOffset + 1] == '{'119
||
Text[SpaceOffset + 1] == '@'57
)) {
132
79
      SpaceOffset = Text.find_last_of(Blanks, SpaceOffset);
133
79
      continue;
134
79
    }
135
136
2.30k
    break;
137
2.38k
  }
138
139
3.81k
  if (SpaceOffset == StringRef::npos ||
140
      // Don't break at leading whitespace.
141
3.81k
      
Text.find_last_not_of(Blanks, SpaceOffset) == StringRef::npos2.30k
) {
142
    // Make sure that we don't break at leading whitespace that
143
    // reaches past MaxSplit.
144
1.54k
    StringRef::size_type FirstNonWhitespace = Text.find_first_not_of(Blanks);
145
1.54k
    if (FirstNonWhitespace == StringRef::npos) {
146
      // If the comment is only whitespace, we cannot split.
147
17
      return BreakableToken::Split(StringRef::npos, 0);
148
17
    }
149
1.52k
    SpaceOffset = Text.find_first_of(
150
1.52k
        Blanks, std::max<unsigned>(MaxSplitBytes, FirstNonWhitespace));
151
1.52k
  }
152
3.80k
  if (SpaceOffset != StringRef::npos && 
SpaceOffset != 02.97k
) {
153
    // adaptStartOfLine will break after lines starting with /** if the comment
154
    // is broken anywhere. Avoid emitting this break twice here.
155
    // Example: in /** longtextcomesherethatbreaks */ (with ColumnLimit 20) will
156
    // insert a break after /**, so this code must not insert the same break.
157
2.97k
    if (SpaceOffset == 1 && 
Text[SpaceOffset - 1] == '*'5
)
158
4
      return BreakableToken::Split(StringRef::npos, 0);
159
2.97k
    StringRef BeforeCut = Text.substr(0, SpaceOffset).rtrim(Blanks);
160
2.97k
    StringRef AfterCut = Text.substr(SpaceOffset);
161
    // Don't trim the leading blanks if it would create a */ after the break.
162
2.97k
    if (!DecorationEndsWithStar || 
AfterCut.size() <= 110
||
AfterCut[1] != '/'10
)
163
2.96k
      AfterCut = AfterCut.ltrim(Blanks);
164
2.97k
    return BreakableToken::Split(BeforeCut.size(),
165
2.97k
                                 AfterCut.begin() - BeforeCut.end());
166
2.97k
  }
167
825
  return BreakableToken::Split(StringRef::npos, 0);
168
3.80k
}
169
170
static BreakableToken::Split
171
getStringSplit(StringRef Text, unsigned UsedColumns, unsigned ColumnLimit,
172
2.25k
               unsigned TabWidth, encoding::Encoding Encoding) {
173
  // FIXME: Reduce unit test case.
174
2.25k
  if (Text.empty())
175
0
    return BreakableToken::Split(StringRef::npos, 0);
176
2.25k
  if (ColumnLimit <= UsedColumns)
177
20
    return BreakableToken::Split(StringRef::npos, 0);
178
2.23k
  unsigned MaxSplit = ColumnLimit - UsedColumns;
179
2.23k
  StringRef::size_type SpaceOffset = 0;
180
2.23k
  StringRef::size_type SlashOffset = 0;
181
2.23k
  StringRef::size_type WordStartOffset = 0;
182
2.23k
  StringRef::size_type SplitPoint = 0;
183
30.4k
  for (unsigned Chars = 0;;) {
184
30.4k
    unsigned Advance;
185
30.4k
    if (Text[0] == '\\') {
186
45
      Advance = encoding::getEscapeSequenceLength(Text);
187
45
      Chars += Advance;
188
30.4k
    } else {
189
30.4k
      Advance = encoding::getCodePointNumBytes(Text[0], Encoding);
190
30.4k
      Chars += encoding::columnWidthWithTabs(
191
30.4k
          Text.substr(0, Advance), UsedColumns + Chars, TabWidth, Encoding);
192
30.4k
    }
193
194
30.4k
    if (Chars > MaxSplit || 
Text.size() <= Advance28.3k
)
195
2.23k
      break;
196
197
28.2k
    if (IsBlank(Text[0]))
198
3.93k
      SpaceOffset = SplitPoint;
199
28.2k
    if (Text[0] == '/')
200
28
      SlashOffset = SplitPoint;
201
28.2k
    if (Advance == 1 && 
!isAlphanumeric(Text[0])28.1k
)
202
4.35k
      WordStartOffset = SplitPoint;
203
204
28.2k
    SplitPoint += Advance;
205
28.2k
    Text = Text.substr(Advance);
206
28.2k
  }
207
208
2.23k
  if (SpaceOffset != 0)
209
1.18k
    return BreakableToken::Split(SpaceOffset + 1, 0);
210
1.05k
  if (SlashOffset != 0)
211
14
    return BreakableToken::Split(SlashOffset + 1, 0);
212
1.03k
  if (WordStartOffset != 0)
213
21
    return BreakableToken::Split(WordStartOffset + 1, 0);
214
1.01k
  if (SplitPoint != 0)
215
981
    return BreakableToken::Split(SplitPoint, 0);
216
35
  return BreakableToken::Split(StringRef::npos, 0);
217
1.01k
}
218
219
31.5k
bool switchesFormatting(const FormatToken &Token) {
220
31.5k
  assert((Token.is(TT_BlockComment) || Token.is(TT_LineComment)) &&
221
31.5k
         "formatting regions are switched by comment tokens");
222
31.5k
  StringRef Content = Token.TokenText.substr(2).ltrim();
223
31.5k
  return Content.startswith("clang-format on") ||
224
31.5k
         
Content.startswith("clang-format off")31.4k
;
225
31.5k
}
226
227
unsigned
228
BreakableToken::getLengthAfterCompression(unsigned RemainingTokenColumns,
229
1.93k
                                          Split Split) const {
230
  // Example: consider the content
231
  // lala  lala
232
  // - RemainingTokenColumns is the original number of columns, 10;
233
  // - Split is (4, 2), denoting the two spaces between the two words;
234
  //
235
  // We compute the number of columns when the split is compressed into a single
236
  // space, like:
237
  // lala lala
238
  //
239
  // FIXME: Correctly measure the length of whitespace in Split.second so it
240
  // works with tabs.
241
1.93k
  return RemainingTokenColumns + 1 - Split.second;
242
1.93k
}
243
244
42.0k
unsigned BreakableStringLiteral::getLineCount() const { return 1; }
245
246
unsigned BreakableStringLiteral::getRangeLength(unsigned LineIndex,
247
                                                unsigned Offset,
248
                                                StringRef::size_type Length,
249
0
                                                unsigned StartColumn) const {
250
0
  llvm_unreachable("Getting the length of a part of the string literal "
251
0
                   "indicates that the code tries to reflow it.");
252
0
}
253
254
unsigned
255
BreakableStringLiteral::getRemainingLength(unsigned LineIndex, unsigned Offset,
256
22.4k
                                           unsigned StartColumn) const {
257
22.4k
  return UnbreakableTailLength + Postfix.size() +
258
22.4k
         encoding::columnWidthWithTabs(Line.substr(Offset), StartColumn,
259
22.4k
                                       Style.TabWidth, Encoding);
260
22.4k
}
261
262
unsigned BreakableStringLiteral::getContentStartColumn(unsigned LineIndex,
263
22.4k
                                                       bool Break) const {
264
22.4k
  return StartColumn + Prefix.size();
265
22.4k
}
266
267
BreakableStringLiteral::BreakableStringLiteral(
268
    const FormatToken &Tok, unsigned StartColumn, StringRef Prefix,
269
    StringRef Postfix, unsigned UnbreakableTailLength, bool InPPDirective,
270
    encoding::Encoding Encoding, const FormatStyle &Style)
271
21.0k
    : BreakableToken(Tok, InPPDirective, Encoding, Style),
272
21.0k
      StartColumn(StartColumn), Prefix(Prefix), Postfix(Postfix),
273
21.0k
      UnbreakableTailLength(UnbreakableTailLength) {
274
21.0k
  assert(Tok.TokenText.startswith(Prefix) && Tok.TokenText.endswith(Postfix));
275
21.0k
  Line = Tok.TokenText.substr(
276
21.0k
      Prefix.size(), Tok.TokenText.size() - Prefix.size() - Postfix.size());
277
21.0k
}
278
279
BreakableToken::Split BreakableStringLiteral::getSplit(
280
    unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
281
2.25k
    unsigned ContentStartColumn, const llvm::Regex &CommentPragmasRegex) const {
282
2.25k
  return getStringSplit(Line.substr(TailOffset), ContentStartColumn,
283
2.25k
                        ColumnLimit - Postfix.size(), Style.TabWidth, Encoding);
284
2.25k
}
285
286
void BreakableStringLiteral::insertBreak(unsigned LineIndex,
287
                                         unsigned TailOffset, Split Split,
288
                                         unsigned ContentIndent,
289
184
                                         WhitespaceManager &Whitespaces) const {
290
184
  Whitespaces.replaceWhitespaceInToken(
291
184
      Tok, Prefix.size() + TailOffset + Split.first, Split.second, Postfix,
292
184
      Prefix, InPPDirective, 1, StartColumn);
293
184
}
294
295
BreakableStringLiteralUsingOperators::BreakableStringLiteralUsingOperators(
296
    const FormatToken &Tok, QuoteStyleType QuoteStyle, bool UnindentPlus,
297
    unsigned StartColumn, unsigned UnbreakableTailLength, bool InPPDirective,
298
    encoding::Encoding Encoding, const FormatStyle &Style)
299
630
    : BreakableStringLiteral(
300
630
          Tok, StartColumn, /*Prefix=*/QuoteStyle == SingleQuotes ? 
"'"0
301
630
                            : QuoteStyle == AtDoubleQuotes        ? 
"@\""16
302
630
                                                                  : 
"\""614
,
303
630
          /*Postfix=*/QuoteStyle == SingleQuotes ? 
"'"0
: "\"",
304
630
          UnbreakableTailLength, InPPDirective, Encoding, Style),
305
630
      BracesNeeded(Tok.isNot(TT_StringInConcatenation)),
306
630
      QuoteStyle(QuoteStyle) {
307
  // Find the replacement text for inserting braces and quotes and line breaks.
308
  // We don't create an allocated string concatenated from parts here because it
309
  // has to outlive the BreakableStringliteral object.  The brace replacements
310
  // include a quote so that WhitespaceManager can tell it apart from whitespace
311
  // replacements between the string and surrounding tokens.
312
313
  // The option is not implemented in JavaScript.
314
630
  bool SignOnNewLine =
315
630
      !Style.isJavaScript() &&
316
630
      Style.BreakBeforeBinaryOperators != FormatStyle::BOS_None;
317
318
630
  if (Style.isVerilog()) {
319
    // In Verilog, all strings are quoted by double quotes, joined by commas,
320
    // and wrapped in braces.  The comma is always before the newline.
321
149
    assert(QuoteStyle == DoubleQuotes);
322
149
    LeftBraceQuote = Style.Cpp11BracedListStyle ? 
"{\""137
:
"{ \""12
;
323
149
    RightBraceQuote = Style.Cpp11BracedListStyle ? 
"\"}"137
:
"\" }"12
;
324
149
    Postfix = "\",";
325
149
    Prefix = "\"";
326
481
  } else {
327
    // The plus sign may be on either line.  And also C# and JavaScript have
328
    // several quoting styles.
329
481
    if (QuoteStyle == SingleQuotes) {
330
0
      LeftBraceQuote = Style.SpacesInParensOptions.Other ? "( '" : "('";
331
0
      RightBraceQuote = Style.SpacesInParensOptions.Other ? "' )" : "')";
332
0
      Postfix = SignOnNewLine ? "'" : "' +";
333
0
      Prefix = SignOnNewLine ? "+ '" : "'";
334
481
    } else {
335
481
      if (QuoteStyle == AtDoubleQuotes) {
336
16
        LeftBraceQuote = Style.SpacesInParensOptions.Other ? 
"( @"0
: "(@";
337
16
        Prefix = SignOnNewLine ? 
"+ @\""0
: "@\"";
338
465
      } else {
339
465
        LeftBraceQuote = Style.SpacesInParensOptions.Other ? 
"( \""0
: "(\"";
340
465
        Prefix = SignOnNewLine ? 
"+ \""311
:
"\""154
;
341
465
      }
342
481
      RightBraceQuote = Style.SpacesInParensOptions.Other ? 
"\" )"0
: "\")";
343
481
      Postfix = SignOnNewLine ? 
"\""311
:
"\" +"170
;
344
481
    }
345
481
  }
346
347
  // Following lines are indented by the width of the brace and space if any.
348
630
  ContinuationIndent = BracesNeeded ? 
LeftBraceQuote.size() - 142
:
0588
;
349
  // The plus sign may need to be unindented depending on the style.
350
  // FIXME: Add support for DontAlign.
351
630
  if (!Style.isVerilog() && 
SignOnNewLine481
&&
!BracesNeeded311
&&
UnindentPlus311
&&
352
630
      
Style.AlignOperands == FormatStyle::OAS_AlignAfterOperator127
) {
353
8
    ContinuationIndent -= 2;
354
8
  }
355
630
}
356
357
unsigned BreakableStringLiteralUsingOperators::getRemainingLength(
358
734
    unsigned LineIndex, unsigned Offset, unsigned StartColumn) const {
359
734
  return UnbreakableTailLength + (BracesNeeded ? 
RightBraceQuote.size()82
:
1652
) +
360
734
         encoding::columnWidthWithTabs(Line.substr(Offset), StartColumn,
361
734
                                       Style.TabWidth, Encoding);
362
734
}
363
364
unsigned
365
BreakableStringLiteralUsingOperators::getContentStartColumn(unsigned LineIndex,
366
734
                                                            bool Break) const {
367
734
  return std::max(
368
734
      0,
369
734
      static_cast<int>(StartColumn) +
370
734
          (Break ? 
ContinuationIndent + static_cast<int>(Prefix.size())104
371
734
                 : 
(630
BracesNeeded630
?
static_cast<int>(LeftBraceQuote.size()) - 142
372
630
                                 : 
0588
) +
373
630
                       (QuoteStyle == AtDoubleQuotes ? 
216
:
1614
)));
374
734
}
375
376
void BreakableStringLiteralUsingOperators::insertBreak(
377
    unsigned LineIndex, unsigned TailOffset, Split Split,
378
22
    unsigned ContentIndent, WhitespaceManager &Whitespaces) const {
379
22
  Whitespaces.replaceWhitespaceInToken(
380
22
      Tok, /*Offset=*/(QuoteStyle == AtDoubleQuotes ? 
22
:
120
) + TailOffset +
381
22
               Split.first,
382
22
      /*ReplaceChars=*/Split.second, /*PreviousPostfix=*/Postfix,
383
22
      /*CurrentPrefix=*/Prefix, InPPDirective, /*NewLines=*/1,
384
      /*Spaces=*/
385
22
      std::max(0, static_cast<int>(StartColumn) + ContinuationIndent));
386
22
}
387
388
void BreakableStringLiteralUsingOperators::updateAfterBroken(
389
16
    WhitespaceManager &Whitespaces) const {
390
  // Add the braces required for breaking the token if they are needed.
391
16
  if (!BracesNeeded)
392
9
    return;
393
394
  // To add a brace or parenthesis, we replace the quote (or the at sign) with a
395
  // brace and another quote.  This is because the rest of the program requires
396
  // one replacement for each source range.  If we replace the empty strings
397
  // around the string, it may conflict with whitespace replacements between the
398
  // string and adjacent tokens.
399
7
  Whitespaces.replaceWhitespaceInToken(
400
7
      Tok, /*Offset=*/0, /*ReplaceChars=*/1, /*PreviousPostfix=*/"",
401
7
      /*CurrentPrefix=*/LeftBraceQuote, InPPDirective, /*NewLines=*/0,
402
7
      /*Spaces=*/0);
403
7
  Whitespaces.replaceWhitespaceInToken(
404
7
      Tok, /*Offset=*/Tok.TokenText.size() - 1, /*ReplaceChars=*/1,
405
7
      /*PreviousPostfix=*/RightBraceQuote,
406
7
      /*CurrentPrefix=*/"", InPPDirective, /*NewLines=*/0, /*Spaces=*/0);
407
7
}
408
409
BreakableComment::BreakableComment(const FormatToken &Token,
410
                                   unsigned StartColumn, bool InPPDirective,
411
                                   encoding::Encoding Encoding,
412
                                   const FormatStyle &Style)
413
28.4k
    : BreakableToken(Token, InPPDirective, Encoding, Style),
414
28.4k
      StartColumn(StartColumn) {}
415
416
56.9k
unsigned BreakableComment::getLineCount() const { return Lines.size(); }
417
418
BreakableToken::Split
419
BreakableComment::getSplit(unsigned LineIndex, unsigned TailOffset,
420
                           unsigned ColumnLimit, unsigned ContentStartColumn,
421
3.48k
                           const llvm::Regex &CommentPragmasRegex) const {
422
  // Don't break lines matching the comment pragmas regex.
423
3.48k
  if (CommentPragmasRegex.match(Content[LineIndex]))
424
0
    return Split(StringRef::npos, 0);
425
3.48k
  return getCommentSplit(Content[LineIndex].substr(TailOffset),
426
3.48k
                         ContentStartColumn, ColumnLimit, Style.TabWidth,
427
3.48k
                         Encoding, Style);
428
3.48k
}
429
430
void BreakableComment::compressWhitespace(
431
    unsigned LineIndex, unsigned TailOffset, Split Split,
432
49
    WhitespaceManager &Whitespaces) const {
433
49
  StringRef Text = Content[LineIndex].substr(TailOffset);
434
  // Text is relative to the content line, but Whitespaces operates relative to
435
  // the start of the corresponding token, so compute the start of the Split
436
  // that needs to be compressed into a single space relative to the start of
437
  // its token.
438
49
  unsigned BreakOffsetInToken =
439
49
      Text.data() - tokenAt(LineIndex).TokenText.data() + Split.first;
440
49
  unsigned CharsToRemove = Split.second;
441
49
  Whitespaces.replaceWhitespaceInToken(
442
49
      tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove, "", "",
443
49
      /*InPPDirective=*/false, /*Newlines=*/0, /*Spaces=*/1);
444
49
}
445
446
4.89k
const FormatToken &BreakableComment::tokenAt(unsigned LineIndex) const {
447
4.89k
  return Tokens[LineIndex] ? 
*Tokens[LineIndex]1.56k
:
Tok3.33k
;
448
4.89k
}
449
450
923
static bool mayReflowContent(StringRef Content) {
451
923
  Content = Content.trim(Blanks);
452
  // Lines starting with '@' commonly have special meaning.
453
  // Lines starting with '-', '-#', '+' or '*' are bulleted/numbered lists.
454
923
  bool hasSpecialMeaningPrefix = false;
455
923
  for (StringRef Prefix :
456
7.25k
       {"@", "TODO", "FIXME", "XXX", "-# ", "- ", "+ ", "* "}) {
457
7.25k
    if (Content.startswith(Prefix)) {
458
30
      hasSpecialMeaningPrefix = true;
459
30
      break;
460
30
    }
461
7.25k
  }
462
463
  // Numbered lists may also start with a number followed by '.'
464
  // To avoid issues if a line starts with a number which is actually the end
465
  // of a previous line, we only consider numbers with up to 2 digits.
466
923
  static const auto kNumberedListRegexp = llvm::Regex("^[1-9][0-9]?\\. ");
467
923
  hasSpecialMeaningPrefix =
468
923
      hasSpecialMeaningPrefix || 
kNumberedListRegexp.match(Content)893
;
469
470
  // Simple heuristic for what to reflow: content should contain at least two
471
  // characters and either the first or second character must be
472
  // non-punctuation.
473
923
  return Content.size() >= 2 && 
!hasSpecialMeaningPrefix773
&&
474
923
         
!Content.endswith("\\")741
&&
475
         // Note that this is UTF-8 safe, since if isPunctuation(Content[0]) is
476
         // true, then the first code point must be 1 byte long.
477
923
         
(741
!isPunctuation(Content[0])741
||
!isPunctuation(Content[1])6
);
478
923
}
479
480
BreakableBlockComment::BreakableBlockComment(
481
    const FormatToken &Token, unsigned StartColumn,
482
    unsigned OriginalStartColumn, bool FirstInLine, bool InPPDirective,
483
    encoding::Encoding Encoding, const FormatStyle &Style, bool UseCRLF)
484
4.49k
    : BreakableComment(Token, StartColumn, InPPDirective, Encoding, Style),
485
4.49k
      DelimitersOnNewline(false),
486
4.49k
      UnbreakableTailLength(Token.UnbreakableTailLength) {
487
4.49k
  assert(Tok.is(TT_BlockComment) &&
488
4.49k
         "block comment section must start with a block comment");
489
490
4.49k
  StringRef TokenText(Tok.TokenText);
491
4.49k
  assert(TokenText.startswith("/*") && TokenText.endswith("*/"));
492
4.49k
  TokenText.substr(2, TokenText.size() - 4)
493
4.49k
      .split(Lines, UseCRLF ? 
"\r\n"20
:
"\n"4.47k
);
494
495
4.49k
  int IndentDelta = StartColumn - OriginalStartColumn;
496
4.49k
  Content.resize(Lines.size());
497
4.49k
  Content[0] = Lines[0];
498
4.49k
  ContentColumn.resize(Lines.size());
499
  // Account for the initial '/*'.
500
4.49k
  ContentColumn[0] = StartColumn + 2;
501
4.49k
  Tokens.resize(Lines.size());
502
6.16k
  for (size_t i = 1; i < Lines.size(); 
++i1.66k
)
503
1.66k
    adjustWhitespace(i, IndentDelta);
504
505
  // Align decorations with the column of the star on the first line,
506
  // that is one column after the start "/*".
507
4.49k
  DecorationColumn = StartColumn + 1;
508
509
  // Account for comment decoration patterns like this:
510
  //
511
  // /*
512
  // ** blah blah blah
513
  // */
514
4.49k
  if (Lines.size() >= 2 && 
Content[1].startswith("**")850
&&
515
4.49k
      
static_cast<unsigned>(ContentColumn[1]) == StartColumn12
) {
516
8
    DecorationColumn = StartColumn;
517
8
  }
518
519
4.49k
  Decoration = "* ";
520
4.49k
  if (Lines.size() == 1 && 
!FirstInLine3.64k
) {
521
    // Comments for which FirstInLine is false can start on arbitrary column,
522
    // and available horizontal space can be too small to align consecutive
523
    // lines with the first one.
524
    // FIXME: We could, probably, align them to current indentation level, but
525
    // now we just wrap them without stars.
526
2.87k
    Decoration = "";
527
2.87k
  }
528
5.45k
  for (size_t i = 1, e = Content.size(); i < e && 
!Decoration.empty()1.48k
;
++i959
) {
529
1.24k
    const StringRef &Text = Content[i];
530
1.24k
    if (i + 1 == e) {
531
      // If the last line is empty, the closing "*/" will have a star.
532
601
      if (Text.empty())
533
281
        break;
534
639
    } else if (!Text.empty() && 
Decoration.startswith(Text)613
) {
535
19
      continue;
536
19
    }
537
1.90k
    
while (940
!Text.startswith(Decoration))
538
966
      Decoration = Decoration.drop_back(1);
539
940
  }
540
541
4.49k
  LastLineNeedsDecoration = true;
542
4.49k
  IndentAtLineBreak = ContentColumn[0] + 1;
543
6.16k
  for (size_t i = 1, e = Lines.size(); i < e; 
++i1.66k
) {
544
1.66k
    if (Content[i].empty()) {
545
549
      if (i + 1 == e) {
546
        // Empty last line means that we already have a star as a part of the
547
        // trailing */. We also need to preserve whitespace, so that */ is
548
        // correctly indented.
549
521
        LastLineNeedsDecoration = false;
550
        // Align the star in the last '*/' with the stars on the previous lines.
551
521
        if (e >= 2 && !Decoration.empty())
552
281
          ContentColumn[i] = DecorationColumn;
553
521
      } else 
if (28
Decoration.empty()28
) {
554
        // For all other lines, set the start column to 0 if they're empty, so
555
        // we do not insert trailing whitespace anywhere.
556
28
        ContentColumn[i] = 0;
557
28
      }
558
549
      continue;
559
549
    }
560
561
    // The first line already excludes the star.
562
    // The last line excludes the star if LastLineNeedsDecoration is false.
563
    // For all other lines, adjust the line to exclude the star and
564
    // (optionally) the first whitespace.
565
1.12k
    unsigned DecorationSize = Decoration.startswith(Content[i])
566
1.12k
                                  ? 
Content[i].size()21
567
1.12k
                                  : 
Decoration.size()1.09k
;
568
1.12k
    if (DecorationSize)
569
490
      ContentColumn[i] = DecorationColumn + DecorationSize;
570
1.12k
    Content[i] = Content[i].substr(DecorationSize);
571
1.12k
    if (!Decoration.startswith(Content[i])) {
572
1.08k
      IndentAtLineBreak =
573
1.08k
          std::min<int>(IndentAtLineBreak, std::max(0, ContentColumn[i]));
574
1.08k
    }
575
1.12k
  }
576
4.49k
  IndentAtLineBreak = std::max<unsigned>(IndentAtLineBreak, Decoration.size());
577
578
  // Detect a multiline jsdoc comment and set DelimitersOnNewline in that case.
579
4.49k
  if (Style.isJavaScript() || 
Style.Language == FormatStyle::LK_Java4.30k
) {
580
210
    if ((Lines[0] == "*" || 
Lines[0].startswith("* ")120
) &&
Lines.size() > 1156
) {
581
      // This is a multiline jsdoc comment.
582
102
      DelimitersOnNewline = true;
583
108
    } else if (Lines[0].startswith("* ") && 
Lines.size() == 154
) {
584
      // Detect a long single-line comment, like:
585
      // /** long long long */
586
      // Below, '2' is the width of '*/'.
587
54
      unsigned EndColumn =
588
54
          ContentColumn[0] +
589
54
          encoding::columnWidthWithTabs(Lines[0], ContentColumn[0],
590
54
                                        Style.TabWidth, Encoding) +
591
54
          2;
592
54
      DelimitersOnNewline = EndColumn > Style.ColumnLimit;
593
54
    }
594
210
  }
595
596
4.49k
  LLVM_DEBUG({
597
4.49k
    llvm::dbgs() << "IndentAtLineBreak " << IndentAtLineBreak << "\n";
598
4.49k
    llvm::dbgs() << "DelimitersOnNewline " << DelimitersOnNewline << "\n";
599
4.49k
    for (size_t i = 0; i < Lines.size(); ++i) {
600
4.49k
      llvm::dbgs() << i << " |" << Content[i] << "| "
601
4.49k
                   << "CC=" << ContentColumn[i] << "| "
602
4.49k
                   << "IN=" << (Content[i].data() - Lines[i].data()) << "\n";
603
4.49k
    }
604
4.49k
  });
605
4.49k
}
606
607
BreakableToken::Split BreakableBlockComment::getSplit(
608
    unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
609
1.25k
    unsigned ContentStartColumn, const llvm::Regex &CommentPragmasRegex) const {
610
  // Don't break lines matching the comment pragmas regex.
611
1.25k
  if (CommentPragmasRegex.match(Content[LineIndex]))
612
6
    return Split(StringRef::npos, 0);
613
1.24k
  return getCommentSplit(Content[LineIndex].substr(TailOffset),
614
1.24k
                         ContentStartColumn, ColumnLimit, Style.TabWidth,
615
1.24k
                         Encoding, Style, Decoration.endswith("*"));
616
1.25k
}
617
618
void BreakableBlockComment::adjustWhitespace(unsigned LineIndex,
619
1.66k
                                             int IndentDelta) {
620
  // When in a preprocessor directive, the trailing backslash in a block comment
621
  // is not needed, but can serve a purpose of uniformity with necessary escaped
622
  // newlines outside the comment. In this case we remove it here before
623
  // trimming the trailing whitespace. The backslash will be re-added later when
624
  // inserting a line break.
625
1.66k
  size_t EndOfPreviousLine = Lines[LineIndex - 1].size();
626
1.66k
  if (InPPDirective && 
Lines[LineIndex - 1].endswith("\\")22
)
627
14
    --EndOfPreviousLine;
628
629
  // Calculate the end of the non-whitespace text in the previous line.
630
1.66k
  EndOfPreviousLine =
631
1.66k
      Lines[LineIndex - 1].find_last_not_of(Blanks, EndOfPreviousLine);
632
1.66k
  if (EndOfPreviousLine == StringRef::npos)
633
376
    EndOfPreviousLine = 0;
634
1.29k
  else
635
1.29k
    ++EndOfPreviousLine;
636
  // Calculate the start of the non-whitespace text in the current line.
637
1.66k
  size_t StartOfLine = Lines[LineIndex].find_first_not_of(Blanks);
638
1.66k
  if (StartOfLine == StringRef::npos)
639
549
    StartOfLine = Lines[LineIndex].size();
640
641
1.66k
  StringRef Whitespace = Lines[LineIndex].substr(0, StartOfLine);
642
  // Adjust Lines to only contain relevant text.
643
1.66k
  size_t PreviousContentOffset =
644
1.66k
      Content[LineIndex - 1].data() - Lines[LineIndex - 1].data();
645
1.66k
  Content[LineIndex - 1] = Lines[LineIndex - 1].substr(
646
1.66k
      PreviousContentOffset, EndOfPreviousLine - PreviousContentOffset);
647
1.66k
  Content[LineIndex] = Lines[LineIndex].substr(StartOfLine);
648
649
  // Adjust the start column uniformly across all lines.
650
1.66k
  ContentColumn[LineIndex] =
651
1.66k
      encoding::columnWidthWithTabs(Whitespace, 0, Style.TabWidth, Encoding) +
652
1.66k
      IndentDelta;
653
1.66k
}
654
655
unsigned BreakableBlockComment::getRangeLength(unsigned LineIndex,
656
                                               unsigned Offset,
657
                                               StringRef::size_type Length,
658
7.88k
                                               unsigned StartColumn) const {
659
7.88k
  return encoding::columnWidthWithTabs(
660
7.88k
      Content[LineIndex].substr(Offset, Length), StartColumn, Style.TabWidth,
661
7.88k
      Encoding);
662
7.88k
}
663
664
unsigned BreakableBlockComment::getRemainingLength(unsigned LineIndex,
665
                                                   unsigned Offset,
666
7.18k
                                                   unsigned StartColumn) const {
667
7.18k
  unsigned LineLength =
668
7.18k
      UnbreakableTailLength +
669
7.18k
      getRangeLength(LineIndex, Offset, StringRef::npos, StartColumn);
670
7.18k
  if (LineIndex + 1 == Lines.size()) {
671
4.97k
    LineLength += 2;
672
    // We never need a decoration when breaking just the trailing "*/" postfix.
673
4.97k
    bool HasRemainingText = Offset < Content[LineIndex].size();
674
4.97k
    if (!HasRemainingText) {
675
854
      bool HasDecoration = Lines[LineIndex].ltrim().startswith(Decoration);
676
854
      if (HasDecoration)
677
545
        LineLength -= Decoration.size();
678
854
    }
679
4.97k
  }
680
7.18k
  return LineLength;
681
7.18k
}
682
683
unsigned BreakableBlockComment::getContentStartColumn(unsigned LineIndex,
684
6.62k
                                                      bool Break) const {
685
6.62k
  if (Break)
686
540
    return IndentAtLineBreak;
687
6.08k
  return std::max(0, ContentColumn[LineIndex]);
688
6.62k
}
689
690
const llvm::StringSet<>
691
    BreakableBlockComment::ContentIndentingJavadocAnnotations = {
692
        "@param", "@return",     "@returns", "@throws",  "@type", "@template",
693
        "@see",   "@deprecated", "@define",  "@exports", "@mods", "@private",
694
};
695
696
591
unsigned BreakableBlockComment::getContentIndent(unsigned LineIndex) const {
697
591
  if (Style.Language != FormatStyle::LK_Java && 
!Style.isJavaScript()569
)
698
504
    return 0;
699
  // The content at LineIndex 0 of a comment like:
700
  // /** line 0 */
701
  // is "* line 0", so we need to skip over the decoration in that case.
702
87
  StringRef ContentWithNoDecoration = Content[LineIndex];
703
87
  if (LineIndex == 0 && 
ContentWithNoDecoration.startswith("*")35
)
704
35
    ContentWithNoDecoration = ContentWithNoDecoration.substr(1).ltrim(Blanks);
705
87
  StringRef FirstWord = ContentWithNoDecoration.substr(
706
87
      0, ContentWithNoDecoration.find_first_of(Blanks));
707
87
  if (ContentIndentingJavadocAnnotations.contains(FirstWord))
708
56
    return Style.ContinuationIndentWidth;
709
31
  return 0;
710
87
}
711
712
void BreakableBlockComment::insertBreak(unsigned LineIndex, unsigned TailOffset,
713
                                        Split Split, unsigned ContentIndent,
714
195
                                        WhitespaceManager &Whitespaces) const {
715
195
  StringRef Text = Content[LineIndex].substr(TailOffset);
716
195
  StringRef Prefix = Decoration;
717
  // We need this to account for the case when we have a decoration "* " for all
718
  // the lines except for the last one, where the star in "*/" acts as a
719
  // decoration.
720
195
  unsigned LocalIndentAtLineBreak = IndentAtLineBreak;
721
195
  if (LineIndex + 1 == Lines.size() &&
722
195
      
Text.size() == Split.first + Split.second85
) {
723
    // For the last line we need to break before "*/", but not to add "* ".
724
17
    Prefix = "";
725
17
    if (LocalIndentAtLineBreak >= 2)
726
17
      LocalIndentAtLineBreak -= 2;
727
17
  }
728
  // The split offset is from the beginning of the line. Convert it to an offset
729
  // from the beginning of the token text.
730
195
  unsigned BreakOffsetInToken =
731
195
      Text.data() - tokenAt(LineIndex).TokenText.data() + Split.first;
732
195
  unsigned CharsToRemove = Split.second;
733
195
  assert(LocalIndentAtLineBreak >= Prefix.size());
734
195
  std::string PrefixWithTrailingIndent = std::string(Prefix);
735
195
  PrefixWithTrailingIndent.append(ContentIndent, ' ');
736
195
  Whitespaces.replaceWhitespaceInToken(
737
195
      tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove, "",
738
195
      PrefixWithTrailingIndent, InPPDirective, /*Newlines=*/1,
739
195
      /*Spaces=*/LocalIndentAtLineBreak + ContentIndent -
740
195
          PrefixWithTrailingIndent.size());
741
195
}
742
743
BreakableToken::Split BreakableBlockComment::getReflowSplit(
744
251
    unsigned LineIndex, const llvm::Regex &CommentPragmasRegex) const {
745
251
  if (!mayReflow(LineIndex, CommentPragmasRegex))
746
150
    return Split(StringRef::npos, 0);
747
748
  // If we're reflowing into a line with content indent, only reflow the next
749
  // line if its starting whitespace matches the content indent.
750
101
  size_t Trimmed = Content[LineIndex].find_first_not_of(Blanks);
751
101
  if (LineIndex) {
752
101
    unsigned PreviousContentIndent = getContentIndent(LineIndex - 1);
753
101
    if (PreviousContentIndent && 
Trimmed != StringRef::npos6
&&
754
101
        
Trimmed != PreviousContentIndent6
) {
755
4
      return Split(StringRef::npos, 0);
756
4
    }
757
101
  }
758
759
97
  return Split(0, Trimmed != StringRef::npos ? Trimmed : 
00
);
760
101
}
761
762
4.49k
bool BreakableBlockComment::introducesBreakBeforeToken() const {
763
  // A break is introduced when we want delimiters on newline.
764
4.49k
  return DelimitersOnNewline &&
765
4.49k
         
Lines[0].substr(1).find_first_not_of(Blanks) != StringRef::npos125
;
766
4.49k
}
767
768
void BreakableBlockComment::reflow(unsigned LineIndex,
769
29
                                   WhitespaceManager &Whitespaces) const {
770
29
  StringRef TrimmedContent = Content[LineIndex].ltrim(Blanks);
771
  // Here we need to reflow.
772
29
  assert(Tokens[LineIndex - 1] == Tokens[LineIndex] &&
773
29
         "Reflowing whitespace within a token");
774
  // This is the offset of the end of the last line relative to the start of
775
  // the token text in the token.
776
29
  unsigned WhitespaceOffsetInToken = Content[LineIndex - 1].data() +
777
29
                                     Content[LineIndex - 1].size() -
778
29
                                     tokenAt(LineIndex).TokenText.data();
779
29
  unsigned WhitespaceLength = TrimmedContent.data() -
780
29
                              tokenAt(LineIndex).TokenText.data() -
781
29
                              WhitespaceOffsetInToken;
782
29
  Whitespaces.replaceWhitespaceInToken(
783
29
      tokenAt(LineIndex), WhitespaceOffsetInToken,
784
29
      /*ReplaceChars=*/WhitespaceLength, /*PreviousPostfix=*/"",
785
29
      /*CurrentPrefix=*/ReflowPrefix, InPPDirective, /*Newlines=*/0,
786
29
      /*Spaces=*/0);
787
29
}
788
789
void BreakableBlockComment::adaptStartOfLine(
790
2.33k
    unsigned LineIndex, WhitespaceManager &Whitespaces) const {
791
2.33k
  if (LineIndex == 0) {
792
1.61k
    if (DelimitersOnNewline) {
793
      // Since we're breaking at index 1 below, the break position and the
794
      // break length are the same.
795
      // Note: this works because getCommentSplit is careful never to split at
796
      // the beginning of a line.
797
55
      size_t BreakLength = Lines[0].substr(1).find_first_not_of(Blanks);
798
55
      if (BreakLength != StringRef::npos) {
799
12
        insertBreak(LineIndex, 0, Split(1, BreakLength), /*ContentIndent=*/0,
800
12
                    Whitespaces);
801
12
      }
802
55
    }
803
1.61k
    return;
804
1.61k
  }
805
  // Here no reflow with the previous line will happen.
806
  // Fix the decoration of the line at LineIndex.
807
716
  StringRef Prefix = Decoration;
808
716
  if (Content[LineIndex].empty()) {
809
258
    if (LineIndex + 1 == Lines.size()) {
810
235
      if (!LastLineNeedsDecoration) {
811
        // If the last line was empty, we don't need a prefix, as the */ will
812
        // line up with the decoration (if it exists).
813
234
        Prefix = "";
814
234
      }
815
235
    } else 
if (23
!Decoration.empty()23
) {
816
      // For other empty lines, if we do have a decoration, adapt it to not
817
      // contain a trailing whitespace.
818
9
      Prefix = Prefix.substr(0, 1);
819
9
    }
820
458
  } else if (ContentColumn[LineIndex] == 1) {
821
    // This line starts immediately after the decorating *.
822
13
    Prefix = Prefix.substr(0, 1);
823
13
  }
824
  // This is the offset of the end of the last line relative to the start of the
825
  // token text in the token.
826
716
  unsigned WhitespaceOffsetInToken = Content[LineIndex - 1].data() +
827
716
                                     Content[LineIndex - 1].size() -
828
716
                                     tokenAt(LineIndex).TokenText.data();
829
716
  unsigned WhitespaceLength = Content[LineIndex].data() -
830
716
                              tokenAt(LineIndex).TokenText.data() -
831
716
                              WhitespaceOffsetInToken;
832
716
  Whitespaces.replaceWhitespaceInToken(
833
716
      tokenAt(LineIndex), WhitespaceOffsetInToken, WhitespaceLength, "", Prefix,
834
716
      InPPDirective, /*Newlines=*/1, ContentColumn[LineIndex] - Prefix.size());
835
716
}
836
837
BreakableToken::Split
838
4.49k
BreakableBlockComment::getSplitAfterLastLine(unsigned TailOffset) const {
839
4.49k
  if (DelimitersOnNewline) {
840
    // Replace the trailing whitespace of the last line with a newline.
841
    // In case the last line is empty, the ending '*/' is already on its own
842
    // line.
843
125
    StringRef Line = Content.back().substr(TailOffset);
844
125
    StringRef TrimmedLine = Line.rtrim(Blanks);
845
125
    if (!TrimmedLine.empty())
846
12
      return Split(TrimmedLine.size(), Line.size() - TrimmedLine.size());
847
125
  }
848
4.48k
  return Split(StringRef::npos, 0);
849
4.49k
}
850
851
bool BreakableBlockComment::mayReflow(
852
251
    unsigned LineIndex, const llvm::Regex &CommentPragmasRegex) const {
853
  // Content[LineIndex] may exclude the indent after the '*' decoration. In that
854
  // case, we compute the start of the comment pragma manually.
855
251
  StringRef IndentContent = Content[LineIndex];
856
251
  if (Lines[LineIndex].ltrim(Blanks).startswith("*"))
857
103
    IndentContent = Lines[LineIndex].ltrim(Blanks).substr(1);
858
251
  return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
859
251
         
mayReflowContent(Content[LineIndex])249
&&
!Tok.Finalized101
&&
860
251
         
!switchesFormatting(tokenAt(LineIndex))101
;
861
251
}
862
863
BreakableLineCommentSection::BreakableLineCommentSection(
864
    const FormatToken &Token, unsigned StartColumn, bool InPPDirective,
865
    encoding::Encoding Encoding, const FormatStyle &Style)
866
23.9k
    : BreakableComment(Token, StartColumn, InPPDirective, Encoding, Style) {
867
23.9k
  assert(Tok.is(TT_LineComment) &&
868
23.9k
         "line comment section must start with a line comment");
869
23.9k
  FormatToken *LineTok = nullptr;
870
23.9k
  const int Minimum = Style.SpacesInLineCommentPrefix.Minimum;
871
  // How many spaces we changed in the first line of the section, this will be
872
  // applied in all following lines
873
23.9k
  int FirstLineSpaceChange = 0;
874
23.9k
  for (const FormatToken *CurrentTok = &Tok;
875
33.6k
       CurrentTok && 
CurrentTok->is(TT_LineComment)26.1k
;
876
26.1k
       
CurrentTok = CurrentTok->Next9.70k
) {
877
26.1k
    LastLineTok = LineTok;
878
26.1k
    StringRef TokenText(CurrentTok->TokenText);
879
26.1k
    assert((TokenText.startswith("//") || TokenText.startswith("#")) &&
880
26.1k
           "unsupported line comment prefix, '//' and '#' are supported");
881
26.1k
    size_t FirstLineIndex = Lines.size();
882
26.1k
    TokenText.split(Lines, "\n");
883
26.1k
    Content.resize(Lines.size());
884
26.1k
    ContentColumn.resize(Lines.size());
885
26.1k
    PrefixSpaceChange.resize(Lines.size());
886
26.1k
    Tokens.resize(Lines.size());
887
26.1k
    Prefix.resize(Lines.size());
888
26.1k
    OriginalPrefix.resize(Lines.size());
889
52.2k
    for (size_t i = FirstLineIndex, e = Lines.size(); i < e; 
++i26.1k
) {
890
26.1k
      Lines[i] = Lines[i].ltrim(Blanks);
891
26.1k
      StringRef IndentPrefix = getLineCommentIndentPrefix(Lines[i], Style);
892
26.1k
      OriginalPrefix[i] = IndentPrefix;
893
26.1k
      const int SpacesInPrefix = llvm::count(IndentPrefix, ' ');
894
895
      // This lambda also considers multibyte character that is not handled in
896
      // functions like isPunctuation provided by CharInfo.
897
26.1k
      const auto NoSpaceBeforeFirstCommentChar = [&]() {
898
21.7k
        assert(Lines[i].size() > IndentPrefix.size());
899
21.7k
        const char FirstCommentChar = Lines[i][IndentPrefix.size()];
900
21.7k
        const unsigned FirstCharByteSize =
901
21.7k
            encoding::getCodePointNumBytes(FirstCommentChar, Encoding);
902
21.7k
        if (encoding::columnWidth(
903
21.7k
                Lines[i].substr(IndentPrefix.size(), FirstCharByteSize),
904
21.7k
                Encoding) != 1) {
905
48
          return false;
906
48
        }
907
        // In C-like comments, add a space before #. For example this is useful
908
        // to preserve the relative indentation when commenting out code with
909
        // #includes.
910
        //
911
        // In languages using # as the comment leader such as proto, don't
912
        // add a space to support patterns like:
913
        // #########
914
        // # section
915
        // #########
916
21.6k
        if (FirstCommentChar == '#' && 
!TokenText.startswith("#")22
)
917
14
          return false;
918
21.6k
        return FirstCommentChar == '\\' || 
isPunctuation(FirstCommentChar)21.6k
||
919
21.6k
               
isHorizontalWhitespace(FirstCommentChar)21.0k
;
920
21.6k
      };
921
922
      // On the first line of the comment section we calculate how many spaces
923
      // are to be added or removed, all lines after that just get only the
924
      // change and we will not look at the maximum anymore. Additionally to the
925
      // actual first line, we calculate that when the non space Prefix changes,
926
      // e.g. from "///" to "//".
927
26.1k
      if (i == 0 || OriginalPrefix[i].rtrim(Blanks) !=
928
24.0k
                        OriginalPrefix[i - 1].rtrim(Blanks)) {
929
24.0k
        if (SpacesInPrefix < Minimum && 
Lines[i].size() > IndentPrefix.size()4.92k
&&
930
24.0k
            
!NoSpaceBeforeFirstCommentChar()280
) {
931
184
          FirstLineSpaceChange = Minimum - SpacesInPrefix;
932
23.8k
        } else if (static_cast<unsigned>(SpacesInPrefix) >
933
23.8k
                   Style.SpacesInLineCommentPrefix.Maximum) {
934
80
          FirstLineSpaceChange =
935
80
              Style.SpacesInLineCommentPrefix.Maximum - SpacesInPrefix;
936
23.7k
        } else {
937
23.7k
          FirstLineSpaceChange = 0;
938
23.7k
        }
939
24.0k
      }
940
941
26.1k
      if (Lines[i].size() != IndentPrefix.size()) {
942
21.4k
        PrefixSpaceChange[i] = FirstLineSpaceChange;
943
944
21.4k
        if (SpacesInPrefix + PrefixSpaceChange[i] < Minimum) {
945
126
          PrefixSpaceChange[i] +=
946
126
              Minimum - (SpacesInPrefix + PrefixSpaceChange[i]);
947
126
        }
948
949
21.4k
        assert(Lines[i].size() > IndentPrefix.size());
950
21.4k
        const auto FirstNonSpace = Lines[i][IndentPrefix.size()];
951
21.4k
        const bool IsFormatComment = LineTok && 
switchesFormatting(*LineTok)2.09k
;
952
21.4k
        const bool LineRequiresLeadingSpace =
953
21.4k
            !NoSpaceBeforeFirstCommentChar() ||
954
21.4k
            
(581
FirstNonSpace == '}'581
&&
FirstLineSpaceChange != 048
);
955
21.4k
        const bool AllowsSpaceChange =
956
21.4k
            !IsFormatComment &&
957
21.4k
            
(21.4k
SpacesInPrefix != 021.4k
||
LineRequiresLeadingSpace389
);
958
959
21.4k
        if (PrefixSpaceChange[i] > 0 && 
AllowsSpaceChange394
) {
960
280
          Prefix[i] = IndentPrefix.str();
961
280
          Prefix[i].append(PrefixSpaceChange[i], ' ');
962
21.1k
        } else if (PrefixSpaceChange[i] < 0 && 
AllowsSpaceChange136
) {
963
136
          Prefix[i] = IndentPrefix
964
136
                          .drop_back(std::min<std::size_t>(
965
136
                              -PrefixSpaceChange[i], SpacesInPrefix))
966
136
                          .str();
967
21.0k
        } else {
968
21.0k
          Prefix[i] = IndentPrefix.str();
969
21.0k
        }
970
21.4k
      } else {
971
        // If the IndentPrefix is the whole line, there is no content and we
972
        // drop just all space
973
4.69k
        Prefix[i] = IndentPrefix.drop_back(SpacesInPrefix).str();
974
4.69k
      }
975
976
26.1k
      Tokens[i] = LineTok;
977
26.1k
      Content[i] = Lines[i].substr(IndentPrefix.size());
978
26.1k
      ContentColumn[i] =
979
26.1k
          StartColumn + encoding::columnWidthWithTabs(Prefix[i], StartColumn,
980
26.1k
                                                      Style.TabWidth, Encoding);
981
982
      // Calculate the end of the non-whitespace text in this line.
983
26.1k
      size_t EndOfLine = Content[i].find_last_not_of(Blanks);
984
26.1k
      if (EndOfLine == StringRef::npos)
985
4.69k
        EndOfLine = Content[i].size();
986
21.4k
      else
987
21.4k
        ++EndOfLine;
988
26.1k
      Content[i] = Content[i].substr(0, EndOfLine);
989
26.1k
    }
990
26.1k
    LineTok = CurrentTok->Next;
991
26.1k
    if (CurrentTok->Next && 
!CurrentTok->Next->ContinuesLineCommentSection18.5k
) {
992
      // A line comment section needs to broken by a line comment that is
993
      // preceded by at least two newlines. Note that we put this break here
994
      // instead of breaking at a previous stage during parsing, since that
995
      // would split the contents of the enum into two unwrapped lines in this
996
      // example, which is undesirable:
997
      // enum A {
998
      //   a, // comment about a
999
      //
1000
      //   // comment about b
1001
      //   b
1002
      // };
1003
      //
1004
      // FIXME: Consider putting separate line comment sections as children to
1005
      // the unwrapped line instead.
1006
16.4k
      break;
1007
16.4k
    }
1008
26.1k
  }
1009
23.9k
}
1010
1011
unsigned
1012
BreakableLineCommentSection::getRangeLength(unsigned LineIndex, unsigned Offset,
1013
                                            StringRef::size_type Length,
1014
30.7k
                                            unsigned StartColumn) const {
1015
30.7k
  return encoding::columnWidthWithTabs(
1016
30.7k
      Content[LineIndex].substr(Offset, Length), StartColumn, Style.TabWidth,
1017
30.7k
      Encoding);
1018
30.7k
}
1019
1020
unsigned
1021
BreakableLineCommentSection::getContentStartColumn(unsigned LineIndex,
1022
26.9k
                                                   bool /*Break*/) const {
1023
26.9k
  return ContentColumn[LineIndex];
1024
26.9k
}
1025
1026
void BreakableLineCommentSection::insertBreak(
1027
    unsigned LineIndex, unsigned TailOffset, Split Split,
1028
510
    unsigned ContentIndent, WhitespaceManager &Whitespaces) const {
1029
510
  StringRef Text = Content[LineIndex].substr(TailOffset);
1030
  // Compute the offset of the split relative to the beginning of the token
1031
  // text.
1032
510
  unsigned BreakOffsetInToken =
1033
510
      Text.data() - tokenAt(LineIndex).TokenText.data() + Split.first;
1034
510
  unsigned CharsToRemove = Split.second;
1035
510
  Whitespaces.replaceWhitespaceInToken(
1036
510
      tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove, "",
1037
510
      Prefix[LineIndex], InPPDirective, /*Newlines=*/1,
1038
510
      /*Spaces=*/ContentColumn[LineIndex] - Prefix[LineIndex].size());
1039
510
}
1040
1041
BreakableComment::Split BreakableLineCommentSection::getReflowSplit(
1042
674
    unsigned LineIndex, const llvm::Regex &CommentPragmasRegex) const {
1043
674
  if (!mayReflow(LineIndex, CommentPragmasRegex))
1044
92
    return Split(StringRef::npos, 0);
1045
1046
582
  size_t Trimmed = Content[LineIndex].find_first_not_of(Blanks);
1047
1048
  // In a line comment section each line is a separate token; thus, after a
1049
  // split we replace all whitespace before the current line comment token
1050
  // (which does not need to be included in the split), plus the start of the
1051
  // line up to where the content starts.
1052
582
  return Split(0, Trimmed != StringRef::npos ? Trimmed : 
00
);
1053
674
}
1054
1055
void BreakableLineCommentSection::reflow(unsigned LineIndex,
1056
226
                                         WhitespaceManager &Whitespaces) const {
1057
226
  if (LineIndex > 0 && Tokens[LineIndex] != Tokens[LineIndex - 1]) {
1058
    // Reflow happens between tokens. Replace the whitespace between the
1059
    // tokens by the empty string.
1060
225
    Whitespaces.replaceWhitespace(
1061
225
        *Tokens[LineIndex], /*Newlines=*/0, /*Spaces=*/0,
1062
225
        /*StartOfTokenColumn=*/StartColumn, /*IsAligned=*/true,
1063
225
        /*InPPDirective=*/false);
1064
225
  } else 
if (1
LineIndex > 01
) {
1065
    // In case we're reflowing after the '\' in:
1066
    //
1067
    //   // line comment \
1068
    //   // line 2
1069
    //
1070
    // the reflow happens inside the single comment token (it is a single line
1071
    // comment with an unescaped newline).
1072
    // Replace the whitespace between the '\' and '//' with the empty string.
1073
    //
1074
    // Offset points to after the '\' relative to start of the token.
1075
1
    unsigned Offset = Lines[LineIndex - 1].data() +
1076
1
                      Lines[LineIndex - 1].size() -
1077
1
                      tokenAt(LineIndex - 1).TokenText.data();
1078
    // WhitespaceLength is the number of chars between the '\' and the '//' on
1079
    // the next line.
1080
1
    unsigned WhitespaceLength =
1081
1
        Lines[LineIndex].data() - tokenAt(LineIndex).TokenText.data() - Offset;
1082
1
    Whitespaces.replaceWhitespaceInToken(*Tokens[LineIndex], Offset,
1083
1
                                         /*ReplaceChars=*/WhitespaceLength,
1084
1
                                         /*PreviousPostfix=*/"",
1085
1
                                         /*CurrentPrefix=*/"",
1086
1
                                         /*InPPDirective=*/false,
1087
1
                                         /*Newlines=*/0,
1088
1
                                         /*Spaces=*/0);
1089
1
  }
1090
  // Replace the indent and prefix of the token with the reflow prefix.
1091
226
  unsigned Offset =
1092
226
      Lines[LineIndex].data() - tokenAt(LineIndex).TokenText.data();
1093
226
  unsigned WhitespaceLength =
1094
226
      Content[LineIndex].data() - Lines[LineIndex].data();
1095
226
  Whitespaces.replaceWhitespaceInToken(*Tokens[LineIndex], Offset,
1096
226
                                       /*ReplaceChars=*/WhitespaceLength,
1097
226
                                       /*PreviousPostfix=*/"",
1098
226
                                       /*CurrentPrefix=*/ReflowPrefix,
1099
226
                                       /*InPPDirective=*/false,
1100
226
                                       /*Newlines=*/0,
1101
226
                                       /*Spaces=*/0);
1102
226
}
1103
1104
void BreakableLineCommentSection::adaptStartOfLine(
1105
5.68k
    unsigned LineIndex, WhitespaceManager &Whitespaces) const {
1106
  // If this is the first line of a token, we need to inform Whitespace Manager
1107
  // about it: either adapt the whitespace range preceding it, or mark it as an
1108
  // untouchable token.
1109
  // This happens for instance here:
1110
  // // line 1 \
1111
  // // line 2
1112
5.68k
  if (LineIndex > 0 && 
Tokens[LineIndex] != Tokens[LineIndex - 1]725
) {
1113
    // This is the first line for the current token, but no reflow with the
1114
    // previous token is necessary. However, we still may need to adjust the
1115
    // start column. Note that ContentColumn[LineIndex] is the expected
1116
    // content column after a possible update to the prefix, hence the prefix
1117
    // length change is included.
1118
722
    unsigned LineColumn =
1119
722
        ContentColumn[LineIndex] -
1120
722
        (Content[LineIndex].data() - Lines[LineIndex].data()) +
1121
722
        (OriginalPrefix[LineIndex].size() - Prefix[LineIndex].size());
1122
1123
    // We always want to create a replacement instead of adding an untouchable
1124
    // token, even if LineColumn is the same as the original column of the
1125
    // token. This is because WhitespaceManager doesn't align trailing
1126
    // comments if they are untouchable.
1127
722
    Whitespaces.replaceWhitespace(*Tokens[LineIndex],
1128
722
                                  /*Newlines=*/1,
1129
722
                                  /*Spaces=*/LineColumn,
1130
722
                                  /*StartOfTokenColumn=*/LineColumn,
1131
722
                                  /*IsAligned=*/true,
1132
722
                                  /*InPPDirective=*/false);
1133
722
  }
1134
5.68k
  if (OriginalPrefix[LineIndex] != Prefix[LineIndex]) {
1135
    // Adjust the prefix if necessary.
1136
189
    const auto SpacesToRemove = -std::min(PrefixSpaceChange[LineIndex], 0);
1137
189
    const auto SpacesToAdd = std::max(PrefixSpaceChange[LineIndex], 0);
1138
189
    Whitespaces.replaceWhitespaceInToken(
1139
189
        tokenAt(LineIndex), OriginalPrefix[LineIndex].size() - SpacesToRemove,
1140
189
        /*ReplaceChars=*/SpacesToRemove, "", "", /*InPPDirective=*/false,
1141
189
        /*Newlines=*/0, /*Spaces=*/SpacesToAdd);
1142
189
  }
1143
5.68k
}
1144
1145
23.9k
void BreakableLineCommentSection::updateNextToken(LineState &State) const {
1146
23.9k
  if (LastLineTok)
1147
989
    State.NextToken = LastLineTok->Next;
1148
23.9k
}
1149
1150
bool BreakableLineCommentSection::mayReflow(
1151
674
    unsigned LineIndex, const llvm::Regex &CommentPragmasRegex) const {
1152
  // Line comments have the indent as part of the prefix, so we need to
1153
  // recompute the start of the line.
1154
674
  StringRef IndentContent = Content[LineIndex];
1155
674
  if (Lines[LineIndex].startswith("//"))
1156
664
    IndentContent = Lines[LineIndex].substr(2);
1157
  // FIXME: Decide whether we want to reflow non-regular indents:
1158
  // Currently, we only reflow when the OriginalPrefix[LineIndex] matches the
1159
  // OriginalPrefix[LineIndex-1]. That means we don't reflow
1160
  // // text that protrudes
1161
  // //    into text with different indent
1162
  // We do reflow in that case in block comments.
1163
674
  return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
1164
674
         mayReflowContent(Content[LineIndex]) && 
!Tok.Finalized638
&&
1165
674
         
!switchesFormatting(tokenAt(LineIndex))638
&&
1166
674
         
OriginalPrefix[LineIndex] == OriginalPrefix[LineIndex - 1]638
;
1167
674
}
1168
1169
} // namespace format
1170
} // namespace clang