Coverage Report

Created: 2023-11-11 10:31

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Format/BreakableToken.h
Line
Count
Source (jump to first uncovered line)
1
//===--- BreakableToken.h - Format C++ code ---------------------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
///
9
/// \file
10
/// Declares BreakableToken, BreakableStringLiteral, BreakableComment,
11
/// BreakableBlockComment and BreakableLineCommentSection classes, that contain
12
/// token type-specific logic to break long lines in tokens and reflow content
13
/// between tokens.
14
///
15
//===----------------------------------------------------------------------===//
16
17
#ifndef LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H
18
#define LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H
19
20
#include "Encoding.h"
21
#include "TokenAnnotator.h"
22
#include "WhitespaceManager.h"
23
#include "llvm/ADT/StringSet.h"
24
#include "llvm/Support/Regex.h"
25
#include <utility>
26
27
namespace clang {
28
namespace format {
29
30
/// Checks if \p Token switches formatting, like /* clang-format off */.
31
/// \p Token must be a comment.
32
bool switchesFormatting(const FormatToken &Token);
33
34
struct FormatStyle;
35
36
/// Base class for tokens / ranges of tokens that can allow breaking
37
/// within the tokens - for example, to avoid whitespace beyond the column
38
/// limit, or to reflow text.
39
///
40
/// Generally, a breakable token consists of logical lines, addressed by a line
41
/// index. For example, in a sequence of line comments, each line comment is its
42
/// own logical line; similarly, for a block comment, each line in the block
43
/// comment is on its own logical line.
44
///
45
/// There are two methods to compute the layout of the token:
46
/// - getRangeLength measures the number of columns needed for a range of text
47
///   within a logical line, and
48
/// - getContentStartColumn returns the start column at which we want the
49
///   content of a logical line to start (potentially after introducing a line
50
///   break).
51
///
52
/// The mechanism to adapt the layout of the breakable token is organised
53
/// around the concept of a \c Split, which is a whitespace range that signifies
54
/// a position of the content of a token where a reformatting might be done.
55
///
56
/// Operating with splits is divided into two operations:
57
/// - getSplit, for finding a split starting at a position,
58
/// - insertBreak, for executing the split using a whitespace manager.
59
///
60
/// There is a pair of operations that are used to compress a long whitespace
61
/// range with a single space if that will bring the line length under the
62
/// column limit:
63
/// - getLineLengthAfterCompression, for calculating the size in columns of the
64
///   line after a whitespace range has been compressed, and
65
/// - compressWhitespace, for executing the whitespace compression using a
66
///   whitespace manager; note that the compressed whitespace may be in the
67
///   middle of the original line and of the reformatted line.
68
///
69
/// For tokens where the whitespace before each line needs to be also
70
/// reformatted, for example for tokens supporting reflow, there are analogous
71
/// operations that might be executed before the main line breaking occurs:
72
/// - getReflowSplit, for finding a split such that the content preceding it
73
///   needs to be specially reflown,
74
/// - reflow, for executing the split using a whitespace manager,
75
/// - introducesBreakBefore, for checking if reformatting the beginning
76
///   of the content introduces a line break before it,
77
/// - adaptStartOfLine, for executing the reflow using a whitespace
78
///   manager.
79
///
80
/// For tokens that require the whitespace after the last line to be
81
/// reformatted, for example in multiline jsdoc comments that require the
82
/// trailing '*/' to be on a line of itself, there are analogous operations
83
/// that might be executed after the last line has been reformatted:
84
/// - getSplitAfterLastLine, for finding a split after the last line that needs
85
///   to be reflown,
86
/// - replaceWhitespaceAfterLastLine, for executing the reflow using a
87
///   whitespace manager.
88
///
89
class BreakableToken {
90
public:
91
  /// Contains starting character index and length of split.
92
  typedef std::pair<StringRef::size_type, unsigned> Split;
93
94
50.9k
  virtual ~BreakableToken() {}
95
96
  /// Returns the number of lines in this token in the original code.
97
  virtual unsigned getLineCount() const = 0;
98
99
  /// Returns the number of columns required to format the text in the
100
  /// byte range [\p Offset, \p Offset \c + \p Length).
101
  ///
102
  /// \p Offset is the byte offset from the start of the content of the line
103
  ///    at \p LineIndex.
104
  ///
105
  /// \p StartColumn is the column at which the text starts in the formatted
106
  ///    file, needed to compute tab stops correctly.
107
  virtual unsigned getRangeLength(unsigned LineIndex, unsigned Offset,
108
                                  StringRef::size_type Length,
109
                                  unsigned StartColumn) const = 0;
110
111
  /// Returns the number of columns required to format the text following
112
  /// the byte \p Offset in the line \p LineIndex, including potentially
113
  /// unbreakable sequences of tokens following after the end of the token.
114
  ///
115
  /// \p Offset is the byte offset from the start of the content of the line
116
  ///    at \p LineIndex.
117
  ///
118
  /// \p StartColumn is the column at which the text starts in the formatted
119
  ///    file, needed to compute tab stops correctly.
120
  ///
121
  /// For breakable tokens that never use extra space at the end of a line, this
122
  /// is equivalent to getRangeLength with a Length of StringRef::npos.
123
  virtual unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,
124
29.9k
                                      unsigned StartColumn) const {
125
29.9k
    return getRangeLength(LineIndex, Offset, StringRef::npos, StartColumn);
126
29.9k
  }
127
128
  /// Returns the column at which content in line \p LineIndex starts,
129
  /// assuming no reflow.
130
  ///
131
  /// If \p Break is true, returns the column at which the line should start
132
  /// after the line break.
133
  /// If \p Break is false, returns the column at which the line itself will
134
  /// start.
135
  virtual unsigned getContentStartColumn(unsigned LineIndex,
136
                                         bool Break) const = 0;
137
138
  /// Returns additional content indent required for the second line after the
139
  /// content at line \p LineIndex is broken.
140
  ///
141
  // (Next lines do not start with `///` since otherwise -Wdocumentation picks
142
  // up the example annotations and generates warnings for them)
143
  // For example, Javadoc @param annotations require and indent of 4 spaces and
144
  // in this example getContentIndex(1) returns 4.
145
  // /**
146
  //  * @param loooooooooooooong line
147
  //  *     continuation
148
  //  */
149
2.96k
  virtual unsigned getContentIndent(unsigned LineIndex) const { return 0; }
150
151
  /// Returns a range (offset, length) at which to break the line at
152
  /// \p LineIndex, if previously broken at \p TailOffset. If possible, do not
153
  /// violate \p ColumnLimit, assuming the text starting at \p TailOffset in
154
  /// the token is formatted starting at ContentStartColumn in the reformatted
155
  /// file.
156
  virtual Split getSplit(unsigned LineIndex, unsigned TailOffset,
157
                         unsigned ColumnLimit, unsigned ContentStartColumn,
158
                         const llvm::Regex &CommentPragmasRegex) const = 0;
159
160
  /// Emits the previously retrieved \p Split via \p Whitespaces.
161
  virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
162
                           unsigned ContentIndent,
163
                           WhitespaceManager &Whitespaces) const = 0;
164
165
  /// Returns the number of columns needed to format
166
  /// \p RemainingTokenColumns, assuming that Split is within the range measured
167
  /// by \p RemainingTokenColumns, and that the whitespace in Split is reduced
168
  /// to a single space.
169
  unsigned getLengthAfterCompression(unsigned RemainingTokenColumns,
170
                                     Split Split) const;
171
172
  /// Replaces the whitespace range described by \p Split with a single
173
  /// space.
174
  virtual void compressWhitespace(unsigned LineIndex, unsigned TailOffset,
175
                                  Split Split,
176
                                  WhitespaceManager &Whitespaces) const = 0;
177
178
  /// Returns whether the token supports reflowing text.
179
2.20k
  virtual bool supportsReflow() const { return false; }
180
181
  /// Returns a whitespace range (offset, length) of the content at \p
182
  /// LineIndex such that the content of that line is reflown to the end of the
183
  /// previous one.
184
  ///
185
  /// Returning (StringRef::npos, 0) indicates reflowing is not possible.
186
  ///
187
  /// The range will include any whitespace preceding the specified line's
188
  /// content.
189
  ///
190
  /// If the split is not contained within one token, for example when reflowing
191
  /// line comments, returns (0, <length>).
192
  virtual Split getReflowSplit(unsigned LineIndex,
193
0
                               const llvm::Regex &CommentPragmasRegex) const {
194
0
    return Split(StringRef::npos, 0);
195
0
  }
196
197
  /// Reflows the current line into the end of the previous one.
198
  virtual void reflow(unsigned LineIndex,
199
0
                      WhitespaceManager &Whitespaces) const {}
200
201
  /// Returns whether there will be a line break at the start of the
202
  /// token.
203
46.4k
  virtual bool introducesBreakBeforeToken() const { return false; }
204
205
  /// Replaces the whitespace between \p LineIndex-1 and \p LineIndex.
206
  virtual void adaptStartOfLine(unsigned LineIndex,
207
2.67k
                                WhitespaceManager &Whitespaces) const {}
208
209
  /// Returns a whitespace range (offset, length) of the content at
210
  /// the last line that needs to be reformatted after the last line has been
211
  /// reformatted.
212
  ///
213
  /// A result having offset == StringRef::npos means that no reformat is
214
  /// necessary.
215
46.4k
  virtual Split getSplitAfterLastLine(unsigned TailOffset) const {
216
46.4k
    return Split(StringRef::npos, 0);
217
46.4k
  }
218
219
  /// Replaces the whitespace from \p SplitAfterLastLine on the last line
220
  /// after the last line has been formatted by performing a reformatting.
221
  void replaceWhitespaceAfterLastLine(unsigned TailOffset,
222
                                      Split SplitAfterLastLine,
223
6
                                      WhitespaceManager &Whitespaces) const {
224
6
    insertBreak(getLineCount() - 1, TailOffset, SplitAfterLastLine,
225
6
                /*ContentIndent=*/0, Whitespaces);
226
6
  }
227
228
  /// Updates the next token of \p State to the next token after this
229
  /// one. This can be used when this token manages a set of underlying tokens
230
  /// as a unit and is responsible for the formatting of the them.
231
25.5k
  virtual void updateNextToken(LineState &State) const {}
232
233
  /// Adds replacements that are needed when the token is broken. Such as
234
  /// wrapping a JavaScript string in parentheses after it gets broken with plus
235
  /// signs.
236
444
  virtual void updateAfterBroken(WhitespaceManager &Whitespaces) const {}
237
238
protected:
239
  BreakableToken(const FormatToken &Tok, bool InPPDirective,
240
                 encoding::Encoding Encoding, const FormatStyle &Style)
241
50.9k
      : Tok(Tok), InPPDirective(InPPDirective), Encoding(Encoding),
242
50.9k
        Style(Style) {}
243
244
  const FormatToken &Tok;
245
  const bool InPPDirective;
246
  const encoding::Encoding Encoding;
247
  const FormatStyle &Style;
248
};
249
250
class BreakableStringLiteral : public BreakableToken {
251
public:
252
  /// Creates a breakable token for a single line string literal.
253
  ///
254
  /// \p StartColumn specifies the column in which the token will start
255
  /// after formatting.
256
  BreakableStringLiteral(const FormatToken &Tok, unsigned StartColumn,
257
                         StringRef Prefix, StringRef Postfix,
258
                         unsigned UnbreakableTailLength, bool InPPDirective,
259
                         encoding::Encoding Encoding, const FormatStyle &Style);
260
261
  Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
262
                 unsigned ContentStartColumn,
263
                 const llvm::Regex &CommentPragmasRegex) const override;
264
  void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
265
                   unsigned ContentIndent,
266
                   WhitespaceManager &Whitespaces) const override;
267
  void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split,
268
0
                          WhitespaceManager &Whitespaces) const override {}
269
  unsigned getLineCount() const override;
270
  unsigned getRangeLength(unsigned LineIndex, unsigned Offset,
271
                          StringRef::size_type Length,
272
                          unsigned StartColumn) const override;
273
  unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,
274
                              unsigned StartColumn) const override;
275
  unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;
276
277
protected:
278
  // The column in which the token starts.
279
  unsigned StartColumn;
280
  // The prefix a line needs after a break in the token.
281
  StringRef Prefix;
282
  // The postfix a line needs before introducing a break.
283
  StringRef Postfix;
284
  // The token text excluding the prefix and postfix.
285
  StringRef Line;
286
  // Length of the sequence of tokens after this string literal that cannot
287
  // contain line breaks.
288
  unsigned UnbreakableTailLength;
289
};
290
291
class BreakableStringLiteralUsingOperators : public BreakableStringLiteral {
292
public:
293
  enum QuoteStyleType {
294
    DoubleQuotes,   // The string is quoted with double quotes.
295
    SingleQuotes,   // The JavaScript string is quoted with single quotes.
296
    AtDoubleQuotes, // The C# verbatim string is quoted with the at sign and
297
                    // double quotes.
298
  };
299
  /// Creates a breakable token for a single line string literal for C#, Java,
300
  /// JavaScript, or Verilog.
301
  ///
302
  /// \p StartColumn specifies the column in which the token will start
303
  /// after formatting.
304
  BreakableStringLiteralUsingOperators(
305
      const FormatToken &Tok, QuoteStyleType QuoteStyle, bool UnindentPlus,
306
      unsigned StartColumn, unsigned UnbreakableTailLength, bool InPPDirective,
307
      encoding::Encoding Encoding, const FormatStyle &Style);
308
  unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,
309
                              unsigned StartColumn) const override;
310
  unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;
311
  void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
312
                   unsigned ContentIndent,
313
                   WhitespaceManager &Whitespaces) const override;
314
  void updateAfterBroken(WhitespaceManager &Whitespaces) const override;
315
316
protected:
317
  // Whether braces or parentheses should be inserted around the string to form
318
  // a concatenation.
319
  bool BracesNeeded;
320
  QuoteStyleType QuoteStyle;
321
  // The braces or parentheses along with the first character which they
322
  // replace, either a quote or at sign.
323
  StringRef LeftBraceQuote;
324
  StringRef RightBraceQuote;
325
  // Width added to the left due to the added brace or parenthesis. Does not
326
  // apply to the first line.
327
  int ContinuationIndent;
328
};
329
330
class BreakableComment : public BreakableToken {
331
protected:
332
  /// Creates a breakable token for a comment.
333
  ///
334
  /// \p StartColumn specifies the column in which the comment will start after
335
  /// formatting.
336
  BreakableComment(const FormatToken &Token, unsigned StartColumn,
337
                   bool InPPDirective, encoding::Encoding Encoding,
338
                   const FormatStyle &Style);
339
340
public:
341
1.93k
  bool supportsReflow() const override { return true; }
342
  unsigned getLineCount() const override;
343
  Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
344
                 unsigned ContentStartColumn,
345
                 const llvm::Regex &CommentPragmasRegex) const override;
346
  void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split,
347
                          WhitespaceManager &Whitespaces) const override;
348
349
protected:
350
  // Returns the token containing the line at LineIndex.
351
  const FormatToken &tokenAt(unsigned LineIndex) const;
352
353
  // Checks if the content of line LineIndex may be reflown with the previous
354
  // line.
355
  virtual bool mayReflow(unsigned LineIndex,
356
                         const llvm::Regex &CommentPragmasRegex) const = 0;
357
358
  // Contains the original text of the lines of the block comment.
359
  //
360
  // In case of a block comments, excludes the leading /* in the first line and
361
  // trailing */ in the last line. In case of line comments, excludes the
362
  // leading // and spaces.
363
  SmallVector<StringRef, 16> Lines;
364
365
  // Contains the text of the lines excluding all leading and trailing
366
  // whitespace between the lines. Note that the decoration (if present) is also
367
  // not considered part of the text.
368
  SmallVector<StringRef, 16> Content;
369
370
  // Tokens[i] contains a reference to the token containing Lines[i] if the
371
  // whitespace range before that token is managed by this block.
372
  // Otherwise, Tokens[i] is a null pointer.
373
  SmallVector<FormatToken *, 16> Tokens;
374
375
  // ContentColumn[i] is the target column at which Content[i] should be.
376
  // Note that this excludes a leading "* " or "*" in case of block comments
377
  // where all lines have a "*" prefix, or the leading "// " or "//" in case of
378
  // line comments.
379
  //
380
  // In block comments, the first line's target column is always positive. The
381
  // remaining lines' target columns are relative to the first line to allow
382
  // correct indentation of comments in \c WhitespaceManager. Thus they can be
383
  // negative as well (in case the first line needs to be unindented more than
384
  // there's actual whitespace in another line).
385
  SmallVector<int, 16> ContentColumn;
386
387
  // The intended start column of the first line of text from this section.
388
  unsigned StartColumn;
389
390
  // The prefix to use in front a line that has been reflown up.
391
  // For example, when reflowing the second line after the first here:
392
  // // comment 1
393
  // // comment 2
394
  // we expect:
395
  // // comment 1 comment 2
396
  // and not:
397
  // // comment 1comment 2
398
  StringRef ReflowPrefix = " ";
399
};
400
401
class BreakableBlockComment : public BreakableComment {
402
public:
403
  BreakableBlockComment(const FormatToken &Token, unsigned StartColumn,
404
                        unsigned OriginalStartColumn, bool FirstInLine,
405
                        bool InPPDirective, encoding::Encoding Encoding,
406
                        const FormatStyle &Style, bool UseCRLF);
407
408
  Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
409
                 unsigned ContentStartColumn,
410
                 const llvm::Regex &CommentPragmasRegex) const override;
411
  unsigned getRangeLength(unsigned LineIndex, unsigned Offset,
412
                          StringRef::size_type Length,
413
                          unsigned StartColumn) const override;
414
  unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,
415
                              unsigned StartColumn) const override;
416
  unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;
417
  unsigned getContentIndent(unsigned LineIndex) const override;
418
  void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
419
                   unsigned ContentIndent,
420
                   WhitespaceManager &Whitespaces) const override;
421
  Split getReflowSplit(unsigned LineIndex,
422
                       const llvm::Regex &CommentPragmasRegex) const override;
423
  void reflow(unsigned LineIndex,
424
              WhitespaceManager &Whitespaces) const override;
425
  bool introducesBreakBeforeToken() const override;
426
  void adaptStartOfLine(unsigned LineIndex,
427
                        WhitespaceManager &Whitespaces) const override;
428
  Split getSplitAfterLastLine(unsigned TailOffset) const override;
429
430
  bool mayReflow(unsigned LineIndex,
431
                 const llvm::Regex &CommentPragmasRegex) const override;
432
433
  // Contains Javadoc annotations that require additional indent when continued
434
  // on multiple lines.
435
  static const llvm::StringSet<> ContentIndentingJavadocAnnotations;
436
437
private:
438
  // Rearranges the whitespace between Lines[LineIndex-1] and Lines[LineIndex].
439
  //
440
  // Updates Content[LineIndex-1] and Content[LineIndex] by stripping off
441
  // leading and trailing whitespace.
442
  //
443
  // Sets ContentColumn to the intended column in which the text at
444
  // Lines[LineIndex] starts (note that the decoration, if present, is not
445
  // considered part of the text).
446
  void adjustWhitespace(unsigned LineIndex, int IndentDelta);
447
448
  // The column at which the text of a broken line should start.
449
  // Note that an optional decoration would go before that column.
450
  // IndentAtLineBreak is a uniform position for all lines in a block comment,
451
  // regardless of their relative position.
452
  // FIXME: Revisit the decision to do this; the main reason was to support
453
  // patterns like
454
  // /**************//**
455
  //  * Comment
456
  // We could also support such patterns by special casing the first line
457
  // instead.
458
  unsigned IndentAtLineBreak;
459
460
  // This is to distinguish between the case when the last line was empty and
461
  // the case when it started with a decoration ("*" or "* ").
462
  bool LastLineNeedsDecoration;
463
464
  // Either "* " if all lines begin with a "*", or empty.
465
  StringRef Decoration;
466
467
  // If this block comment has decorations, this is the column of the start of
468
  // the decorations.
469
  unsigned DecorationColumn;
470
471
  // If true, make sure that the opening '/**' and the closing '*/' ends on a
472
  // line of itself. Styles like jsdoc require this for multiline comments.
473
  bool DelimitersOnNewline;
474
475
  // Length of the sequence of tokens after this string literal that cannot
476
  // contain line breaks.
477
  unsigned UnbreakableTailLength;
478
};
479
480
class BreakableLineCommentSection : public BreakableComment {
481
public:
482
  BreakableLineCommentSection(const FormatToken &Token, unsigned StartColumn,
483
                              bool InPPDirective, encoding::Encoding Encoding,
484
                              const FormatStyle &Style);
485
486
  unsigned getRangeLength(unsigned LineIndex, unsigned Offset,
487
                          StringRef::size_type Length,
488
                          unsigned StartColumn) const override;
489
  unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;
490
  void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
491
                   unsigned ContentIndent,
492
                   WhitespaceManager &Whitespaces) const override;
493
  Split getReflowSplit(unsigned LineIndex,
494
                       const llvm::Regex &CommentPragmasRegex) const override;
495
  void reflow(unsigned LineIndex,
496
              WhitespaceManager &Whitespaces) const override;
497
  void adaptStartOfLine(unsigned LineIndex,
498
                        WhitespaceManager &Whitespaces) const override;
499
  void updateNextToken(LineState &State) const override;
500
  bool mayReflow(unsigned LineIndex,
501
                 const llvm::Regex &CommentPragmasRegex) const override;
502
503
private:
504
  // OriginalPrefix[i] contains the original prefix of line i, including
505
  // trailing whitespace before the start of the content. The indentation
506
  // preceding the prefix is not included.
507
  // For example, if the line is:
508
  // // content
509
  // then the original prefix is "// ".
510
  SmallVector<StringRef, 16> OriginalPrefix;
511
512
  /// Prefix[i] + SpacesToAdd[i] contains the intended leading "//" with
513
  /// trailing spaces to account for the indentation of content within the
514
  /// comment at line i after formatting. It can be different than the original
515
  /// prefix.
516
  /// When the original line starts like this:
517
  /// //content
518
  /// Then the OriginalPrefix[i] is "//", but the Prefix[i] is "// " in the LLVM
519
  /// style.
520
  /// When the line starts like:
521
  /// // content
522
  /// And we want to remove the spaces the OriginalPrefix[i] is "// " and
523
  /// Prefix[i] is "//".
524
  SmallVector<std::string, 16> Prefix;
525
526
  /// How many spaces are added or removed from the OriginalPrefix to form
527
  /// Prefix.
528
  SmallVector<int, 16> PrefixSpaceChange;
529
530
  /// The token to which the last line of this breakable token belongs
531
  /// to; nullptr if that token is the initial token.
532
  ///
533
  /// The distinction is because if the token of the last line of this breakable
534
  /// token is distinct from the initial token, this breakable token owns the
535
  /// whitespace before the token of the last line, and the whitespace manager
536
  /// must be able to modify it.
537
  FormatToken *LastLineTok = nullptr;
538
};
539
} // namespace format
540
} // namespace clang
541
542
#endif