Coverage Report

Created: 2023-05-31 04:38

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Format/FormatTokenSource.h
Line
Count
Source (jump to first uncovered line)
1
//===--- FormatTokenSource.h - Format C++ code ------------------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
///
9
/// \file
10
/// This file defines the \c FormatTokenSource interface, which provides a token
11
/// stream as well as the ability to manipulate the token stream.
12
///
13
//===----------------------------------------------------------------------===//
14
15
#ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H
16
#define LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H
17
18
#include "FormatToken.h"
19
#include "UnwrappedLineParser.h"
20
#include "llvm/ADT/DenseMap.h"
21
#include <cstddef>
22
23
#define DEBUG_TYPE "format-token-source"
24
25
namespace clang {
26
namespace format {
27
28
// Navigate a token stream.
29
//
30
// Enables traversal of a token stream, resetting the position in a token
31
// stream, as well as inserting new tokens.
32
class FormatTokenSource {
33
public:
34
90.5k
  virtual ~FormatTokenSource() {}
35
36
  // Returns the next token in the token stream.
37
  virtual FormatToken *getNextToken() = 0;
38
39
  // Returns the token preceding the token returned by the last call to
40
  // getNextToken() in the token stream, or nullptr if no such token exists.
41
  //
42
  // Must not be called directly at the position directly after insertTokens()
43
  // is called.
44
  virtual FormatToken *getPreviousToken() = 0;
45
46
  // Returns the token that would be returned by the next call to
47
  // getNextToken().
48
  virtual FormatToken *peekNextToken(bool SkipComment = false) = 0;
49
50
  // Returns whether we are at the end of the file.
51
  // This can be different from whether getNextToken() returned an eof token
52
  // when the FormatTokenSource is a view on a part of the token stream.
53
  virtual bool isEOF() = 0;
54
55
  // Gets the current position in the token stream, to be used by setPosition().
56
  //
57
  // Note that the value of the position is not meaningful, and specifically
58
  // should not be used to get relative token positions.
59
  virtual unsigned getPosition() = 0;
60
61
  // Resets the token stream to the state it was in when getPosition() returned
62
  // Position, and return the token at that position in the stream.
63
  virtual FormatToken *setPosition(unsigned Position) = 0;
64
65
  // Insert the given tokens before the current position.
66
  // Returns the first token in \c Tokens.
67
  // The next returned token will be the second token in \c Tokens.
68
  // Requires the last token in Tokens to be EOF; once the EOF token is reached,
69
  // the next token will be the last token returned by getNextToken();
70
  //
71
  // For example, given the token sequence 'a1 a2':
72
  // getNextToken() -> a1
73
  // insertTokens('b1 b2') -> b1
74
  // getNextToken() -> b2
75
  // getNextToken() -> a1
76
  // getNextToken() -> a2
77
  virtual FormatToken *insertTokens(ArrayRef<FormatToken *> Tokens) = 0;
78
};
79
80
class IndexedTokenSource : public FormatTokenSource {
81
public:
82
  IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
83
74.7k
      : Tokens(Tokens), Position(-1) {}
84
85
1.43M
  FormatToken *getNextToken() override {
86
1.43M
    if (Position >= 0 && 
isEOF()1.35M
) {
87
2
      LLVM_DEBUG({
88
2
        llvm::dbgs() << "Next ";
89
2
        dbgToken(Position);
90
2
      });
91
2
      return Tokens[Position];
92
2
    }
93
1.43M
    Position = successor(Position);
94
1.43M
    LLVM_DEBUG({
95
1.43M
      llvm::dbgs() << "Next ";
96
1.43M
      dbgToken(Position);
97
1.43M
    });
98
1.43M
    return Tokens[Position];
99
1.43M
  }
100
101
2.42k
  FormatToken *getPreviousToken() override {
102
2.42k
    assert(Position <= 0 || !Tokens[Position - 1]->is(tok::eof));
103
2.42k
    return Position > 0 ? 
Tokens[Position - 1]2.19k
:
nullptr228
;
104
2.42k
  }
105
106
77.9k
  FormatToken *peekNextToken(bool SkipComment = false) override {
107
77.9k
    if (isEOF())
108
2
      return Tokens[Position];
109
77.9k
    int Next = successor(Position);
110
77.9k
    if (SkipComment)
111
36.6k
      
while (35.6k
Tokens[Next]->is(tok::comment))
112
994
        Next = successor(Next);
113
77.9k
    LLVM_DEBUG({
114
77.9k
      llvm::dbgs() << "Peeking ";
115
77.9k
      dbgToken(Next);
116
77.9k
    });
117
77.9k
    return Tokens[Next];
118
77.9k
  }
119
120
1.43M
  bool isEOF() override {
121
1.43M
    return Position == -1 ? 
false2
:
Tokens[Position]->is(tok::eof)1.43M
;
122
1.43M
  }
123
124
24.2k
  unsigned getPosition() override {
125
24.2k
    LLVM_DEBUG(llvm::dbgs() << "Getting Position: " << Position << "\n");
126
24.2k
    assert(Position >= 0);
127
24.2k
    return Position;
128
24.2k
  }
129
130
23.7k
  FormatToken *setPosition(unsigned P) override {
131
23.7k
    LLVM_DEBUG(llvm::dbgs() << "Setting Position: " << P << "\n");
132
23.7k
    Position = P;
133
23.7k
    return Tokens[Position];
134
23.7k
  }
135
136
326
  FormatToken *insertTokens(ArrayRef<FormatToken *> New) override {
137
326
    assert(Position != -1);
138
326
    assert((*New.rbegin())->Tok.is(tok::eof));
139
326
    int Next = Tokens.size();
140
326
    Tokens.append(New.begin(), New.end());
141
326
    LLVM_DEBUG({
142
326
      llvm::dbgs() << "Inserting:\n";
143
326
      for (int I = Next, E = Tokens.size(); I != E; ++I)
144
326
        dbgToken(I, "  ");
145
326
      llvm::dbgs() << "  Jump from: " << (Tokens.size() - 1) << " -> "
146
326
                   << Position << "\n";
147
326
    });
148
326
    Jumps[Tokens.size() - 1] = Position;
149
326
    Position = Next;
150
326
    LLVM_DEBUG({
151
326
      llvm::dbgs() << "At inserted token ";
152
326
      dbgToken(Position);
153
326
    });
154
326
    return Tokens[Position];
155
326
  }
156
157
75.3k
  void reset() { Position = -1; }
158
159
private:
160
1.51M
  int successor(int Current) const {
161
1.51M
    int Next = Current + 1;
162
1.51M
    auto it = Jumps.find(Next);
163
1.51M
    if (it != Jumps.end()) {
164
359
      Next = it->second;
165
359
      assert(!Jumps.contains(Next));
166
359
    }
167
1.51M
    return Next;
168
1.51M
  }
169
170
0
  void dbgToken(int Position, llvm::StringRef Indent = "") {
171
0
    FormatToken *Tok = Tokens[Position];
172
0
    llvm::dbgs() << Indent << "[" << Position
173
0
                 << "] Token: " << Tok->Tok.getName() << " / " << Tok->TokenText
174
0
                 << ", Macro: " << !!Tok->MacroCtx << "\n";
175
0
  }
176
177
  SmallVector<FormatToken *> Tokens;
178
  int Position;
179
180
  // Maps from position a to position b, so that when we reach a, the token
181
  // stream continues at position b instead.
182
  llvm::DenseMap<int, int> Jumps;
183
};
184
185
class ScopedMacroState : public FormatTokenSource {
186
public:
187
  ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
188
                   FormatToken *&ResetToken)
189
      : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
190
        PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
191
15.8k
        Token(nullptr), PreviousToken(nullptr) {
192
15.8k
    FakeEOF.Tok.startToken();
193
15.8k
    FakeEOF.Tok.setKind(tok::eof);
194
15.8k
    TokenSource = this;
195
15.8k
    Line.Level = 0;
196
15.8k
    Line.InPPDirective = true;
197
    // InMacroBody gets set after the `#define x` part.
198
15.8k
  }
199
200
15.8k
  ~ScopedMacroState() override {
201
15.8k
    TokenSource = PreviousTokenSource;
202
15.8k
    ResetToken = Token;
203
15.8k
    Line.InPPDirective = false;
204
15.8k
    Line.InMacroBody = false;
205
15.8k
    Line.Level = PreviousLineLevel;
206
15.8k
  }
207
208
71.3k
  FormatToken *getNextToken() override {
209
    // The \c UnwrappedLineParser guards against this by never calling
210
    // \c getNextToken() after it has encountered the first eof token.
211
71.3k
    assert(!eof());
212
71.3k
    PreviousToken = Token;
213
71.3k
    Token = PreviousTokenSource->getNextToken();
214
71.3k
    if (eof())
215
12.9k
      return &FakeEOF;
216
58.3k
    return Token;
217
71.3k
  }
218
219
6
  FormatToken *getPreviousToken() override {
220
6
    return PreviousTokenSource->getPreviousToken();
221
6
  }
222
223
667
  FormatToken *peekNextToken(bool SkipComment) override {
224
667
    if (eof())
225
0
      return &FakeEOF;
226
667
    return PreviousTokenSource->peekNextToken(SkipComment);
227
667
  }
228
229
60
  bool isEOF() override { return PreviousTokenSource->isEOF(); }
230
231
288
  unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
232
233
288
  FormatToken *setPosition(unsigned Position) override {
234
288
    PreviousToken = nullptr;
235
288
    Token = PreviousTokenSource->setPosition(Position);
236
288
    return Token;
237
288
  }
238
239
0
  FormatToken *insertTokens(ArrayRef<FormatToken *> Tokens) override {
240
0
    llvm_unreachable("Cannot insert tokens while parsing a macro.");
241
0
    return nullptr;
242
0
  }
243
244
private:
245
143k
  bool eof() {
246
143k
    return Token && 
Token->HasUnescapedNewline127k
&&
247
143k
           !continuesLineComment(*Token, PreviousToken,
248
13.0k
                                 /*MinColumnToken=*/PreviousToken);
249
143k
  }
250
251
  FormatToken FakeEOF;
252
  UnwrappedLine &Line;
253
  FormatTokenSource *&TokenSource;
254
  FormatToken *&ResetToken;
255
  unsigned PreviousLineLevel;
256
  FormatTokenSource *PreviousTokenSource;
257
258
  FormatToken *Token;
259
  FormatToken *PreviousToken;
260
};
261
262
} // namespace format
263
} // namespace clang
264
265
#undef DEBUG_TYPE
266
267
#endif