Coverage Report

Created: 2022-05-17 06:19

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H
10
#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H
11
12
#include "clang/Basic/LLVM.h"
13
#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
14
#include "llvm/ADT/DenseSet.h"
15
#include "llvm/ADT/StringMap.h"
16
#include "llvm/Support/Allocator.h"
17
#include "llvm/Support/ErrorOr.h"
18
#include "llvm/Support/VirtualFileSystem.h"
19
#include <mutex>
20
21
namespace clang {
22
namespace tooling {
23
namespace dependencies {
24
25
/// Original and minimized contents of a cached file entry. Single instance can
26
/// be shared between multiple entries.
27
struct CachedFileContents {
28
  CachedFileContents(std::unique_ptr<llvm::MemoryBuffer> Original)
29
3.77k
      : Original(std::move(Original)), MinimizedAccess(nullptr) {}
30
31
  /// Owning storage for the minimized contents.
32
  std::unique_ptr<llvm::MemoryBuffer> Original;
33
34
  /// The mutex that must be locked before mutating minimized contents.
35
  std::mutex ValueLock;
36
  /// Owning storage for the minimized contents.
37
  std::unique_ptr<llvm::MemoryBuffer> MinimizedStorage;
38
  /// Accessor to the minimized contents that's atomic to avoid data races.
39
  std::atomic<llvm::MemoryBuffer *> MinimizedAccess;
40
  /// Skipped range mapping of the minimized contents.
41
  /// This is initialized iff `MinimizedAccess != nullptr`.
42
  PreprocessorSkippedRangeMapping PPSkippedRangeMapping;
43
};
44
45
/// An in-memory representation of a file system entity that is of interest to
46
/// the dependency scanning filesystem.
47
///
48
/// It represents one of the following:
49
/// - opened file with original contents and a stat value,
50
/// - opened file with original contents, minimized contents and a stat value,
51
/// - directory entry with its stat value,
52
/// - filesystem error.
53
///
54
/// Single instance of this class can be shared across different filenames (e.g.
55
/// a regular file and a symlink). For this reason the status filename is empty
56
/// and is only materialized by \c EntryRef that knows the requested filename.
57
class CachedFileSystemEntry {
58
public:
59
  /// Creates an entry without contents: either a filesystem error or
60
  /// a directory with stat value.
61
  CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat)
62
120
      : MaybeStat(std::move(Stat)), Contents(nullptr) {
63
120
    clearStatName();
64
120
  }
65
66
  /// Creates an entry representing a file with contents.
67
  CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat,
68
                        CachedFileContents *Contents)
69
4.58k
      : MaybeStat(std::move(Stat)), Contents(std::move(Contents)) {
70
4.58k
    clearStatName();
71
4.58k
  }
72
73
  /// \returns True if the entry is a filesystem error.
74
47.6k
  bool isError() const { return !MaybeStat; }
75
76
  /// \returns True if the current entry represents a directory.
77
11.5k
  bool isDirectory() const { return !isError() && MaybeStat->isDirectory(); }
78
79
  /// \returns Original contents of the file.
80
1.34k
  StringRef getOriginalContents() const {
81
1.34k
    assert(!isError() && "error");
82
0
    assert(!MaybeStat->isDirectory() && "not a file");
83
0
    assert(Contents && "contents not initialized");
84
0
    return Contents->Original->getBuffer();
85
1.34k
  }
86
87
  /// \returns Minimized contents of the file.
88
4.99k
  StringRef getMinimizedContents() const {
89
4.99k
    assert(!isError() && "error");
90
0
    assert(!MaybeStat->isDirectory() && "not a file");
91
0
    assert(Contents && "contents not initialized");
92
0
    llvm::MemoryBuffer *Buffer = Contents->MinimizedAccess.load();
93
4.99k
    assert(Buffer && "not minimized");
94
0
    return Buffer->getBuffer();
95
4.99k
  }
96
97
  /// \returns The error.
98
160
  std::error_code getError() const { return MaybeStat.getError(); }
99
100
  /// \returns The entry status with empty filename.
101
6.85k
  llvm::vfs::Status getStatus() const {
102
6.85k
    assert(!isError() && "error");
103
0
    assert(MaybeStat->getName().empty() && "stat name must be empty");
104
0
    return *MaybeStat;
105
6.85k
  }
106
107
  /// \returns The unique ID of the entry.
108
4.90k
  llvm::sys::fs::UniqueID getUniqueID() const {
109
4.90k
    assert(!isError() && "error");
110
0
    return MaybeStat->getUniqueID();
111
4.90k
  }
112
113
  /// \returns The mapping between location -> distance that is used to speed up
114
  /// the block skipping in the preprocessor.
115
328
  const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const {
116
328
    assert(!isError() && "error");
117
0
    assert(!isDirectory() && "not a file");
118
0
    assert(Contents && "contents not initialized");
119
0
    return Contents->PPSkippedRangeMapping;
120
328
  }
121
122
  /// \returns The data structure holding both original and minimized contents.
123
4.34k
  CachedFileContents *getContents() const {
124
4.34k
    assert(!isError() && "error");
125
0
    assert(!isDirectory() && "not a file");
126
0
    return Contents;
127
4.34k
  }
128
129
private:
130
4.70k
  void clearStatName() {
131
4.70k
    if (MaybeStat)
132
4.58k
      MaybeStat = llvm::vfs::Status::copyWithNewName(*MaybeStat, "");
133
4.70k
  }
134
135
  /// Either the filesystem error or status of the entry.
136
  /// The filename is empty and only materialized by \c EntryRef.
137
  llvm::ErrorOr<llvm::vfs::Status> MaybeStat;
138
139
  /// Non-owning pointer to the file contents.
140
  ///
141
  /// We're using pointer here to keep the size of this class small. Instances
142
  /// representing directories and filesystem errors don't hold any contents
143
  /// anyway.
144
  CachedFileContents *Contents;
145
};
146
147
/// This class is a shared cache, that caches the 'stat' and 'open' calls to the
148
/// underlying real file system. It distinguishes between minimized and original
149
/// files.
150
///
151
/// It is sharded based on the hash of the key to reduce the lock contention for
152
/// the worker threads.
153
class DependencyScanningFilesystemSharedCache {
154
public:
155
  struct CacheShard {
156
    /// The mutex that needs to be locked before mutation of any member.
157
    mutable std::mutex CacheLock;
158
159
    /// Map from filenames to cached entries.
160
    llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>
161
        EntriesByFilename;
162
163
    /// Map from unique IDs to cached entries.
164
    llvm::DenseMap<llvm::sys::fs::UniqueID, const CachedFileSystemEntry *>
165
        EntriesByUID;
166
167
    /// The backing storage for cached entries.
168
    llvm::SpecificBumpPtrAllocator<CachedFileSystemEntry> EntryStorage;
169
170
    /// The backing storage for cached contents.
171
    llvm::SpecificBumpPtrAllocator<CachedFileContents> ContentsStorage;
172
173
    /// Returns entry associated with the filename or nullptr if none is found.
174
    const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const;
175
176
    /// Returns entry associated with the unique ID or nullptr if none is found.
177
    const CachedFileSystemEntry *
178
    findEntryByUID(llvm::sys::fs::UniqueID UID) const;
179
180
    /// Returns entry associated with the filename if there is some. Otherwise,
181
    /// constructs new one with the given status, associates it with the
182
    /// filename and returns the result.
183
    const CachedFileSystemEntry &
184
    getOrEmplaceEntryForFilename(StringRef Filename,
185
                                 llvm::ErrorOr<llvm::vfs::Status> Stat);
186
187
    /// Returns entry associated with the unique ID if there is some. Otherwise,
188
    /// constructs new one with the given status and contents, associates it
189
    /// with the unique ID and returns the result.
190
    const CachedFileSystemEntry &
191
    getOrEmplaceEntryForUID(llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat,
192
                            std::unique_ptr<llvm::MemoryBuffer> Contents);
193
194
    /// Returns entry associated with the filename if there is some. Otherwise,
195
    /// associates the given entry with the filename and returns it.
196
    const CachedFileSystemEntry &
197
    getOrInsertEntryForFilename(StringRef Filename,
198
                                const CachedFileSystemEntry &Entry);
199
  };
200
201
  DependencyScanningFilesystemSharedCache();
202
203
  /// Returns shard for the given key.
204
  CacheShard &getShardForFilename(StringRef Filename) const;
205
  CacheShard &getShardForUID(llvm::sys::fs::UniqueID UID) const;
206
207
private:
208
  std::unique_ptr<CacheShard[]> CacheShards;
209
  unsigned NumShards;
210
};
211
212
/// This class is a local cache, that caches the 'stat' and 'open' calls to the
213
/// underlying real file system. It distinguishes between minimized and original
214
/// files.
215
class DependencyScanningFilesystemLocalCache {
216
  llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
217
218
public:
219
  /// Returns entry associated with the filename or nullptr if none is found.
220
8.15k
  const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const {
221
8.15k
    auto It = Cache.find(Filename);
222
8.15k
    return It == Cache.end() ? 
nullptr6.88k
:
It->getValue()1.27k
;
223
8.15k
  }
224
225
  /// Associates the given entry with the filename and returns the given entry
226
  /// pointer (for convenience).
227
  const CachedFileSystemEntry &
228
  insertEntryForFilename(StringRef Filename,
229
5.04k
                         const CachedFileSystemEntry &Entry) {
230
5.04k
    const auto *InsertedEntry = Cache.insert({Filename, &Entry}).first->second;
231
5.04k
    assert(InsertedEntry == &Entry && "entry already present");
232
0
    return *InsertedEntry;
233
5.04k
  }
234
};
235
236
/// Reference to a CachedFileSystemEntry.
237
/// If the underlying entry is an opened file, this wrapper returns the correct
238
/// contents (original or minimized) and ensures consistency with file size
239
/// reported by status.
240
class EntryRef {
241
  /// For entry that is an opened file, this bit signifies whether its contents
242
  /// are minimized.
243
  bool Minimized;
244
245
  /// The filename used to access this entry.
246
  std::string Filename;
247
248
  /// The underlying cached entry.
249
  const CachedFileSystemEntry &Entry;
250
251
public:
252
  EntryRef(bool Minimized, StringRef Name, const CachedFileSystemEntry &Entry)
253
6.32k
      : Minimized(Minimized), Filename(Name), Entry(Entry) {}
254
255
6.85k
  llvm::vfs::Status getStatus() const {
256
6.85k
    llvm::vfs::Status Stat = Entry.getStatus();
257
6.85k
    if (!Stat.isDirectory())
258
5.64k
      Stat = llvm::vfs::Status::copyWithNewSize(Stat, getContents().size());
259
6.85k
    return llvm::vfs::Status::copyWithNewName(Stat, Filename);
260
6.85k
  }
261
262
7.02k
  bool isError() const { return Entry.isError(); }
263
698
  bool isDirectory() const { return Entry.isDirectory(); }
264
265
  /// If the cached entry represents an error, promotes it into `ErrorOr`.
266
6.32k
  llvm::ErrorOr<EntryRef> unwrapError() const {
267
6.32k
    if (isError())
268
160
      return Entry.getError();
269
6.16k
    return *this;
270
6.32k
  }
271
272
6.33k
  StringRef getContents() const {
273
6.33k
    return Minimized ? 
Entry.getMinimizedContents()4.99k
274
6.33k
                     : 
Entry.getOriginalContents()1.34k
;
275
6.33k
  }
276
277
696
  const PreprocessorSkippedRangeMapping *getPPSkippedRangeMapping() const {
278
696
    return Minimized ? 
&Entry.getPPSkippedRangeMapping()328
:
nullptr368
;
279
696
  }
280
};
281
282
/// A virtual file system optimized for the dependency discovery.
283
///
284
/// It is primarily designed to work with source files whose contents was was
285
/// preprocessed to remove any tokens that are unlikely to affect the dependency
286
/// computation.
287
///
288
/// This is not a thread safe VFS. A single instance is meant to be used only in
289
/// one thread. Multiple instances are allowed to service multiple threads
290
/// running in parallel.
291
class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
292
public:
293
  DependencyScanningWorkerFilesystem(
294
      DependencyScanningFilesystemSharedCache &SharedCache,
295
      IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
296
      ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings)
297
      : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
298
427
        PPSkipMappings(PPSkipMappings) {}
299
300
  llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
301
  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
302
  openFileForRead(const Twine &Path) override;
303
304
  /// Disable minimization of the given file.
305
  void disableMinimization(StringRef Filename);
306
  /// Enable minimization of all files.
307
144
  void enableMinimizationOfAllFiles() { NotToBeMinimized.clear(); }
308
309
private:
310
  /// Check whether the file should be minimized.
311
  bool shouldMinimize(StringRef Filename, llvm::sys::fs::UniqueID UID);
312
313
  /// Returns entry for the given filename.
314
  ///
315
  /// Attempts to use the local and shared caches first, then falls back to
316
  /// using the underlying filesystem.
317
  llvm::ErrorOr<EntryRef>
318
  getOrCreateFileSystemEntry(StringRef Filename,
319
                             bool DisableMinimization = false);
320
321
  /// For a filename that's not yet associated with any entry in the caches,
322
  /// uses the underlying filesystem to either look up the entry based in the
323
  /// shared cache indexed by unique ID, or creates new entry from scratch.
324
  llvm::ErrorOr<const CachedFileSystemEntry &>
325
  computeAndStoreResult(StringRef Filename);
326
327
  /// Minimizes the given entry if necessary and returns a wrapper object with
328
  /// reference semantics.
329
  EntryRef minimizeIfNecessary(const CachedFileSystemEntry &Entry,
330
                               StringRef Filename, bool Disable);
331
332
  /// Represents a filesystem entry that has been stat-ed (and potentially read)
333
  /// and that's about to be inserted into the cache as `CachedFileSystemEntry`.
334
  struct TentativeEntry {
335
    llvm::vfs::Status Status;
336
    std::unique_ptr<llvm::MemoryBuffer> Contents;
337
338
    TentativeEntry(llvm::vfs::Status Status,
339
                   std::unique_ptr<llvm::MemoryBuffer> Contents = nullptr)
340
4.64k
        : Status(std::move(Status)), Contents(std::move(Contents)) {}
341
  };
342
343
  /// Reads file at the given path. Enforces consistency between the file size
344
  /// in status and size of read contents.
345
  llvm::ErrorOr<TentativeEntry> readFile(StringRef Filename);
346
347
  /// Returns entry associated with the unique ID of the given tentative entry
348
  /// if there is some in the shared cache. Otherwise, constructs new one,
349
  /// associates it with the unique ID and returns the result.
350
  const CachedFileSystemEntry &
351
  getOrEmplaceSharedEntryForUID(TentativeEntry TEntry);
352
353
  /// Returns entry associated with the filename or nullptr if none is found.
354
  ///
355
  /// Returns entry from local cache if there is some. Otherwise, if the entry
356
  /// is found in the shared cache, writes it through the local cache and
357
  /// returns it. Otherwise returns nullptr.
358
  const CachedFileSystemEntry *
359
  findEntryByFilenameWithWriteThrough(StringRef Filename);
360
361
  /// Returns entry associated with the unique ID in the shared cache or nullptr
362
  /// if none is found.
363
  const CachedFileSystemEntry *
364
4.81k
  findSharedEntryByUID(llvm::vfs::Status Stat) const {
365
4.81k
    return SharedCache.getShardForUID(Stat.getUniqueID())
366
4.81k
        .findEntryByUID(Stat.getUniqueID());
367
4.81k
  }
368
369
  /// Associates the given entry with the filename in the local cache and
370
  /// returns it.
371
  const CachedFileSystemEntry &
372
  insertLocalEntryForFilename(StringRef Filename,
373
4.95k
                              const CachedFileSystemEntry &Entry) {
374
4.95k
    return LocalCache.insertEntryForFilename(Filename, Entry);
375
4.95k
  }
376
377
  /// Returns entry associated with the filename in the shared cache if there is
378
  /// some. Otherwise, constructs new one with the given error code, associates
379
  /// it with the filename and returns the result.
380
  const CachedFileSystemEntry &
381
137
  getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) {
382
137
    return SharedCache.getShardForFilename(Filename)
383
137
        .getOrEmplaceEntryForFilename(Filename, EC);
384
137
  }
385
386
  /// Returns entry associated with the filename in the shared cache if there is
387
  /// some. Otherwise, associates the given entry with the filename and returns
388
  /// it.
389
  const CachedFileSystemEntry &
390
  getOrInsertSharedEntryForFilename(StringRef Filename,
391
4.64k
                                    const CachedFileSystemEntry &Entry) {
392
4.64k
    return SharedCache.getShardForFilename(Filename)
393
4.64k
        .getOrInsertEntryForFilename(Filename, Entry);
394
4.64k
  }
395
396
  /// The global cache shared between worker threads.
397
  DependencyScanningFilesystemSharedCache &SharedCache;
398
  /// The local cache is used by the worker thread to cache file system queries
399
  /// locally instead of querying the global cache every time.
400
  DependencyScanningFilesystemLocalCache LocalCache;
401
  /// The mapping structure which records information about the
402
  /// excluded conditional directive skip mappings that are used by the
403
  /// currently active preprocessor.
404
  ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings;
405
  /// The set of files that should not be minimized.
406
  llvm::DenseSet<llvm::sys::fs::UniqueID> NotToBeMinimized;
407
};
408
409
} // end namespace dependencies
410
} // end namespace tooling
411
} // end namespace clang
412
413
#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H