Coverage Report

Created: 2022-07-16 07:03

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H
10
#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H
11
12
#include "clang/Basic/LLVM.h"
13
#include "clang/Lex/DependencyDirectivesScanner.h"
14
#include "llvm/ADT/DenseSet.h"
15
#include "llvm/ADT/StringMap.h"
16
#include "llvm/Support/Allocator.h"
17
#include "llvm/Support/ErrorOr.h"
18
#include "llvm/Support/VirtualFileSystem.h"
19
#include <mutex>
20
21
namespace clang {
22
namespace tooling {
23
namespace dependencies {
24
25
using DependencyDirectivesTy =
26
    SmallVector<dependency_directives_scan::Directive, 20>;
27
28
/// Contents and directive tokens of a cached file entry. Single instance can
29
/// be shared between multiple entries.
30
struct CachedFileContents {
31
  CachedFileContents(std::unique_ptr<llvm::MemoryBuffer> Contents)
32
3.81k
      : Original(std::move(Contents)), DepDirectives(nullptr) {}
33
34
  /// Owning storage for the original contents.
35
  std::unique_ptr<llvm::MemoryBuffer> Original;
36
37
  /// The mutex that must be locked before mutating directive tokens.
38
  std::mutex ValueLock;
39
  SmallVector<dependency_directives_scan::Token, 10> DepDirectiveTokens;
40
  /// Accessor to the directive tokens that's atomic to avoid data races.
41
  /// \p CachedFileContents has ownership of the pointer.
42
  std::atomic<const Optional<DependencyDirectivesTy> *> DepDirectives;
43
44
3.81k
  ~CachedFileContents() { delete DepDirectives.load(); }
45
};
46
47
/// An in-memory representation of a file system entity that is of interest to
48
/// the dependency scanning filesystem.
49
///
50
/// It represents one of the following:
51
/// - opened file with contents and a stat value,
52
/// - opened file with contents, directive tokens and a stat value,
53
/// - directory entry with its stat value,
54
/// - filesystem error.
55
///
56
/// Single instance of this class can be shared across different filenames (e.g.
57
/// a regular file and a symlink). For this reason the status filename is empty
58
/// and is only materialized by \c EntryRef that knows the requested filename.
59
class CachedFileSystemEntry {
60
public:
61
  /// Creates an entry without contents: either a filesystem error or
62
  /// a directory with stat value.
63
  CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat)
64
120
      : MaybeStat(std::move(Stat)), Contents(nullptr) {
65
120
    clearStatName();
66
120
  }
67
68
  /// Creates an entry representing a file with contents.
69
  CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat,
70
                        CachedFileContents *Contents)
71
4.65k
      : MaybeStat(std::move(Stat)), Contents(std::move(Contents)) {
72
4.65k
    clearStatName();
73
4.65k
  }
74
75
  /// \returns True if the entry is a filesystem error.
76
43.9k
  bool isError() const { return !MaybeStat; }
77
78
  /// \returns True if the current entry represents a directory.
79
12.0k
  bool isDirectory() const { return !isError() && MaybeStat->isDirectory(); }
80
81
  /// \returns Original contents of the file.
82
6.42k
  StringRef getOriginalContents() const {
83
6.42k
    assert(!isError() && "error");
84
0
    assert(!MaybeStat->isDirectory() && "not a file");
85
0
    assert(Contents && "contents not initialized");
86
0
    return Contents->Original->getBuffer();
87
6.42k
  }
88
89
  /// \returns The scanned preprocessor directive tokens of the file that are
90
  /// used to speed up preprocessing, if available.
91
  Optional<ArrayRef<dependency_directives_scan::Directive>>
92
321
  getDirectiveTokens() const {
93
321
    assert(!isError() && "error");
94
0
    assert(!isDirectory() && "not a file");
95
0
    assert(Contents && "contents not initialized");
96
321
    if (auto *Directives = Contents->DepDirectives.load()) {
97
321
      if (Directives->has_value())
98
321
        return ArrayRef<dependency_directives_scan::Directive>(
99
321
            Directives->value());
100
321
    }
101
0
    return None;
102
321
  }
103
104
  /// \returns The error.
105
160
  std::error_code getError() const { return MaybeStat.getError(); }
106
107
  /// \returns The entry status with empty filename.
108
6.71k
  llvm::vfs::Status getStatus() const {
109
6.71k
    assert(!isError() && "error");
110
0
    assert(MaybeStat->getName().empty() && "stat name must be empty");
111
0
    return *MaybeStat;
112
6.71k
  }
113
114
  /// \returns The unique ID of the entry.
115
0
  llvm::sys::fs::UniqueID getUniqueID() const {
116
0
    assert(!isError() && "error");
117
0
    return MaybeStat->getUniqueID();
118
0
  }
119
120
  /// \returns The data structure holding both contents and directive tokens.
121
4.72k
  CachedFileContents *getCachedContents() const {
122
4.72k
    assert(!isError() && "error");
123
0
    assert(!isDirectory() && "not a file");
124
0
    return Contents;
125
4.72k
  }
126
127
private:
128
4.77k
  void clearStatName() {
129
4.77k
    if (MaybeStat)
130
4.65k
      MaybeStat = llvm::vfs::Status::copyWithNewName(*MaybeStat, "");
131
4.77k
  }
132
133
  /// Either the filesystem error or status of the entry.
134
  /// The filename is empty and only materialized by \c EntryRef.
135
  llvm::ErrorOr<llvm::vfs::Status> MaybeStat;
136
137
  /// Non-owning pointer to the file contents.
138
  ///
139
  /// We're using pointer here to keep the size of this class small. Instances
140
  /// representing directories and filesystem errors don't hold any contents
141
  /// anyway.
142
  CachedFileContents *Contents;
143
};
144
145
/// This class is a shared cache, that caches the 'stat' and 'open' calls to the
146
/// underlying real file system, and the scanned preprocessor directives of
147
/// files.
148
///
149
/// It is sharded based on the hash of the key to reduce the lock contention for
150
/// the worker threads.
151
class DependencyScanningFilesystemSharedCache {
152
public:
153
  struct CacheShard {
154
    /// The mutex that needs to be locked before mutation of any member.
155
    mutable std::mutex CacheLock;
156
157
    /// Map from filenames to cached entries.
158
    llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>
159
        EntriesByFilename;
160
161
    /// Map from unique IDs to cached entries.
162
    llvm::DenseMap<llvm::sys::fs::UniqueID, const CachedFileSystemEntry *>
163
        EntriesByUID;
164
165
    /// The backing storage for cached entries.
166
    llvm::SpecificBumpPtrAllocator<CachedFileSystemEntry> EntryStorage;
167
168
    /// The backing storage for cached contents.
169
    llvm::SpecificBumpPtrAllocator<CachedFileContents> ContentsStorage;
170
171
    /// Returns entry associated with the filename or nullptr if none is found.
172
    const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const;
173
174
    /// Returns entry associated with the unique ID or nullptr if none is found.
175
    const CachedFileSystemEntry *
176
    findEntryByUID(llvm::sys::fs::UniqueID UID) const;
177
178
    /// Returns entry associated with the filename if there is some. Otherwise,
179
    /// constructs new one with the given status, associates it with the
180
    /// filename and returns the result.
181
    const CachedFileSystemEntry &
182
    getOrEmplaceEntryForFilename(StringRef Filename,
183
                                 llvm::ErrorOr<llvm::vfs::Status> Stat);
184
185
    /// Returns entry associated with the unique ID if there is some. Otherwise,
186
    /// constructs new one with the given status and contents, associates it
187
    /// with the unique ID and returns the result.
188
    const CachedFileSystemEntry &
189
    getOrEmplaceEntryForUID(llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat,
190
                            std::unique_ptr<llvm::MemoryBuffer> Contents);
191
192
    /// Returns entry associated with the filename if there is some. Otherwise,
193
    /// associates the given entry with the filename and returns it.
194
    const CachedFileSystemEntry &
195
    getOrInsertEntryForFilename(StringRef Filename,
196
                                const CachedFileSystemEntry &Entry);
197
  };
198
199
  DependencyScanningFilesystemSharedCache();
200
201
  /// Returns shard for the given key.
202
  CacheShard &getShardForFilename(StringRef Filename) const;
203
  CacheShard &getShardForUID(llvm::sys::fs::UniqueID UID) const;
204
205
private:
206
  std::unique_ptr<CacheShard[]> CacheShards;
207
  unsigned NumShards;
208
};
209
210
/// This class is a local cache, that caches the 'stat' and 'open' calls to the
211
/// underlying real file system.
212
class DependencyScanningFilesystemLocalCache {
213
  llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
214
215
public:
216
  /// Returns entry associated with the filename or nullptr if none is found.
217
8.11k
  const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const {
218
8.11k
    auto It = Cache.find(Filename);
219
8.11k
    return It == Cache.end() ? 
nullptr6.71k
:
It->getValue()1.40k
;
220
8.11k
  }
221
222
  /// Associates the given entry with the filename and returns the given entry
223
  /// pointer (for convenience).
224
  const CachedFileSystemEntry &
225
  insertEntryForFilename(StringRef Filename,
226
5.05k
                         const CachedFileSystemEntry &Entry) {
227
5.05k
    const auto *InsertedEntry = Cache.insert({Filename, &Entry}).first->second;
228
5.05k
    assert(InsertedEntry == &Entry && "entry already present");
229
0
    return *InsertedEntry;
230
5.05k
  }
231
};
232
233
/// Reference to a CachedFileSystemEntry.
234
/// If the underlying entry is an opened file, this wrapper returns the file
235
/// contents and the scanned preprocessor directives.
236
class EntryRef {
237
  /// The filename used to access this entry.
238
  std::string Filename;
239
240
  /// The underlying cached entry.
241
  const CachedFileSystemEntry &Entry;
242
243
public:
244
  EntryRef(StringRef Name, const CachedFileSystemEntry &Entry)
245
6.45k
      : Filename(Name), Entry(Entry) {}
246
247
6.71k
  llvm::vfs::Status getStatus() const {
248
6.71k
    llvm::vfs::Status Stat = Entry.getStatus();
249
6.71k
    if (!Stat.isDirectory())
250
5.69k
      Stat = llvm::vfs::Status::copyWithNewSize(Stat, getContents().size());
251
6.71k
    return llvm::vfs::Status::copyWithNewName(Stat, Filename);
252
6.71k
  }
253
254
7.20k
  bool isError() const { return Entry.isError(); }
255
740
  bool isDirectory() const { return Entry.isDirectory(); }
256
257
  /// If the cached entry represents an error, promotes it into `ErrorOr`.
258
6.46k
  llvm::ErrorOr<EntryRef> unwrapError() const {
259
6.46k
    if (isError())
260
160
      return Entry.getError();
261
6.30k
    return *this;
262
6.46k
  }
263
264
6.42k
  StringRef getContents() const { return Entry.getOriginalContents(); }
265
266
  Optional<ArrayRef<dependency_directives_scan::Directive>>
267
321
  getDirectiveTokens() const {
268
321
    return Entry.getDirectiveTokens();
269
321
  }
270
};
271
272
/// A virtual file system optimized for the dependency discovery.
273
///
274
/// It is primarily designed to work with source files whose contents was was
275
/// preprocessed to remove any tokens that are unlikely to affect the dependency
276
/// computation.
277
///
278
/// This is not a thread safe VFS. A single instance is meant to be used only in
279
/// one thread. Multiple instances are allowed to service multiple threads
280
/// running in parallel.
281
class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
282
public:
283
  DependencyScanningWorkerFilesystem(
284
      DependencyScanningFilesystemSharedCache &SharedCache,
285
      IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
286
554
      : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache) {}
287
288
  llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
289
  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
290
  openFileForRead(const Twine &Path) override;
291
292
  /// Returns entry for the given filename.
293
  ///
294
  /// Attempts to use the local and shared caches first, then falls back to
295
  /// using the underlying filesystem.
296
  llvm::ErrorOr<EntryRef>
297
  getOrCreateFileSystemEntry(StringRef Filename,
298
                             bool DisableDirectivesScanning = false);
299
300
private:
301
  /// Check whether the file should be scanned for preprocessor directives.
302
  bool shouldScanForDirectives(StringRef Filename);
303
304
  /// For a filename that's not yet associated with any entry in the caches,
305
  /// uses the underlying filesystem to either look up the entry based in the
306
  /// shared cache indexed by unique ID, or creates new entry from scratch.
307
  llvm::ErrorOr<const CachedFileSystemEntry &>
308
  computeAndStoreResult(StringRef Filename);
309
310
  /// Scan for preprocessor directives for the given entry if necessary and
311
  /// returns a wrapper object with reference semantics.
312
  EntryRef scanForDirectivesIfNecessary(const CachedFileSystemEntry &Entry,
313
                                        StringRef Filename, bool Disable);
314
315
  /// Represents a filesystem entry that has been stat-ed (and potentially read)
316
  /// and that's about to be inserted into the cache as `CachedFileSystemEntry`.
317
  struct TentativeEntry {
318
    llvm::vfs::Status Status;
319
    std::unique_ptr<llvm::MemoryBuffer> Contents;
320
321
    TentativeEntry(llvm::vfs::Status Status,
322
                   std::unique_ptr<llvm::MemoryBuffer> Contents = nullptr)
323
4.70k
        : Status(std::move(Status)), Contents(std::move(Contents)) {}
324
  };
325
326
  /// Reads file at the given path. Enforces consistency between the file size
327
  /// in status and size of read contents.
328
  llvm::ErrorOr<TentativeEntry> readFile(StringRef Filename);
329
330
  /// Returns entry associated with the unique ID of the given tentative entry
331
  /// if there is some in the shared cache. Otherwise, constructs new one,
332
  /// associates it with the unique ID and returns the result.
333
  const CachedFileSystemEntry &
334
  getOrEmplaceSharedEntryForUID(TentativeEntry TEntry);
335
336
  /// Returns entry associated with the filename or nullptr if none is found.
337
  ///
338
  /// Returns entry from local cache if there is some. Otherwise, if the entry
339
  /// is found in the shared cache, writes it through the local cache and
340
  /// returns it. Otherwise returns nullptr.
341
  const CachedFileSystemEntry *
342
  findEntryByFilenameWithWriteThrough(StringRef Filename);
343
344
  /// Returns entry associated with the unique ID in the shared cache or nullptr
345
  /// if none is found.
346
  const CachedFileSystemEntry *
347
4.81k
  findSharedEntryByUID(llvm::vfs::Status Stat) const {
348
4.81k
    return SharedCache.getShardForUID(Stat.getUniqueID())
349
4.81k
        .findEntryByUID(Stat.getUniqueID());
350
4.81k
  }
351
352
  /// Associates the given entry with the filename in the local cache and
353
  /// returns it.
354
  const CachedFileSystemEntry &
355
  insertLocalEntryForFilename(StringRef Filename,
356
4.95k
                              const CachedFileSystemEntry &Entry) {
357
4.95k
    return LocalCache.insertEntryForFilename(Filename, Entry);
358
4.95k
  }
359
360
  /// Returns entry associated with the filename in the shared cache if there is
361
  /// some. Otherwise, constructs new one with the given error code, associates
362
  /// it with the filename and returns the result.
363
  const CachedFileSystemEntry &
364
137
  getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) {
365
137
    return SharedCache.getShardForFilename(Filename)
366
137
        .getOrEmplaceEntryForFilename(Filename, EC);
367
137
  }
368
369
  /// Returns entry associated with the filename in the shared cache if there is
370
  /// some. Otherwise, associates the given entry with the filename and returns
371
  /// it.
372
  const CachedFileSystemEntry &
373
  getOrInsertSharedEntryForFilename(StringRef Filename,
374
4.70k
                                    const CachedFileSystemEntry &Entry) {
375
4.70k
    return SharedCache.getShardForFilename(Filename)
376
4.70k
        .getOrInsertEntryForFilename(Filename, Entry);
377
4.70k
  }
378
379
  /// The global cache shared between worker threads.
380
  DependencyScanningFilesystemSharedCache &SharedCache;
381
  /// The local cache is used by the worker thread to cache file system queries
382
  /// locally instead of querying the global cache every time.
383
  DependencyScanningFilesystemLocalCache LocalCache;
384
};
385
386
} // end namespace dependencies
387
} // end namespace tooling
388
} // end namespace clang
389
390
#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H