/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
Line | Count | Source (jump to first uncovered line) |
1 | | //===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | |
9 | | #ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H |
10 | | #define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H |
11 | | |
12 | | #include "clang/Basic/LLVM.h" |
13 | | #include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h" |
14 | | #include "llvm/ADT/DenseSet.h" |
15 | | #include "llvm/ADT/StringMap.h" |
16 | | #include "llvm/Support/Allocator.h" |
17 | | #include "llvm/Support/ErrorOr.h" |
18 | | #include "llvm/Support/VirtualFileSystem.h" |
19 | | #include <mutex> |
20 | | |
21 | | namespace clang { |
22 | | namespace tooling { |
23 | | namespace dependencies { |
24 | | |
25 | | /// Original and minimized contents of a cached file entry. Single instance can |
26 | | /// be shared between multiple entries. |
27 | | struct CachedFileContents { |
28 | | CachedFileContents(std::unique_ptr<llvm::MemoryBuffer> Original) |
29 | 3.77k | : Original(std::move(Original)), MinimizedAccess(nullptr) {} |
30 | | |
31 | | /// Owning storage for the minimized contents. |
32 | | std::unique_ptr<llvm::MemoryBuffer> Original; |
33 | | |
34 | | /// The mutex that must be locked before mutating minimized contents. |
35 | | std::mutex ValueLock; |
36 | | /// Owning storage for the minimized contents. |
37 | | std::unique_ptr<llvm::MemoryBuffer> MinimizedStorage; |
38 | | /// Accessor to the minimized contents that's atomic to avoid data races. |
39 | | std::atomic<llvm::MemoryBuffer *> MinimizedAccess; |
40 | | /// Skipped range mapping of the minimized contents. |
41 | | /// This is initialized iff `MinimizedAccess != nullptr`. |
42 | | PreprocessorSkippedRangeMapping PPSkippedRangeMapping; |
43 | | }; |
44 | | |
45 | | /// An in-memory representation of a file system entity that is of interest to |
46 | | /// the dependency scanning filesystem. |
47 | | /// |
48 | | /// It represents one of the following: |
49 | | /// - opened file with original contents and a stat value, |
50 | | /// - opened file with original contents, minimized contents and a stat value, |
51 | | /// - directory entry with its stat value, |
52 | | /// - filesystem error. |
53 | | /// |
54 | | /// Single instance of this class can be shared across different filenames (e.g. |
55 | | /// a regular file and a symlink). For this reason the status filename is empty |
56 | | /// and is only materialized by \c EntryRef that knows the requested filename. |
57 | | class CachedFileSystemEntry { |
58 | | public: |
59 | | /// Creates an entry without contents: either a filesystem error or |
60 | | /// a directory with stat value. |
61 | | CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat) |
62 | 120 | : MaybeStat(std::move(Stat)), Contents(nullptr) { |
63 | 120 | clearStatName(); |
64 | 120 | } |
65 | | |
66 | | /// Creates an entry representing a file with contents. |
67 | | CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat, |
68 | | CachedFileContents *Contents) |
69 | 4.58k | : MaybeStat(std::move(Stat)), Contents(std::move(Contents)) { |
70 | 4.58k | clearStatName(); |
71 | 4.58k | } |
72 | | |
73 | | /// \returns True if the entry is a filesystem error. |
74 | 47.6k | bool isError() const { return !MaybeStat; } |
75 | | |
76 | | /// \returns True if the current entry represents a directory. |
77 | 11.5k | bool isDirectory() const { return !isError() && MaybeStat->isDirectory(); } |
78 | | |
79 | | /// \returns Original contents of the file. |
80 | 1.34k | StringRef getOriginalContents() const { |
81 | 1.34k | assert(!isError() && "error"); |
82 | 0 | assert(!MaybeStat->isDirectory() && "not a file"); |
83 | 0 | assert(Contents && "contents not initialized"); |
84 | 0 | return Contents->Original->getBuffer(); |
85 | 1.34k | } |
86 | | |
87 | | /// \returns Minimized contents of the file. |
88 | 4.99k | StringRef getMinimizedContents() const { |
89 | 4.99k | assert(!isError() && "error"); |
90 | 0 | assert(!MaybeStat->isDirectory() && "not a file"); |
91 | 0 | assert(Contents && "contents not initialized"); |
92 | 0 | llvm::MemoryBuffer *Buffer = Contents->MinimizedAccess.load(); |
93 | 4.99k | assert(Buffer && "not minimized"); |
94 | 0 | return Buffer->getBuffer(); |
95 | 4.99k | } |
96 | | |
97 | | /// \returns The error. |
98 | 160 | std::error_code getError() const { return MaybeStat.getError(); } |
99 | | |
100 | | /// \returns The entry status with empty filename. |
101 | 6.85k | llvm::vfs::Status getStatus() const { |
102 | 6.85k | assert(!isError() && "error"); |
103 | 0 | assert(MaybeStat->getName().empty() && "stat name must be empty"); |
104 | 0 | return *MaybeStat; |
105 | 6.85k | } |
106 | | |
107 | | /// \returns The unique ID of the entry. |
108 | 4.90k | llvm::sys::fs::UniqueID getUniqueID() const { |
109 | 4.90k | assert(!isError() && "error"); |
110 | 0 | return MaybeStat->getUniqueID(); |
111 | 4.90k | } |
112 | | |
113 | | /// \returns The mapping between location -> distance that is used to speed up |
114 | | /// the block skipping in the preprocessor. |
115 | 328 | const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const { |
116 | 328 | assert(!isError() && "error"); |
117 | 0 | assert(!isDirectory() && "not a file"); |
118 | 0 | assert(Contents && "contents not initialized"); |
119 | 0 | return Contents->PPSkippedRangeMapping; |
120 | 328 | } |
121 | | |
122 | | /// \returns The data structure holding both original and minimized contents. |
123 | 4.34k | CachedFileContents *getContents() const { |
124 | 4.34k | assert(!isError() && "error"); |
125 | 0 | assert(!isDirectory() && "not a file"); |
126 | 0 | return Contents; |
127 | 4.34k | } |
128 | | |
129 | | private: |
130 | 4.70k | void clearStatName() { |
131 | 4.70k | if (MaybeStat) |
132 | 4.58k | MaybeStat = llvm::vfs::Status::copyWithNewName(*MaybeStat, ""); |
133 | 4.70k | } |
134 | | |
135 | | /// Either the filesystem error or status of the entry. |
136 | | /// The filename is empty and only materialized by \c EntryRef. |
137 | | llvm::ErrorOr<llvm::vfs::Status> MaybeStat; |
138 | | |
139 | | /// Non-owning pointer to the file contents. |
140 | | /// |
141 | | /// We're using pointer here to keep the size of this class small. Instances |
142 | | /// representing directories and filesystem errors don't hold any contents |
143 | | /// anyway. |
144 | | CachedFileContents *Contents; |
145 | | }; |
146 | | |
147 | | /// This class is a shared cache, that caches the 'stat' and 'open' calls to the |
148 | | /// underlying real file system. It distinguishes between minimized and original |
149 | | /// files. |
150 | | /// |
151 | | /// It is sharded based on the hash of the key to reduce the lock contention for |
152 | | /// the worker threads. |
153 | | class DependencyScanningFilesystemSharedCache { |
154 | | public: |
155 | | struct CacheShard { |
156 | | /// The mutex that needs to be locked before mutation of any member. |
157 | | mutable std::mutex CacheLock; |
158 | | |
159 | | /// Map from filenames to cached entries. |
160 | | llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> |
161 | | EntriesByFilename; |
162 | | |
163 | | /// Map from unique IDs to cached entries. |
164 | | llvm::DenseMap<llvm::sys::fs::UniqueID, const CachedFileSystemEntry *> |
165 | | EntriesByUID; |
166 | | |
167 | | /// The backing storage for cached entries. |
168 | | llvm::SpecificBumpPtrAllocator<CachedFileSystemEntry> EntryStorage; |
169 | | |
170 | | /// The backing storage for cached contents. |
171 | | llvm::SpecificBumpPtrAllocator<CachedFileContents> ContentsStorage; |
172 | | |
173 | | /// Returns entry associated with the filename or nullptr if none is found. |
174 | | const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const; |
175 | | |
176 | | /// Returns entry associated with the unique ID or nullptr if none is found. |
177 | | const CachedFileSystemEntry * |
178 | | findEntryByUID(llvm::sys::fs::UniqueID UID) const; |
179 | | |
180 | | /// Returns entry associated with the filename if there is some. Otherwise, |
181 | | /// constructs new one with the given status, associates it with the |
182 | | /// filename and returns the result. |
183 | | const CachedFileSystemEntry & |
184 | | getOrEmplaceEntryForFilename(StringRef Filename, |
185 | | llvm::ErrorOr<llvm::vfs::Status> Stat); |
186 | | |
187 | | /// Returns entry associated with the unique ID if there is some. Otherwise, |
188 | | /// constructs new one with the given status and contents, associates it |
189 | | /// with the unique ID and returns the result. |
190 | | const CachedFileSystemEntry & |
191 | | getOrEmplaceEntryForUID(llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat, |
192 | | std::unique_ptr<llvm::MemoryBuffer> Contents); |
193 | | |
194 | | /// Returns entry associated with the filename if there is some. Otherwise, |
195 | | /// associates the given entry with the filename and returns it. |
196 | | const CachedFileSystemEntry & |
197 | | getOrInsertEntryForFilename(StringRef Filename, |
198 | | const CachedFileSystemEntry &Entry); |
199 | | }; |
200 | | |
201 | | DependencyScanningFilesystemSharedCache(); |
202 | | |
203 | | /// Returns shard for the given key. |
204 | | CacheShard &getShardForFilename(StringRef Filename) const; |
205 | | CacheShard &getShardForUID(llvm::sys::fs::UniqueID UID) const; |
206 | | |
207 | | private: |
208 | | std::unique_ptr<CacheShard[]> CacheShards; |
209 | | unsigned NumShards; |
210 | | }; |
211 | | |
212 | | /// This class is a local cache, that caches the 'stat' and 'open' calls to the |
213 | | /// underlying real file system. It distinguishes between minimized and original |
214 | | /// files. |
215 | | class DependencyScanningFilesystemLocalCache { |
216 | | llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache; |
217 | | |
218 | | public: |
219 | | /// Returns entry associated with the filename or nullptr if none is found. |
220 | 8.15k | const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const { |
221 | 8.15k | auto It = Cache.find(Filename); |
222 | 8.15k | return It == Cache.end() ? nullptr6.88k : It->getValue()1.27k ; |
223 | 8.15k | } |
224 | | |
225 | | /// Associates the given entry with the filename and returns the given entry |
226 | | /// pointer (for convenience). |
227 | | const CachedFileSystemEntry & |
228 | | insertEntryForFilename(StringRef Filename, |
229 | 5.04k | const CachedFileSystemEntry &Entry) { |
230 | 5.04k | const auto *InsertedEntry = Cache.insert({Filename, &Entry}).first->second; |
231 | 5.04k | assert(InsertedEntry == &Entry && "entry already present"); |
232 | 0 | return *InsertedEntry; |
233 | 5.04k | } |
234 | | }; |
235 | | |
236 | | /// Reference to a CachedFileSystemEntry. |
237 | | /// If the underlying entry is an opened file, this wrapper returns the correct |
238 | | /// contents (original or minimized) and ensures consistency with file size |
239 | | /// reported by status. |
240 | | class EntryRef { |
241 | | /// For entry that is an opened file, this bit signifies whether its contents |
242 | | /// are minimized. |
243 | | bool Minimized; |
244 | | |
245 | | /// The filename used to access this entry. |
246 | | std::string Filename; |
247 | | |
248 | | /// The underlying cached entry. |
249 | | const CachedFileSystemEntry &Entry; |
250 | | |
251 | | public: |
252 | | EntryRef(bool Minimized, StringRef Name, const CachedFileSystemEntry &Entry) |
253 | 6.32k | : Minimized(Minimized), Filename(Name), Entry(Entry) {} |
254 | | |
255 | 6.85k | llvm::vfs::Status getStatus() const { |
256 | 6.85k | llvm::vfs::Status Stat = Entry.getStatus(); |
257 | 6.85k | if (!Stat.isDirectory()) |
258 | 5.64k | Stat = llvm::vfs::Status::copyWithNewSize(Stat, getContents().size()); |
259 | 6.85k | return llvm::vfs::Status::copyWithNewName(Stat, Filename); |
260 | 6.85k | } |
261 | | |
262 | 7.02k | bool isError() const { return Entry.isError(); } |
263 | 698 | bool isDirectory() const { return Entry.isDirectory(); } |
264 | | |
265 | | /// If the cached entry represents an error, promotes it into `ErrorOr`. |
266 | 6.32k | llvm::ErrorOr<EntryRef> unwrapError() const { |
267 | 6.32k | if (isError()) |
268 | 160 | return Entry.getError(); |
269 | 6.16k | return *this; |
270 | 6.32k | } |
271 | | |
272 | 6.33k | StringRef getContents() const { |
273 | 6.33k | return Minimized ? Entry.getMinimizedContents()4.99k |
274 | 6.33k | : Entry.getOriginalContents()1.34k ; |
275 | 6.33k | } |
276 | | |
277 | 696 | const PreprocessorSkippedRangeMapping *getPPSkippedRangeMapping() const { |
278 | 696 | return Minimized ? &Entry.getPPSkippedRangeMapping()328 : nullptr368 ; |
279 | 696 | } |
280 | | }; |
281 | | |
282 | | /// A virtual file system optimized for the dependency discovery. |
283 | | /// |
284 | | /// It is primarily designed to work with source files whose contents was was |
285 | | /// preprocessed to remove any tokens that are unlikely to affect the dependency |
286 | | /// computation. |
287 | | /// |
288 | | /// This is not a thread safe VFS. A single instance is meant to be used only in |
289 | | /// one thread. Multiple instances are allowed to service multiple threads |
290 | | /// running in parallel. |
291 | | class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem { |
292 | | public: |
293 | | DependencyScanningWorkerFilesystem( |
294 | | DependencyScanningFilesystemSharedCache &SharedCache, |
295 | | IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, |
296 | | ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings) |
297 | | : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache), |
298 | 427 | PPSkipMappings(PPSkipMappings) {} |
299 | | |
300 | | llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override; |
301 | | llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> |
302 | | openFileForRead(const Twine &Path) override; |
303 | | |
304 | | /// Disable minimization of the given file. |
305 | | void disableMinimization(StringRef Filename); |
306 | | /// Enable minimization of all files. |
307 | 144 | void enableMinimizationOfAllFiles() { NotToBeMinimized.clear(); } |
308 | | |
309 | | private: |
310 | | /// Check whether the file should be minimized. |
311 | | bool shouldMinimize(StringRef Filename, llvm::sys::fs::UniqueID UID); |
312 | | |
313 | | /// Returns entry for the given filename. |
314 | | /// |
315 | | /// Attempts to use the local and shared caches first, then falls back to |
316 | | /// using the underlying filesystem. |
317 | | llvm::ErrorOr<EntryRef> |
318 | | getOrCreateFileSystemEntry(StringRef Filename, |
319 | | bool DisableMinimization = false); |
320 | | |
321 | | /// For a filename that's not yet associated with any entry in the caches, |
322 | | /// uses the underlying filesystem to either look up the entry based in the |
323 | | /// shared cache indexed by unique ID, or creates new entry from scratch. |
324 | | llvm::ErrorOr<const CachedFileSystemEntry &> |
325 | | computeAndStoreResult(StringRef Filename); |
326 | | |
327 | | /// Minimizes the given entry if necessary and returns a wrapper object with |
328 | | /// reference semantics. |
329 | | EntryRef minimizeIfNecessary(const CachedFileSystemEntry &Entry, |
330 | | StringRef Filename, bool Disable); |
331 | | |
332 | | /// Represents a filesystem entry that has been stat-ed (and potentially read) |
333 | | /// and that's about to be inserted into the cache as `CachedFileSystemEntry`. |
334 | | struct TentativeEntry { |
335 | | llvm::vfs::Status Status; |
336 | | std::unique_ptr<llvm::MemoryBuffer> Contents; |
337 | | |
338 | | TentativeEntry(llvm::vfs::Status Status, |
339 | | std::unique_ptr<llvm::MemoryBuffer> Contents = nullptr) |
340 | 4.64k | : Status(std::move(Status)), Contents(std::move(Contents)) {} |
341 | | }; |
342 | | |
343 | | /// Reads file at the given path. Enforces consistency between the file size |
344 | | /// in status and size of read contents. |
345 | | llvm::ErrorOr<TentativeEntry> readFile(StringRef Filename); |
346 | | |
347 | | /// Returns entry associated with the unique ID of the given tentative entry |
348 | | /// if there is some in the shared cache. Otherwise, constructs new one, |
349 | | /// associates it with the unique ID and returns the result. |
350 | | const CachedFileSystemEntry & |
351 | | getOrEmplaceSharedEntryForUID(TentativeEntry TEntry); |
352 | | |
353 | | /// Returns entry associated with the filename or nullptr if none is found. |
354 | | /// |
355 | | /// Returns entry from local cache if there is some. Otherwise, if the entry |
356 | | /// is found in the shared cache, writes it through the local cache and |
357 | | /// returns it. Otherwise returns nullptr. |
358 | | const CachedFileSystemEntry * |
359 | | findEntryByFilenameWithWriteThrough(StringRef Filename); |
360 | | |
361 | | /// Returns entry associated with the unique ID in the shared cache or nullptr |
362 | | /// if none is found. |
363 | | const CachedFileSystemEntry * |
364 | 4.81k | findSharedEntryByUID(llvm::vfs::Status Stat) const { |
365 | 4.81k | return SharedCache.getShardForUID(Stat.getUniqueID()) |
366 | 4.81k | .findEntryByUID(Stat.getUniqueID()); |
367 | 4.81k | } |
368 | | |
369 | | /// Associates the given entry with the filename in the local cache and |
370 | | /// returns it. |
371 | | const CachedFileSystemEntry & |
372 | | insertLocalEntryForFilename(StringRef Filename, |
373 | 4.95k | const CachedFileSystemEntry &Entry) { |
374 | 4.95k | return LocalCache.insertEntryForFilename(Filename, Entry); |
375 | 4.95k | } |
376 | | |
377 | | /// Returns entry associated with the filename in the shared cache if there is |
378 | | /// some. Otherwise, constructs new one with the given error code, associates |
379 | | /// it with the filename and returns the result. |
380 | | const CachedFileSystemEntry & |
381 | 137 | getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) { |
382 | 137 | return SharedCache.getShardForFilename(Filename) |
383 | 137 | .getOrEmplaceEntryForFilename(Filename, EC); |
384 | 137 | } |
385 | | |
386 | | /// Returns entry associated with the filename in the shared cache if there is |
387 | | /// some. Otherwise, associates the given entry with the filename and returns |
388 | | /// it. |
389 | | const CachedFileSystemEntry & |
390 | | getOrInsertSharedEntryForFilename(StringRef Filename, |
391 | 4.64k | const CachedFileSystemEntry &Entry) { |
392 | 4.64k | return SharedCache.getShardForFilename(Filename) |
393 | 4.64k | .getOrInsertEntryForFilename(Filename, Entry); |
394 | 4.64k | } |
395 | | |
396 | | /// The global cache shared between worker threads. |
397 | | DependencyScanningFilesystemSharedCache &SharedCache; |
398 | | /// The local cache is used by the worker thread to cache file system queries |
399 | | /// locally instead of querying the global cache every time. |
400 | | DependencyScanningFilesystemLocalCache LocalCache; |
401 | | /// The mapping structure which records information about the |
402 | | /// excluded conditional directive skip mappings that are used by the |
403 | | /// currently active preprocessor. |
404 | | ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings; |
405 | | /// The set of files that should not be minimized. |
406 | | llvm::DenseSet<llvm::sys::fs::UniqueID> NotToBeMinimized; |
407 | | }; |
408 | | |
409 | | } // end namespace dependencies |
410 | | } // end namespace tooling |
411 | | } // end namespace clang |
412 | | |
413 | | #endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H |