/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | |
9 | | #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" |
10 | | #include "llvm/Support/MemoryBuffer.h" |
11 | | #include "llvm/Support/SmallVectorMemoryBuffer.h" |
12 | | #include "llvm/Support/Threading.h" |
13 | | |
14 | | using namespace clang; |
15 | | using namespace tooling; |
16 | | using namespace dependencies; |
17 | | |
18 | | llvm::ErrorOr<DependencyScanningWorkerFilesystem::TentativeEntry> |
19 | 3.85k | DependencyScanningWorkerFilesystem::readFile(StringRef Filename) { |
20 | | // Load the file and its content from the file system. |
21 | 3.85k | auto MaybeFile = getUnderlyingFS().openFileForRead(Filename); |
22 | 3.85k | if (!MaybeFile) |
23 | 0 | return MaybeFile.getError(); |
24 | 3.85k | auto File = std::move(*MaybeFile); |
25 | | |
26 | 3.85k | auto MaybeStat = File->status(); |
27 | 3.85k | if (!MaybeStat) |
28 | 0 | return MaybeStat.getError(); |
29 | 3.85k | auto Stat = std::move(*MaybeStat); |
30 | | |
31 | 3.85k | auto MaybeBuffer = File->getBuffer(Stat.getName()); |
32 | 3.85k | if (!MaybeBuffer) |
33 | 0 | return MaybeBuffer.getError(); |
34 | 3.85k | auto Buffer = std::move(*MaybeBuffer); |
35 | | |
36 | | // If the file size changed between read and stat, pretend it didn't. |
37 | 3.85k | if (Stat.getSize() != Buffer->getBufferSize()) |
38 | 0 | Stat = llvm::vfs::Status::copyWithNewSize(Stat, Buffer->getBufferSize()); |
39 | | |
40 | 3.85k | return TentativeEntry(Stat, std::move(Buffer)); |
41 | 3.85k | } |
42 | | |
43 | | EntryRef DependencyScanningWorkerFilesystem::scanForDirectivesIfNecessary( |
44 | 6.46k | const CachedFileSystemEntry &Entry, StringRef Filename, bool Disable) { |
45 | 6.46k | if (Entry.isError() || Entry.isDirectory()6.30k || Disable5.27k || |
46 | 6.46k | !shouldScanForDirectives(Filename)5.27k ) |
47 | 1.73k | return EntryRef(Filename, Entry); |
48 | | |
49 | 4.73k | CachedFileContents *Contents = Entry.getCachedContents(); |
50 | 4.73k | assert(Contents && "contents not initialized"); |
51 | | |
52 | | // Double-checked locking. |
53 | 4.73k | if (Contents->DepDirectives.load()) |
54 | 1.15k | return EntryRef(Filename, Entry); |
55 | | |
56 | 3.57k | std::lock_guard<std::mutex> GuardLock(Contents->ValueLock); |
57 | | |
58 | | // Double-checked locking. |
59 | 3.57k | if (Contents->DepDirectives.load()) |
60 | 12 | return EntryRef(Filename, Entry); |
61 | | |
62 | 3.56k | SmallVector<dependency_directives_scan::Directive, 64> Directives; |
63 | | // Scan the file for preprocessor directives that might affect the |
64 | | // dependencies. |
65 | 3.56k | if (scanSourceForDependencyDirectives(Contents->Original->getBuffer(), |
66 | 3.56k | Contents->DepDirectiveTokens, |
67 | 3.56k | Directives)) { |
68 | 0 | Contents->DepDirectiveTokens.clear(); |
69 | | // FIXME: Propagate the diagnostic if desired by the client. |
70 | 0 | Contents->DepDirectives.store(new Optional<DependencyDirectivesTy>()); |
71 | 0 | return EntryRef(Filename, Entry); |
72 | 0 | } |
73 | | |
74 | | // This function performed double-checked locking using `DepDirectives`. |
75 | | // Assigning it must be the last thing this function does, otherwise other |
76 | | // threads may skip the |
77 | | // critical section (`DepDirectives != nullptr`), leading to a data race. |
78 | 3.56k | Contents->DepDirectives.store( |
79 | 3.56k | new Optional<DependencyDirectivesTy>(std::move(Directives))); |
80 | 3.56k | return EntryRef(Filename, Entry); |
81 | 3.56k | } |
82 | | |
83 | | DependencyScanningFilesystemSharedCache:: |
84 | 88 | DependencyScanningFilesystemSharedCache() { |
85 | | // This heuristic was chosen using a empirical testing on a |
86 | | // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache |
87 | | // sharding gives a performance edge by reducing the lock contention. |
88 | | // FIXME: A better heuristic might also consider the OS to account for |
89 | | // the different cost of lock contention on different OSes. |
90 | 88 | NumShards = |
91 | 88 | std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4); |
92 | 88 | CacheShards = std::make_unique<CacheShard[]>(NumShards); |
93 | 88 | } |
94 | | |
95 | | DependencyScanningFilesystemSharedCache::CacheShard & |
96 | | DependencyScanningFilesystemSharedCache::getShardForFilename( |
97 | 11.5k | StringRef Filename) const { |
98 | 11.5k | return CacheShards[llvm::hash_value(Filename) % NumShards]; |
99 | 11.5k | } |
100 | | |
101 | | DependencyScanningFilesystemSharedCache::CacheShard & |
102 | | DependencyScanningFilesystemSharedCache::getShardForUID( |
103 | 9.52k | llvm::sys::fs::UniqueID UID) const { |
104 | 9.52k | auto Hash = llvm::hash_combine(UID.getDevice(), UID.getFile()); |
105 | 9.52k | return CacheShards[Hash % NumShards]; |
106 | 9.52k | } |
107 | | |
108 | | const CachedFileSystemEntry * |
109 | | DependencyScanningFilesystemSharedCache::CacheShard::findEntryByFilename( |
110 | 6.70k | StringRef Filename) const { |
111 | 6.70k | std::lock_guard<std::mutex> LockGuard(CacheLock); |
112 | 6.70k | auto It = EntriesByFilename.find(Filename); |
113 | 6.70k | return It == EntriesByFilename.end() ? nullptr6.61k : It->getValue()97 ; |
114 | 6.70k | } |
115 | | |
116 | | const CachedFileSystemEntry * |
117 | | DependencyScanningFilesystemSharedCache::CacheShard::findEntryByUID( |
118 | 4.81k | llvm::sys::fs::UniqueID UID) const { |
119 | 4.81k | std::lock_guard<std::mutex> LockGuard(CacheLock); |
120 | 4.81k | auto It = EntriesByUID.find(UID); |
121 | 4.81k | return It == EntriesByUID.end() ? nullptr4.70k : It->getSecond()109 ; |
122 | 4.81k | } |
123 | | |
124 | | const CachedFileSystemEntry & |
125 | | DependencyScanningFilesystemSharedCache::CacheShard:: |
126 | | getOrEmplaceEntryForFilename(StringRef Filename, |
127 | 137 | llvm::ErrorOr<llvm::vfs::Status> Stat) { |
128 | 137 | std::lock_guard<std::mutex> LockGuard(CacheLock); |
129 | 137 | auto Insertion = EntriesByFilename.insert({Filename, nullptr}); |
130 | 137 | if (Insertion.second) |
131 | 120 | Insertion.first->second = |
132 | 120 | new (EntryStorage.Allocate()) CachedFileSystemEntry(std::move(Stat)); |
133 | 137 | return *Insertion.first->second; |
134 | 137 | } |
135 | | |
136 | | const CachedFileSystemEntry & |
137 | | DependencyScanningFilesystemSharedCache::CacheShard::getOrEmplaceEntryForUID( |
138 | | llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat, |
139 | 4.70k | std::unique_ptr<llvm::MemoryBuffer> Contents) { |
140 | 4.70k | std::lock_guard<std::mutex> LockGuard(CacheLock); |
141 | 4.70k | auto Insertion = EntriesByUID.insert({UID, nullptr}); |
142 | 4.70k | if (Insertion.second) { |
143 | 4.65k | CachedFileContents *StoredContents = nullptr; |
144 | 4.65k | if (Contents) |
145 | 3.81k | StoredContents = new (ContentsStorage.Allocate()) |
146 | 3.81k | CachedFileContents(std::move(Contents)); |
147 | 4.65k | Insertion.first->second = new (EntryStorage.Allocate()) |
148 | 4.65k | CachedFileSystemEntry(std::move(Stat), StoredContents); |
149 | 4.65k | } |
150 | 4.70k | return *Insertion.first->second; |
151 | 4.70k | } |
152 | | |
153 | | const CachedFileSystemEntry & |
154 | | DependencyScanningFilesystemSharedCache::CacheShard:: |
155 | | getOrInsertEntryForFilename(StringRef Filename, |
156 | 4.70k | const CachedFileSystemEntry &Entry) { |
157 | 4.70k | std::lock_guard<std::mutex> LockGuard(CacheLock); |
158 | 4.70k | return *EntriesByFilename.insert({Filename, &Entry}).first->getValue(); |
159 | 4.70k | } |
160 | | |
161 | | /// Whitelist file extensions that should be minimized, treating no extension as |
162 | | /// a source file that should be minimized. |
163 | | /// |
164 | | /// This is kinda hacky, it would be better if we knew what kind of file Clang |
165 | | /// was expecting instead. |
166 | 6.57k | static bool shouldScanForDirectivesBasedOnExtension(StringRef Filename) { |
167 | 6.57k | StringRef Ext = llvm::sys::path::extension(Filename); |
168 | 6.57k | if (Ext.empty()) |
169 | 12 | return true; // C++ standard library |
170 | 6.56k | return llvm::StringSwitch<bool>(Ext) |
171 | 6.56k | .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true) |
172 | 6.56k | .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true) |
173 | 6.56k | .CasesLower(".m", ".mm", true) |
174 | 6.56k | .CasesLower(".i", ".ii", ".mi", ".mmi", true) |
175 | 6.56k | .CasesLower(".def", ".inc", true) |
176 | 6.56k | .Default(false); |
177 | 6.57k | } |
178 | | |
179 | 1.79k | static bool shouldCacheStatFailures(StringRef Filename) { |
180 | 1.79k | StringRef Ext = llvm::sys::path::extension(Filename); |
181 | 1.79k | if (Ext.empty()) |
182 | 490 | return false; // This may be the module cache directory. |
183 | | // Only cache stat failures on source files. |
184 | 1.30k | return shouldScanForDirectivesBasedOnExtension(Filename); |
185 | 1.79k | } |
186 | | |
187 | | bool DependencyScanningWorkerFilesystem::shouldScanForDirectives( |
188 | 5.27k | StringRef Filename) { |
189 | 5.27k | return shouldScanForDirectivesBasedOnExtension(Filename); |
190 | 5.27k | } |
191 | | |
192 | | const CachedFileSystemEntry & |
193 | | DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID( |
194 | 4.70k | TentativeEntry TEntry) { |
195 | 4.70k | auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID()); |
196 | 4.70k | return Shard.getOrEmplaceEntryForUID(TEntry.Status.getUniqueID(), |
197 | 4.70k | std::move(TEntry.Status), |
198 | 4.70k | std::move(TEntry.Contents)); |
199 | 4.70k | } |
200 | | |
201 | | const CachedFileSystemEntry * |
202 | | DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough( |
203 | 8.11k | StringRef Filename) { |
204 | 8.11k | if (const auto *Entry = LocalCache.findEntryByFilename(Filename)) |
205 | 1.40k | return Entry; |
206 | 6.71k | auto &Shard = SharedCache.getShardForFilename(Filename); |
207 | 6.71k | if (const auto *Entry = Shard.findEntryByFilename(Filename)) |
208 | 101 | return &LocalCache.insertEntryForFilename(Filename, *Entry); |
209 | 6.61k | return nullptr; |
210 | 6.71k | } |
211 | | |
212 | | llvm::ErrorOr<const CachedFileSystemEntry &> |
213 | 6.61k | DependencyScanningWorkerFilesystem::computeAndStoreResult(StringRef Filename) { |
214 | 6.61k | llvm::ErrorOr<llvm::vfs::Status> Stat = getUnderlyingFS().status(Filename); |
215 | 6.61k | if (!Stat) { |
216 | 1.79k | if (!shouldCacheStatFailures(Filename)) |
217 | 1.65k | return Stat.getError(); |
218 | 137 | const auto &Entry = |
219 | 137 | getOrEmplaceSharedEntryForFilename(Filename, Stat.getError()); |
220 | 137 | return insertLocalEntryForFilename(Filename, Entry); |
221 | 1.79k | } |
222 | | |
223 | 4.82k | if (const auto *Entry = findSharedEntryByUID(*Stat)) |
224 | 112 | return insertLocalEntryForFilename(Filename, *Entry); |
225 | | |
226 | 4.70k | auto TEntry = |
227 | 4.70k | Stat->isDirectory() ? TentativeEntry(*Stat)848 : readFile(Filename)3.86k ; |
228 | | |
229 | 4.70k | const CachedFileSystemEntry *SharedEntry = [&]() { |
230 | 4.70k | if (TEntry) { |
231 | 4.70k | const auto &UIDEntry = getOrEmplaceSharedEntryForUID(std::move(*TEntry)); |
232 | 4.70k | return &getOrInsertSharedEntryForFilename(Filename, UIDEntry); |
233 | 4.70k | } |
234 | 0 | return &getOrEmplaceSharedEntryForFilename(Filename, TEntry.getError()); |
235 | 4.70k | }(); |
236 | | |
237 | 4.70k | return insertLocalEntryForFilename(Filename, *SharedEntry); |
238 | 4.82k | } |
239 | | |
240 | | llvm::ErrorOr<EntryRef> |
241 | | DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( |
242 | 8.11k | StringRef Filename, bool DisableDirectivesScanning) { |
243 | 8.11k | if (const auto *Entry = findEntryByFilenameWithWriteThrough(Filename)) |
244 | 1.50k | return scanForDirectivesIfNecessary(*Entry, Filename, |
245 | 1.50k | DisableDirectivesScanning) |
246 | 1.50k | .unwrapError(); |
247 | 6.60k | auto MaybeEntry = computeAndStoreResult(Filename); |
248 | 6.60k | if (!MaybeEntry) |
249 | 1.65k | return MaybeEntry.getError(); |
250 | 4.95k | return scanForDirectivesIfNecessary(*MaybeEntry, Filename, |
251 | 4.95k | DisableDirectivesScanning) |
252 | 4.95k | .unwrapError(); |
253 | 6.60k | } |
254 | | |
255 | | llvm::ErrorOr<llvm::vfs::Status> |
256 | 6.92k | DependencyScanningWorkerFilesystem::status(const Twine &Path) { |
257 | 6.92k | SmallString<256> OwnedFilename; |
258 | 6.92k | StringRef Filename = Path.toStringRef(OwnedFilename); |
259 | | |
260 | 6.92k | llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename); |
261 | 6.92k | if (!Result) |
262 | 1.67k | return Result.getError(); |
263 | 5.24k | return Result->getStatus(); |
264 | 6.92k | } |
265 | | |
266 | | namespace { |
267 | | |
268 | | /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using |
269 | | /// this subclass. |
270 | | class DepScanFile final : public llvm::vfs::File { |
271 | | public: |
272 | | DepScanFile(std::unique_ptr<llvm::MemoryBuffer> Buffer, |
273 | | llvm::vfs::Status Stat) |
274 | 737 | : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} |
275 | | |
276 | | static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> create(EntryRef Entry); |
277 | | |
278 | 784 | llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; } |
279 | | |
280 | | llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> |
281 | | getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, |
282 | 626 | bool IsVolatile) override { |
283 | 626 | return std::move(Buffer); |
284 | 626 | } |
285 | | |
286 | 0 | std::error_code close() override { return {}; } |
287 | | |
288 | | private: |
289 | | std::unique_ptr<llvm::MemoryBuffer> Buffer; |
290 | | llvm::vfs::Status Stat; |
291 | | }; |
292 | | |
293 | | } // end anonymous namespace |
294 | | |
295 | | llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> |
296 | 740 | DepScanFile::create(EntryRef Entry) { |
297 | 740 | assert(!Entry.isError() && "error"); |
298 | | |
299 | 740 | if (Entry.isDirectory()) |
300 | 2 | return std::make_error_code(std::errc::is_a_directory); |
301 | | |
302 | 738 | auto Result = std::make_unique<DepScanFile>( |
303 | 738 | llvm::MemoryBuffer::getMemBuffer(Entry.getContents(), |
304 | 738 | Entry.getStatus().getName(), |
305 | 738 | /*RequiresNullTerminator=*/false), |
306 | 738 | Entry.getStatus()); |
307 | | |
308 | 738 | return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( |
309 | 738 | std::unique_ptr<llvm::vfs::File>(std::move(Result))); |
310 | 740 | } |
311 | | |
312 | | llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> |
313 | 875 | DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) { |
314 | 875 | SmallString<256> OwnedFilename; |
315 | 875 | StringRef Filename = Path.toStringRef(OwnedFilename); |
316 | | |
317 | 875 | llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename); |
318 | 875 | if (!Result) |
319 | 136 | return Result.getError(); |
320 | 739 | return DepScanFile::create(Result.get()); |
321 | 875 | } |