/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | |
9 | | #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" |
10 | | #include "clang/Lex/DependencyDirectivesSourceMinimizer.h" |
11 | | #include "llvm/Support/MemoryBuffer.h" |
12 | | #include "llvm/Support/Threading.h" |
13 | | |
14 | | using namespace clang; |
15 | | using namespace tooling; |
16 | | using namespace dependencies; |
17 | | |
18 | | CachedFileSystemEntry CachedFileSystemEntry::createFileEntry( |
19 | 96 | StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) { |
20 | | // Load the file and its content from the file system. |
21 | 96 | llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile = |
22 | 96 | FS.openFileForRead(Filename); |
23 | 96 | if (!MaybeFile) |
24 | 0 | return MaybeFile.getError(); |
25 | 96 | llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status(); |
26 | 96 | if (!Stat) |
27 | 0 | return Stat.getError(); |
28 | | |
29 | 96 | llvm::vfs::File &F = **MaybeFile; |
30 | 96 | llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer = |
31 | 96 | F.getBuffer(Stat->getName()); |
32 | 96 | if (!MaybeBuffer) |
33 | 0 | return MaybeBuffer.getError(); |
34 | | |
35 | 96 | llvm::SmallString<1024> MinimizedFileContents; |
36 | | // Minimize the file down to directives that might affect the dependencies. |
37 | 96 | const auto &Buffer = *MaybeBuffer; |
38 | 96 | SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens; |
39 | 96 | if (!Minimize || minimizeSourceToDependencyDirectives( |
40 | 17 | Buffer->getBuffer(), MinimizedFileContents, Tokens)) { |
41 | | // Use the original file unless requested otherwise, or |
42 | | // if the minimization failed. |
43 | | // FIXME: Propage the diagnostic if desired by the client. |
44 | 17 | CachedFileSystemEntry Result; |
45 | 17 | Result.MaybeStat = std::move(*Stat); |
46 | 17 | Result.Contents.reserve(Buffer->getBufferSize() + 1); |
47 | 17 | Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd()); |
48 | | // Implicitly null terminate the contents for Clang's lexer. |
49 | 17 | Result.Contents.push_back('\0'); |
50 | 17 | Result.Contents.pop_back(); |
51 | 17 | return Result; |
52 | 17 | } |
53 | | |
54 | 79 | CachedFileSystemEntry Result; |
55 | 79 | size_t Size = MinimizedFileContents.size(); |
56 | 79 | Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(), |
57 | 79 | Stat->getLastModificationTime(), |
58 | 79 | Stat->getUser(), Stat->getGroup(), Size, |
59 | 79 | Stat->getType(), Stat->getPermissions()); |
60 | | // The contents produced by the minimizer must be null terminated. |
61 | 79 | assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' && |
62 | 79 | "not null terminated contents"); |
63 | | // Even though there's an implicit null terminator in the minimized contents, |
64 | | // we want to temporarily make it explicit. This will ensure that the |
65 | | // std::move will preserve it even if it needs to do a copy if the |
66 | | // SmallString still has the small capacity. |
67 | 79 | MinimizedFileContents.push_back('\0'); |
68 | 79 | Result.Contents = std::move(MinimizedFileContents); |
69 | | // Now make the null terminator implicit again, so that Clang's lexer can find |
70 | | // it right where the buffer ends. |
71 | 79 | Result.Contents.pop_back(); |
72 | | |
73 | | // Compute the skipped PP ranges that speedup skipping over inactive |
74 | | // preprocessor blocks. |
75 | 79 | llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32> |
76 | 79 | SkippedRanges; |
77 | 79 | minimize_source_to_dependency_directives::computeSkippedRanges(Tokens, |
78 | 79 | SkippedRanges); |
79 | 79 | PreprocessorSkippedRangeMapping Mapping; |
80 | 30 | for (const auto &Range : SkippedRanges) { |
81 | 30 | if (Range.Length < 16) { |
82 | | // Ignore small ranges as non-profitable. |
83 | | // FIXME: This is a heuristic, its worth investigating the tradeoffs |
84 | | // when it should be applied. |
85 | 4 | continue; |
86 | 4 | } |
87 | 26 | Mapping[Range.Offset] = Range.Length; |
88 | 26 | } |
89 | 79 | Result.PPSkippedRangeMapping = std::move(Mapping); |
90 | | |
91 | 79 | return Result; |
92 | 79 | } |
93 | | |
94 | | CachedFileSystemEntry |
95 | 180 | CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) { |
96 | 180 | assert(Stat.isDirectory() && "not a directory!"); |
97 | 180 | auto Result = CachedFileSystemEntry(); |
98 | 180 | Result.MaybeStat = std::move(Stat); |
99 | 180 | return Result; |
100 | 180 | } |
101 | | |
102 | | DependencyScanningFilesystemSharedCache:: |
103 | 31 | DependencyScanningFilesystemSharedCache() { |
104 | | // This heuristic was chosen using a empirical testing on a |
105 | | // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache |
106 | | // sharding gives a performance edge by reducing the lock contention. |
107 | | // FIXME: A better heuristic might also consider the OS to account for |
108 | | // the different cost of lock contention on different OSes. |
109 | 31 | NumShards = |
110 | 31 | std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4); |
111 | 31 | CacheShards = std::make_unique<CacheShard[]>(NumShards); |
112 | 31 | } |
113 | | |
114 | | /// Returns a cache entry for the corresponding key. |
115 | | /// |
116 | | /// A new cache entry is created if the key is not in the cache. This is a |
117 | | /// thread safe call. |
118 | | DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & |
119 | 524 | DependencyScanningFilesystemSharedCache::get(StringRef Key) { |
120 | 524 | CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards]; |
121 | 524 | std::unique_lock<std::mutex> LockGuard(Shard.CacheLock); |
122 | 524 | auto It = Shard.Cache.try_emplace(Key); |
123 | 524 | return It.first->getValue(); |
124 | 524 | } |
125 | | |
126 | | /// Whitelist file extensions that should be minimized, treating no extension as |
127 | | /// a source file that should be minimized. |
128 | | /// |
129 | | /// This is kinda hacky, it would be better if we knew what kind of file Clang |
130 | | /// was expecting instead. |
131 | 480 | static bool shouldMinimize(StringRef Filename) { |
132 | 480 | StringRef Ext = llvm::sys::path::extension(Filename); |
133 | 480 | if (Ext.empty()) |
134 | 207 | return true; // C++ standard library |
135 | 273 | return llvm::StringSwitch<bool>(Ext) |
136 | 273 | .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true) |
137 | 273 | .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true) |
138 | 273 | .CasesLower(".m", ".mm", true) |
139 | 273 | .CasesLower(".i", ".ii", ".mi", ".mmi", true) |
140 | 273 | .CasesLower(".def", ".inc", true) |
141 | 273 | .Default(false); |
142 | 273 | } |
143 | | |
144 | | |
145 | 178 | static bool shouldCacheStatFailures(StringRef Filename) { |
146 | 178 | StringRef Ext = llvm::sys::path::extension(Filename); |
147 | 178 | if (Ext.empty()) |
148 | 118 | return false; // This may be the module cache directory. |
149 | 60 | return shouldMinimize(Filename); // Only cache stat failures on source files. |
150 | 60 | } |
151 | | |
152 | | llvm::ErrorOr<const CachedFileSystemEntry *> |
153 | | DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( |
154 | 622 | const StringRef Filename) { |
155 | 622 | if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) { |
156 | 94 | return Entry; |
157 | 94 | } |
158 | | |
159 | | // FIXME: Handle PCM/PCH files. |
160 | | // FIXME: Handle module map files. |
161 | | |
162 | 528 | bool KeepOriginalSource = IgnoredFiles.count(Filename) || |
163 | 420 | !shouldMinimize(Filename); |
164 | 528 | DependencyScanningFilesystemSharedCache::SharedFileSystemEntry |
165 | 528 | &SharedCacheEntry = SharedCache.get(Filename); |
166 | 528 | const CachedFileSystemEntry *Result; |
167 | 528 | { |
168 | 528 | std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock); |
169 | 528 | CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value; |
170 | | |
171 | 528 | if (!CacheEntry.isValid()) { |
172 | 454 | llvm::vfs::FileSystem &FS = getUnderlyingFS(); |
173 | 454 | auto MaybeStatus = FS.status(Filename); |
174 | 454 | if (!MaybeStatus) { |
175 | 178 | if (!shouldCacheStatFailures(Filename)) |
176 | | // HACK: We need to always restat non source files if the stat fails. |
177 | | // This is because Clang first looks up the module cache and module |
178 | | // files before building them, and then looks for them again. If we |
179 | | // cache the stat failure, it won't see them the second time. |
180 | 157 | return MaybeStatus.getError(); |
181 | 21 | else |
182 | 21 | CacheEntry = CachedFileSystemEntry(MaybeStatus.getError()); |
183 | 276 | } else if (MaybeStatus->isDirectory()) |
184 | 180 | CacheEntry = CachedFileSystemEntry::createDirectoryEntry( |
185 | 180 | std::move(*MaybeStatus)); |
186 | 96 | else |
187 | 96 | CacheEntry = CachedFileSystemEntry::createFileEntry( |
188 | 96 | Filename, FS, !KeepOriginalSource); |
189 | 454 | } |
190 | | |
191 | 371 | Result = &CacheEntry; |
192 | 371 | } |
193 | | |
194 | | // Store the result in the local cache. |
195 | 371 | setCachedEntry(Filename, Result); |
196 | 371 | return Result; |
197 | 528 | } |
198 | | |
199 | | llvm::ErrorOr<llvm::vfs::Status> |
200 | 421 | DependencyScanningWorkerFilesystem::status(const Twine &Path) { |
201 | 421 | SmallString<256> OwnedFilename; |
202 | 421 | StringRef Filename = Path.toStringRef(OwnedFilename); |
203 | 421 | const llvm::ErrorOr<const CachedFileSystemEntry *> Result = |
204 | 421 | getOrCreateFileSystemEntry(Filename); |
205 | 421 | if (!Result) |
206 | 152 | return Result.getError(); |
207 | 269 | return (*Result)->getStatus(); |
208 | 269 | } |
209 | | |
210 | | namespace { |
211 | | |
212 | | /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using |
213 | | /// this subclass. |
214 | | class MinimizedVFSFile final : public llvm::vfs::File { |
215 | | public: |
216 | | MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer, |
217 | | llvm::vfs::Status Stat) |
218 | 165 | : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} |
219 | | |
220 | | static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> |
221 | | create(const CachedFileSystemEntry *Entry, |
222 | | ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings); |
223 | | |
224 | 188 | llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; } |
225 | | |
226 | | llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> |
227 | | getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, |
228 | 147 | bool IsVolatile) override { |
229 | 147 | return std::move(Buffer); |
230 | 147 | } |
231 | | |
232 | 0 | std::error_code close() override { return {}; } |
233 | | |
234 | | private: |
235 | | std::unique_ptr<llvm::MemoryBuffer> Buffer; |
236 | | llvm::vfs::Status Stat; |
237 | | }; |
238 | | |
239 | | } // end anonymous namespace |
240 | | |
241 | | llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create( |
242 | | const CachedFileSystemEntry *Entry, |
243 | 196 | ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) { |
244 | 196 | if (Entry->isDirectory()) |
245 | 2 | return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( |
246 | 2 | std::make_error_code(std::errc::is_a_directory)); |
247 | 194 | llvm::ErrorOr<StringRef> Contents = Entry->getContents(); |
248 | 194 | if (!Contents) |
249 | 29 | return Contents.getError(); |
250 | 165 | auto Result = std::make_unique<MinimizedVFSFile>( |
251 | 165 | llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(), |
252 | 165 | /*RequiresNullTerminator=*/false), |
253 | 165 | *Entry->getStatus()); |
254 | 165 | if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings42 ) |
255 | 39 | (*PPSkipMappings)[Result->Buffer->getBufferStart()] = |
256 | 39 | &Entry->getPPSkippedRangeMapping(); |
257 | 165 | return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( |
258 | 165 | std::unique_ptr<llvm::vfs::File>(std::move(Result))); |
259 | 165 | } |
260 | | |
261 | | llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> |
262 | 201 | DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) { |
263 | 201 | SmallString<256> OwnedFilename; |
264 | 201 | StringRef Filename = Path.toStringRef(OwnedFilename); |
265 | | |
266 | 201 | const llvm::ErrorOr<const CachedFileSystemEntry *> Result = |
267 | 201 | getOrCreateFileSystemEntry(Filename); |
268 | 201 | if (!Result) |
269 | 5 | return Result.getError(); |
270 | 196 | return MinimizedVFSFile::create(Result.get(), PPSkipMappings); |
271 | 196 | } |