Coverage Report

Created: 2021-08-24 07:12

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10
#include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
11
#include "llvm/Support/MemoryBuffer.h"
12
#include "llvm/Support/Threading.h"
13
14
using namespace clang;
15
using namespace tooling;
16
using namespace dependencies;
17
18
CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
19
1.65k
    StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
20
  // Load the file and its content from the file system.
21
1.65k
  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
22
1.65k
      FS.openFileForRead(Filename);
23
1.65k
  if (!MaybeFile)
24
0
    return MaybeFile.getError();
25
1.65k
  llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status();
26
1.65k
  if (!Stat)
27
0
    return Stat.getError();
28
29
1.65k
  llvm::vfs::File &F = **MaybeFile;
30
1.65k
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer =
31
1.65k
      F.getBuffer(Stat->getName());
32
1.65k
  if (!MaybeBuffer)
33
0
    return MaybeBuffer.getError();
34
35
1.65k
  llvm::SmallString<1024> MinimizedFileContents;
36
  // Minimize the file down to directives that might affect the dependencies.
37
1.65k
  const auto &Buffer = *MaybeBuffer;
38
1.65k
  SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
39
1.65k
  if (!Minimize || minimizeSourceToDependencyDirectives(
40
1.53k
                       Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
41
    // Use the original file unless requested otherwise, or
42
    // if the minimization failed.
43
    // FIXME: Propage the diagnostic if desired by the client.
44
113
    CachedFileSystemEntry Result;
45
113
    Result.MaybeStat = std::move(*Stat);
46
113
    Result.Contents.reserve(Buffer->getBufferSize() + 1);
47
113
    Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
48
    // Implicitly null terminate the contents for Clang's lexer.
49
113
    Result.Contents.push_back('\0');
50
113
    Result.Contents.pop_back();
51
113
    return Result;
52
113
  }
53
54
1.53k
  CachedFileSystemEntry Result;
55
1.53k
  size_t Size = MinimizedFileContents.size();
56
1.53k
  Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
57
1.53k
                                       Stat->getLastModificationTime(),
58
1.53k
                                       Stat->getUser(), Stat->getGroup(), Size,
59
1.53k
                                       Stat->getType(), Stat->getPermissions());
60
  // The contents produced by the minimizer must be null terminated.
61
1.53k
  assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
62
1.53k
         "not null terminated contents");
63
  // Even though there's an implicit null terminator in the minimized contents,
64
  // we want to temporarily make it explicit. This will ensure that the
65
  // std::move will preserve it even if it needs to do a copy if the
66
  // SmallString still has the small capacity.
67
0
  MinimizedFileContents.push_back('\0');
68
1.53k
  Result.Contents = std::move(MinimizedFileContents);
69
  // Now make the null terminator implicit again, so that Clang's lexer can find
70
  // it right where the buffer ends.
71
1.53k
  Result.Contents.pop_back();
72
73
  // Compute the skipped PP ranges that speedup skipping over inactive
74
  // preprocessor blocks.
75
1.53k
  llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
76
1.53k
      SkippedRanges;
77
1.53k
  minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
78
1.53k
                                                                 SkippedRanges);
79
1.53k
  PreprocessorSkippedRangeMapping Mapping;
80
26.3k
  for (const auto &Range : SkippedRanges) {
81
26.3k
    if (Range.Length < 16) {
82
      // Ignore small ranges as non-profitable.
83
      // FIXME: This is a heuristic, its worth investigating the tradeoffs
84
      // when it should be applied.
85
64
      continue;
86
64
    }
87
26.3k
    Mapping[Range.Offset] = Range.Length;
88
26.3k
  }
89
1.53k
  Result.PPSkippedRangeMapping = std::move(Mapping);
90
91
1.53k
  return Result;
92
1.65k
}
93
94
CachedFileSystemEntry
95
452
CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) {
96
452
  assert(Stat.isDirectory() && "not a directory!");
97
0
  auto Result = CachedFileSystemEntry();
98
452
  Result.MaybeStat = std::move(Stat);
99
452
  return Result;
100
452
}
101
102
110
DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() {
103
  // This heuristic was chosen using a empirical testing on a
104
  // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
105
  // sharding gives a performance edge by reducing the lock contention.
106
  // FIXME: A better heuristic might also consider the OS to account for
107
  // the different cost of lock contention on different OSes.
108
110
  NumShards =
109
110
      std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
110
110
  CacheShards = std::make_unique<CacheShard[]>(NumShards);
111
110
}
112
113
DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
114
2.96k
DependencyScanningFilesystemSharedCache::SingleCache::get(StringRef Key) {
115
2.96k
  CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
116
2.96k
  std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
117
2.96k
  auto It = Shard.Cache.try_emplace(Key);
118
2.96k
  return It.first->getValue();
119
2.96k
}
120
121
DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
122
2.96k
DependencyScanningFilesystemSharedCache::get(StringRef Key, bool Minimized) {
123
2.96k
  SingleCache &Cache = Minimized ? 
CacheMinimized2.18k
:
CacheOriginal779
;
124
2.96k
  return Cache.get(Key);
125
2.96k
}
126
127
/// Whitelist file extensions that should be minimized, treating no extension as
128
/// a source file that should be minimized.
129
///
130
/// This is kinda hacky, it would be better if we knew what kind of file Clang
131
/// was expecting instead.
132
3.85k
static bool shouldMinimize(StringRef Filename) {
133
3.85k
  StringRef Ext = llvm::sys::path::extension(Filename);
134
3.85k
  if (Ext.empty())
135
555
    return true; // C++ standard library
136
3.30k
  return llvm::StringSwitch<bool>(Ext)
137
3.30k
    .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
138
3.30k
    .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
139
3.30k
    .CasesLower(".m", ".mm", true)
140
3.30k
    .CasesLower(".i", ".ii", ".mi", ".mmi", true)
141
3.30k
    .CasesLower(".def", ".inc", true)
142
3.30k
    .Default(false);
143
3.85k
}
144
145
146
656
static bool shouldCacheStatFailures(StringRef Filename) {
147
656
  StringRef Ext = llvm::sys::path::extension(Filename);
148
656
  if (Ext.empty())
149
195
    return false; // This may be the module cache directory.
150
461
  return shouldMinimize(Filename); // Only cache stat failures on source files.
151
656
}
152
153
377
void DependencyScanningWorkerFilesystem::ignoreFile(StringRef RawFilename) {
154
377
  llvm::SmallString<256> Filename;
155
377
  llvm::sys::path::native(RawFilename, Filename);
156
377
  IgnoredFiles.insert(Filename);
157
377
}
158
159
bool DependencyScanningWorkerFilesystem::shouldIgnoreFile(
160
3.58k
    StringRef RawFilename) {
161
3.58k
  llvm::SmallString<256> Filename;
162
3.58k
  llvm::sys::path::native(RawFilename, Filename);
163
3.58k
  return IgnoredFiles.contains(Filename);
164
3.58k
}
165
166
llvm::ErrorOr<const CachedFileSystemEntry *>
167
DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
168
3.58k
    const StringRef Filename) {
169
3.58k
  bool ShouldMinimize = !shouldIgnoreFile(Filename) && 
shouldMinimize(Filename)3.39k
;
170
171
3.58k
  if (const auto *Entry = Cache.getCachedEntry(Filename, ShouldMinimize))
172
623
    return Entry;
173
174
  // FIXME: Handle PCM/PCH files.
175
  // FIXME: Handle module map files.
176
177
2.96k
  DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
178
2.96k
      &SharedCacheEntry = SharedCache.get(Filename, ShouldMinimize);
179
2.96k
  const CachedFileSystemEntry *Result;
180
2.96k
  {
181
2.96k
    std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
182
2.96k
    CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
183
184
2.96k
    if (!CacheEntry.isValid()) {
185
2.75k
      llvm::vfs::FileSystem &FS = getUnderlyingFS();
186
2.75k
      auto MaybeStatus = FS.status(Filename);
187
2.75k
      if (!MaybeStatus) {
188
656
        if (!shouldCacheStatFailures(Filename))
189
          // HACK: We need to always restat non source files if the stat fails.
190
          //   This is because Clang first looks up the module cache and module
191
          //   files before building them, and then looks for them again. If we
192
          //   cache the stat failure, it won't see them the second time.
193
598
          return MaybeStatus.getError();
194
58
        else
195
58
          CacheEntry = CachedFileSystemEntry(MaybeStatus.getError());
196
2.10k
      } else if (MaybeStatus->isDirectory())
197
452
        CacheEntry = CachedFileSystemEntry::createDirectoryEntry(
198
452
            std::move(*MaybeStatus));
199
1.65k
      else
200
1.65k
        CacheEntry = CachedFileSystemEntry::createFileEntry(Filename, FS,
201
1.65k
                                                            ShouldMinimize);
202
2.75k
    }
203
204
2.36k
    Result = &CacheEntry;
205
2.36k
  }
206
207
  // Store the result in the local cache.
208
0
  Cache.setCachedEntry(Filename, ShouldMinimize, Result);
209
2.36k
  return Result;
210
2.96k
}
211
212
llvm::ErrorOr<llvm::vfs::Status>
213
3.08k
DependencyScanningWorkerFilesystem::status(const Twine &Path) {
214
3.08k
  SmallString<256> OwnedFilename;
215
3.08k
  StringRef Filename = Path.toStringRef(OwnedFilename);
216
3.08k
  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
217
3.08k
      getOrCreateFileSystemEntry(Filename);
218
3.08k
  if (!Result)
219
586
    return Result.getError();
220
2.49k
  return (*Result)->getStatus();
221
3.08k
}
222
223
namespace {
224
225
/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
226
/// this subclass.
227
class MinimizedVFSFile final : public llvm::vfs::File {
228
public:
229
  MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
230
                   llvm::vfs::Status Stat)
231
438
      : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
232
233
  static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
234
  create(const CachedFileSystemEntry *Entry,
235
         ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings);
236
237
420
  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
238
239
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
240
  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
241
389
            bool IsVolatile) override {
242
389
    return std::move(Buffer);
243
389
  }
244
245
0
  std::error_code close() override { return {}; }
246
247
private:
248
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
249
  llvm::vfs::Status Stat;
250
};
251
252
} // end anonymous namespace
253
254
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create(
255
    const CachedFileSystemEntry *Entry,
256
496
    ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
257
496
  if (Entry->isDirectory())
258
2
    return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
259
2
        std::make_error_code(std::errc::is_a_directory));
260
494
  llvm::ErrorOr<StringRef> Contents = Entry->getContents();
261
494
  if (!Contents)
262
56
    return Contents.getError();
263
438
  auto Result = std::make_unique<MinimizedVFSFile>(
264
438
      llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
265
438
                                       /*RequiresNullTerminator=*/false),
266
438
      *Entry->getStatus());
267
438
  if (!Entry->getPPSkippedRangeMapping().empty() && 
PPSkipMappings78
)
268
72
    (*PPSkipMappings)[Result->Buffer->getBufferStart()] =
269
72
        &Entry->getPPSkippedRangeMapping();
270
438
  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
271
438
      std::unique_ptr<llvm::vfs::File>(std::move(Result)));
272
494
}
273
274
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
275
508
DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
276
508
  SmallString<256> OwnedFilename;
277
508
  StringRef Filename = Path.toStringRef(OwnedFilename);
278
279
508
  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
280
508
      getOrCreateFileSystemEntry(Filename);
281
508
  if (!Result)
282
12
    return Result.getError();
283
496
  return MinimizedVFSFile::create(Result.get(), PPSkipMappings);
284
508
}