Coverage Report

Created: 2021-01-23 06:44

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10
#include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
11
#include "llvm/Support/MemoryBuffer.h"
12
#include "llvm/Support/Threading.h"
13
14
using namespace clang;
15
using namespace tooling;
16
using namespace dependencies;
17
18
CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
19
96
    StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
20
  // Load the file and its content from the file system.
21
96
  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
22
96
      FS.openFileForRead(Filename);
23
96
  if (!MaybeFile)
24
0
    return MaybeFile.getError();
25
96
  llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status();
26
96
  if (!Stat)
27
0
    return Stat.getError();
28
29
96
  llvm::vfs::File &F = **MaybeFile;
30
96
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer =
31
96
      F.getBuffer(Stat->getName());
32
96
  if (!MaybeBuffer)
33
0
    return MaybeBuffer.getError();
34
35
96
  llvm::SmallString<1024> MinimizedFileContents;
36
  // Minimize the file down to directives that might affect the dependencies.
37
96
  const auto &Buffer = *MaybeBuffer;
38
96
  SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
39
96
  if (!Minimize || minimizeSourceToDependencyDirectives(
40
17
                       Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
41
    // Use the original file unless requested otherwise, or
42
    // if the minimization failed.
43
    // FIXME: Propage the diagnostic if desired by the client.
44
17
    CachedFileSystemEntry Result;
45
17
    Result.MaybeStat = std::move(*Stat);
46
17
    Result.Contents.reserve(Buffer->getBufferSize() + 1);
47
17
    Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
48
    // Implicitly null terminate the contents for Clang's lexer.
49
17
    Result.Contents.push_back('\0');
50
17
    Result.Contents.pop_back();
51
17
    return Result;
52
17
  }
53
54
79
  CachedFileSystemEntry Result;
55
79
  size_t Size = MinimizedFileContents.size();
56
79
  Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
57
79
                                       Stat->getLastModificationTime(),
58
79
                                       Stat->getUser(), Stat->getGroup(), Size,
59
79
                                       Stat->getType(), Stat->getPermissions());
60
  // The contents produced by the minimizer must be null terminated.
61
79
  assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
62
79
         "not null terminated contents");
63
  // Even though there's an implicit null terminator in the minimized contents,
64
  // we want to temporarily make it explicit. This will ensure that the
65
  // std::move will preserve it even if it needs to do a copy if the
66
  // SmallString still has the small capacity.
67
79
  MinimizedFileContents.push_back('\0');
68
79
  Result.Contents = std::move(MinimizedFileContents);
69
  // Now make the null terminator implicit again, so that Clang's lexer can find
70
  // it right where the buffer ends.
71
79
  Result.Contents.pop_back();
72
73
  // Compute the skipped PP ranges that speedup skipping over inactive
74
  // preprocessor blocks.
75
79
  llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
76
79
      SkippedRanges;
77
79
  minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
78
79
                                                                 SkippedRanges);
79
79
  PreprocessorSkippedRangeMapping Mapping;
80
30
  for (const auto &Range : SkippedRanges) {
81
30
    if (Range.Length < 16) {
82
      // Ignore small ranges as non-profitable.
83
      // FIXME: This is a heuristic, its worth investigating the tradeoffs
84
      // when it should be applied.
85
4
      continue;
86
4
    }
87
26
    Mapping[Range.Offset] = Range.Length;
88
26
  }
89
79
  Result.PPSkippedRangeMapping = std::move(Mapping);
90
91
79
  return Result;
92
79
}
93
94
CachedFileSystemEntry
95
180
CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) {
96
180
  assert(Stat.isDirectory() && "not a directory!");
97
180
  auto Result = CachedFileSystemEntry();
98
180
  Result.MaybeStat = std::move(Stat);
99
180
  return Result;
100
180
}
101
102
DependencyScanningFilesystemSharedCache::
103
31
    DependencyScanningFilesystemSharedCache() {
104
  // This heuristic was chosen using a empirical testing on a
105
  // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
106
  // sharding gives a performance edge by reducing the lock contention.
107
  // FIXME: A better heuristic might also consider the OS to account for
108
  // the different cost of lock contention on different OSes.
109
31
  NumShards =
110
31
      std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
111
31
  CacheShards = std::make_unique<CacheShard[]>(NumShards);
112
31
}
113
114
/// Returns a cache entry for the corresponding key.
115
///
116
/// A new cache entry is created if the key is not in the cache. This is a
117
/// thread safe call.
118
DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
119
524
DependencyScanningFilesystemSharedCache::get(StringRef Key) {
120
524
  CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
121
524
  std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
122
524
  auto It = Shard.Cache.try_emplace(Key);
123
524
  return It.first->getValue();
124
524
}
125
126
/// Whitelist file extensions that should be minimized, treating no extension as
127
/// a source file that should be minimized.
128
///
129
/// This is kinda hacky, it would be better if we knew what kind of file Clang
130
/// was expecting instead.
131
480
static bool shouldMinimize(StringRef Filename) {
132
480
  StringRef Ext = llvm::sys::path::extension(Filename);
133
480
  if (Ext.empty())
134
207
    return true; // C++ standard library
135
273
  return llvm::StringSwitch<bool>(Ext)
136
273
    .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
137
273
    .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
138
273
    .CasesLower(".m", ".mm", true)
139
273
    .CasesLower(".i", ".ii", ".mi", ".mmi", true)
140
273
    .CasesLower(".def", ".inc", true)
141
273
    .Default(false);
142
273
}
143
144
145
178
static bool shouldCacheStatFailures(StringRef Filename) {
146
178
  StringRef Ext = llvm::sys::path::extension(Filename);
147
178
  if (Ext.empty())
148
118
    return false; // This may be the module cache directory.
149
60
  return shouldMinimize(Filename); // Only cache stat failures on source files.
150
60
}
151
152
llvm::ErrorOr<const CachedFileSystemEntry *>
153
DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
154
622
    const StringRef Filename) {
155
622
  if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) {
156
94
    return Entry;
157
94
  }
158
159
  // FIXME: Handle PCM/PCH files.
160
  // FIXME: Handle module map files.
161
162
528
  bool KeepOriginalSource = IgnoredFiles.count(Filename) ||
163
420
                            !shouldMinimize(Filename);
164
528
  DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
165
528
      &SharedCacheEntry = SharedCache.get(Filename);
166
528
  const CachedFileSystemEntry *Result;
167
528
  {
168
528
    std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
169
528
    CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
170
171
528
    if (!CacheEntry.isValid()) {
172
454
      llvm::vfs::FileSystem &FS = getUnderlyingFS();
173
454
      auto MaybeStatus = FS.status(Filename);
174
454
      if (!MaybeStatus) {
175
178
        if (!shouldCacheStatFailures(Filename))
176
          // HACK: We need to always restat non source files if the stat fails.
177
          //   This is because Clang first looks up the module cache and module
178
          //   files before building them, and then looks for them again. If we
179
          //   cache the stat failure, it won't see them the second time.
180
157
          return MaybeStatus.getError();
181
21
        else
182
21
          CacheEntry = CachedFileSystemEntry(MaybeStatus.getError());
183
276
      } else if (MaybeStatus->isDirectory())
184
180
        CacheEntry = CachedFileSystemEntry::createDirectoryEntry(
185
180
            std::move(*MaybeStatus));
186
96
      else
187
96
        CacheEntry = CachedFileSystemEntry::createFileEntry(
188
96
            Filename, FS, !KeepOriginalSource);
189
454
    }
190
191
371
    Result = &CacheEntry;
192
371
  }
193
194
  // Store the result in the local cache.
195
371
  setCachedEntry(Filename, Result);
196
371
  return Result;
197
528
}
198
199
llvm::ErrorOr<llvm::vfs::Status>
200
421
DependencyScanningWorkerFilesystem::status(const Twine &Path) {
201
421
  SmallString<256> OwnedFilename;
202
421
  StringRef Filename = Path.toStringRef(OwnedFilename);
203
421
  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
204
421
      getOrCreateFileSystemEntry(Filename);
205
421
  if (!Result)
206
152
    return Result.getError();
207
269
  return (*Result)->getStatus();
208
269
}
209
210
namespace {
211
212
/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
213
/// this subclass.
214
class MinimizedVFSFile final : public llvm::vfs::File {
215
public:
216
  MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
217
                   llvm::vfs::Status Stat)
218
165
      : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
219
220
  static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
221
  create(const CachedFileSystemEntry *Entry,
222
         ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings);
223
224
188
  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
225
226
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
227
  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
228
147
            bool IsVolatile) override {
229
147
    return std::move(Buffer);
230
147
  }
231
232
0
  std::error_code close() override { return {}; }
233
234
private:
235
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
236
  llvm::vfs::Status Stat;
237
};
238
239
} // end anonymous namespace
240
241
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create(
242
    const CachedFileSystemEntry *Entry,
243
196
    ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
244
196
  if (Entry->isDirectory())
245
2
    return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
246
2
        std::make_error_code(std::errc::is_a_directory));
247
194
  llvm::ErrorOr<StringRef> Contents = Entry->getContents();
248
194
  if (!Contents)
249
29
    return Contents.getError();
250
165
  auto Result = std::make_unique<MinimizedVFSFile>(
251
165
      llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
252
165
                                       /*RequiresNullTerminator=*/false),
253
165
      *Entry->getStatus());
254
165
  if (!Entry->getPPSkippedRangeMapping().empty() && 
PPSkipMappings42
)
255
39
    (*PPSkipMappings)[Result->Buffer->getBufferStart()] =
256
39
        &Entry->getPPSkippedRangeMapping();
257
165
  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
258
165
      std::unique_ptr<llvm::vfs::File>(std::move(Result)));
259
165
}
260
261
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
262
201
DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
263
201
  SmallString<256> OwnedFilename;
264
201
  StringRef Filename = Path.toStringRef(OwnedFilename);
265
266
201
  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
267
201
      getOrCreateFileSystemEntry(Filename);
268
201
  if (!Result)
269
5
    return Result.getError();
270
196
  return MinimizedVFSFile::create(Result.get(), PPSkipMappings);
271
196
}