Coverage Report

Created: 2020-02-25 14:32

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10
#include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
11
#include "llvm/Support/MemoryBuffer.h"
12
#include "llvm/Support/Threading.h"
13
14
using namespace clang;
15
using namespace tooling;
16
using namespace dependencies;
17
18
CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
19
84
    StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
20
84
  // Load the file and its content from the file system.
21
84
  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
22
84
      FS.openFileForRead(Filename);
23
84
  if (!MaybeFile)
24
0
    return MaybeFile.getError();
25
84
  llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status();
26
84
  if (!Stat)
27
0
    return Stat.getError();
28
84
29
84
  llvm::vfs::File &F = **MaybeFile;
30
84
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer =
31
84
      F.getBuffer(Stat->getName());
32
84
  if (!MaybeBuffer)
33
0
    return MaybeBuffer.getError();
34
84
35
84
  llvm::SmallString<1024> MinimizedFileContents;
36
84
  // Minimize the file down to directives that might affect the dependencies.
37
84
  const auto &Buffer = *MaybeBuffer;
38
84
  SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
39
84
  if (!Minimize || minimizeSourceToDependencyDirectives(
40
66
                       Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
41
17
    // Use the original file unless requested otherwise, or
42
17
    // if the minimization failed.
43
17
    // FIXME: Propage the diagnostic if desired by the client.
44
17
    CachedFileSystemEntry Result;
45
17
    Result.MaybeStat = std::move(*Stat);
46
17
    Result.Contents.reserve(Buffer->getBufferSize() + 1);
47
17
    Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
48
17
    // Implicitly null terminate the contents for Clang's lexer.
49
17
    Result.Contents.push_back('\0');
50
17
    Result.Contents.pop_back();
51
17
    return Result;
52
17
  }
53
67
54
67
  CachedFileSystemEntry Result;
55
67
  size_t Size = MinimizedFileContents.size();
56
67
  Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
57
67
                                       Stat->getLastModificationTime(),
58
67
                                       Stat->getUser(), Stat->getGroup(), Size,
59
67
                                       Stat->getType(), Stat->getPermissions());
60
67
  // The contents produced by the minimizer must be null terminated.
61
67
  assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
62
67
         "not null terminated contents");
63
67
  // Even though there's an implicit null terminator in the minimized contents,
64
67
  // we want to temporarily make it explicit. This will ensure that the
65
67
  // std::move will preserve it even if it needs to do a copy if the
66
67
  // SmallString still has the small capacity.
67
67
  MinimizedFileContents.push_back('\0');
68
67
  Result.Contents = std::move(MinimizedFileContents);
69
67
  // Now make the null terminator implicit again, so that Clang's lexer can find
70
67
  // it right where the buffer ends.
71
67
  Result.Contents.pop_back();
72
67
73
67
  // Compute the skipped PP ranges that speedup skipping over inactive
74
67
  // preprocessor blocks.
75
67
  llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
76
67
      SkippedRanges;
77
67
  minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
78
67
                                                                 SkippedRanges);
79
67
  PreprocessorSkippedRangeMapping Mapping;
80
67
  for (const auto &Range : SkippedRanges) {
81
30
    if (Range.Length < 16) {
82
4
      // Ignore small ranges as non-profitable.
83
4
      // FIXME: This is a heuristic, its worth investigating the tradeoffs
84
4
      // when it should be applied.
85
4
      continue;
86
4
    }
87
26
    Mapping[Range.Offset] = Range.Length;
88
26
  }
89
67
  Result.PPSkippedRangeMapping = std::move(Mapping);
90
67
91
67
  return Result;
92
67
}
93
94
CachedFileSystemEntry
95
129
CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) {
96
129
  assert(Stat.isDirectory() && "not a directory!");
97
129
  auto Result = CachedFileSystemEntry();
98
129
  Result.MaybeStat = std::move(Stat);
99
129
  return Result;
100
129
}
101
102
DependencyScanningFilesystemSharedCache::
103
27
    DependencyScanningFilesystemSharedCache() {
104
27
  // This heuristic was chosen using a empirical testing on a
105
27
  // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
106
27
  // sharding gives a performance edge by reducing the lock contention.
107
27
  // FIXME: A better heuristic might also consider the OS to account for
108
27
  // the different cost of lock contention on different OSes.
109
27
  NumShards =
110
27
      std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
111
27
  CacheShards = std::make_unique<CacheShard[]>(NumShards);
112
27
}
113
114
/// Returns a cache entry for the corresponding key.
115
///
116
/// A new cache entry is created if the key is not in the cache. This is a
117
/// thread safe call.
118
DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
119
536
DependencyScanningFilesystemSharedCache::get(StringRef Key) {
120
536
  CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
121
536
  std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
122
536
  auto It = Shard.Cache.try_emplace(Key);
123
536
  return It.first->getValue();
124
536
}
125
126
/// Whitelist file extensions that should be minimized, treating no extension as
127
/// a source file that should be minimized.
128
///
129
/// This is kinda hacky, it would be better if we knew what kind of file Clang
130
/// was expecting instead.
131
484
static bool shouldMinimize(StringRef Filename) {
132
484
  StringRef Ext = llvm::sys::path::extension(Filename);
133
484
  if (Ext.empty())
134
226
    return true; // C++ standard library
135
258
  return llvm::StringSwitch<bool>(Ext)
136
258
    .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
137
258
    .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
138
258
    .CasesLower(".m", ".mm", true)
139
258
    .CasesLower(".i", ".ii", ".mi", ".mmi", true)
140
258
    .CasesLower(".def", ".inc", true)
141
258
    .Default(false);
142
258
}
143
144
145
260
static bool shouldCacheStatFailures(StringRef Filename) {
146
260
  StringRef Ext = llvm::sys::path::extension(Filename);
147
260
  if (Ext.empty())
148
202
    return false; // This may be the module cache directory.
149
58
  return shouldMinimize(Filename); // Only cache stat failures on source files.
150
58
}
151
152
llvm::ErrorOr<const CachedFileSystemEntry *>
153
DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
154
628
    const StringRef Filename) {
155
628
  if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) {
156
88
    return Entry;
157
88
  }
158
540
159
540
  // FIXME: Handle PCM/PCH files.
160
540
  // FIXME: Handle module map files.
161
540
162
540
  bool KeepOriginalSource = IgnoredFiles.count(Filename) ||
163
540
                            
!shouldMinimize(Filename)426
;
164
540
  DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
165
540
      &SharedCacheEntry = SharedCache.get(Filename);
166
540
  const CachedFileSystemEntry *Result;
167
540
  {
168
540
    std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
169
540
    CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
170
540
171
540
    if (!CacheEntry.isValid()) {
172
473
      llvm::vfs::FileSystem &FS = getUnderlyingFS();
173
473
      auto MaybeStatus = FS.status(Filename);
174
473
      if (!MaybeStatus) {
175
260
        if (!shouldCacheStatFailures(Filename))
176
239
          // HACK: We need to always restat non source files if the stat fails.
177
239
          //   This is because Clang first looks up the module cache and module
178
239
          //   files before building them, and then looks for them again. If we
179
239
          //   cache the stat failure, it won't see them the second time.
180
239
          return MaybeStatus.getError();
181
21
        else
182
21
          CacheEntry = CachedFileSystemEntry(MaybeStatus.getError());
183
260
      } else 
if (213
MaybeStatus->isDirectory()213
)
184
129
        CacheEntry = CachedFileSystemEntry::createDirectoryEntry(
185
129
            std::move(*MaybeStatus));
186
84
      else
187
84
        CacheEntry = CachedFileSystemEntry::createFileEntry(
188
84
            Filename, FS, !KeepOriginalSource);
189
473
    }
190
540
191
540
    Result = &CacheEntry;
192
301
  }
193
301
194
301
  // Store the result in the local cache.
195
301
  setCachedEntry(Filename, Result);
196
301
  return Result;
197
540
}
198
199
llvm::ErrorOr<llvm::vfs::Status>
200
445
DependencyScanningWorkerFilesystem::status(const Twine &Path) {
201
445
  SmallString<256> OwnedFilename;
202
445
  StringRef Filename = Path.toStringRef(OwnedFilename);
203
445
  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
204
445
      getOrCreateFileSystemEntry(Filename);
205
445
  if (!Result)
206
236
    return Result.getError();
207
209
  return (*Result)->getStatus();
208
209
}
209
210
namespace {
211
212
/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
213
/// this subclass.
214
class MinimizedVFSFile final : public llvm::vfs::File {
215
public:
216
  MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
217
                   llvm::vfs::Status Stat)
218
150
      : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
219
220
167
  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
221
222
39
  const llvm::MemoryBuffer *getBufferPtr() const { return Buffer.get(); }
223
224
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
225
  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
226
132
            bool IsVolatile) override {
227
132
    return std::move(Buffer);
228
132
  }
229
230
0
  std::error_code close() override { return {}; }
231
232
private:
233
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
234
  llvm::vfs::Status Stat;
235
};
236
237
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
238
createFile(const CachedFileSystemEntry *Entry,
239
181
           ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
240
181
  if (Entry->isDirectory())
241
2
    return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
242
2
        std::make_error_code(std::errc::is_a_directory));
243
179
  llvm::ErrorOr<StringRef> Contents = Entry->getContents();
244
179
  if (!Contents)
245
29
    return Contents.getError();
246
150
  auto Result = std::make_unique<MinimizedVFSFile>(
247
150
      llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
248
150
                                       /*RequiresNullTerminator=*/false),
249
150
      *Entry->getStatus());
250
150
  if (!Entry->getPPSkippedRangeMapping().empty() && 
PPSkipMappings42
)
251
39
    (*PPSkipMappings)[Result->getBufferPtr()] =
252
39
        &Entry->getPPSkippedRangeMapping();
253
150
  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
254
150
      std::unique_ptr<llvm::vfs::File>(std::move(Result)));
255
150
}
256
257
} // end anonymous namespace
258
259
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
260
184
DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
261
184
  SmallString<256> OwnedFilename;
262
184
  StringRef Filename = Path.toStringRef(OwnedFilename);
263
184
264
184
  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
265
184
      getOrCreateFileSystemEntry(Filename);
266
184
  if (!Result)
267
3
    return Result.getError();
268
181
  return createFile(Result.get(), PPSkipMappings);
269
181
}