Coverage Report

Created: 2022-05-17 06:19

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11
#include "clang/Frontend/CompilerInstance.h"
12
#include "clang/Frontend/CompilerInvocation.h"
13
#include "clang/Frontend/FrontendActions.h"
14
#include "clang/Frontend/TextDiagnosticPrinter.h"
15
#include "clang/Frontend/Utils.h"
16
#include "clang/Lex/PreprocessorOptions.h"
17
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19
#include "clang/Tooling/Tooling.h"
20
21
using namespace clang;
22
using namespace tooling;
23
using namespace dependencies;
24
25
namespace {
26
27
/// Forwards the gatherered dependencies to the consumer.
28
class DependencyConsumerForwarder : public DependencyFileGenerator {
29
public:
30
  DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31
                              DependencyConsumer &C)
32
132
      : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33
34
132
  void finishedMainFile(DiagnosticsEngine &Diags) override {
35
132
    C.handleDependencyOutputOpts(*Opts);
36
132
    llvm::SmallString<256> CanonPath;
37
376
    for (const auto &File : getDependencies()) {
38
376
      CanonPath = File;
39
376
      llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40
376
      C.handleFileDependency(CanonPath);
41
376
    }
42
132
  }
43
44
private:
45
  std::unique_ptr<DependencyOutputOptions> Opts;
46
  DependencyConsumer &C;
47
};
48
49
using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
50
51
/// A listener that collects the imported modules and optionally the input
52
/// files.
53
class PrebuiltModuleListener : public ASTReaderListener {
54
public:
55
  PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
56
                         llvm::StringSet<> &InputFiles, bool VisitInputFiles,
57
                         llvm::SmallVector<std::string> &NewModuleFiles)
58
      : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
59
6
        VisitInputFiles(VisitInputFiles), NewModuleFiles(NewModuleFiles) {}
60
61
33
  bool needsImportVisitation() const override { return true; }
62
33
  bool needsInputFileVisitation() override { return VisitInputFiles; }
63
33
  bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
64
65
27
  void visitImport(StringRef ModuleName, StringRef Filename) override {
66
27
    if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
67
27
      NewModuleFiles.push_back(Filename.str());
68
27
  }
69
70
  bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
71
61
                      bool isExplicitModule) override {
72
61
    InputFiles.insert(Filename);
73
61
    return true;
74
61
  }
75
76
private:
77
  PrebuiltModuleFilesT &PrebuiltModuleFiles;
78
  llvm::StringSet<> &InputFiles;
79
  bool VisitInputFiles;
80
  llvm::SmallVector<std::string> &NewModuleFiles;
81
};
82
83
/// Visit the given prebuilt module and collect all of the modules it
84
/// transitively imports and contributing input files.
85
static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
86
                                CompilerInstance &CI,
87
                                PrebuiltModuleFilesT &ModuleFiles,
88
                                llvm::StringSet<> &InputFiles,
89
6
                                bool VisitInputFiles) {
90
  // List of module files to be processed.
91
6
  llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()};
92
6
  PrebuiltModuleListener Listener(ModuleFiles, InputFiles, VisitInputFiles,
93
6
                                  Worklist);
94
95
39
  while (!Worklist.empty())
96
33
    ASTReader::readASTFileControlBlock(
97
33
        Worklist.pop_back_val(), CI.getFileManager(),
98
33
        CI.getPCHContainerReader(),
99
33
        /*FindModuleFileExtensions=*/false, Listener,
100
33
        /*ValidateDiagnosticOptions=*/false);
101
6
}
102
103
/// Transform arbitrary file name into an object-like file name.
104
65
static std::string makeObjFileName(StringRef FileName) {
105
65
  SmallString<128> ObjFileName(FileName);
106
65
  llvm::sys::path::replace_extension(ObjFileName, "o");
107
65
  return std::string(ObjFileName.str());
108
65
}
109
110
/// Deduce the dependency target based on the output file and input files.
111
static std::string
112
deduceDepTarget(const std::string &OutputFile,
113
149
                const SmallVectorImpl<FrontendInputFile> &InputFiles) {
114
149
  if (OutputFile != "-")
115
84
    return OutputFile;
116
117
65
  if (InputFiles.empty() || !InputFiles.front().isFile())
118
0
    return "clang-scan-deps\\ dependency";
119
120
65
  return makeObjFileName(InputFiles.front().getFile());
121
65
}
122
123
/// Sanitize diagnostic options for dependency scan.
124
361
static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
125
  // Don't print 'X warnings and Y errors generated'.
126
361
  DiagOpts.ShowCarets = false;
127
  // Don't write out diagnostic file.
128
361
  DiagOpts.DiagnosticSerializationFile.clear();
129
  // Don't treat warnings as errors.
130
361
  DiagOpts.Warnings.push_back("no-error");
131
361
}
132
133
/// A clang tool that runs the preprocessor in a mode that's optimized for
134
/// dependency scanning for the given compiler invocation.
135
class DependencyScanningAction : public tooling::ToolAction {
136
public:
137
  DependencyScanningAction(
138
      StringRef WorkingDirectory, DependencyConsumer &Consumer,
139
      llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
140
      ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings,
141
      ScanningOutputFormat Format, bool OptimizeArgs,
142
      llvm::Optional<StringRef> ModuleName = None)
143
      : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
144
        DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
145
182
        OptimizeArgs(OptimizeArgs), ModuleName(ModuleName) {}
146
147
  bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
148
                     FileManager *FileMgr,
149
                     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
150
178
                     DiagnosticConsumer *DiagConsumer) override {
151
    // Make a deep copy of the original Clang invocation.
152
178
    CompilerInvocation OriginalInvocation(*Invocation);
153
154
    // Create a compiler instance to handle the actual work.
155
178
    CompilerInstance ScanInstance(std::move(PCHContainerOps));
156
178
    ScanInstance.setInvocation(std::move(Invocation));
157
158
    // Create the compiler's actual diagnostics engine.
159
178
    sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
160
178
    ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
161
178
    if (!ScanInstance.hasDiagnostics())
162
0
      return false;
163
164
178
    ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
165
178
        true;
166
167
178
    ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
168
178
    ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
169
170
178
    FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
171
178
    ScanInstance.setFileManager(FileMgr);
172
178
    ScanInstance.createSourceManager(*FileMgr);
173
174
178
    llvm::StringSet<> PrebuiltModulesInputFiles;
175
    // Store the list of prebuilt module files into header search options. This
176
    // will prevent the implicit build to create duplicate modules and will
177
    // force reuse of the existing prebuilt module files instead.
178
178
    if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
179
6
      visitPrebuiltModule(
180
6
          ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
181
6
          ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
182
6
          PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
183
184
    // Use the dependency scanning optimized file system if requested to do so.
185
178
    if (DepFS) {
186
142
      DepFS->enableMinimizationOfAllFiles();
187
      // Don't minimize any files that contributed to prebuilt modules. The
188
      // implicit build validates the modules by comparing the reported sizes of
189
      // their inputs to the current state of the filesystem. Minimization would
190
      // throw this mechanism off.
191
142
      for (const auto &File : PrebuiltModulesInputFiles)
192
40
        DepFS->disableMinimization(File.getKey());
193
      // Don't minimize any files that were explicitly passed in the build
194
      // settings and that might be opened.
195
142
      for (const auto &E : ScanInstance.getHeaderSearchOpts().UserEntries)
196
488
        DepFS->disableMinimization(E.Path);
197
142
      for (const auto &F : ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
198
3
        DepFS->disableMinimization(F);
199
200
      // Support for virtual file system overlays on top of the caching
201
      // filesystem.
202
142
      FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
203
142
          ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
204
205
      // Pass the skip mappings which should speed up excluded conditional block
206
      // skipping in the preprocessor.
207
142
      ScanInstance.getPreprocessorOpts()
208
142
          .ExcludedConditionalDirectiveSkipMappings = &PPSkipMappings;
209
142
    }
210
211
    // Create the dependency collector that will collect the produced
212
    // dependencies.
213
    //
214
    // This also moves the existing dependency output options from the
215
    // invocation to the collector. The options in the invocation are reset,
216
    // which ensures that the compiler won't create new dependency collectors,
217
    // and thus won't write out the extra '.d' files to disk.
218
178
    auto Opts = std::make_unique<DependencyOutputOptions>();
219
178
    std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
220
    // We need at least one -MT equivalent for the generator of make dependency
221
    // files to work.
222
178
    if (Opts->Targets.empty())
223
149
      Opts->Targets = {
224
149
          deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
225
149
                          ScanInstance.getFrontendOpts().Inputs)};
226
178
    Opts->IncludeSystemHeaders = true;
227
228
178
    switch (Format) {
229
132
    case ScanningOutputFormat::Make:
230
132
      ScanInstance.addDependencyCollector(
231
132
          std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
232
132
                                                        Consumer));
233
132
      break;
234
48
    case ScanningOutputFormat::Full:
235
48
      ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
236
48
          std::move(Opts), ScanInstance, Consumer,
237
48
          std::move(OriginalInvocation), OptimizeArgs));
238
48
      break;
239
178
    }
240
241
    // Consider different header search and diagnostic options to create
242
    // different modules. This avoids the unsound aliasing of module PCMs.
243
    //
244
    // TODO: Implement diagnostic bucketing to reduce the impact of strict
245
    // context hashing.
246
180
    ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
247
248
180
    std::unique_ptr<FrontendAction> Action;
249
250
180
    if (ModuleName.hasValue())
251
4
      Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
252
176
    else
253
176
      Action = std::make_unique<ReadPCHAndPreprocessAction>();
254
255
180
    const bool Result = ScanInstance.ExecuteAction(*Action);
256
180
    if (!DepFS)
257
38
      FileMgr->clearStatCache();
258
180
    return Result;
259
178
  }
260
261
private:
262
  StringRef WorkingDirectory;
263
  DependencyConsumer &Consumer;
264
  llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
265
  ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings;
266
  ScanningOutputFormat Format;
267
  bool OptimizeArgs;
268
  llvm::Optional<StringRef> ModuleName;
269
};
270
271
} // end anonymous namespace
272
273
DependencyScanningWorker::DependencyScanningWorker(
274
    DependencyScanningService &Service)
275
445
    : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
276
445
  PCHContainerOps = std::make_shared<PCHContainerOperations>();
277
445
  PCHContainerOps->registerReader(
278
445
      std::make_unique<ObjectFilePCHContainerReader>());
279
  // We don't need to write object files, but the current PCH implementation
280
  // requires the writer to be registered as well.
281
445
  PCHContainerOps->registerWriter(
282
445
      std::make_unique<ObjectFilePCHContainerWriter>());
283
284
445
  auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
285
445
      llvm::vfs::createPhysicalFileSystem());
286
445
  InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
287
445
  OverlayFS->pushOverlay(InMemoryFS);
288
445
  RealFS = OverlayFS;
289
290
445
  if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
291
425
    DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(),
292
425
                                                   RealFS, PPSkipMappings);
293
445
  if (Service.canReuseFileManager())
294
443
    Files = new FileManager(FileSystemOptions(), RealFS);
295
445
}
296
297
static llvm::Error
298
runWithDiags(DiagnosticOptions *DiagOpts,
299
             llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
300
181
                 BodyShouldSucceed) {
301
181
  sanitizeDiagOpts(*DiagOpts);
302
303
  // Capture the emitted diagnostics and report them to the client
304
  // in the case of a failure.
305
181
  std::string DiagnosticOutput;
306
181
  llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
307
181
  TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
308
309
181
  if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
310
175
    return llvm::Error::success();
311
6
  return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
312
6
                                             llvm::inconvertibleErrorCode());
313
181
}
314
315
llvm::Error DependencyScanningWorker::computeDependencies(
316
    StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
317
182
    DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
318
  // Reset what might have been modified in the previous worker invocation.
319
182
  RealFS->setCurrentWorkingDirectory(WorkingDirectory);
320
182
  if (Files)
321
178
    Files->setVirtualFileSystem(RealFS);
322
323
182
  llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
324
182
      Files ? 
Files178
:
new FileManager(FileSystemOptions(), RealFS)4
;
325
326
182
  Optional<std::vector<std::string>> ModifiedCommandLine;
327
182
  if (ModuleName.hasValue()) {
328
4
    ModifiedCommandLine = CommandLine;
329
4
    InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
330
4
    ModifiedCommandLine->emplace_back(*ModuleName);
331
4
  }
332
333
182
  const std::vector<std::string> &FinalCommandLine =
334
182
      ModifiedCommandLine ? 
*ModifiedCommandLine4
:
CommandLine178
;
335
336
182
  std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
337
182
  llvm::transform(CommandLine, FinalCCommandLine.begin(),
338
1.67k
                  [](const std::string &Str) { return Str.c_str(); });
339
340
182
  return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
341
182
                      [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
342
182
                        DependencyScanningAction Action(
343
182
                            WorkingDirectory, Consumer, DepFS, PPSkipMappings,
344
182
                            Format, OptimizeArgs, ModuleName);
345
                        // Create an invocation that uses the underlying file
346
                        // system to ensure that any file system requests that
347
                        // are made by the driver do not go through the
348
                        // dependency scanning filesystem.
349
182
                        ToolInvocation Invocation(FinalCommandLine, &Action,
350
182
                                                  CurrentFiles.get(),
351
182
                                                  PCHContainerOps);
352
182
                        Invocation.setDiagnosticConsumer(&DC);
353
182
                        Invocation.setDiagnosticOptions(&DiagOpts);
354
182
                        return Invocation.run();
355
182
                      });
356
182
}